def pass_netcdf_checker(netcdf_file_path, tests=['cf:latest', 'imos:latest']):
"""Calls the netcdf checker and run the IMOS and CF latests version tests
by default.
Returns True if passes, False otherwise
"""
from compliance_checker.runner import ComplianceChecker, CheckSuite
import tempfile
import os
tmp_json_checker_output = tempfile.mkstemp()
return_values = []
had_errors = []
CheckSuite.load_all_available_checkers()
for test in tests:
# creation of a tmp json file. Only way (with html) to create an output not displayed to stdin by default
return_value, errors = ComplianceChecker.run_checker(
netcdf_file_path, [test],
1,
'normal',
output_filename=tmp_json_checker_output[1],
output_format='json')
had_errors.append(errors)
return_values.append(return_value)
os.close(tmp_json_checker_output[0])
os.remove(
tmp_json_checker_output[1]
) #file object needs to be closed or can end up with too many open files
if any(had_errors):
return False # checker exceptions
if all(return_values):
return True # all tests passed
return False # at least one did not pass
async def cc_report(args):
if args.verbose > 1:
print(f'Checking OPeNDAP URL: {url}')
if args.format == 'summary':
cs = CheckSuite()
if args.criteria == 'normal':
limit = 2
elif args.criteria == 'strict':
limit = 1
elif args.criteria == 'lenient':
limit = 3
ds = cs.load_dataset(url)
skip_checks = ()
score_groups = cs.run(ds, skip_checks, *args.test)
# Always use sorted test (groups) so they print in correct order
reports = {}
for checker, rpair in sorted(score_groups.items()):
groups, _ = rpair
_, points, out_of = cs.get_points(groups, limit)
reports[checker] = (100 * float(points) / float(out_of))
print((report_fmt).format(url, *[reports[t] for t in sorted(args.test)]))
sys.stdout.flush()
else:
# Send the compliance report to stdout
ComplianceChecker.run_checker(url, args.test, args.verbose, args.criteria,
args.output, args.format)
def profile_compliance(filepath):
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
_, outfile = tempfile.mkstemp()
try:
return_value, errors = ComplianceChecker.run_checker(
ds_loc=filepath,
checker_names=['gliderdac'],
verbose=True,
criteria='normal',
output_format='json',
output_filename=outfile
)
assert errors is False
return True
except AssertionError:
with open(outfile, 'rt') as f:
ers = json.loads(f.read())
for k, v in ers.items():
if isinstance(v, list):
for x in v:
if 'msgs' in x and x['msgs']:
logger.debug(x['msgs'])
return False
except BaseException as e:
logger.warning(e)
return False
finally:
if os.path.isfile(outfile):
os.remove(outfile)
def _handler(self, request, response):
dataset = None
if 'dataset_opendap' in request.inputs:
dataset = request.inputs['dataset_opendap'][0].url
LOGGER.debug("opendap dataset url: {}".format(dataset))
elif 'dataset' in request.inputs:
dataset = request.inputs['dataset'][0].file
LOGGER.debug("opendap dataset file: {}".format(dataset))
if not dataset:
raise ProcessError("You need to provide a Dataset.")
output_format = request.inputs['format'][0].data
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
output_file = os.path.join(
self.workdir,
"check_report.{}".format(output_format))
LOGGER.info("checking dataset {}".format(dataset))
ComplianceChecker.run_checker(
dataset,
checker_names=[checker.data for checker in request.inputs['test']],
verbose=True,
criteria=request.inputs['criteria'][0].data,
output_filename=output_file,
output_format=output_format)
response.outputs['output'].file = output_file
response.update_status("compliance checker finshed.", 100)
return response
def _handler(self, request, response):
dataset = None
if 'dataset_opendap' in request.inputs:
dataset = request.inputs['dataset_opendap'][0].url
LOGGER.debug("opendap dataset url: {}".format(dataset))
elif 'dataset' in request.inputs:
dataset = request.inputs['dataset'][0].file
LOGGER.debug("opendap dataset file: {}".format(dataset))
if not dataset:
raise ProcessError("You need to provide a Dataset.")
output_format = request.inputs['format'][0].data
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
if not request.inputs['test'][0].data in check_suite.checkers:
raise ProcessError("Test {} is not available.".format(request.inputs['test'][0].data))
output_file = os.path.join(
self.workdir,
"check_report.{}".format(output_format))
LOGGER.info("checking dataset {}".format(dataset))
ComplianceChecker.run_checker(
dataset,
checker_names=[checker.data for checker in request.inputs['test']],
verbose=True,
criteria=request.inputs['criteria'][0].data,
output_filename=output_file,
output_format=output_format)
response.outputs['output'].file = output_file
response.update_status("compliance checker finshed.", 100)
return response
def compliance_check(job_id, dataset, checker):
try:
redis = get_current_connection()
cs = CheckSuite()
if dataset.startswith('http'):
dataset = check_redirect(dataset)
ds = cs.load_dataset(dataset)
score_groups = cs.run(ds, [], checker)
rpair = score_groups[checker]
groups, errors = rpair
aggregates = cs.build_structure(checker, groups, dataset)
aggregates = cs.serialize(aggregates)
aggregates['all_priorities'] = sorted(aggregates['all_priorities'], key=lambda x: x['weight'], reverse=True)
# We use b64 to keep the filenames safe but it's helpful to the user to see
# the filename they uploaded
if not aggregates['source_name'].startswith('http'):
decoded = base64.b64decode(aggregates['source_name'].split('/')[-1])
if isinstance(decoded, str):
aggregates['source_name'] = decoded
else:
aggregates['source_name'] = decoded.decode('utf-8')
aggregates['ncdump'] = ncdump(dataset)
buf = json.dumps(aggregates)
redis.set('processing:job:%s' % job_id, buf, 3600)
return True
except Exception as e:
redis.set('processing:job:%s' % job_id, json.dumps({"error":type(e).__name__, "message":e.message}), 3600)
return False
def main():
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
parser = argparse.ArgumentParser()
parser.add_argument('--test', '-t', '--test=', '-t=', default=('acdd',),
nargs='+',
choices=sorted(check_suite.checkers.keys()),
help="Select the Checks you want to perform. Defaults to 'acdd' if unspecified")
parser.add_argument('--criteria', '-c',
help="Define the criteria for the checks. Either Strict, Normal, or Lenient. Defaults to Normal.",
nargs='?', default='normal',
choices = ['lenient', 'normal', 'strict'])
parser.add_argument('--verbose', '-v',
help="Increase output. May be specified up to three times.",
action="count",
default=0)
parser.add_argument('-f', '--format', default='text',
choices=['text', 'html', 'json'], help='Output format')
parser.add_argument('-o', '--output', default='-', action='store',
help='Output filename')
parser.add_argument('-V', '--version', action='store_true',
help='Display the IOOS Compliance Checker version information.')
parser.add_argument('dataset_location', nargs='*',
help="Defines the location of the dataset to be checked.")
args = parser.parse_args()
if args.version:
print("IOOS compliance checker version %s" % __version__)
return 0
return_values = []
had_errors = []
for dataset in args.dataset_location:
if args.format != 'json':
print("Running Compliance Checker on the dataset from: {}".format(dataset), file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(args.dataset_location[0],
args.test,
args.verbose,
args.criteria,
args.output,
args.format)
return_values.append(return_value)
had_errors.append(errors)
if any(had_errors):
return 2
if all(return_values):
return 0
return 1
def pass_netcdf_checker(netcdf_file_path,
tests=['cf:latest', 'imos:latest'],
criteria='normal',
skip_checks="",
output_format='json',
keep_outfile=False):
"""Calls the netcdf checker and run the IMOS and CF latests version tests
by default.
Returns True if passes, False otherwise
default arguments explanation:
* keep_outfile=False -> not keeping the output file generated by the checker
"""
from compliance_checker.runner import ComplianceChecker, CheckSuite
import tempfile
import os
tmp_json_checker_output_fd, tmp_json_checker_output_filename = tempfile.mkstemp(
)
return_values = []
had_errors = []
CheckSuite.load_all_available_checkers()
for test in tests:
# creation of a tmp json file. Only way (with html) to create an output not displayed to stdin by default
return_value, errors = ComplianceChecker.run_checker(
netcdf_file_path, [test],
1,
criteria,
skip_checks=skip_checks,
output_filename=tmp_json_checker_output_filename,
output_format=output_format)
had_errors.append(errors)
return_values.append(return_value)
os.close(
tmp_json_checker_output_fd
) # file object needs to be closed or can end up with too many open files
if keep_outfile: # optional output
if any(had_errors):
return False, tmp_json_checker_output_filename # checker exceptions
if all(return_values):
return True, tmp_json_checker_output_filename # all tests passed
return False, tmp_json_checker_output_filename # at least one did not pass
else:
os.remove(tmp_json_checker_output_filename)
if any(had_errors):
return False # checker exceptions
if all(return_values):
return True # all tests passed
return False # at least one did not pass
def check_cf_compliance(dataset):
try:
from compliance_checker.runner import CheckSuite, ComplianceChecker
except ImportError:
warnings.warn('compliance_checker unavailable, skipping NetCDF-CF Compliance Checks')
return
cs = CheckSuite()
cs.load_all_available_checkers()
score_groups = cs.run(dataset, 'cf')
groups = ComplianceChecker.stdout_output(cs, score_groups, verbose=1, limit=COMPLIANCE_CHECKER_NORMAL_LIMIT)
assert cs.passtree(groups, limit=COMPLIANCE_CHECKER_NORMAL_LIMIT)
def main(args):
if args.format == 'summary':
hdr_fmt = '{},' * len(args.test)
rpt_fmt = '{:.1f},' * len(args.test)
report_fmt = '{},' + rpt_fmt[:-1]
print(('{},' + hdr_fmt[:-1]).format('url', *sorted(args.test)))
for cat in args.catalog_urls:
if args.verbose > 1:
print(f'Opening catalog_url: {cat}')
for url in get_opendap_urls(cat):
if args.verbose > 1:
print(f'Checking OPeNDAP URL: {url}')
if args.format == 'summary':
cs = CheckSuite()
if args.criteria == 'normal':
limit = 2
elif args.criteria == 'strict':
limit = 1
elif args.criteria == 'lenient':
limit = 3
try:
ds = cs.load_dataset(url)
except ValueError as e:
print(f'Failed to get report for {url}')
print(str(e))
continue
skip_checks = ()
score_groups = cs.run(ds, skip_checks, *args.test)
# Always use sorted test (groups) so they print in correct order
reports = {}
for checker, rpair in sorted(score_groups.items()):
groups, _ = rpair
_, points, out_of = cs.get_points(groups, limit)
reports[checker] = (100 * float(points) / float(out_of))
print((report_fmt).format(url, *[reports[t] for t in sorted(args.test)]))
sys.stdout.flush()
else:
# Send the compliance report to stdout
ComplianceChecker.run_checker(url, args.test, args.verbose, args.criteria,
args.output, args.format)
def check_dataset(args):
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
outhandle, outfile = tempfile.mkstemp()
def show_messages(jn, log):
out_messages = []
for k, v in jn.items():
if isinstance(v, list):
for x in v:
if 'msgs' in x and x['msgs']:
out_messages += x['msgs']
log(
'{}:\n{}'.format(args.file, '\n'.join([' * {}'.format(
m) for m in out_messages ])
)
)
try:
return_value, errors = ComplianceChecker.run_checker(
ds_loc=args.file,
checker_names=['gliderdac'],
verbose=True,
criteria='normal',
output_format='json',
output_filename=outfile
)
except BaseException as e:
L.warning('{} - {}'.format(args.file, e))
return 1
else:
if errors is False:
return_value = 0
log = L.debug
else:
return_value = 1
log = L.warning
with open(outfile, 'rt') as f:
show_messages(json.loads(f.read())['gliderdac'], log)
return return_value
finally:
os.close(outhandle)
if os.path.isfile(outfile):
os.remove(outfile)
def _handler(self, request, response):
if 'dataset_opendap' in request.inputs:
dataset = request.inputs['dataset_opendap'][0].url
elif 'dataset' in request.inputs:
dataset = request.inputs['dataset'][0].file
else:
raise ProcessError("You need to provide a Dataset.")
checker = request.inputs['test'][0].data
with open(os.path.join(self.workdir, "nc_dump.txt"), 'w') as fp:
response.outputs['ncdump'].file = fp.name
fp.writelines(ncdump(dataset))
response.update_status('ncdump done.', 10)
if 'CF' in checker:
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
with open(os.path.join(self.workdir, "report.html"), 'w') as fp:
response.update_status("cfchecker ...", 20)
response.outputs['output'].file = fp.name
ComplianceChecker.run_checker(
dataset,
checker_names=['cf'],
verbose=True,
criteria="normal",
output_filename=fp.name,
output_format="html")
elif 'CMIP6' in checker:
with open(os.path.join(self.workdir, "cmip6-cmor.txt"), 'w') as fp:
response.outputs['output'].file = fp.name
response.update_status("cmip6 checker ...", 20)
cmor_checker(dataset, output_filename=fp.name)
else:
response.update_status("qa checker ...", 20)
from hummingbird.processing import hdh_qa_checker
logfile, _ = hdh_qa_checker(dataset, project=request.inputs['test'][0].data)
response.outputs['output'].file = logfile
response.update_status('spotchecker done.', 100)
return response
def check_compliance(netcdf_file):
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
_, outfile = tempfile.mkstemp(prefix='scobs-cc-', suffix='.json')
try:
return_value, errors = ComplianceChecker.run_checker(
ds_loc=netcdf_file,
checker_names=['cf', 'acdd'],
verbose=True,
criteria='strict',
output_format='json',
output_filename=outfile
)
z = []
score = namedtuple('ComplianceScore', 'name passed score possible')
with open(outfile, 'rt') as f:
for checker, results in json.load(f).items():
z.append(
score(
name=checker,
passed=not errors,
score=results['scored_points'],
possible=results['possible_points']
)
)
for s in z:
level = 'info'
if not s.passed:
level = 'error'
getattr(logger, level)(s)
return z
except BaseException as e:
logger.warning(e)
finally:
if os.path.isfile(outfile):
os.remove(outfile)
def main():
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
parser = argparse.ArgumentParser()
parser.add_argument('--test', '-t', '--test=', '-t=', action='append', help= "Select the Checks you want to perform.", choices=check_suite.checkers.keys())
parser.add_argument('--criteria', '-c', help="Define the criteria for the checks. Either Strict, Normal, or Lenient. Defaults to Normal.", nargs='?', default='normal', choices = ['lenient', 'normal', 'strict'])
parser.add_argument('--verbose' , '-v', help="Increase output. May be specified up to three times.", action="count")
parser.add_argument('-f', '--format', default='text', choices=['text', 'html', 'json'], help='Output format')
parser.add_argument('-o', '--output', default='-', action='store', help='Output filename')
parser.add_argument('-V', '--version', action='store_true', help='Display the IOOS Compliance Checker version information.')
parser.add_argument('dataset_location', nargs='*', help= "Defines the location of the dataset to be checked.")
args = parser.parse_args()
args.test = args.test or ['acdd']
if args.version:
print("IOOS compliance checker version %s" % __version__)
return 0
return_values = []
had_errors = []
for dataset in args.dataset_location:
if args.format != 'json':
print("Running Compliance Checker on the dataset from: %s" % dataset, file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(args.dataset_location[0],
args.test,
args.verbose,
args.criteria,
args.output,
args.format)
return_values.append(return_value)
had_errors.append(errors)
if any(had_errors):
return 2
if all(return_values):
return 0
return 1
def test_list_checks(self):
"""
Tests listing of both old-style, deprecated checkers using .name
attributes, and newer ones which use ._cc_spec and _cc_spec_version
attributes
"""
# hack: use argparse.Namespace to mock checker object with attributes
# since SimpleNamespace is Python 3.3+ only
CheckSuite.checkers.clear()
# need to mock setuptools entrypoints here in order to load in checkers
def checker_1():
return Namespace(name='checker_1')
def checker_2():
return Namespace(_cc_spec='checker_2',
_cc_spec_version='2.2')
mock_checkers = [Namespace(resolve=checker_1),
Namespace(resolve=checker_2)]
with pytest.warns(DeprecationWarning):
CheckSuite._load_checkers(mock_checkers)
cs = CheckSuite()
saved = sys.stdout
try:
# ugly! consider refactoring to use py.test capsys
fake_stdout = io.StringIO()
sys.stdout = fake_stdout
cs._print_suites()
assert (fake_stdout.getvalue() ==
' - checker_1:unknown\n - checker_2:2.2\n')
finally:
sys.stdout = saved
fake_stdout.close()
def check_cf_compliance(dataset):
try:
from compliance_checker.runner import CheckSuite, ComplianceChecker
import compliance_checker
except ImportError:
warnings.warn('compliance_checker unavailable, skipping NetCDF-CF Compliance Checks')
return
if compliance_checker.__version__ < '4.0.0':
warnings.warn('Please upgrade compliance-checker to 4+ version')
warnings.warn('compliance_checker version is too old, skipping NetCDF-CF Compliance Checks')
return
skip = ['check_dimension_order',
'check_all_features_are_same_type',
'check_conventions_version',
'check_appendix_a']
cs = CheckSuite()
cs.load_all_available_checkers()
score_groups = cs.run(dataset, skip, 'cf')
score_dict = {dataset.filepath(): score_groups}
groups = ComplianceChecker.stdout_output(cs, score_dict, verbose=1, limit=COMPLIANCE_CHECKER_NORMAL_LIMIT)
assert cs.passtree(groups, limit=COMPLIANCE_CHECKER_NORMAL_LIMIT)
def compliance_check(job_id, dataset, checker, path=None):
'''
Performs the Check Suite for the specified checker and sets the result in a
redis result for the job_id
:param str job_id: ID for the rq job
:param dataset: Dataset handle
:param str checker: Check Suite ID for a checker
:param str path: Full path to dataset directory (OPeNDAP only)
'''
try:
redis = get_current_connection()
cs = CheckSuite()
if dataset.startswith('http'):
dataset = check_redirect(dataset)
ds = cs.load_dataset(dataset)
score_groups = cs.run(ds, [], checker)
rpair = score_groups[checker]
groups, errors = rpair
aggregates = cs.build_structure(checker, groups, dataset)
aggregates = cs.serialize(aggregates)
aggregates['all_priorities'] = sorted(aggregates['all_priorities'],
key=lambda x: x['weight'],
reverse=True)
# We use b64 to keep the filenames safe but it's helpful to the user to see
# the filename they uploaded
if not aggregates['source_name'].startswith('http'):
decoded = base64.b64decode(
aggregates['source_name'].split('/')[-1])
if isinstance(decoded, str):
aggregates['source_name'] = decoded
else:
aggregates['source_name'] = decoded.decode('utf-8')
aggregates['ncdump'] = ncdump(dataset)
buf = json.dumps(aggregates)
# Write the report to a text file for download
if path is None:
# Must be a local file, get the path from the dataset
path = os.path.dirname(dataset)
fname = 'compliance_{}.txt'.format(job_id)
output_filename = os.path.join(path, fname)
with io.open(output_filename, 'w', encoding='utf-8') as f:
with stdout_redirector(f):
stdout_output(cs, score_groups, aggregates['source_name'])
redis.set('processing:job:%s' % job_id, buf, 3600)
return True
except Exception as e:
logger.exception("Failed to process job")
error_message = {"error": type(e).__name__, "message": e.message}
redis.set('processing:job:%s' % job_id, json.dumps(error_message),
3600)
return False
def check_compliance(nc_file, checker_names=["cf:1.6"]):
from compliance_checker.runner import ComplianceChecker, CheckSuite
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
verbose = 1
criteria = "normal"
output_filename = nc_file + ".cfcheck.txt"
output_format = "text"
print(
f"*** Checking CF compliance. Please view contents of {output_filename} for any compliance issues."
)
return_value, errors = ComplianceChecker.run_checker(
nc_file,
checker_names,
verbose,
criteria,
output_filename=output_filename,
output_format=output_format,
)
def __init__(self, config, logger, check_params=None):
super().__init__(config, logger)
if check_params is None:
check_params = {}
self.checks = check_params.get('checks', None)
self.verbosity = check_params.get('verbosity', 0)
self.criteria = check_params.get('criteria', 'normal')
self.skip_checks = check_params.get('skip_checks', None)
self.output_format = check_params.get('output_format', 'text')
if not self.checks:
raise InvalidCheckSuiteError('compliance check requested but no check suite specified')
CheckSuite.load_all_available_checkers()
# workaround a possible bug in the compliance checker where invalid check suites are ignored
available_checkers = set(CheckSuite.checkers)
these_checkers = set(self.checks)
if not these_checkers.issubset(available_checkers):
invalid_suites = list(these_checkers.difference(available_checkers))
raise InvalidCheckSuiteError(
'invalid compliance check suites: {invalid_suites}'.format(invalid_suites=invalid_suites))
def check_compliance(netcdf_file):
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
_, outfile = tempfile.mkstemp(prefix='scobs-cc-', suffix='.json')
try:
return_value, errors = ComplianceChecker.run_checker(
ds_loc=netcdf_file,
checker_names=['cf', 'acdd'],
verbose=True,
criteria='strict',
output_format='json',
output_filename=outfile)
z = []
score = namedtuple('ComplianceScore', 'name passed score possible')
with open(outfile, 'rt') as f:
for checker, results in json.load(f).items():
z.append(
score(name=checker,
passed=not errors,
score=results['scored_points'],
possible=results['possible_points']))
for s in z:
level = 'info'
if not s.passed:
level = 'error'
getattr(logger, level)(s)
return z
except BaseException as e:
logger.warning(e)
finally:
if os.path.isfile(outfile):
os.remove(outfile)
def check_cf_compliance(dataset):
try:
from compliance_checker.runner import CheckSuite, ComplianceChecker
except ImportError:
warnings.warn('compliance_checker unavailable, skipping NetCDF-CF Compliance Checks')
return
cs = CheckSuite()
cs.load_all_available_checkers()
score_groups = cs.run(dataset, 'cf')
groups = ComplianceChecker.stdout_output(cs, score_groups, verbose=1, limit=COMPLIANCE_CHECKER_NORMAL_LIMIT)
assert cs.passtree(groups, limit=COMPLIANCE_CHECKER_NORMAL_LIMIT)
def check_cf_compliance(dataset):
try:
from compliance_checker.runner import CheckSuite, ComplianceChecker
import compliance_checker
except ImportError:
warnings.warn('compliance_checker unavailable, skipping NetCDF-CF Compliance Checks')
return
cs = CheckSuite()
cs.load_all_available_checkers()
if compliance_checker.__version__ >= '2.3.0':
# This skips a failing compliance check. Our files don't contain all the lats/lons
# as an auxiliary cordinate var as it's unnecessary for any software we've tried.
# It may be added at some point in the future, and this check should be re-enabled.
score_groups = cs.run(dataset, ['check_dimension_order'], 'cf')
else:
warnings.warn('Please upgrade to compliance-checker 2.3.0 or higher.')
score_groups = cs.run(dataset, 'cf')
groups = ComplianceChecker.stdout_output(cs, score_groups, verbose=1, limit=COMPLIANCE_CHECKER_NORMAL_LIMIT)
assert cs.passtree(groups, limit=COMPLIANCE_CHECKER_NORMAL_LIMIT)
def compliance_check(job_id, dataset, checker):
try:
redis = get_current_connection()
cs = CheckSuite()
if dataset.startswith('http'):
dataset = check_redirect(dataset)
ds = cs.load_dataset(dataset)
score_groups = cs.run(ds, checker)
rpair = score_groups[checker]
groups, errors = rpair
aggregates = cs.build_structure(checker, groups, dataset)
aggregates = cs.serialize(aggregates)
aggregates['all_priorities'] = sorted(aggregates['all_priorities'],
key=lambda x: x['weight'],
reverse=True)
# We use b64 to keep the filenames safe but it's helpful to the user to see
# the filename they uploaded
if not aggregates['source_name'].startswith('http'):
decoded = base64.b64decode(
aggregates['source_name'].split('/')[-1])
if isinstance(decoded, str):
aggregates['source_name'] = decoded
else:
aggregates['source_name'] = decoded.decode('utf-8')
aggregates['ncdump'] = ncdump(dataset)
buf = json.dumps(aggregates)
redis.set('processing:job:%s' % job_id, buf, 3600)
return True
except Exception as e:
redis.set(
'processing:job:%s' % job_id,
json.dumps({
"error": type(e).__name__,
"message": e.message
}), 3600)
return False
def main():
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
parser = argparse.ArgumentParser()
parser.add_argument(
'--test',
'-t',
'--test=',
'-t=',
default=[],
action='append',
help=(
"Select the Checks you want to perform. Defaults to 'acdd'"
" if unspecified. Versions of standards can be specified via "
"`-t <test_standard>:<version>`. If `<version>` is omitted, or "
"is \"latest\", the latest version of the test standard is used."))
parser.add_argument(
'--criteria',
'-c',
help=("Define the criteria for the checks. "
"Either Strict, Normal, or Lenient. Defaults to Normal."),
nargs='?',
default='normal',
choices=['lenient', 'normal', 'strict'])
parser.add_argument(
'--verbose',
'-v',
help="Increase output. May be specified up to three times.",
action="count",
default=0)
parser.add_argument('--skip-checks',
'-s',
help="Specifies tests to skip",
action='append')
parser.add_argument(
'-f',
'--format',
default=[],
action='append',
help=(
"Output format(s). Options are 'text', 'html', 'json', 'json_new'."
" The difference between the 'json' and the 'json_new'"
" formats is that the 'json' format has the check as the top level"
" key, whereas the 'json_new' format has the dataset name(s) as the"
" main key in the output follow by any checks as subkeys. Also, "
"'json' format can be only be run against one input file, whereas "
"'json_new' can be run against multiple files."),
choices=['text', 'html', 'json', 'json_new'])
parser.add_argument(
'-o',
'--output',
default=[],
action='append',
help=("Output filename(s). If '-' is supplied, output to stdout."
" Can either be one or many files. If one file is supplied,"
" but the checker is run against many files, all the output"
" from the checks goes to that file (does not presently work "
"with 'json' format). If more than one output file is "
"supplied, the number of input datasets supplied must match "
"the number of output files."))
parser.add_argument(
'-V',
'--version',
action='store_true',
help='Display the IOOS Compliance Checker version information.')
parser.add_argument(
'dataset_location',
nargs='*',
help="Defines the location of the dataset to be checked.")
parser.add_argument('-l',
'--list-tests',
action='store_true',
help='List the available tests')
parser.add_argument('-d',
'--download-standard-names',
help=("Specify a version of the cf standard name table"
" to download as packaged version"))
# Add command line args from generator plugins
check_suite.add_plugin_args(parser)
args = parser.parse_args()
check_suite.load_generated_checkers(args)
if args.version:
print("IOOS compliance checker version %s" % __version__)
return 0
if args.list_tests:
print("IOOS compliance checker available checker suites:")
for checker in sorted(check_suite.checkers.keys()):
version = getattr(check_suite.checkers[checker],
'_cc_checker_version', "???")
if args.verbose:
print(" - {} (v{})".format(checker, version))
elif ':' in checker and not checker.endswith(
':latest'): # Skip the "latest" output
print(" - {}".format(checker))
return 0
if args.download_standard_names:
download_cf_standard_name_table(args.download_standard_names)
if len(args.dataset_location) == 0:
parser.print_help()
return 1
# Check the number of output files
if not args.output:
args.output = '-'
output_len = len(args.output)
if not (output_len == 1 or output_len == len(args.dataset_location)):
print(
'The number of output files must either be one or the same as the number of datasets',
file=sys.stderr)
sys.exit(2)
# Run the compliance checker
# 2 modes, concatenated output file or multiple output files
return_values = []
had_errors = []
if output_len == 1:
if args.format != 'json':
print("Running Compliance Checker on the datasets from: {}".format(
args.dataset_location),
file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(
args.dataset_location, args.test or ['acdd'], args.verbose,
args.criteria, args.skip_checks, args.output[0], args.format
or ['text'])
return_values.append(return_value)
had_errors.append(errors)
else:
for output, dataset in zip(args.output, args.dataset_location):
if args.format != 'json':
print("Running Compliance Checker on the dataset from: {}".
format(dataset),
file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(
[dataset], args.test or ['acdd'], args.verbose, args.criteria,
args.skip_checks, output, args.format or ['text'])
return_values.append(return_value)
had_errors.append(errors)
if any(had_errors):
return 2
if all(return_values):
return 0
return 1
ds['time'].attrs['long_name'] = 'time of measurement'
ds['time'].encoding['dtype'] = 'i4'
ds['time'].attrs['standard_name'] = 'time'
# check for 0-length attributes (this causes problems with ERDDAP) and set to an empty string
for k in ds.attrs:
if isinstance(ds.attrs[k], np.ndarray) and not ds.attrs[k].size:
print(k, ds.attrs[k].dtype, ds.attrs[k].size)
print(ds.attrs[k])
ds.attrs[k] = ''
ds['wh_4061']
ds.to_netcdf(fildir + 'clean/' + filnam)
# %%
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
path = fildir + 'clean/' + filnam
print(path)
# path = '/Users/dnowacki/Downloads/NMB14M1T01awWvs-p.nc'
# path = '/Users/dnowacki/Downloads/NMB15SCW01rbrWvs-p.nc'
checker_names = [
'cf:1.6',
#'acdd'
]
return_value, errors = ComplianceChecker.run_checker(
path, checker_names, 0, 'normal'
# verbose,
# criteria,
# output_filename=output_filename,
def main():
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
parser = argparse.ArgumentParser()
parser.add_argument(
'--test',
'-t',
'--test=',
'-t=',
default=[],
action='append',
help=
"Select the Checks you want to perform. Defaults to 'acdd' if unspecified"
)
parser.add_argument(
'--criteria',
'-c',
help=
"Define the criteria for the checks. Either Strict, Normal, or Lenient. Defaults to Normal.",
nargs='?',
default='normal',
choices=['lenient', 'normal', 'strict'])
parser.add_argument(
'--verbose',
'-v',
help="Increase output. May be specified up to three times.",
action="count",
default=0)
parser.add_argument('--skip-checks',
'-s',
help="Specifies tests to skip",
action='append')
parser.add_argument('-f',
'--format',
default='text',
choices=['text', 'html', 'json'],
help='Output format')
parser.add_argument('-o',
'--output',
default='-',
action='store',
help='Output filename')
parser.add_argument(
'-V',
'--version',
action='store_true',
help='Display the IOOS Compliance Checker version information.')
parser.add_argument(
'dataset_location',
nargs='*',
help="Defines the location of the dataset to be checked.")
parser.add_argument('-l',
'--list-tests',
action='store_true',
help='List the available tests')
parser.add_argument(
'-d',
'--download-standard-names',
help=
'Specify a version of the cf standard name table to download as packaged version'
)
args = parser.parse_args()
if args.version:
print("IOOS compliance checker version %s" % __version__)
return 0
if args.list_tests:
print(
"IOOS compliance checker available checker suites (code version):")
for checker in sorted(check_suite.checkers.keys()):
version = getattr(check_suite.checkers[checker],
'_cc_checker_version', "???")
print(" - {} ({})".format(checker, version))
return 0
if args.download_standard_names:
download_cf_standard_name_table(args.download_standard_names)
return_values = []
had_errors = []
for dataset in args.dataset_location:
if args.format != 'json':
print("Running Compliance Checker on the dataset from: {}".format(
dataset),
file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(
args.dataset_location[0], args.test or ['acdd'], args.verbose,
args.criteria, args.skip_checks, args.output, args.format)
return_values.append(return_value)
had_errors.append(errors)
if any(had_errors):
return 2
if all(return_values):
return 0
return 1
def main():
if hasattr(CheckSuite, 'templates_root'):
CheckSuite.templates_root = "xsf_checker.library"
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
parser = argparse.ArgumentParser()
parser.add_argument('-t', '--test', default=None, action='append',
help='Select the test (defaults: [\"xsf\", \"cf\"]). '
'Version can be specified via \"-t <test_standard>:<version>\". '
'If `<version>` is omitted (or \"latest\"), the latest version is used.')
parser.add_argument('-c', '--criteria', nargs='?', default='normal',
choices=['lenient', 'normal', 'strict'],
help='Define the checking criteria (default: \"normal\").')
parser.add_argument('-s', '--skip-checks', action='append',
help='Specify the tests to skip')
parser.add_argument('-f', '--format', default=None, action='append', choices=['text', 'html', 'json_new'],
help='Set output formats (default: \"text\").')
parser.add_argument('-o', '--output', default=[], action='append',
help=('Set output filenames. '
'If \"-\", output to stdout. '
'If 1 file is supplied, all the output goes in that file. '
'If more outputs are supplied, their number must match the inputs'))
parser.add_argument('-v', '--verbose', action='count', default=0,
help='Increase output verbosity (up to 3 times).')
parser.add_argument('--version', action='store_true',
help='Display the XSF Checker version information.')
parser.add_argument('--ioos-version', action='store_true',
help='Display the IOOS Compliance Checker version information.')
parser.add_argument('input', nargs='*',
help="Define the input dataset to be checked.")
parser.add_argument('-l', '--list-tests', action='store_true',
help='List the available tests')
parser.add_argument('-d', '--download-standard-names',
help='Specify the CF version of standard name table to download')
# Add command line args from generator plugins
check_suite.add_plugin_args(parser)
args = parser.parse_args()
check_suite.load_generated_checkers(args)
if args.version:
print("XSF Checker version %s" % xsf_checker_version)
return 0
if args.ioos_version:
print("IOOS compliance checker version %s" % ioos_checker_version)
return 0
if args.list_tests:
print("Available checker suites:")
for checker in sorted(check_suite.checkers.keys()):
version = getattr(check_suite.checkers[checker],
'_cc_checker_version', "???")
if args.verbose:
print(" - {} (v{})".format(checker, version))
elif ':' in checker and not checker.endswith(':latest'): # Skip the "latest" output
print(" - {}".format(checker))
return 0
if args.download_standard_names:
download_cf_standard_name_table(args.download_standard_names)
nr_inputs = len(args.input)
if nr_inputs == 0:
parser.print_help()
return 1
# Check the number of output files
if not args.output:
args.output = '-'
nr_outputs = len(args.output)
if (nr_outputs != 1) and (nr_outputs != nr_inputs):
print('The number of outputs must either be 1 or the same as the number of inputs:\n'
'- current inputs: %d\n'
'- current outputs: %d' % (nr_inputs, nr_outputs),
file=sys.stderr)
sys.exit(2)
if args.test is None:
args.test = ['cf', 'xsf']
if args.format is None:
args.format = ['text', ]
print("Parameters:\n"
"- Tests: %s\n"
"- Checks to skip: %s\n"
"- Criteria: %s\n"
"- Output format: %s\n"
"- Verbosity: %s"
% (args.test, args.skip_checks, args.criteria, args.format, args.verbose)
)
# Run the compliance checker
# 2 modes, concatenated output file or multiple output files
return_values = []
had_errors = []
if nr_outputs == 1:
if args.format != 'json':
print("Running on input: %s" % args.input, file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(
args.input,
args.test,
args.verbose,
args.criteria,
args.skip_checks,
args.output[0],
args.format)
return_values.append(return_value)
had_errors.append(errors)
else:
for output, input in zip(args.output, args.input):
if args.format != 'json':
print("Running on input: %s" % args.input, file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(
[input],
args.test,
args.verbose,
args.criteria,
args.skip_checks,
output,
args.format)
return_values.append(return_value)
had_errors.append(errors)
if any(had_errors):
return 2
if all(return_values):
return 0
return 1
def main():
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
parser = argparse.ArgumentParser()
parser.add_argument('--test', '-t', '--test=', '-t=', default=[],
action='append',
help=("Select the Checks you want to perform. Defaults to 'acdd'"
" if unspecified. Versions of standards can be specified via "
"`-t <test_standard>:<version>`. If `<version>` is omitted, or "
"is \"latest\", the latest version of the test standard is used."))
parser.add_argument('--criteria', '-c',
help=("Define the criteria for the checks. "
"Either Strict, Normal, or Lenient. Defaults to Normal."),
nargs='?', default='normal',
choices=['lenient', 'normal', 'strict'])
parser.add_argument('--verbose', '-v',
help="Increase output. May be specified up to three times.",
action="count",
default=0)
parser.add_argument('--skip-checks', '-s',
help=dedent("""
Specifies tests to skip. Can take the form
of either `<check_name>` or
`<check_name>:<skip_level>`. The first
form skips any checks matching the name.
In the second form <skip_level> may be
specified as "A", "M", or "L". "A" skips
all checks and is equivalent to calling
the first form. "M" will only show high
priority output from the given check and
will skip medium and low. "L" will show
both high and medium priority issues, while
skipping low priority issues.
"""),
action='append')
parser.add_argument('-f', '--format', default=[], action='append',
help=("Output format(s). Options are 'text', 'html', 'json', 'json_new'."
" The difference between the 'json' and the 'json_new'"
" formats is that the 'json' format has the check as the top level"
" key, whereas the 'json_new' format has the dataset name(s) as the"
" main key in the output follow by any checks as subkeys. Also, "
"'json' format can be only be run against one input file, whereas "
"'json_new' can be run against multiple files."),
choices=['text', 'html', 'json', 'json_new'])
parser.add_argument('-o', '--output', default=[], action='append',
help=("Output filename(s). If '-' is supplied, output to stdout."
" Can either be one or many files. If one file is supplied,"
" but the checker is run against many files, all the output"
" from the checks goes to that file (does not presently work "
"with 'json' format). If more than one output file is "
"supplied, the number of input datasets supplied must match "
"the number of output files."))
parser.add_argument('-V', '--version', action='store_true',
help='Display the IOOS Compliance Checker version information.')
parser.add_argument('dataset_location', nargs='*',
help="Defines the location of the dataset to be checked.")
parser.add_argument('-l', '--list-tests', action='store_true',
help='List the available tests')
parser.add_argument('-d', '--download-standard-names',
help=("Specify a version of the cf standard name table"
" to download as packaged version"))
# Add command line args from generator plugins
check_suite.add_plugin_args(parser)
args = parser.parse_args()
check_suite.load_generated_checkers(args)
if args.version:
print("IOOS compliance checker version %s" % __version__)
return 0
if args.list_tests:
print("IOOS compliance checker available checker suites:")
for checker in sorted(check_suite.checkers.keys()):
version = getattr(check_suite.checkers[checker],
'_cc_checker_version', "???")
if args.verbose:
print(" - {} (v{})".format(checker, version))
elif ':' in checker and not checker.endswith(':latest'): # Skip the "latest" output
print(" - {}".format(checker))
return 0
if args.download_standard_names:
download_cf_standard_name_table(args.download_standard_names)
if len(args.dataset_location) == 0:
parser.print_help()
return 1
# Check the number of output files
if not args.output:
args.output = '-'
output_len = len(args.output)
if not (output_len == 1 or output_len == len(args.dataset_location)):
print('The number of output files must either be one or the same as the number of datasets', file=sys.stderr)
sys.exit(2)
# Run the compliance checker
# 2 modes, concatenated output file or multiple output files
return_values = []
had_errors = []
if output_len == 1:
if args.format != 'json':
print("Running Compliance Checker on the datasets from: {}".format(args.dataset_location), file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(args.dataset_location,
args.test or ['acdd'],
args.verbose,
args.criteria,
args.skip_checks,
args.output[0],
args.format or ['text'])
return_values.append(return_value)
had_errors.append(errors)
else:
for output, dataset in zip(args.output, args.dataset_location):
if args.format != 'json':
print("Running Compliance Checker on the dataset from: {}".format(dataset), file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker([dataset],
args.test or ['acdd'],
args.verbose,
args.criteria,
args.skip_checks,
output,
args.format or ['text'])
return_values.append(return_value)
had_errors.append(errors)
if any(had_errors):
return 2
if all(return_values):
return 0
return 1
def setUp(self):
self.fid, self.path = tempfile.mkstemp()
# why is the class being written to
CheckSuite.checkers.clear()
CheckSuite.load_all_available_checkers()
def main():
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
parser = argparse.ArgumentParser()
parser.add_argument(
'--test',
'-t',
'--test=',
'-t=',
default=[],
action='append',
help=(
"Select the Checks you want to perform. Defaults to 'acdd'"
" if unspecified. Versions of standards can be specified via "
"`-t <test_standard>:<version>`. If `<version>` is omitted, or "
"is \"latest\", the latest version of the test standard is used."))
parser.add_argument(
'--criteria',
'-c',
help=("Define the criteria for the checks. "
"Either Strict, Normal, or Lenient. Defaults to Normal."),
nargs='?',
default='normal',
choices=['lenient', 'normal', 'strict'])
parser.add_argument(
'--verbose',
'-v',
help="Increase output. May be specified up to three times.",
action="count",
default=0)
parser.add_argument('--describe-checks',
'-D',
help=("Describes checks for checkers specified using "
"`-t`. If `-t` is not specified, lists checks "
"from all available checkers."),
action='store_true')
parser.add_argument('--skip-checks',
'-s',
help=dedent("""
Specifies tests to skip. Can take the form
of either `<check_name>` or
`<check_name>:<skip_level>`. The first
form skips any checks matching the name.
In the second form <skip_level> may be
specified as "A", "M", or "L". "A" skips
all checks and is equivalent to calling
the first form. "M" will only show high
priority output from the given check and
will skip medium and low. "L" will show
both high and medium priority issues, while
skipping low priority issues.
"""),
action='append')
parser.add_argument(
'-f',
'--format',
default=[],
action='append',
help=(
"Output format(s). Options are 'text', 'html', 'json', 'json_new'."
" The difference between the 'json' and the 'json_new'"
" formats is that the 'json' format has the check as the top level"
" key, whereas the 'json_new' format has the dataset name(s) as the"
" main key in the output follow by any checks as subkeys. Also, "
"'json' format can be only be run against one input file, whereas "
"'json_new' can be run against multiple files."),
choices=['text', 'html', 'json', 'json_new'])
parser.add_argument(
'-o',
'--output',
default=[],
action='append',
help=("Output filename(s). If '-' is supplied, output to stdout."
" Can either be one or many files. If one file is supplied,"
" but the checker is run against many files, all the output"
" from the checks goes to that file (does not presently work "
"with 'json' format). If more than one output file is "
"supplied, the number of input datasets supplied must match "
"the number of output files."))
parser.add_argument(
'-V',
'--version',
action='store_true',
help='Display the IOOS Compliance Checker version information.')
parser.add_argument(
'dataset_location',
nargs='*',
help="Defines the location of the dataset to be checked.")
parser.add_argument('-l',
'--list-tests',
action='store_true',
help='List the available tests')
parser.add_argument('-d',
'--download-standard-names',
help=("Specify a version of the cf standard name table"
" to download as packaged version"))
# Add command line args from generator plugins
check_suite.add_plugin_args(parser)
args = parser.parse_args()
check_suite.load_generated_checkers(args)
if args.version:
print("IOOS compliance checker version %s" % __version__)
sys.exit(0)
if args.describe_checks:
error_stat = 0
if args.test:
checker_names = set(args.test)
else:
# skip "latest" meta-versions (":latest" or no explicit version
# specifier)
checker_names = [
c for c in check_suite.checkers
if ':' in c and not c.endswith(':latest')
]
for checker_name in sorted(checker_names):
if checker_name not in check_suite.checkers:
print("Cannot find checker '{}' with which to "
"describe checks".format(checker_name),
file=sys.stderr)
error_stat = 1
else:
_print_checker_name_header(checker_name)
check_suite._print_checker(check_suite.checkers[checker_name])
sys.exit(error_stat)
if args.list_tests:
print("IOOS compliance checker available checker suites:")
check_suite._print_suites(args.verbose)
return 0
if args.download_standard_names:
download_cf_standard_name_table(args.download_standard_names)
if len(args.dataset_location) == 0:
parser.print_help()
sys.exit(1)
# Check the number of output files
if not args.output:
args.output = '-'
output_len = len(args.output)
if not (output_len == 1 or output_len == len(args.dataset_location)):
print(
'The number of output files must either be one or the same as the number of datasets',
file=sys.stderr)
sys.exit(2)
# Run the compliance checker
# 2 modes, concatenated output file or multiple output files
return_values = []
had_errors = []
if output_len == 1:
if args.format != 'json':
print("Running Compliance Checker on the datasets from: {}".format(
args.dataset_location),
file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(
args.dataset_location, args.test or ['acdd'], args.verbose,
args.criteria, args.skip_checks, args.output[0], args.format
or ['text'])
return_values.append(return_value)
had_errors.append(errors)
else:
for output, dataset in zip(args.output, args.dataset_location):
if args.format != 'json':
print("Running Compliance Checker on the dataset from: {}".
format(dataset),
file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(
[dataset], args.test or ['acdd'], args.verbose, args.criteria,
args.skip_checks, output, args.format or ['text'])
return_values.append(return_value)
had_errors.append(errors)
if any(had_errors):
sys.exit(2)
if all(return_values):
sys.exit(0)
sys.exit(1)
# -*- coding: utf-8 -*-
'''
Tests for command line output and parsing
'''
import io
import sys
import json
from unittest import TestCase
from compliance_checker.runner import ComplianceChecker, CheckSuite
from compliance_checker.tests.resources import STATIC_FILES
import tempfile
import os
CheckSuite.load_all_available_checkers()
class TestCLI(TestCase):
'''
Tests various functions and aspects of the command line tool and runner
'''
def setUp(self):
self.fid, self.path = tempfile.mkstemp()
def tearDown(self):
if os.path.isfile(self.path):
os.close(self.fid)
os.remove(self.path)
from rq import Queue
app.queue = Queue('default', connection=app.redis)
@app.context_processor
def url_process():
def url_root():
return url_for('.show_root')
return {'url_root': url_root}
from cchecker_web.utils import setup_uploads
setup_uploads(app)
from cchecker_web import cache
cache.init_app(app, config=app.config['CACHE'])
# The compliance checker needs to load all plugins at runtime
from compliance_checker.runner import CheckSuite
CheckSuite.load_all_available_checkers()
if __name__ == '__main__':
app.run(host=app.config['HOST'],
port=app.config['PORT'],
debug=app.config['DEBUG'])
def main():
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
parser = argparse.ArgumentParser()
parser.add_argument(
"--test",
"-t",
"--test=",
"-t=",
default=[],
action="append",
help=("Select the Checks you want to perform. Defaults to 'acdd'"
" if unspecified. Versions of standards can be specified via "
"`-t <test_standard>:<version>`. If `<version>` is omitted, or "
'is "latest", the latest version of the test standard is used.'),
)
parser.add_argument(
"--criteria",
"-c",
help=("Define the criteria for the checks. "
"Either Strict, Normal, or Lenient. Defaults to Normal."),
nargs="?",
default="normal",
choices=["lenient", "normal", "strict"],
)
parser.add_argument(
"--verbose",
"-v",
help="Increase output. May be specified up to three times.",
action="count",
default=0,
)
parser.add_argument(
"--describe-checks",
"-D",
help=("Describes checks for checkers specified using "
"`-t`. If `-t` is not specified, lists checks "
"from all available checkers."),
action="store_true",
)
parser.add_argument(
"--skip-checks",
"-s",
help=dedent("""
Specifies tests to skip. Can take the form
of either `<check_name>` or
`<check_name>:<skip_level>`. The first
form skips any checks matching the name.
In the second form <skip_level> may be
specified as "A", "M", or "L". "A" skips
all checks and is equivalent to calling
the first form. "M" will only show high
priority output from the given check and
will skip medium and low. "L" will show
both high and medium priority issues, while
skipping low priority issues.
"""),
action="append",
)
parser.add_argument(
"-f",
"--format",
default=[],
action="append",
help=(
"Output format(s). Options are 'text', 'html', 'json', 'json_new'."
" The difference between the 'json' and the 'json_new'"
" formats is that the 'json' format has the check as the top level"
" key, whereas the 'json_new' format has the dataset name(s) as the"
" main key in the output follow by any checks as subkeys. Also, "
"'json' format can be only be run against one input file, whereas "
"'json_new' can be run against multiple files."),
choices=["text", "html", "json", "json_new"],
)
parser.add_argument(
"-o",
"--output",
default=[],
action="append",
help=("Output filename(s). If '-' is supplied, output to stdout."
" Can either be one or many files. If one file is supplied,"
" but the checker is run against many files, all the output"
" from the checks goes to that file (does not presently work "
"with 'json' format). If more than one output file is "
"supplied, the number of input datasets supplied must match "
"the number of output files."),
)
parser.add_argument(
"-O",
"--option",
default=[],
action="append",
help=dedent("""
Additional options to be passed to the
checkers. Multiple options can be specified
via multiple invocations of this switch.
Options should be prefixed with a the
checker name followed by the option, e.g.
'<checker>:<option_name>'
Available options:
'cf:enable_appendix_a_checks' - Allow check
results against CF Appendix A for attribute
location and data types.
"""),
)
parser.add_argument(
"-V",
"--version",
action="store_true",
help="Display the IOOS Compliance Checker version information.",
)
parser.add_argument(
"dataset_location",
nargs="*",
help=(
"Defines the location of the dataset to be checked. The location "
"can be a local netCDF file, a remote OPeNDAP endpoint, a remote "
"netCDF file which returns content-type header of "
"'application/x-netcdf', or an ERDDAP TableDAP endpoint. "
"Note that the ERDDAP TableDAP endpoint will currently attempt "
"to fetch the entire TableDAP dataset."))
parser.add_argument("-l",
"--list-tests",
action="store_true",
help="List the available tests")
parser.add_argument(
"-d",
"--download-standard-names",
help=("Specify a version of the cf standard name table"
" to download as packaged version. Either specify"
' a version number (e.g. "72") to fetch a '
'specific version or "latest" to get the '
"latest CF standard name table."),
)
# Add command line args from generator plugins
check_suite.add_plugin_args(parser)
args = parser.parse_args()
check_suite.load_generated_checkers(args)
if args.version:
print("IOOS compliance checker version %s" % __version__)
sys.exit(0)
options_dict = parse_options(
args.option) if args.option else defaultdict(set)
if args.describe_checks:
error_stat = 0
if args.test:
checker_names = set(args.test)
else:
# skip "latest" meta-versions (":latest" or no explicit version
# specifier)
checker_names = [
c for c in check_suite.checkers
if ":" in c and not c.endswith(":latest")
]
for checker_name in sorted(checker_names):
if checker_name not in check_suite.checkers:
print(
"Cannot find checker '{}' with which to "
"describe checks".format(checker_name),
file=sys.stderr,
)
error_stat = 1
else:
_print_checker_name_header(checker_name)
check_suite._print_checker(check_suite.checkers[checker_name])
sys.exit(error_stat)
if args.list_tests:
print("IOOS compliance checker available checker suites:")
check_suite._print_suites(args.verbose)
return 0
if args.download_standard_names:
download_cf_standard_name_table(args.download_standard_names)
if len(args.dataset_location) == 0:
parser.print_help()
sys.exit(1)
# Check the number of output files
if not args.output:
args.output = "-"
output_len = len(args.output)
if not (output_len == 1 or output_len == len(args.dataset_location)):
print(
"The number of output files must either be one or the same as the number of datasets",
file=sys.stderr,
)
sys.exit(2)
# Run the compliance checker
# 2 modes, concatenated output file or multiple output files
return_values = []
had_errors = []
if output_len == 1:
if args.format != "json":
print(
"Running Compliance Checker on the datasets from: {}".format(
args.dataset_location),
file=sys.stderr,
)
return_value, errors = ComplianceChecker.run_checker(
args.dataset_location,
args.test or ["acdd"],
args.verbose,
args.criteria,
args.skip_checks,
args.output[0],
args.format or ["text"],
options=options_dict,
)
return_values.append(return_value)
had_errors.append(errors)
else:
for output, dataset in zip(args.output, args.dataset_location):
if args.format != "json":
print(
"Running Compliance Checker on the dataset from: {}".
format(dataset),
file=sys.stderr,
)
return_value, errors = ComplianceChecker.run_checker(
[dataset],
args.test or ["acdd"],
args.verbose,
args.criteria,
args.skip_checks,
output,
args.format or ["text"],
options=options_dict,
)
return_values.append(return_value)
had_errors.append(errors)
if any(had_errors):
sys.exit(2)
if all(return_values):
sys.exit(0)
sys.exit(1)
import logging
import os
import tempfile
from pathlib import Path
from osgeo import gdal
from compliance_checker.runner import ComplianceChecker, CheckSuite
from digitalearthau import paths
# prevent aux.xml write
os.environ["GDAL_PAM_ENABLED"] = "NO"
# The 'load' method actually loads it globally, not on the specific instance.
CHECK_SUITE = CheckSuite()
CHECK_SUITE.load_all_available_checkers()
def validate_dataset(md_path: Path, log: logging.Logger):
base_path, all_files = paths.get_dataset_paths(md_path)
for file in all_files:
if file.suffix.lower() in ('.nc', '.tif'):
if not validate_image(file, log):
return False
return True
def validate_image(file: Path, log: logging.Logger, compliance_check=False):
try:
storage_unit = gdal.Open(str(file), gdal.gdalconst.GA_ReadOnly)
def main():
# Load all available checker classes
check_suite = CheckSuite()
check_suite.load_all_available_checkers()
parser = argparse.ArgumentParser()
parser.add_argument('--test', '-t', '--test=', '-t=', default=[],
action='append',
help="Select the Checks you want to perform. Defaults to 'acdd' if unspecified")
parser.add_argument('--criteria', '-c',
help="Define the criteria for the checks. Either Strict, Normal, or Lenient. Defaults to Normal.",
nargs='?', default='normal',
choices=['lenient', 'normal', 'strict'])
parser.add_argument('--verbose', '-v',
help="Increase output. May be specified up to three times.",
action="count",
default=0)
parser.add_argument('--skip-checks', '-s',
help="Specifies tests to skip",
action='append')
parser.add_argument('-f', '--format', default='text',
choices=['text', 'html', 'json'], help='Output format')
parser.add_argument('-o', '--output', default='-', action='store',
help='Output filename')
parser.add_argument('-V', '--version', action='store_true',
help='Display the IOOS Compliance Checker version information.')
parser.add_argument('dataset_location', nargs='*',
help="Defines the location of the dataset to be checked.")
parser.add_argument('-l', '--list-tests', action='store_true', help='List the available tests')
parser.add_argument('-d', '--download-standard-names', help='Specify a version of the cf standard name table to download as packaged version')
args = parser.parse_args()
if args.version:
print("IOOS compliance checker version %s" % __version__)
return 0
if args.list_tests:
print("IOOS compliance checker available checker suites (code version):")
for checker in sorted(check_suite.checkers.keys()):
version = getattr(check_suite.checkers[checker], '_cc_checker_version', "???")
print(" - {} ({})".format(checker, version))
return 0
if args.download_standard_names:
download_cf_standard_name_table(args.download_standard_names)
return_values = []
had_errors = []
for dataset in args.dataset_location:
if args.format != 'json':
print("Running Compliance Checker on the dataset from: {}".format(dataset), file=sys.stderr)
return_value, errors = ComplianceChecker.run_checker(args.dataset_location[0],
args.test or ['acdd'],
args.verbose,
args.criteria,
args.skip_checks,
args.output,
args.format)
return_values.append(return_value)
had_errors.append(errors)
if any(had_errors):
return 2
if all(return_values):
return 0
return 1
