def compliance_check(job_id, dataset, checker):
try:
redis = get_current_connection()
cs = CheckSuite()
if dataset.startswith('http'):
dataset = check_redirect(dataset)
ds = cs.load_dataset(dataset)
score_groups = cs.run(ds, [], checker)
rpair = score_groups[checker]
groups, errors = rpair
aggregates = cs.build_structure(checker, groups, dataset)
aggregates = cs.serialize(aggregates)
aggregates['all_priorities'] = sorted(aggregates['all_priorities'], key=lambda x: x['weight'], reverse=True)
# We use b64 to keep the filenames safe but it's helpful to the user to see
# the filename they uploaded
if not aggregates['source_name'].startswith('http'):
decoded = base64.b64decode(aggregates['source_name'].split('/')[-1])
if isinstance(decoded, str):
aggregates['source_name'] = decoded
else:
aggregates['source_name'] = decoded.decode('utf-8')
aggregates['ncdump'] = ncdump(dataset)
buf = json.dumps(aggregates)
redis.set('processing:job:%s' % job_id, buf, 3600)
return True
except Exception as e:
redis.set('processing:job:%s' % job_id, json.dumps({"error":type(e).__name__, "message":e.message}), 3600)
return False
def compliance_check(job_id, dataset, checker, path=None):
'''
Performs the Check Suite for the specified checker and sets the result in a
redis result for the job_id
:param str job_id: ID for the rq job
:param dataset: Dataset handle
:param str checker: Check Suite ID for a checker
:param str path: Full path to dataset directory (OPeNDAP only)
'''
try:
redis = get_current_connection()
cs = CheckSuite()
if dataset.startswith('http'):
dataset = check_redirect(dataset)
ds = cs.load_dataset(dataset)
score_groups = cs.run(ds, [], checker)
rpair = score_groups[checker]
groups, errors = rpair
aggregates = cs.build_structure(checker, groups, dataset)
aggregates = cs.serialize(aggregates)
aggregates['all_priorities'] = sorted(aggregates['all_priorities'],
key=lambda x: x['weight'],
reverse=True)
# We use b64 to keep the filenames safe but it's helpful to the user to see
# the filename they uploaded
if not aggregates['source_name'].startswith('http'):
decoded = base64.b64decode(
aggregates['source_name'].split('/')[-1])
if isinstance(decoded, str):
aggregates['source_name'] = decoded
else:
aggregates['source_name'] = decoded.decode('utf-8')
aggregates['ncdump'] = ncdump(dataset)
buf = json.dumps(aggregates)
# Write the report to a text file for download
if path is None:
# Must be a local file, get the path from the dataset
path = os.path.dirname(dataset)
fname = 'compliance_{}.txt'.format(job_id)
output_filename = os.path.join(path, fname)
with io.open(output_filename, 'w', encoding='utf-8') as f:
with stdout_redirector(f):
stdout_output(cs, score_groups, aggregates['source_name'])
redis.set('processing:job:%s' % job_id, buf, 3600)
return True
except Exception as e:
logger.exception("Failed to process job")
error_message = {"error": type(e).__name__, "message": e.message}
redis.set('processing:job:%s' % job_id, json.dumps(error_message),
3600)
return False
def compliance_check(job_id, dataset, checker):
try:
redis = get_current_connection()
cs = CheckSuite()
if dataset.startswith('http'):
dataset = check_redirect(dataset)
ds = cs.load_dataset(dataset)
score_groups = cs.run(ds, checker)
rpair = score_groups[checker]
groups, errors = rpair
aggregates = cs.build_structure(checker, groups, dataset)
aggregates = cs.serialize(aggregates)
aggregates['all_priorities'] = sorted(aggregates['all_priorities'],
key=lambda x: x['weight'],
reverse=True)
# We use b64 to keep the filenames safe but it's helpful to the user to see
# the filename they uploaded
if not aggregates['source_name'].startswith('http'):
decoded = base64.b64decode(
aggregates['source_name'].split('/')[-1])
if isinstance(decoded, str):
aggregates['source_name'] = decoded
else:
aggregates['source_name'] = decoded.decode('utf-8')
aggregates['ncdump'] = ncdump(dataset)
buf = json.dumps(aggregates)
redis.set('processing:job:%s' % job_id, buf, 3600)
return True
except Exception as e:
redis.set(
'processing:job:%s' % job_id,
json.dumps({
"error": type(e).__name__,
"message": e.message
}), 3600)
return False
