def generate_empty_report(hpo_id, bucket, folder_prefix):
"""
Generate an empty report with a "validation failed" error
Also write processed.txt to folder to prevent processing in the future
:param hpo_id: identifies the HPO site
:param bucket: name of the bucket with the submission
:param folder_prefix: folder containing the submission
:return: report_data: dict whose keys are params in resources/templates/hpo_report.html
"""
report_data = dict()
processed_datetime_str = datetime.datetime.now().strftime(
'%Y-%m-%dT%H:%M:%S')
report_data[report_consts.HPO_NAME_REPORT_KEY] = get_hpo_name(hpo_id)
report_data[report_consts.FOLDER_REPORT_KEY] = folder_prefix
report_data[report_consts.TIMESTAMP_REPORT_KEY] = processed_datetime_str
report_data[
report_consts.
SUBMISSION_ERROR_REPORT_KEY] = f'Submission folder name {folder_prefix} does not follow the ' \
f'naming convention {consts.FOLDER_NAMING_CONVENTION}, where vN represents ' \
f'the version number for the day, starting at v1 each day. ' \
f'Please resubmit the files in a new folder with the correct naming convention'
logging.info(
'Processing skipped. Reason: Folder %s does not follow naming convention %s. '
'Saving timestamp %s to `gs://%s/%s`.', folder_prefix,
consts.FOLDER_NAMING_CONVENTION, processed_datetime_str, bucket,
folder_prefix + common.PROCESSED_TXT)
_write_string_to_file(bucket, folder_prefix + common.PROCESSED_TXT,
processed_datetime_str)
results_html = hpo_report.render(report_data)
_write_string_to_file(bucket, folder_prefix + common.RESULTS_HTML,
results_html)
return report_data
def test_render(self):
report_data = {
consts.HPO_NAME_REPORT_KEY: self.hpo_name,
consts.TIMESTAMP_REPORT_KEY: self.timestamp,
consts.FOLDER_REPORT_KEY: self.folder,
consts.RESULTS_REPORT_KEY: self.results,
consts.ERRORS_REPORT_KEY: [],
consts.WARNINGS_REPORT_KEY: [],
consts.NONUNIQUE_KEY_METRICS_REPORT_KEY: [],
consts.DRUG_CLASS_METRICS_REPORT_KEY: self.drug_class_metrics,
consts.HEEL_ERRORS_REPORT_KEY: self.heel_errors,
consts.COMPLETENESS_REPORT_KEY: self.completeness
}
# report ok
render_output = hpo_report.render(report_data)
self.save_render_output(render_output)
self.assert_report_data_in_output(report_data, render_output)
# submission error
report_data[
consts.SUBMISSION_ERROR_REPORT_KEY] = 'Required files are missing'
render_output = hpo_report.render(report_data)
self.save_render_output(render_output)
self.assert_report_data_in_output(report_data, render_output)
# error occurred
report_data[consts.ERROR_OCCURRED_REPORT_KEY] = True
render_output = hpo_report.render(report_data)
self.save_render_output(render_output)
self.assert_report_data_in_output(report_data, render_output)
report_data[consts.ERRORS_REPORT_KEY] = [{
'file_name': 'visit_occurrence.csv',
'message': 'Fake error'
}]
render_output = hpo_report.render(report_data)
self.save_render_output(render_output)
self.assert_report_data_in_output(report_data, render_output)
def perform_reporting(hpo_id, report_data, folder_items, bucket,
folder_prefix):
"""
Generate html report, upload to GCS and send email if possible
:param hpo_id: identifies the hpo site
:param report_data: dictionary containing items for populating hpo_report.html
:param folder_items: items in the folder without folder prefix
:param bucket: bucket containing the folder
:param folder_prefix: submission folder
:return:
"""
processed_time_str = get_eastern_time()
report_data[report_consts.TIMESTAMP_REPORT_KEY] = processed_time_str
results_html = hpo_report.render(report_data)
results_html_path = folder_prefix + common.RESULTS_HTML
logging.info(f"Saving file {common.RESULTS_HTML} to "
f"gs://{bucket}/{results_html_path}.")
upload_string_to_gcs(bucket, results_html_path, results_html)
processed_txt_path = folder_prefix + common.PROCESSED_TXT
logging.info(f"Saving timestamp {processed_time_str} to "
f"gs://{bucket}/{processed_txt_path}.")
upload_string_to_gcs(bucket, processed_txt_path, processed_time_str)
folder_uri = f"gs://{bucket}/{folder_prefix}"
if folder_items and is_first_validation_run(folder_items):
logging.info(f"Attempting to send report via email for {hpo_id}")
email_msg = en.generate_email_message(hpo_id, results_html, folder_uri,
report_data)
if email_msg is None:
logging.info(
f"Not enough info in contact list to send emails for hpo_id {hpo_id}"
)
else:
result = en.send_email(email_msg)
if result is None:
logging.info(
'Mandrill error occurred. Please check logs for more details'
)
else:
result_ids = ', '.join(
[result_item['_id'] for result_item in result])
logging.info(
f"Sending emails for hpo_id {hpo_id} with Mandrill tracking ids: {result_ids}"
)
logging.info(f"Reporting complete")
return
def generate_metrics(hpo_id, bucket, folder_prefix, summary):
"""
Generate metrics regarding a submission
:param hpo_id: identifies the HPO site
:param bucket: name of the bucket with the submission
:param folder_prefix: folder containing the submission
:param summary: file summary from validation
{results: [(file_name, found, parsed, loaded)],
errors: [(file_name, message)],
warnings: [(file_name, message)]}
:return:
"""
report_data = summary.copy()
processed_datetime_str = datetime.datetime.now().strftime(
'%Y-%m-%dT%H:%M:%S')
error_occurred = False
# TODO separate query generation, query execution, writing to GCS
gcs_path = 'gs://%s/%s' % (bucket, folder_prefix)
report_data[report_consts.HPO_NAME_REPORT_KEY] = get_hpo_name(hpo_id)
report_data[report_consts.FOLDER_REPORT_KEY] = folder_prefix
report_data[report_consts.TIMESTAMP_REPORT_KEY] = processed_datetime_str
results = report_data['results']
try:
# TODO modify achilles to run successfully when tables are empty
# achilles queries will raise exceptions (e.g. division by zero) if files not present
if all_required_files_loaded(results):
logging.info('Running achilles on %s.', folder_prefix)
run_achilles(hpo_id)
run_export(datasource_id=hpo_id, folder_prefix=folder_prefix)
logging.info('Uploading achilles index files to `%s`.', gcs_path)
_upload_achilles_files(hpo_id, folder_prefix)
heel_error_query = get_heel_error_query(hpo_id)
report_data[report_consts.HEEL_ERRORS_REPORT_KEY] = query_rows(
heel_error_query)
else:
report_data[
report_consts.
SUBMISSION_ERROR_REPORT_KEY] = 'Required files are missing'
logging.info(
'Required files are missing in %s. Skipping achilles.',
gcs_path)
# non-unique key metrics
logging.info('Getting non-unique key stats for %s...' % hpo_id)
nonunique_metrics_query = get_duplicate_counts_query(hpo_id)
report_data[
report_consts.NONUNIQUE_KEY_METRICS_REPORT_KEY] = query_rows(
nonunique_metrics_query)
# drug class metrics
logging.info('Getting drug class for %s...' % hpo_id)
drug_class_metrics_query = get_drug_class_counts_query(hpo_id)
report_data[report_consts.DRUG_CLASS_METRICS_REPORT_KEY] = query_rows(
drug_class_metrics_query)
# missing PII
logging.info('Getting missing record stats for %s...' % hpo_id)
missing_pii_query = get_hpo_missing_pii_query(hpo_id)
missing_pii_results = query_rows(missing_pii_query)
report_data[report_consts.MISSING_PII_KEY] = missing_pii_results
# completeness
logging.info('Getting completeness stats for %s...' % hpo_id)
completeness_query = completeness.get_hpo_completeness_query(hpo_id)
report_data[report_consts.COMPLETENESS_REPORT_KEY] = query_rows(
completeness_query)
# lab concept metrics
logging.info('Getting lab concepts for %s...' % hpo_id)
lab_concept_metrics_query = required_labs.get_lab_concept_summary_query(
hpo_id)
report_data[report_consts.LAB_CONCEPT_METRICS_REPORT_KEY] = query_rows(
lab_concept_metrics_query)
logging.info(
'Processing complete. Saving timestamp %s to `gs://%s/%s`.',
processed_datetime_str, bucket,
folder_prefix + common.PROCESSED_TXT)
_write_string_to_file(bucket, folder_prefix + common.PROCESSED_TXT,
processed_datetime_str)
except HttpError as err:
# cloud error occurred- log details for troubleshooting
logging.exception(
'Failed to generate full report due to the following cloud error:\n\n%s'
% err.content)
error_occurred = True
# re-raise error
raise err
finally:
# report all results collected (attempt even if cloud error occurred)
report_data[report_consts.ERROR_OCCURRED_REPORT_KEY] = error_occurred
results_html = hpo_report.render(report_data)
_write_string_to_file(bucket, folder_prefix + common.RESULTS_HTML,
results_html)
return report_data
