def test_hpo_query(self):
with patch('validation.metrics.completeness.get_cols') as mock_get_cols:
nyc_cu_cols = self.get_nyc_cu_cols()
mock_get_cols.return_value = self.get_nyc_cu_cols()
query = completeness.get_hpo_completeness_query(self.hpo_id)
# For now checking for expected column expressions
# TODO find more robust way to test output
for nyc_cu_col in nyc_cu_cols:
column_exp = "'%s' AS column_name" % nyc_cu_col[
consts.COLUMN_NAME]
self.assertTrue(column_exp in query)
def generate_metrics(hpo_id, bucket, folder_prefix, summary):
"""
Generate metrics regarding a submission
:param hpo_id: identifies the HPO site
:param bucket: name of the bucket with the submission
:param folder_prefix: folder containing the submission
:param summary: file summary from validation
{results: [(file_name, found, parsed, loaded)],
errors: [(file_name, message)],
warnings: [(file_name, message)]}
:return:
"""
report_data = summary.copy()
processed_datetime_str = datetime.datetime.now().strftime(
'%Y-%m-%dT%H:%M:%S')
error_occurred = False
# TODO separate query generation, query execution, writing to GCS
gcs_path = 'gs://%s/%s' % (bucket, folder_prefix)
report_data[report_consts.HPO_NAME_REPORT_KEY] = get_hpo_name(hpo_id)
report_data[report_consts.FOLDER_REPORT_KEY] = folder_prefix
report_data[report_consts.TIMESTAMP_REPORT_KEY] = processed_datetime_str
results = report_data['results']
try:
# TODO modify achilles to run successfully when tables are empty
# achilles queries will raise exceptions (e.g. division by zero) if files not present
if all_required_files_loaded(results):
logging.info('Running achilles on %s.', folder_prefix)
run_achilles(hpo_id)
run_export(datasource_id=hpo_id, folder_prefix=folder_prefix)
logging.info('Uploading achilles index files to `%s`.', gcs_path)
_upload_achilles_files(hpo_id, folder_prefix)
heel_error_query = get_heel_error_query(hpo_id)
report_data[report_consts.HEEL_ERRORS_REPORT_KEY] = query_rows(
heel_error_query)
else:
report_data[
report_consts.
SUBMISSION_ERROR_REPORT_KEY] = 'Required files are missing'
logging.info(
'Required files are missing in %s. Skipping achilles.',
gcs_path)
# non-unique key metrics
logging.info('Getting non-unique key stats for %s...' % hpo_id)
nonunique_metrics_query = get_duplicate_counts_query(hpo_id)
report_data[
report_consts.NONUNIQUE_KEY_METRICS_REPORT_KEY] = query_rows(
nonunique_metrics_query)
# drug class metrics
logging.info('Getting drug class for %s...' % hpo_id)
drug_class_metrics_query = get_drug_class_counts_query(hpo_id)
report_data[report_consts.DRUG_CLASS_METRICS_REPORT_KEY] = query_rows(
drug_class_metrics_query)
# missing PII
logging.info('Getting missing record stats for %s...' % hpo_id)
missing_pii_query = get_hpo_missing_pii_query(hpo_id)
missing_pii_results = query_rows(missing_pii_query)
report_data[report_consts.MISSING_PII_KEY] = missing_pii_results
# completeness
logging.info('Getting completeness stats for %s...' % hpo_id)
completeness_query = completeness.get_hpo_completeness_query(hpo_id)
report_data[report_consts.COMPLETENESS_REPORT_KEY] = query_rows(
completeness_query)
# lab concept metrics
logging.info('Getting lab concepts for %s...' % hpo_id)
lab_concept_metrics_query = required_labs.get_lab_concept_summary_query(
hpo_id)
report_data[report_consts.LAB_CONCEPT_METRICS_REPORT_KEY] = query_rows(
lab_concept_metrics_query)
logging.info(
'Processing complete. Saving timestamp %s to `gs://%s/%s`.',
processed_datetime_str, bucket,
folder_prefix + common.PROCESSED_TXT)
_write_string_to_file(bucket, folder_prefix + common.PROCESSED_TXT,
processed_datetime_str)
except HttpError as err:
# cloud error occurred- log details for troubleshooting
logging.exception(
'Failed to generate full report due to the following cloud error:\n\n%s'
% err.content)
error_occurred = True
# re-raise error
raise err
finally:
# report all results collected (attempt even if cloud error occurred)
report_data[report_consts.ERROR_OCCURRED_REPORT_KEY] = error_occurred
results_html = hpo_report.render(report_data)
_write_string_to_file(bucket, folder_prefix + common.RESULTS_HTML,
results_html)
return report_data
def generate_metrics(hpo_id, bucket, folder_prefix, summary):
"""
Generate metrics regarding a submission
:param hpo_id: identifies the HPO site
:param bucket: name of the bucket with the submission
:param folder_prefix: folder containing the submission
:param summary: file summary from validation
{results: [(file_name, found, parsed, loaded)],
errors: [(file_name, message)],
warnings: [(file_name, message)]}
:return:
"""
report_data = summary.copy()
error_occurred = False
# TODO separate query generation, query execution, writing to GCS
gcs_path = f"gs://{bucket}/{folder_prefix}"
report_data[report_consts.HPO_NAME_REPORT_KEY] = get_hpo_name(hpo_id)
report_data[report_consts.FOLDER_REPORT_KEY] = folder_prefix
results = report_data['results']
try:
# TODO modify achilles to run successfully when tables are empty
# achilles queries will raise exceptions (e.g. division by zero) if files not present
if all_required_files_loaded(results):
logging.info(f"Running achilles on {folder_prefix}.")
run_achilles(hpo_id)
run_export(datasource_id=hpo_id, folder_prefix=folder_prefix)
logging.info(f"Uploading achilles index files to '{gcs_path}'.")
_upload_achilles_files(hpo_id, folder_prefix)
heel_error_query = get_heel_error_query(hpo_id)
report_data[report_consts.HEEL_ERRORS_REPORT_KEY] = query_rows(
heel_error_query)
else:
report_data[
report_consts.
SUBMISSION_ERROR_REPORT_KEY] = "Required files are missing"
logging.info(
f"Required files are missing in {gcs_path}. Skipping achilles.")
# non-unique key metrics
logging.info(f"Getting non-unique key stats for {hpo_id}")
nonunique_metrics_query = get_duplicate_counts_query(hpo_id)
report_data[
report_consts.NONUNIQUE_KEY_METRICS_REPORT_KEY] = query_rows(
nonunique_metrics_query)
# drug class metrics
logging.info(f"Getting drug class for {hpo_id}")
drug_class_metrics_query = get_drug_class_counts_query(hpo_id)
report_data[report_consts.DRUG_CLASS_METRICS_REPORT_KEY] = query_rows(
drug_class_metrics_query)
# missing PII
logging.info(f"Getting missing record stats for {hpo_id}")
missing_pii_query = get_hpo_missing_pii_query(hpo_id)
missing_pii_results = query_rows(missing_pii_query)
report_data[report_consts.MISSING_PII_KEY] = missing_pii_results
# completeness
logging.info(f"Getting completeness stats for {hpo_id}")
completeness_query = completeness.get_hpo_completeness_query(hpo_id)
report_data[report_consts.COMPLETENESS_REPORT_KEY] = query_rows(
completeness_query)
# lab concept metrics
logging.info(f"Getting lab concepts for {hpo_id}")
lab_concept_metrics_query = required_labs.get_lab_concept_summary_query(
hpo_id)
report_data[report_consts.LAB_CONCEPT_METRICS_REPORT_KEY] = query_rows(
lab_concept_metrics_query)
logging.info(f"Processing complete.")
except HttpError as err:
# cloud error occurred- log details for troubleshooting
logging.exception(
f"Failed to generate full report due to the following cloud error:\n\n{err.content}"
)
error_occurred = True
finally:
# report all results collected (attempt even if cloud error occurred)
report_data[report_consts.ERROR_OCCURRED_REPORT_KEY] = error_occurred
return report_data
