def test_curation_report_ignored(self, mock_check_cron):
exclude_file_list = ["person.csv"]
exclude_file_list = [
self.folder_prefix + item for item in exclude_file_list
]
expected_result_items = []
for file_name in exclude_file_list:
test_util.write_cloud_str(self.hpo_bucket, file_name, ".")
main.app.testing = True
with main.app.test_client() as c:
c.get(test_util.VALIDATE_HPO_FILES_URL)
# check content of the bucket is correct
expected_bucket_items = exclude_file_list + [
self.folder_prefix + item for item in resources.IGNORE_LIST
]
list_bucket_result = gcs_utils.list_bucket(self.hpo_bucket)
actual_bucket_items = [item['name'] for item in list_bucket_result]
actual_bucket_items = [
item for item in actual_bucket_items if
not main._is_string_excluded_file(item[len(self.folder_prefix):])
]
self.assertSetEqual(set(expected_bucket_items),
set(actual_bucket_items))
# check that the errors file is empty
bucket_items = gcs_utils.list_bucket(self.hpo_bucket)
r = main.validate_submission(self.hpo_id, self.hpo_bucket,
bucket_items, self.folder_prefix)
self.assertListEqual(expected_result_items, r['errors'])
def test_all_files_unparseable_output(self):
# TODO possible bug: if no pre-existing table, results in bq table not found error
for cdm_table in common.SUBMISSION_FILES:
test_util.write_cloud_str(self.hpo_bucket, self.folder_prefix + cdm_table, ".\n .")
bucket_items = gcs_utils.list_bucket(self.hpo_bucket)
expected_results = [(f, 1, 0, 0) for f in common.SUBMISSION_FILES]
r = main.validate_submission(self.hpo_id, self.hpo_bucket, bucket_items, self.folder_prefix)
self.assertSetEqual(set(expected_results), set(r['results']))
def test_check_processed(self):
test_util.write_cloud_str(self.hpo_bucket, self.folder_prefix + 'person.csv', '\n')
test_util.write_cloud_str(self.hpo_bucket, self.folder_prefix + common.PROCESSED_TXT, '\n')
bucket_items = gcs_utils.list_bucket(self.hpo_bucket)
result = main._get_submission_folder(self.hpo_bucket, bucket_items, force_process=False)
self.assertIsNone(result)
result = main._get_submission_folder(self.hpo_bucket, bucket_items, force_process=True)
self.assertEqual(result, self.folder_prefix)
def test_bad_file_names(self):
bad_file_names = ["avisit_occurrence.csv",
"condition_occurence.csv", # misspelled
"person_final.csv",
"procedure_occurrence.tsv"] # unsupported file extension
expected_warnings = []
for file_name in bad_file_names:
test_util.write_cloud_str(self.hpo_bucket, self.folder_prefix + file_name, ".")
expected_item = (file_name, common.UNKNOWN_FILE)
expected_warnings.append(expected_item)
bucket_items = gcs_utils.list_bucket(self.hpo_bucket)
r = main.validate_submission(self.hpo_id, self.hpo_bucket, bucket_items, self.folder_prefix)
self.assertListEqual(expected_warnings, r['warnings'])
def test_html_report_person_only(self, mock_check_cron):
folder_prefix = '2019-01-01/'
test_util.write_cloud_str(self.hpo_bucket, folder_prefix + 'person.csv', ".\n .,.,.")
with open(test_util.PERSON_ONLY_RESULTS_FILE, 'r') as f:
expected_result_file = self._remove_timestamp_tags_from_results(f.read())
main.app.testing = True
with main.app.test_client() as c:
c.get(test_util.VALIDATE_HPO_FILES_URL)
actual_result = test_util.read_cloud_file(self.hpo_bucket, folder_prefix + common.RESULTS_HTML)
actual_result_file = self._remove_timestamp_tags_from_results(StringIO.StringIO(actual_result).getvalue())
self.assertEqual(expected_result_file, actual_result_file)
def test_folder_list(self):
folder_prefix_1 = '2018-03-22-v1/'
folder_prefix_2 = '2018-03-22-v2/'
folder_prefix_3 = '2018-03-22-v3/'
file_list = [folder_prefix_1 + 'person.csv',
folder_prefix_2 + 'blah.csv',
folder_prefix_3 + 'visit_occurrence.csv',
'person.csv']
for filename in file_list:
test_util.write_cloud_str(self.hpo_bucket, filename, ".\n .")
bucket_items = gcs_utils.list_bucket(self.hpo_bucket)
folder_prefix = main._get_submission_folder(self.hpo_bucket, bucket_items)
self.assertEqual(folder_prefix, folder_prefix_3)