def test_curation_report_ignored(self, mock_check_cron): exclude_file_list = ["person.csv"] exclude_file_list = [ self.folder_prefix + item for item in exclude_file_list ] expected_result_items = [] for file_name in exclude_file_list: test_util.write_cloud_str(self.hpo_bucket, file_name, ".") main.app.testing = True with main.app.test_client() as c: c.get(test_util.VALIDATE_HPO_FILES_URL) # check content of the bucket is correct expected_bucket_items = exclude_file_list + [ self.folder_prefix + item for item in resources.IGNORE_LIST ] list_bucket_result = gcs_utils.list_bucket(self.hpo_bucket) actual_bucket_items = [item['name'] for item in list_bucket_result] actual_bucket_items = [ item for item in actual_bucket_items if not main._is_string_excluded_file(item[len(self.folder_prefix):]) ] self.assertSetEqual(set(expected_bucket_items), set(actual_bucket_items)) # check that the errors file is empty bucket_items = gcs_utils.list_bucket(self.hpo_bucket) r = main.validate_submission(self.hpo_id, self.hpo_bucket, bucket_items, self.folder_prefix) self.assertListEqual(expected_result_items, r['errors'])
def test_upload_object(self): bucket_items = gcs_utils.list_bucket(self.hpo_bucket) self.assertEqual(len(bucket_items), 0) with open(FIVE_PERSONS_PERSON_CSV, 'rb') as fp: gcs_utils.upload_object(self.hpo_bucket, 'person.csv', fp) bucket_items = gcs_utils.list_bucket(self.hpo_bucket) self.assertEqual(len(bucket_items), 1) bucket_item = bucket_items[0] self.assertEqual(bucket_item['name'], 'person.csv')
def bucket_access_configured(bucket_name): """ Determine if the service account has appropriate permissions on the bucket :param bucket_name: identifies the GCS bucket :return: True if the service account has appropriate permissions, False otherwise :raises HttpError if accessing bucket fails """ try: gcs_utils.list_bucket(bucket_name) return True except HttpError: raise
def tearDown(self): self._empty_bucket() to_delete_list = gcs_utils.list_bucket(gcs_utils.get_drc_bucket()) for bucket_item in to_delete_list: gcs_utils.delete_object(gcs_utils.get_drc_bucket(), bucket_item['name']) self.testbed.deactivate()
def test_pii_files_loaded(self, mock_check_cron): # tests if pii files are loaded test_file_paths = [ test_util.PII_NAME_FILE, test_util.PII_MRN_BAD_PERSON_ID_FILE ] test_file_names = [os.path.basename(f) for f in test_file_paths] test_util.write_cloud_file(self.hpo_bucket, test_util.PII_NAME_FILE, prefix=self.folder_prefix) test_util.write_cloud_file(self.hpo_bucket, test_util.PII_MRN_BAD_PERSON_ID_FILE, prefix=self.folder_prefix) rs = resources.csv_to_list(test_util.PII_FILE_LOAD_RESULT_CSV) expected_results = [(r['file_name'], int(r['found']), int(r['parsed']), int(r['loaded'])) for r in rs] for f in common.SUBMISSION_FILES: if f not in test_file_names: expected_result = (f, 0, 0, 0) expected_results.append(expected_result) bucket_items = gcs_utils.list_bucket(self.hpo_bucket) folder_items = main.get_folder_items(bucket_items, self.folder_prefix) r = main.validate_submission(self.hpo_id, self.hpo_bucket, folder_items, self.folder_prefix) self.assertSetEqual(set(expected_results), set(r['results']))
def test_errors_csv(self, mock_check_cron): folder_prefix = 'dummy-prefix-2018-03-22/' test_util.write_cloud_str(self.hpo_bucket, folder_prefix + 'person.csv', ".\n .,.,.") main.app.testing = True with main.app.test_client() as c: c.get(test_util.VALIDATE_HPO_FILES_URL) # check the result file was put in bucket list_bucket_result = gcs_utils.list_bucket(self.hpo_bucket) bucket_item_names = [ item['name'] for item in list_bucket_result if item['name'].startswith(folder_prefix) ] expected_items = ['person.csv'] + common.IGNORE_LIST expected_items = [folder_prefix + item for item in expected_items] self.assertSetEqual(set(bucket_item_names), set(expected_items)) # check content of the file is correct actual_result = test_util.read_cloud_file( self.hpo_bucket, folder_prefix + common.ERRORS_CSV) actual = resources._csv_file_to_list( StringIO.StringIO(actual_result)) for row in actual: row.pop('message', None) expected = [{'file_name': 'person.csv', 'type': 'error'}] self.assertEqual(actual, expected)
def test_errors_csv(self, mock_check_cron): folder_prefix = 'dummy-prefix-2018-03-22/' test_util.write_cloud_str(self.hpo_bucket, folder_prefix + 'person.csv', ".\n .,.,.") main.app.testing = True with main.app.test_client() as c: c.get(test_util.VALIDATE_HPO_FILES_URL) # check the result file was put in bucket list_bucket_result = gcs_utils.list_bucket(self.hpo_bucket) bucket_item_names = [ item['name'] for item in list_bucket_result if item['name'].startswith(folder_prefix) ] expected_items = ['person.csv'] + common.IGNORE_LIST expected_items = [folder_prefix + item for item in expected_items] self.assertSetEqual(set(bucket_item_names), set(expected_items)) # check content of the file is correct actual_result = test_util.read_cloud_file( self.hpo_bucket, folder_prefix + common.ERRORS_CSV) with open(test_util.BAD_PERSON_FILE_BQ_LOAD_ERRORS_CSV, 'r') as f: expected = f.read() self.assertEqual(expected, actual_result)
def test_run_export(self): folder_prefix = 'dummy-prefix-2018-03-24/' main._upload_achilles_files(test_util.FAKE_HPO_ID, folder_prefix) main.run_export(hpo_id=test_util.FAKE_HPO_ID, folder_prefix=folder_prefix) bucket_objects = gcs_utils.list_bucket(self.hpo_bucket) actual_object_names = [obj['name'] for obj in bucket_objects] for report in common.ALL_REPORT_FILES: prefix = folder_prefix + common.ACHILLES_EXPORT_PREFIX_STRING + test_util.FAKE_HPO_ID + '/' expected_object_name = prefix + report self.assertIn(expected_object_name, actual_object_names) datasources_json_path = folder_prefix + common.ACHILLES_EXPORT_DATASOURCES_JSON self.assertIn(datasources_json_path, actual_object_names) datasources_json = gcs_utils.get_object(self.hpo_bucket, datasources_json_path) datasources_actual = json.loads(datasources_json) datasources_expected = { 'datasources': [{ 'name': test_util.FAKE_HPO_ID, 'folder': test_util.FAKE_HPO_ID, 'cdmVersion': 5 }] } self.assertDictEqual(datasources_expected, datasources_actual)
def test_run_export_with_target_bucket(self): folder_prefix = 'dummy-prefix-2018-03-24/' bucket_nyc = gcs_utils.get_hpo_bucket('nyc') test_util.get_synpuf_results_files() test_util.populate_achilles(self.hpo_bucket, hpo_id=None) main.run_export(folder_prefix=folder_prefix, target_bucket=bucket_nyc) bucket_objects = gcs_utils.list_bucket(bucket_nyc) actual_object_names = [obj['name'] for obj in bucket_objects] for report in common.ALL_REPORT_FILES: expected_object_name = folder_prefix + common.ACHILLES_EXPORT_PREFIX_STRING + 'default' + '/' + report self.assertIn(expected_object_name, actual_object_names) datasources_json_path = folder_prefix + common.ACHILLES_EXPORT_DATASOURCES_JSON self.assertIn(datasources_json_path, actual_object_names) datasources_json = gcs_utils.get_object(bucket_nyc, datasources_json_path) datasources_actual = json.loads(datasources_json) datasources_expected = { 'datasources': [{ 'name': 'default', 'folder': 'default', 'cdmVersion': 5 }] } self.assertDictEqual(datasources_expected, datasources_actual)
def test_validate_five_persons_success(self, mock_check_cron): expected_results = [] test_file_names = [ os.path.basename(f) for f in test_util.FIVE_PERSONS_FILES ] for cdm_file in common.SUBMISSION_FILES: if cdm_file in test_file_names: expected_result = (cdm_file, 1, 1, 1) test_file = os.path.join(test_util.FIVE_PERSONS_PATH, cdm_file) test_util.write_cloud_file(self.hpo_bucket, test_file, prefix=self.folder_prefix) else: expected_result = (cdm_file, 0, 0, 0) expected_results.append(expected_result) bucket_items = gcs_utils.list_bucket(self.hpo_bucket) folder_items = main.get_folder_items(bucket_items, self.folder_prefix) r = main.validate_submission(self.hpo_id, self.hpo_bucket, folder_items, self.folder_prefix) self.assertSetEqual(set(r['results']), set(expected_results)) # check tables exist and are clustered as expected for table in resources.CDM_TABLES + common.PII_TABLES: fields_file = os.path.join(resources.fields_path, table + '.json') table_id = bq_utils.get_table_id(test_util.FAKE_HPO_ID, table) table_info = bq_utils.get_table_info(table_id) with open(fields_file, 'r') as fp: fields = json.load(fp) field_names = [field['name'] for field in fields] if 'person_id' in field_names: self.table_has_clustering(table_info)
def test_run_export_with_target_bucket_and_datasource_id( self, mock_is_hpo_id): # validation/main.py INTEGRATION TEST mock_is_hpo_id.return_value = True folder_prefix = 'dummy-prefix-2018-03-24/' bucket_nyc = gcs_utils.get_hpo_bucket('nyc') main.run_export(datasource_id=FAKE_HPO_ID, folder_prefix=folder_prefix, target_bucket=bucket_nyc) bucket_objects = gcs_utils.list_bucket(bucket_nyc) actual_object_names = [obj['name'] for obj in bucket_objects] for report in common.ALL_REPORT_FILES: prefix = folder_prefix + common.ACHILLES_EXPORT_PREFIX_STRING + FAKE_HPO_ID + '/' expected_object_name = prefix + report self.assertIn(expected_object_name, actual_object_names) datasources_json_path = folder_prefix + common.ACHILLES_EXPORT_DATASOURCES_JSON self.assertIn(datasources_json_path, actual_object_names) datasources_json = gcs_utils.get_object(bucket_nyc, datasources_json_path) datasources_actual = json.loads(datasources_json) datasources_expected = { 'datasources': [{ 'name': FAKE_HPO_ID, 'folder': FAKE_HPO_ID, 'cdmVersion': 5 }] } self.assertDictEqual(datasources_expected, datasources_actual)
def test_bad_file_names(self, mock_check_cron): folder_prefix = 'dummy-prefix-2018-03-22/' exclude_file_list = ["person_final.csv", "condition_occurence.csv", # misspelled "avisit_occurrence.csv", "procedure_occurrence.tsv"] # unsupported file extension exclude_file_list = [folder_prefix + item for item in exclude_file_list] expected_result_items = [] for file_name in exclude_file_list: test_util.write_cloud_str(self.hpo_bucket, file_name, ".") expected_item = dict(file_name=file_name.split('/')[1], message=main.UNKNOWN_FILE) expected_result_items.append(expected_item) main.app.testing = True with main.app.test_client() as c: c.get(test_util.VALIDATE_HPO_FILES_URL) # check content of the bucket is correct expected_bucket_items = exclude_file_list + [folder_prefix + item for item in common.IGNORE_LIST] # [common.RESULT_CSV, common.WARNINGS_CSV] list_bucket_result = gcs_utils.list_bucket(self.hpo_bucket) actual_bucket_items = [item['name'] for item in list_bucket_result] self.assertSetEqual(set(expected_bucket_items), set(actual_bucket_items)) # check content of the warnings file is correct actual_result = test_util.read_cloud_file(self.hpo_bucket, folder_prefix + common.WARNINGS_CSV) actual_result_file = StringIO.StringIO(actual_result) actual_result_items = resources._csv_file_to_list(actual_result_file) # sort in order to compare expected_result_items.sort() actual_result_items.sort() self.assertListEqual(expected_result_items, actual_result_items)
def test_all_files_unparseable_output(self, mock_check_cron): # TODO possible bug: if no pre-existing table, results in bq table not found error folder_prefix = 'dummy-prefix-2018-03-22/' for cdm_table in common.CDM_FILES: test_util.write_cloud_str(self.hpo_bucket, folder_prefix + cdm_table, ".\n .") main.app.testing = True with main.app.test_client() as c: c.get(test_util.VALIDATE_HPO_FILES_URL) # check the result file was put in bucket list_bucket_result = gcs_utils.list_bucket(self.hpo_bucket) bucket_item_names = [ item['name'] for item in list_bucket_result if item['name'].startswith(folder_prefix) ] expected_items = common.CDM_FILES + common.IGNORE_LIST expected_items = [ folder_prefix + item_name for item_name in expected_items ] self.assertSetEqual(set(bucket_item_names), set(expected_items)) # check content of the file is correct actual_result = test_util.read_cloud_file( self.hpo_bucket, folder_prefix + common.RESULT_CSV) actual_result = resources._csv_file_to_list( StringIO.StringIO(actual_result)) expected = [{ 'cdm_file_name': cdm_file_name, 'found': '1', 'parsed': '0', 'loaded': '0' } for cdm_file_name in common.CDM_FILES] self.assertEqual(expected, actual_result)
def test_copy_five_persons(self, mock_check_cron): # upload all five_persons files for cdm_file in test_util.FIVE_PERSONS_FILES: test_util.write_cloud_file(self.hpo_bucket, cdm_file, prefix=self.folder_prefix) test_util.write_cloud_file(self.hpo_bucket, cdm_file, prefix=self.folder_prefix + self.folder_prefix) main.app.testing = True with main.app.test_client() as c: c.get(test_util.COPY_HPO_FILES_URL) prefix = test_util.FAKE_HPO_ID + '/' + self.hpo_bucket + '/' + self.folder_prefix expected_bucket_items = [ prefix + item.split(os.sep)[-1] for item in test_util.FIVE_PERSONS_FILES ] expected_bucket_items.extend([ prefix + self.folder_prefix + item.split(os.sep)[-1] for item in test_util.FIVE_PERSONS_FILES ]) list_bucket_result = gcs_utils.list_bucket( gcs_utils.get_drc_bucket()) actual_bucket_items = [item['name'] for item in list_bucket_result] self.assertSetEqual(set(expected_bucket_items), set(actual_bucket_items))
def test_copy_five_persons(self, mock_check_cron): # upload all five_persons files for cdm_pathfile in test_util.FIVE_PERSONS_FILES: test_filename: str = os.path.basename(cdm_pathfile) blob_name: str = f'{self.folder_prefix}{test_filename}' test_blob = self.storage_bucket.blob(blob_name) test_blob.upload_from_filename(cdm_pathfile) blob_name: str = f'{self.folder_prefix}{self.folder_prefix}{test_filename}' test_blob = self.storage_bucket.blob(blob_name) test_blob.upload_from_filename(cdm_pathfile) main.app.testing = True with main.app.test_client() as c: c.get(test_util.COPY_HPO_FILES_URL) prefix = test_util.FAKE_HPO_ID + '/' + self.hpo_bucket + '/' + self.folder_prefix expected_bucket_items = [ prefix + item.split(os.sep)[-1] for item in test_util.FIVE_PERSONS_FILES ] expected_bucket_items.extend([ prefix + self.folder_prefix + item.split(os.sep)[-1] for item in test_util.FIVE_PERSONS_FILES ]) list_bucket_result = gcs_utils.list_bucket( gcs_utils.get_drc_bucket()) actual_bucket_items = [item['name'] for item in list_bucket_result] self.assertSetEqual(set(expected_bucket_items), set(actual_bucket_items))
def test_check_processed(self, mock_updated_datetime_object): mock_updated_datetime_object.return_value = datetime.datetime.today( ) - datetime.timedelta(minutes=7) for fname in common.AOU_REQUIRED_FILES: blob_name: str = f'{self.folder_prefix}{fname}' test_blob = self.storage_bucket.blob(blob_name) test_blob.upload_from_string('\n') sleep(1) blob_name: str = f'{self.folder_prefix}{common.PROCESSED_TXT}' test_blob = self.storage_bucket.blob(blob_name) test_blob.upload_from_string('\n') bucket_items = gcs_utils.list_bucket(self.hpo_bucket) result = main._get_submission_folder(self.hpo_bucket, bucket_items, force_process=False) self.assertIsNone(result) result = main._get_submission_folder(self.hpo_bucket, bucket_items, force_process=True) self.assertEqual(result, self.folder_prefix)
def test_validate_five_persons_success(self, mock_check_cron): expected_results: list = [] test_file_names: list = [ os.path.basename(f) for f in test_util.FIVE_PERSONS_FILES ] for cdm_filename in common.SUBMISSION_FILES: if cdm_filename in test_file_names: expected_result: tuple = (cdm_filename, 1, 1, 1) test_filepath: str = os.path.join(test_util.FIVE_PERSONS_PATH, cdm_filename) test_blob = self.storage_bucket.blob( f'{self.folder_prefix}{cdm_filename}') test_blob.upload_from_filename(test_filepath) else: expected_result: tuple = (cdm_filename, 0, 0, 0) expected_results.append(expected_result) bucket_items = gcs_utils.list_bucket(self.hpo_bucket) folder_items = main.get_folder_items(bucket_items, self.folder_prefix) r = main.validate_submission(self.hpo_id, self.hpo_bucket, folder_items, self.folder_prefix) self.assertSetEqual(set(r['results']), set(expected_results)) # check tables exist and are clustered as expected for table in resources.CDM_TABLES + common.PII_TABLES: table_id = bq_utils.get_table_id(test_util.FAKE_HPO_ID, table) table_info = bq_utils.get_table_info(table_id) fields = resources.fields_for(table) field_names = [field['name'] for field in fields] if 'person_id' in field_names: self.table_has_clustering(table_info)
def _test_validate_missing_files_output(self, mock_check_cron): # enable exception propagation as described at https://goo.gl/LqDgnj folder_prefix = 'dummy-prefix-2018-03-22/' main.app.testing = True with main.app.test_client() as c: c.get(test_util.VALIDATE_HPO_FILES_URL) # check the result files were placed in bucket bucket_items = gcs_utils.list_bucket(self.hpo_bucket) item_names = [] for item in bucket_items: item_names.append(item['name']) for ignore_file in common.IGNORE_LIST: self.assertIn(folder_prefix + ignore_file, item_names) # check content of result.csv is correct # TODO fix this for all cdm files and use object comparison actual_result = test_util.read_cloud_file( self.hpo_bucket, folder_prefix + common.RESULT_CSV) actual = resources._csv_file_to_list( StringIO.StringIO(actual_result)) expected = [{ 'cdm_file_name': cdm_file_name, 'found': '0', 'parsed': '0', 'loaded': '0' } for cdm_file_name in common.REQUIRED_FILES] self.assertEqual(expected, actual) self.assertFalse( main.all_required_files_loaded(test_util.FAKE_HPO_ID, folder_prefix))
def test_all_files_unparseable_output(self): # TODO possible bug: if no pre-existing table, results in bq table not found error for cdm_table in common.SUBMISSION_FILES: test_util.write_cloud_str(self.hpo_bucket, self.folder_prefix + cdm_table, ".\n .") bucket_items = gcs_utils.list_bucket(self.hpo_bucket) expected_results = [(f, 1, 0, 0) for f in common.SUBMISSION_FILES] r = main.validate_submission(self.hpo_id, self.hpo_bucket, bucket_items, self.folder_prefix) self.assertSetEqual(set(expected_results), set(r['results']))
def _empty_bucket(self, bucket): try: # TODO : catch specific errors bucket_items = gcs_utils.list_bucket(bucket) except: return for bucket_item in bucket_items: gcs_utils.delete_object(bucket, bucket_item['name'])
def test_check_processed(self): test_util.write_cloud_str(self.hpo_bucket, self.folder_prefix + 'person.csv', '\n') test_util.write_cloud_str(self.hpo_bucket, self.folder_prefix + common.PROCESSED_TXT, '\n') bucket_items = gcs_utils.list_bucket(self.hpo_bucket) result = main._get_submission_folder(self.hpo_bucket, bucket_items, force_process=False) self.assertIsNone(result) result = main._get_submission_folder(self.hpo_bucket, bucket_items, force_process=True) self.assertEqual(result, self.folder_prefix)
def test_target_bucket_upload(self): bucket_nyc = gcs_utils.get_hpo_bucket('nyc') folder_prefix = 'test-folder-fake/' test_util.empty_bucket(bucket_nyc) main._upload_achilles_files(hpo_id=None, folder_prefix=folder_prefix, target_bucket=bucket_nyc) actual_bucket_files = set([item['name'] for item in gcs_utils.list_bucket(bucket_nyc)]) expected_bucket_files = set(['test-folder-fake/' + item for item in common.ALL_ACHILLES_INDEX_FILES]) self.assertSetEqual(expected_bucket_files, actual_bucket_files)
def test_validate_five_persons_success(self, mock_check_cron): prefix = 'dummy-prefix-2018-03-22/' expected_result_items = resources._csv_to_list( test_util.FIVE_PERSONS_SUCCESS_RESULT_CSV) json_export_files = self.get_json_export_files(test_util.FAKE_HPO_ID) # upload all five_persons files for cdm_file in test_util.FIVE_PERSONS_FILES: test_util.write_cloud_file(self.hpo_bucket, cdm_file, prefix=prefix) expected_tables = [ 'person', 'visit_occurrence', 'condition_occurrence', 'procedure_occurrence', 'drug_exposure', 'measurement' ] cdm_files = [table + '.csv' for table in expected_tables] main.app.testing = True with main.app.test_client() as c: c.get(test_util.VALIDATE_HPO_FILES_URL) # check the result file was put in bucket expected_object_names = cdm_files + common.IGNORE_LIST + json_export_files expected_objects = [ prefix + item for item in expected_object_names ] list_bucket_result = gcs_utils.list_bucket(self.hpo_bucket) actual_objects = [item['name'] for item in list_bucket_result] self.assertSetEqual(set(expected_objects), set(actual_objects)) # result says file found, parsed, loaded actual_result = test_util.read_cloud_file( self.hpo_bucket, prefix + common.RESULT_CSV) actual_result_file = StringIO.StringIO(actual_result) actual_result_items = resources._csv_file_to_list( actual_result_file) expected_result_items.sort() actual_result_items.sort() self.assertListEqual(expected_result_items, actual_result_items) self.assertTrue( main.all_required_files_loaded(test_util.FAKE_HPO_ID, folder_prefix=prefix)) # check tables exist and are clustered as expected for table in expected_tables: fields_file = os.path.join(resources.fields_path, table + '.json') table_id = bq_utils.get_table_id(test_util.FAKE_HPO_ID, table) table_info = bq_utils.get_table_info(table_id) with open(fields_file, 'r') as fp: fields = json.load(fp) field_names = [field['name'] for field in fields] if 'person_id' in field_names: self.table_has_clustering(table_info)
def list_bucket(bucket): try: return gcs_utils.list_bucket(bucket) except HttpError as err: if err.resp.status == 404: raise BucketDoesNotExistError('Failed to list objects in bucket ', bucket) raise except Exception: raise
def test_check_processed(self): folder_prefix = 'folder/' test_util.write_cloud_str(self.hpo_bucket, folder_prefix + 'person.csv', '\n') test_util.write_cloud_str(self.hpo_bucket, folder_prefix + common.PROCESSED_TXT, '\n') bucket_items = gcs_utils.list_bucket(self.hpo_bucket) result = main._get_to_process_list(self.hpo_bucket, bucket_items, force_process=False) self.assertListEqual([], result) result = main._get_to_process_list(self.hpo_bucket, bucket_items, force_process=True) self.assertListEqual(result, [folder_prefix])
def list_bucket(bucket): try: return gcs_utils.list_bucket(bucket) except HttpError as err: if err.resp.status == 404: raise BucketDoesNotExistError( f"Failed to list objects in bucket {bucket}", bucket) raise except Exception as e: msg = getattr(e, 'message', repr(e)) logging.exception(f"Unknown error {msg}") raise
def test_bad_file_names(self): bad_file_names = ["avisit_occurrence.csv", "condition_occurence.csv", # misspelled "person_final.csv", "procedure_occurrence.tsv"] # unsupported file extension expected_warnings = [] for file_name in bad_file_names: test_util.write_cloud_str(self.hpo_bucket, self.folder_prefix + file_name, ".") expected_item = (file_name, common.UNKNOWN_FILE) expected_warnings.append(expected_item) bucket_items = gcs_utils.list_bucket(self.hpo_bucket) r = main.validate_submission(self.hpo_id, self.hpo_bucket, bucket_items, self.folder_prefix) self.assertListEqual(expected_warnings, r['warnings'])
def test_all_files_unparseable_output(self): # TODO possible bug: if no pre-existing table, results in bq table not found error for cdm_table in common.SUBMISSION_FILES: cdm_blob = self.storage_bucket.blob( f'{self.folder_prefix}{cdm_table}') cdm_blob.upload_from_string('.\n .') bucket_items = gcs_utils.list_bucket(self.hpo_bucket) folder_items = main.get_folder_items(bucket_items, self.folder_prefix) expected_results = [(f, 1, 0, 0) for f in common.SUBMISSION_FILES] r = main.validate_submission(self.hpo_id, self.hpo_bucket, folder_items, self.folder_prefix) self.assertSetEqual(set(expected_results), set(r['results']))
def test_folder_list(self): folder_prefix_1 = 'dummy-prefix-2018-03-22-v1/' folder_prefix_2 = 'dummy-prefix-2018-03-22-v2/' folder_prefix_3 = 'dummy-prefix-2018-03-22-v3/' file_list = [ folder_prefix_1 + 'person.csv', folder_prefix_2 + 'blah.csv', folder_prefix_3 + 'visit_occurrence.csv', 'person.csv' ] for filename in file_list: test_util.write_cloud_str(self.hpo_bucket, filename, ".\n .") bucket_items = gcs_utils.list_bucket(self.hpo_bucket) folder_list = main._get_to_process_list(self.hpo_bucket, bucket_items) self.assertListEqual(folder_list, [folder_prefix_3])
def _test_site_generation(self, mock_check_cron): self._empty_drc_bucket() self._empty_hpo_buckets() with main.app.test_request_context(): result = main._generate_site() self.assertEquals(result, 'okay') # verify that page worked bucket = gcs_utils.get_drc_bucket() expected_files = map(lambda n: n + '.html', main.PAGE_NAMES) + [common.LOG_JSON] file_count = 0 for stat in gcs_utils.list_bucket(bucket): filename = stat['name'] self.assertIn(filename, expected_files) file_count += 1 self.assertEquals(file_count, len(expected_files))