def test_run_export(self): folder_prefix = 'dummy-prefix-2018-03-24/' main._upload_achilles_files(test_util.FAKE_HPO_ID, folder_prefix) main.run_export(hpo_id=test_util.FAKE_HPO_ID, folder_prefix=folder_prefix) bucket_objects = gcs_utils.list_bucket(self.hpo_bucket) actual_object_names = [obj['name'] for obj in bucket_objects] for report in common.ALL_REPORT_FILES: prefix = folder_prefix + common.ACHILLES_EXPORT_PREFIX_STRING + test_util.FAKE_HPO_ID + '/' expected_object_name = prefix + report self.assertIn(expected_object_name, actual_object_names) datasources_json_path = folder_prefix + common.ACHILLES_EXPORT_DATASOURCES_JSON self.assertIn(datasources_json_path, actual_object_names) datasources_json = gcs_utils.get_object(self.hpo_bucket, datasources_json_path) datasources_actual = json.loads(datasources_json) datasources_expected = { 'datasources': [{ 'name': test_util.FAKE_HPO_ID, 'folder': test_util.FAKE_HPO_ID, 'cdmVersion': 5 }] } self.assertDictEqual(datasources_expected, datasources_actual)
def test_run_export(self, mock_is_hpo_id): # validation/main.py INTEGRATION TEST mock_is_hpo_id.return_value = True folder_prefix: str = 'dummy-prefix-2018-03-24/' main._upload_achilles_files(FAKE_HPO_ID, folder_prefix) main.run_export(datasource_id=FAKE_HPO_ID, folder_prefix=folder_prefix) storage_bucket = self.storage_client.get_bucket(self.hpo_bucket) bucket_objects = storage_bucket.list_blobs() actual_object_names: list = [obj.name for obj in bucket_objects] for report in common.ALL_REPORT_FILES: prefix: str = f'{folder_prefix}{common.ACHILLES_EXPORT_PREFIX_STRING}{FAKE_HPO_ID}/' expected_object_name: str = f'{prefix}{report}' self.assertIn(expected_object_name, actual_object_names) datasources_json_path: str = folder_prefix + common.ACHILLES_EXPORT_DATASOURCES_JSON self.assertIn(datasources_json_path, actual_object_names) datasources_blob = storage_bucket.blob(datasources_json_path) datasources_json: str = datasources_blob.download_as_bytes().decode() datasources_actual: dict = json.loads(datasources_json) datasources_expected: dict = { 'datasources': [{ 'name': FAKE_HPO_ID, 'folder': FAKE_HPO_ID, 'cdmVersion': 5 }] } self.assertDictEqual(datasources_expected, datasources_actual)
def main(args): folder = args.folder target_bucket = args.bucket folder_prefix = folder + '/' _run_achilles() _run_export(folder_prefix=folder_prefix, target_bucket=target_bucket) _upload_achilles_files(folder_prefix=folder_prefix, target_bucket=target_bucket)
def test_target_bucket_upload(self): bucket_nyc = gcs_utils.get_hpo_bucket('nyc') folder_prefix = 'test-folder-fake/' test_util.empty_bucket(bucket_nyc) main._upload_achilles_files(hpo_id=None, folder_prefix=folder_prefix, target_bucket=bucket_nyc) actual_bucket_files = set([item['name'] for item in gcs_utils.list_bucket(bucket_nyc)]) expected_bucket_files = set(['test-folder-fake/' + item for item in common.ALL_ACHILLES_INDEX_FILES]) self.assertSetEqual(expected_bucket_files, actual_bucket_files)
def test_run_export(self): folder_prefix = 'dummy-prefix-2018-03-24/' main._upload_achilles_files(test_util.FAKE_HPO_ID, folder_prefix) main.run_export(test_util.FAKE_HPO_ID, folder_prefix) for report in common.ALL_REPORT_FILES: _reports_prefix = folder_prefix + common.ACHILLES_EXPORT_PREFIX_STRING + test_util.FAKE_HPO_ID + '/' _exist_check = gcs_utils.get_metadata(self.hpo_bucket, _reports_prefix + report) self.assertIsNotNone(_exist_check)
def main(args): dataset_id = get_dataset_id() target_bucket = args.bucket folder_prefix = args.folder + '/' _run_achilles() _run_export(datasource_id=dataset_id, folder_prefix=folder_prefix, target_bucket=target_bucket) _upload_achilles_files(folder_prefix=folder_prefix, target_bucket=target_bucket)
def main(args): hpo_id = args.hpo_id folder = args.folder folder_prefix = folder + '/' for table_name in common.CDM_TABLES: table_id = hpo_id + '_' + table_name if bq_utils.table_exists(table_id): print table_id, ' exists' else: print table_id, ' being created' bq_utils.create_standard_table(table_name, table_id, False) _run_achilles(hpo_id) _run_export(hpo_id, folder_prefix) _upload_achilles_files(hpo_id, folder_prefix)
def main(args): folder = args.folder target_bucket = args.bucket folder_prefix = folder + '/' for table_name in common.CDM_TABLES: table_id = table_name if bq_utils.table_exists(table_id): print table_id, ' exists' else: print table_id, ' being created' bq_utils.create_standard_table(table_name, table_id, False) _run_achilles() _run_export(folder_prefix=folder_prefix, target_bucket=target_bucket) _upload_achilles_files(folder_prefix=folder_prefix, target_bucket=target_bucket)