def test_run_export(self):
folder_prefix = 'dummy-prefix-2018-03-24/'
main._upload_achilles_files(test_util.FAKE_HPO_ID, folder_prefix)
main.run_export(hpo_id=test_util.FAKE_HPO_ID,
folder_prefix=folder_prefix)
bucket_objects = gcs_utils.list_bucket(self.hpo_bucket)
actual_object_names = [obj['name'] for obj in bucket_objects]
for report in common.ALL_REPORT_FILES:
prefix = folder_prefix + common.ACHILLES_EXPORT_PREFIX_STRING + test_util.FAKE_HPO_ID + '/'
expected_object_name = prefix + report
self.assertIn(expected_object_name, actual_object_names)
datasources_json_path = folder_prefix + common.ACHILLES_EXPORT_DATASOURCES_JSON
self.assertIn(datasources_json_path, actual_object_names)
datasources_json = gcs_utils.get_object(self.hpo_bucket,
datasources_json_path)
datasources_actual = json.loads(datasources_json)
datasources_expected = {
'datasources': [{
'name': test_util.FAKE_HPO_ID,
'folder': test_util.FAKE_HPO_ID,
'cdmVersion': 5
}]
}
self.assertDictEqual(datasources_expected, datasources_actual)
def test_run_export(self, mock_is_hpo_id):
# validation/main.py INTEGRATION TEST
mock_is_hpo_id.return_value = True
folder_prefix: str = 'dummy-prefix-2018-03-24/'
main._upload_achilles_files(FAKE_HPO_ID, folder_prefix)
main.run_export(datasource_id=FAKE_HPO_ID, folder_prefix=folder_prefix)
storage_bucket = self.storage_client.get_bucket(self.hpo_bucket)
bucket_objects = storage_bucket.list_blobs()
actual_object_names: list = [obj.name for obj in bucket_objects]
for report in common.ALL_REPORT_FILES:
prefix: str = f'{folder_prefix}{common.ACHILLES_EXPORT_PREFIX_STRING}{FAKE_HPO_ID}/'
expected_object_name: str = f'{prefix}{report}'
self.assertIn(expected_object_name, actual_object_names)
datasources_json_path: str = folder_prefix + common.ACHILLES_EXPORT_DATASOURCES_JSON
self.assertIn(datasources_json_path, actual_object_names)
datasources_blob = storage_bucket.blob(datasources_json_path)
datasources_json: str = datasources_blob.download_as_bytes().decode()
datasources_actual: dict = json.loads(datasources_json)
datasources_expected: dict = {
'datasources': [{
'name': FAKE_HPO_ID,
'folder': FAKE_HPO_ID,
'cdmVersion': 5
}]
}
self.assertDictEqual(datasources_expected, datasources_actual)
def main(args):
folder = args.folder
target_bucket = args.bucket
folder_prefix = folder + '/'
_run_achilles()
_run_export(folder_prefix=folder_prefix, target_bucket=target_bucket)
_upload_achilles_files(folder_prefix=folder_prefix,
target_bucket=target_bucket)
def test_target_bucket_upload(self):
bucket_nyc = gcs_utils.get_hpo_bucket('nyc')
folder_prefix = 'test-folder-fake/'
test_util.empty_bucket(bucket_nyc)
main._upload_achilles_files(hpo_id=None, folder_prefix=folder_prefix, target_bucket=bucket_nyc)
actual_bucket_files = set([item['name'] for item in gcs_utils.list_bucket(bucket_nyc)])
expected_bucket_files = set(['test-folder-fake/' + item for item in common.ALL_ACHILLES_INDEX_FILES])
self.assertSetEqual(expected_bucket_files, actual_bucket_files)
def test_run_export(self):
folder_prefix = 'dummy-prefix-2018-03-24/'
main._upload_achilles_files(test_util.FAKE_HPO_ID, folder_prefix)
main.run_export(test_util.FAKE_HPO_ID, folder_prefix)
for report in common.ALL_REPORT_FILES:
_reports_prefix = folder_prefix + common.ACHILLES_EXPORT_PREFIX_STRING + test_util.FAKE_HPO_ID + '/'
_exist_check = gcs_utils.get_metadata(self.hpo_bucket,
_reports_prefix + report)
self.assertIsNotNone(_exist_check)
def main(args):
dataset_id = get_dataset_id()
target_bucket = args.bucket
folder_prefix = args.folder + '/'
_run_achilles()
_run_export(datasource_id=dataset_id,
folder_prefix=folder_prefix,
target_bucket=target_bucket)
_upload_achilles_files(folder_prefix=folder_prefix,
target_bucket=target_bucket)
def main(args):
hpo_id = args.hpo_id
folder = args.folder
folder_prefix = folder + '/'
for table_name in common.CDM_TABLES:
table_id = hpo_id + '_' + table_name
if bq_utils.table_exists(table_id):
print table_id, ' exists'
else:
print table_id, ' being created'
bq_utils.create_standard_table(table_name, table_id, False)
_run_achilles(hpo_id)
_run_export(hpo_id, folder_prefix)
_upload_achilles_files(hpo_id, folder_prefix)
def main(args):
folder = args.folder
target_bucket = args.bucket
folder_prefix = folder + '/'
for table_name in common.CDM_TABLES:
table_id = table_name
if bq_utils.table_exists(table_id):
print table_id, ' exists'
else:
print table_id, ' being created'
bq_utils.create_standard_table(table_name, table_id, False)
_run_achilles()
_run_export(folder_prefix=folder_prefix, target_bucket=target_bucket)
_upload_achilles_files(folder_prefix=folder_prefix,
target_bucket=target_bucket)