def setUp(self): self.app_id = app_identity.get_application_id() self.dataset_id = bq_utils.get_dataset_id() self.bucket = gcs_utils.get_drc_bucket() test_util.empty_bucket(self.bucket) test_util.delete_all_tables(self.dataset_id) self.load_test_data(hpo_id=HPO_NYC)
def test_target_bucket_upload(self): bucket_nyc = gcs_utils.get_hpo_bucket('nyc') folder_prefix = 'test-folder-fake/' test_util.empty_bucket(bucket_nyc) main._upload_achilles_files(hpo_id=None, folder_prefix=folder_prefix, target_bucket=bucket_nyc) actual_bucket_files = set( [item['name'] for item in gcs_utils.list_bucket(bucket_nyc)]) expected_bucket_files = set([ 'test-folder-fake/' + item for item in resources.ALL_ACHILLES_INDEX_FILES ]) self.assertSetEqual(expected_bucket_files, actual_bucket_files)
def setUp(self): self.hpo_bucket = gcs_utils.get_hpo_bucket(FAKE_HPO_ID) self.project_id = app_identity.get_application_id() self.dataset_id = bq_utils.get_dataset_id() self.rdr_dataset_id = bq_utils.get_rdr_dataset_id() self.folder_prefix = '2019-01-01/' test_util.delete_all_tables(self.dataset_id) test_util.empty_bucket(self.hpo_bucket) mock_get_hpo_name = mock.patch('validation.main.get_hpo_name') self.mock_get_hpo_name = mock_get_hpo_name.start() self.mock_get_hpo_name.return_value = 'Fake HPO' self.addCleanup(mock_get_hpo_name.stop) self._load_data()
def load_dataset_from_files(dataset_id, path, mappings=False): bucket = gcs_utils.get_hpo_bucket(test_util.FAKE_HPO_ID) test_util.empty_bucket(bucket) job_ids = [] for table in resources.CDM_TABLES: job_ids.append( CombineEhrRdrTest._upload_file_to_bucket( bucket, dataset_id, path, table)) if mappings and table in DOMAIN_TABLES: mapping_table = '_mapping_{table}'.format(table=table) job_ids.append( CombineEhrRdrTest._upload_file_to_bucket( bucket, dataset_id, path, mapping_table)) incomplete_jobs = bq_utils.wait_on_jobs(job_ids) if len(incomplete_jobs) > 0: message = "Job id(s) %s failed to complete" % incomplete_jobs raise RuntimeError(message) test_util.empty_bucket(bucket)
def tearDown(self): self._empty_bucket() bucket_nyc = gcs_utils.get_hpo_bucket('nyc') test_util.empty_bucket(bucket_nyc) test_util.empty_bucket(gcs_utils.get_drc_bucket()) test_util.delete_all_tables(self.bigquery_dataset_id)
def _empty_hpo_buckets(self): for hpo_id in self.hpo_ids: bucket = gcs_utils.get_hpo_bucket(hpo_id) test_util.empty_bucket(bucket)
def tearDown(self): self._empty_bucket() test_util.empty_bucket(self.bucket)
def tearDown(self): test_util.delete_all_tables(bq_utils.get_dataset_id()) test_util.empty_bucket(self.hpo_bucket)
def setUp(self): self.hpo_bucket = gcs_utils.get_hpo_bucket(test_util.FAKE_HPO_ID) test_util.empty_bucket(self.hpo_bucket) test_util.delete_all_tables(bq_utils.get_dataset_id())
def tearDown(self): self._empty_bucket() bucket_nyc = gcs_utils.get_hpo_bucket('nyc') test_util.empty_bucket(bucket_nyc)
def tearDown(self): test_util.empty_bucket(self.bucket) test_util.delete_all_tables(self.dataset_id)