def _drop_tables(self):
result = bq_utils.list_tables()
if result['totalItems'] > 0:
for table in result['tables']:
table_id = table['tableReference']['tableId']
if table_id not in common.VOCABULARY_TABLES:
bq_utils.delete_table(table_id)
def test_integration_create_drug_route_mappings_table(self):
if bq_utils.table_exists(populate_route_ids.DRUG_ROUTES_TABLE_ID,
dataset_id=self.dataset_id):
bq_utils.delete_table(populate_route_ids.DRUG_ROUTES_TABLE_ID,
dataset_id=self.dataset_id)
if not bq_utils.table_exists(
populate_route_ids.DOSE_FORM_ROUTES_TABLE_ID,
dataset_id=self.dataset_id):
populate_route_ids.create_dose_form_route_mappings_table(
self.project_id, self.dataset_id)
populate_route_ids.create_drug_route_mappings_table(
self.project_id, self.dataset_id,
populate_route_ids.DOSE_FORM_ROUTES_TABLE_ID,
self.route_mapping_prefix)
time.sleep(10)
query = ("SELECT COUNT(*) AS n "
"FROM `{project_id}.{dataset_id}.{table_id}`").format(
project_id=self.project_id,
dataset_id=self.dataset_id,
table_id=populate_route_ids.DRUG_ROUTES_TABLE_ID)
result = bq_utils.query(query)
actual = bq_utils.response2rows(result)
self.assertGreater(actual[0]["n"], 0)
def run_analyses(hpo_id):
"""
Run the achilles analyses
:param hpo_id:
:return:
"""
commands = _get_run_analysis_commands(hpo_id)
for command in commands:
logging.debug(' ---- Running `%s`...\n' % command)
if sql_wrangle.is_to_temp_table(command):
table_id = sql_wrangle.get_temp_table_name(command)
query = sql_wrangle.get_temp_table_query(command)
insert_query_job_result = bq_utils.query(query, False, table_id)
query_job_id = insert_query_job_result['jobReference']['jobId']
incomplete_jobs = bq_utils.wait_on_jobs([query_job_id])
if len(incomplete_jobs) > 0:
logging.critical('tempresults doesnt get created in 15 secs')
raise RuntimeError('Tempresults taking too long to create')
elif sql_wrangle.is_truncate(command):
table_id = sql_wrangle.get_truncate_table_name(command)
if bq_utils.table_exists(table_id):
bq_utils.delete_table(table_id)
elif sql_wrangle.is_drop(command):
table_id = sql_wrangle.get_drop_table_name(command)
if bq_utils.table_exists(table_id):
bq_utils.delete_table(table_id)
else:
bq_utils.query(command)
def run_heel(hpo_id):
"""
Run heel commands
:param hpo_id: string name for the hpo identifier
:returns: None
:raises RuntimeError: Raised if BigQuery takes longer than 30 seconds
to complete a job on a temporary table
"""
# very long test
commands = _get_heel_commands(hpo_id)
count = 0
for command in commands:
count = count + 1
logging.debug(' ---- running query # {}'.format(count))
logging.debug(' ---- Running `%s`...\n' % command)
if sql_wrangle.is_to_temp_table(command):
table_id = sql_wrangle.get_temp_table_name(command)
query = sql_wrangle.get_temp_table_query(command)
insert_query_job_result = bq_utils.query(query, False, table_id)
query_job_id = insert_query_job_result['jobReference']['jobId']
incomplete_jobs = bq_utils.wait_on_jobs([query_job_id])
if len(incomplete_jobs) > 0:
logging.critical('tempresults doesnt get created in 30 secs')
raise RuntimeError('Tempresults taking too long to create')
elif sql_wrangle.is_truncate(command):
table_id = sql_wrangle.get_truncate_table_name(command)
query = 'DELETE FROM %s WHERE TRUE' % table_id
bq_utils.query(query)
elif sql_wrangle.is_drop(command):
table_id = sql_wrangle.get_drop_table_name(command)
bq_utils.delete_table(table_id)
else:
bq_utils.query(command)
def run_heel(hpo_id):
# very long test
commands = _get_heel_commands(hpo_id)
count = 0
for command in commands:
count = count + 1
logging.debug(' ---- running query # {}'.format(count))
logging.debug(' ---- Running `%s`...\n' % command)
if sql_wrangle.is_to_temp_table(command):
table_id = sql_wrangle.get_temp_table_name(command)
query = sql_wrangle.get_temp_table_query(command)
insert_query_job_result = bq_utils.query(query, False, table_id)
query_job_id = insert_query_job_result['jobReference']['jobId']
incomplete_jobs = bq_utils.wait_on_jobs([query_job_id])
if len(incomplete_jobs) > 0:
logging.critical('tempresults doesnt get created in 30 secs')
raise RuntimeError('Tempresults taking too long to create')
elif sql_wrangle.is_truncate(command):
table_id = sql_wrangle.get_truncate_table_name(command)
query = 'DELETE FROM %s WHERE TRUE' % table_id
bq_utils.query(query)
elif sql_wrangle.is_drop(command):
table_id = sql_wrangle.get_drop_table_name(command)
bq_utils.delete_table(table_id)
else:
bq_utils.query(command)
time.sleep(1)
def tearDown(self):
delete_list = ['visit_id_mapping_table'] + [
'unioned_ehr_' + table_name for table_name in common.CDM_TABLES
]
existing_tables = bq_utils.list_dataset_contents(
bq_utils.get_dataset_id())
for table_id in delete_list:
if table_id not in common.VOCABULARY_TABLES and table_id in existing_tables:
bq_utils.delete_table(table_id)
self._empty_bucket(self.hpo_bucket)
self.testbed.deactivate()
def drop_or_truncate_table(command):
"""
Deletes or truncates table
:param command: query to run
:return: None
"""
if sql_wrangle.is_truncate(command):
table_id = sql_wrangle.get_truncate_table_name(command)
query = 'DELETE FROM %s WHERE TRUE' % table_id
bq_utils.query(query)
else:
table_id = sql_wrangle.get_drop_table_name(command)
bq_utils.delete_table(table_id)
def drop_or_truncate_table(command):
"""
Deletes or truncates table
Previously, deletion was used for both truncate and drop, and this function retains the behavior
:param command: query to run
:return: None
"""
if sql_wrangle.is_truncate(command):
table_id = sql_wrangle.get_truncate_table_name(command)
else:
table_id = sql_wrangle.get_drop_table_name(command)
if bq_utils.table_exists(table_id):
bq_utils.delete_table(table_id)
def delete_all_tables(dataset_id):
"""
Remove all non-vocabulary tables from a dataset
:param dataset_id: ID of the dataset with the tables to delete
:return: list of deleted tables
"""
deleted = []
table_infos = bq_utils.list_tables(dataset_id)
table_ids = [table['tableReference']['tableId'] for table in table_infos]
for table_id in table_ids:
if table_id not in common.VOCABULARY_TABLES:
bq_utils.delete_table(table_id, dataset_id)
deleted.append(table_id)
return deleted
def delete_all_tables(dataset_id):
"""
Remove all non-vocabulary tables from a dataset
:param dataset_id: ID of the dataset with the tables to delete
:return: list of deleted tables
"""
import bq_utils
deleted = []
result = bq_utils.list_tables(dataset_id)
tables = result.get('tables', [])
for table in tables:
table_id = table['tableReference']['tableId']
if table_id not in common.VOCABULARY_TABLES:
bq_utils.delete_table(table_id, dataset_id)
deleted.append(table_id)
return deleted
def _drop_tables(self):
tables = bq_utils.list_tables()
for table in tables:
table_id = table['tableReference']['tableId']
if table_id not in common.VOCABULARY_TABLES:
bq_utils.delete_table(table_id)
def tearDown(self):
bq_utils.delete_table(gen_ext.SITE_TABLE_ID, self.bq_dataset_id)
