def _drop_tables(self): result = bq_utils.list_tables() if result['totalItems'] > 0: for table in result['tables']: table_id = table['tableReference']['tableId'] if table_id not in common.VOCABULARY_TABLES: bq_utils.delete_table(table_id)
def test_integration_create_drug_route_mappings_table(self): if bq_utils.table_exists(populate_route_ids.DRUG_ROUTES_TABLE_ID, dataset_id=self.dataset_id): bq_utils.delete_table(populate_route_ids.DRUG_ROUTES_TABLE_ID, dataset_id=self.dataset_id) if not bq_utils.table_exists( populate_route_ids.DOSE_FORM_ROUTES_TABLE_ID, dataset_id=self.dataset_id): populate_route_ids.create_dose_form_route_mappings_table( self.project_id, self.dataset_id) populate_route_ids.create_drug_route_mappings_table( self.project_id, self.dataset_id, populate_route_ids.DOSE_FORM_ROUTES_TABLE_ID, self.route_mapping_prefix) time.sleep(10) query = ("SELECT COUNT(*) AS n " "FROM `{project_id}.{dataset_id}.{table_id}`").format( project_id=self.project_id, dataset_id=self.dataset_id, table_id=populate_route_ids.DRUG_ROUTES_TABLE_ID) result = bq_utils.query(query) actual = bq_utils.response2rows(result) self.assertGreater(actual[0]["n"], 0)
def run_analyses(hpo_id): """ Run the achilles analyses :param hpo_id: :return: """ commands = _get_run_analysis_commands(hpo_id) for command in commands: logging.debug(' ---- Running `%s`...\n' % command) if sql_wrangle.is_to_temp_table(command): table_id = sql_wrangle.get_temp_table_name(command) query = sql_wrangle.get_temp_table_query(command) insert_query_job_result = bq_utils.query(query, False, table_id) query_job_id = insert_query_job_result['jobReference']['jobId'] incomplete_jobs = bq_utils.wait_on_jobs([query_job_id]) if len(incomplete_jobs) > 0: logging.critical('tempresults doesnt get created in 15 secs') raise RuntimeError('Tempresults taking too long to create') elif sql_wrangle.is_truncate(command): table_id = sql_wrangle.get_truncate_table_name(command) if bq_utils.table_exists(table_id): bq_utils.delete_table(table_id) elif sql_wrangle.is_drop(command): table_id = sql_wrangle.get_drop_table_name(command) if bq_utils.table_exists(table_id): bq_utils.delete_table(table_id) else: bq_utils.query(command)
def run_heel(hpo_id): """ Run heel commands :param hpo_id: string name for the hpo identifier :returns: None :raises RuntimeError: Raised if BigQuery takes longer than 30 seconds to complete a job on a temporary table """ # very long test commands = _get_heel_commands(hpo_id) count = 0 for command in commands: count = count + 1 logging.debug(' ---- running query # {}'.format(count)) logging.debug(' ---- Running `%s`...\n' % command) if sql_wrangle.is_to_temp_table(command): table_id = sql_wrangle.get_temp_table_name(command) query = sql_wrangle.get_temp_table_query(command) insert_query_job_result = bq_utils.query(query, False, table_id) query_job_id = insert_query_job_result['jobReference']['jobId'] incomplete_jobs = bq_utils.wait_on_jobs([query_job_id]) if len(incomplete_jobs) > 0: logging.critical('tempresults doesnt get created in 30 secs') raise RuntimeError('Tempresults taking too long to create') elif sql_wrangle.is_truncate(command): table_id = sql_wrangle.get_truncate_table_name(command) query = 'DELETE FROM %s WHERE TRUE' % table_id bq_utils.query(query) elif sql_wrangle.is_drop(command): table_id = sql_wrangle.get_drop_table_name(command) bq_utils.delete_table(table_id) else: bq_utils.query(command)
def run_heel(hpo_id): # very long test commands = _get_heel_commands(hpo_id) count = 0 for command in commands: count = count + 1 logging.debug(' ---- running query # {}'.format(count)) logging.debug(' ---- Running `%s`...\n' % command) if sql_wrangle.is_to_temp_table(command): table_id = sql_wrangle.get_temp_table_name(command) query = sql_wrangle.get_temp_table_query(command) insert_query_job_result = bq_utils.query(query, False, table_id) query_job_id = insert_query_job_result['jobReference']['jobId'] incomplete_jobs = bq_utils.wait_on_jobs([query_job_id]) if len(incomplete_jobs) > 0: logging.critical('tempresults doesnt get created in 30 secs') raise RuntimeError('Tempresults taking too long to create') elif sql_wrangle.is_truncate(command): table_id = sql_wrangle.get_truncate_table_name(command) query = 'DELETE FROM %s WHERE TRUE' % table_id bq_utils.query(query) elif sql_wrangle.is_drop(command): table_id = sql_wrangle.get_drop_table_name(command) bq_utils.delete_table(table_id) else: bq_utils.query(command) time.sleep(1)
def tearDown(self): delete_list = ['visit_id_mapping_table'] + [ 'unioned_ehr_' + table_name for table_name in common.CDM_TABLES ] existing_tables = bq_utils.list_dataset_contents( bq_utils.get_dataset_id()) for table_id in delete_list: if table_id not in common.VOCABULARY_TABLES and table_id in existing_tables: bq_utils.delete_table(table_id) self._empty_bucket(self.hpo_bucket) self.testbed.deactivate()
def drop_or_truncate_table(command): """ Deletes or truncates table :param command: query to run :return: None """ if sql_wrangle.is_truncate(command): table_id = sql_wrangle.get_truncate_table_name(command) query = 'DELETE FROM %s WHERE TRUE' % table_id bq_utils.query(query) else: table_id = sql_wrangle.get_drop_table_name(command) bq_utils.delete_table(table_id)
def drop_or_truncate_table(command): """ Deletes or truncates table Previously, deletion was used for both truncate and drop, and this function retains the behavior :param command: query to run :return: None """ if sql_wrangle.is_truncate(command): table_id = sql_wrangle.get_truncate_table_name(command) else: table_id = sql_wrangle.get_drop_table_name(command) if bq_utils.table_exists(table_id): bq_utils.delete_table(table_id)
def delete_all_tables(dataset_id): """ Remove all non-vocabulary tables from a dataset :param dataset_id: ID of the dataset with the tables to delete :return: list of deleted tables """ deleted = [] table_infos = bq_utils.list_tables(dataset_id) table_ids = [table['tableReference']['tableId'] for table in table_infos] for table_id in table_ids: if table_id not in common.VOCABULARY_TABLES: bq_utils.delete_table(table_id, dataset_id) deleted.append(table_id) return deleted
def delete_all_tables(dataset_id): """ Remove all non-vocabulary tables from a dataset :param dataset_id: ID of the dataset with the tables to delete :return: list of deleted tables """ import bq_utils deleted = [] result = bq_utils.list_tables(dataset_id) tables = result.get('tables', []) for table in tables: table_id = table['tableReference']['tableId'] if table_id not in common.VOCABULARY_TABLES: bq_utils.delete_table(table_id, dataset_id) deleted.append(table_id) return deleted
def _drop_tables(self): tables = bq_utils.list_tables() for table in tables: table_id = table['tableReference']['tableId'] if table_id not in common.VOCABULARY_TABLES: bq_utils.delete_table(table_id)
def tearDown(self): bq_utils.delete_table(gen_ext.SITE_TABLE_ID, self.bq_dataset_id)