def test_create_cdm_tables(self):
# Sanity check
tables_before = bq_utils.list_tables(self.combined_dataset_id)
table_names_before = [t['tableReference']['tableId'] for t in tables_before]
for table in resources.CDM_TABLES:
self.assertNotIn(table, table_names_before)
create_cdm_tables()
tables_after = bq_utils.list_tables(self.combined_dataset_id)
table_names_after = [t['tableReference']['tableId'] for t in tables_after]
for table in resources.CDM_TABLES:
self.assertIn(table, table_names_after)
def _drop_tables(self):
result = bq_utils.list_tables()
if result['totalItems'] > 0:
for table in result['tables']:
table_id = table['tableReference']['tableId']
if table_id not in common.VOCABULARY_TABLES:
bq_utils.delete_table(table_id)
def construct_query(app_id, dataset_id, all_hpo=False):
"""
Construct query to retrieve most prevalent errors from achilles heel results table(s)
Construct an appropriate and executable query.
:param app_id: Identifies the google cloud project containing the dataset
:param dataset_id: Identifies the dataset where from achilles heel results
should be obtained
:param all_hpo: If `True` query <hpo_id>_achilles_heel_results, otherwise
just achilles_heel_results (default)
:return: The query or :None if no achilles heel results table is found whatsoever
"""
query = None
# Resulting query should only reference existing tables in dataset
all_tables = bq_utils.list_tables(dataset_id)
all_table_ids = [r['tableReference']['tableId'] for r in all_tables]
if all_hpo:
# Fetch and union results from all <hpo_id>_achilles_heel_results tables
subqueries = get_hpo_subqueries(app_id, dataset_id, all_table_ids)
enclosed = ['(%s)' % s for s in subqueries]
query = UNION_ALL.join(enclosed)
else:
# Fetch from achilles_heel_results table
table_id = common.ACHILLES_HEEL_RESULTS
if table_id in all_table_ids:
query = QUERY_FORMAT(dataset_name=dataset_id,
app_id=app_id,
dataset_id=dataset_id,
table_id=table_id)
return query
def list_existing_tables(project_id, dataset_id):
existing_tables = []
all_table_objs = bq_utils.list_tables(project_id=project_id,
dataset_id=dataset_id)
for table_obj in all_table_objs:
table_id = bq_utils.get_table_id_from_obj(table_obj)
existing_tables.append(table_id)
return existing_tables
def _dataset_tables(self, dataset_id):
"""
Get names of existing tables in specified dataset
:param dataset_id: identifies the dataset
:return: list of table_ids
"""
tables = bq_utils.list_tables(dataset_id)
return [table['tableReference']['tableId'] for table in tables]
def get_mapping_table_ids(project_id, combined_dataset_id):
"""
returns all the mapping table ids found in the dataset
:param project_id: project_id containing the dataset
:param combined_dataset_id: dataset_id containing mapping tables
:return: returns mapping table ids
"""
table_objs = bq_utils.list_tables(combined_dataset_id,
project_id=project_id)
mapping_table_ids = []
for table_obj in table_objs:
table_id = bq_utils.get_table_id_from_obj(table_obj)
if MAPPING_PREFIX in table_id:
mapping_table_ids.append(table_id)
return mapping_table_ids
def delete_all_tables(dataset_id):
"""
Remove all non-vocabulary tables from a dataset
:param dataset_id: ID of the dataset with the tables to delete
:return: list of deleted tables
"""
deleted = []
table_infos = bq_utils.list_tables(dataset_id)
table_ids = [table['tableReference']['tableId'] for table in table_infos]
for table_id in table_ids:
if table_id not in common.VOCABULARY_TABLES:
bq_utils.delete_table(table_id, dataset_id)
deleted.append(table_id)
return deleted
def delete_all_tables(dataset_id):
"""
Remove all non-vocabulary tables from a dataset
:param dataset_id: ID of the dataset with the tables to delete
:return: list of deleted tables
"""
import bq_utils
deleted = []
result = bq_utils.list_tables(dataset_id)
tables = result.get('tables', [])
for table in tables:
table_id = table['tableReference']['tableId']
if table_id not in common.VOCABULARY_TABLES:
bq_utils.delete_table(table_id, dataset_id)
deleted.append(table_id)
return deleted
def _drop_tables(self):
tables = bq_utils.list_tables()
for table in tables:
table_id = table['tableReference']['tableId']
if table_id not in common.VOCABULARY_TABLES:
bq_utils.delete_table(table_id)
def _list_all_table_ids(dataset_id):
tables = bq_utils.list_tables(dataset_id)
return [table['tableReference']['tableId'] for table in tables]
def _list_all_table_ids(dataset_id):
result = bq_utils.list_tables(dataset_id)
tables = result.get('tables', [])
return [table['tableReference']['tableId'] for table in tables]