def test_exist_domain_mappings(self):
self.assertTrue(
domain_mapping.exist_domain_mappings(self.condition_table,
self.procedure_table))
self.assertTrue(
domain_mapping.exist_domain_mappings(self.procedure_table,
self.condition_table))
self.assertFalse(
domain_mapping.exist_domain_mappings(self.procedure_table,
self.measurement_table))
self.assertFalse(
domain_mapping.exist_domain_mappings(self.condition_table,
self.measurement_table))
def get_reroute_domain_mapping_queries(project_id, dataset_id):
"""
The functions generates a list of query dicts for rerouting the mapping records to the
approapriate domain.
:param project_id: the project_id in which the query is run
:param dataset_id: the dataset_id in which the query is run
:return: a list of query dicts for rerouting the mapping records to the corresponding mapping
table
"""
queries = []
for dest_table in domain_mapping.DOMAIN_TABLE_NAMES:
# Figure out all possible rerouting source tables for a given destination table
src_tables = [
src_table for src_table in domain_mapping.DOMAIN_TABLE_NAMES
if (src_table == dest_table)
or domain_mapping.exist_domain_mappings(src_table, dest_table)
]
queries.append({
cdr_consts.QUERY:
REROUTE_DOMAIN_MAPPING_RECORD_QUERY.render(project_id=project_id,
dataset_id=dataset_id,
src_tables=src_tables,
dest_table=dest_table),
cdr_consts.DESTINATION_TABLE:
mapping_table_for(dest_table),
cdr_consts.DISPOSITION:
bq_consts.WRITE_TRUNCATE,
cdr_consts.DESTINATION_DATASET:
dataset_id
})
return queries
def parse_reroute_domain_query(project_id, dataset_id, dest_table):
"""
This function generates a query that reroutes the records from all domain tables for the given dest_table.
It uses _mapping_alignment_table to determine in which domain table the records should land.
:param project_id: the project_id in which the query is run
:param dataset_id: the dataset_id in which the query is run
:param dest_table: the destination CDM table for rerouting
:return: a query that reroutes the records from all domain tables for the given dest_table
"""
union_query = EMPTY_STRING
for src_table in domain_mapping.DOMAIN_TABLE_NAMES:
if src_table == dest_table or domain_mapping.exist_domain_mappings(
src_table, dest_table):
src_domain_id_field = get_domain_id_field(src_table)
dest_domain_id_field = get_domain_id_field(dest_table)
field_mapping_expr = resolve_field_mappings(src_table, dest_table)
if union_query != EMPTY_STRING:
union_query += UNION_ALL
union_query += REROUTE_DOMAIN_RECORD_QUERY.format(
project_id=project_id,
dataset_id=dataset_id,
src_table=src_table,
dest_table=dest_table,
src_domain_id_field=src_domain_id_field,
dest_domain_id_field=dest_domain_id_field,
field_mapping_expr=field_mapping_expr)
return union_query
def parse_domain_mapping_query_cross_domain(project_id, dataset_id,
dest_table):
"""
This function creates a query that generates id mappings in _logging_domain_alignment
for the rerouting records for dest_table
:param project_id: the project_id in which the query is run
:param dataset_id: the dataset_id in which the query is run
:param dest_table: the destination table to which the records are rerouted
:return: the query that generates id mappings for the rerouting records
"""
union_query = EMPTY_STRING
domain = resources.get_domain(dest_table)
dest_id_field = resources.get_domain_id_field(dest_table)
for src_table in domain_mapping.DOMAIN_TABLE_NAMES:
if src_table != dest_table and domain_mapping.exist_domain_mappings(
src_table, dest_table):
src_id_field = resources.get_domain_id_field(src_table)
domain_concept_id = resources.get_domain_concept_id(src_table)
if union_query != EMPTY_STRING:
union_query += UNION_ALL
union_query += DOMAIN_REROUTE_INCLUDED_INNER_QUERY.render(
project_id=project_id,
dataset_id=dataset_id,
src_table=src_table,
dest_table=dest_table,
src_id=src_id_field,
dest_id=NULL_VALUE,
domain_concept_id=domain_concept_id,
domain='\'{}\''.format(domain))
criteria = domain_mapping.get_rerouting_criteria(
src_table, dest_table)
if criteria != EMPTY_STRING:
union_query += AND + criteria
output_query = EMPTY_STRING
if union_query != EMPTY_STRING:
# the query to get the max id for the dest table
domain_query = MAXIMUM_DOMAIN_ID_QUERY.render(
project_id=project_id,
dataset_id=dataset_id,
domain_table=dest_table,
domain_id_field=dest_id_field)
output_query = DOMAIN_MAPPING_OUTER_QUERY.render(
union_query=union_query, domain_query=domain_query)
return output_query
def parse_reroute_domain_query(project_id, dataset_id, dest_table):
"""
This function generates a query that reroutes the records from all domain tables for the given dest_table.
It uses _mapping_alignment_table to determine in which domain table the records should land.
:param project_id: the project_id in which the query is run
:param dataset_id: the dataset_id in which the query is run
:param dest_table: the destination CDM table for rerouting
:return: a query that reroutes the records from all domain tables for the given dest_table
"""
union_queries = []
for src_table in domain_mapping.DOMAIN_TABLE_NAMES:
src_domain_id_field = get_domain_id_field(src_table)
dest_domain_id_field = get_domain_id_field(dest_table)
field_mapping_expr = resolve_field_mappings(src_table, dest_table)
if src_table == dest_table:
# We are doing this to make sure the schema doesn't change and also keep all the
# records in the domain table for later rerouting to the other domains
union_queries.append(
SELECT_DOMAIN_RECORD_QUERY.render(
project_id=project_id,
dataset_id=dataset_id,
dest_table=dest_table,
field_mapping_expr=field_mapping_expr,
dest_domain_id_field=dest_domain_id_field))
elif domain_mapping.exist_domain_mappings(src_table, dest_table):
# We are only rerouting the records between domain tables that are not the same
union_queries.append(
REROUTE_DOMAIN_RECORD_QUERY.render(
project_id=project_id,
dataset_id=dataset_id,
src_table=src_table,
dest_table=dest_table,
src_domain_id_field=src_domain_id_field,
dest_domain_id_field=dest_domain_id_field,
field_mapping_expr=field_mapping_expr))
return UNION_ALL.join(union_queries)
"""
fields = CDM_TABLE_SCHEMAS[_domain_table]
for field in fields:
if field[NAME_FIELD] == field_name:
return field[FIELD_MODE] == FIELD_REQUIRED
return False
if __name__ == '__main__':
with open(resources.field_mappings_replaced_path, 'w') as fr:
fr.write(FIELD_MAPPING_HEADER)
field_dict = create_domain_field_dict()
for src_table in DOMAIN_TABLE_NAMES:
for dest_table in DOMAIN_TABLE_NAMES:
if src_table == dest_table or exist_domain_mappings(
src_table, dest_table):
field_mappings = generate_field_mappings(
src_table, dest_table, field_dict[src_table],
field_dict[dest_table])
for dest_field, src_field in field_mappings.items():
translation = 1 if TYPE_CONCEPT_SUFFIX in src_field \
and TYPE_CONCEPT_SUFFIX in dest_field \
and src_table != dest_table else 0
field_mapping = ATTRIBUTE_MAPPING_TEMPLATE.format(
src_table=src_table,
dest_table=dest_table,
src_field=src_field,
dest_field=dest_field,
translation=translation)
fr.write(field_mapping)