def test_get_domain_fields(self):
with patch.dict(
'cdr_cleaner.cleaning_rules.field_mapping.CDM_TABLE_SCHEMAS',
self.cdm_schemas):
fields = field_mapping.get_domain_fields(self.condition_table)
self.assertEqual(fields, self.condition_occurrence_fields)
self.assertEqual(
field_mapping.get_domain_fields(self.procedure_table),
self.procedure_occurrence_fields)
def generate_field_expr(table_id, year_threshold):
"""
This function generates the select statements for the table. For the nullable date fields, it sets the value to NULL
if the nullable date field fails the threshold criteria
:param table_id:
:param year_threshold:
:return:
"""
col_expression_list = []
nullable_date_field_names = [
field for field in get_date_fields(table_id)
if not field_mapping.is_field_required(table_id, field)
]
for field_name in field_mapping.get_domain_fields(table_id):
if field_name in nullable_date_field_names:
col_expression_list.append(
NULLABLE_DATE_FIELD_EXPRESSION.format(
date_field_name=field_name, year_threshold=year_threshold))
else:
col_expression_list.append(field_name)
return ','.join(col_expression_list)
def test_get_cols(self):
# pre conditions
self.assertEqual(
self.query_class.get_affected_datasets(),
[cdr_consts.RDR, cdr_consts.UNIONED, cdr_consts.COMBINED])
for table in TABLE_DATES:
# test
result_list = self.query_class.get_cols(table)
# post conditions
table_fields = field_mapping.get_domain_fields(table)
expected_list = []
for field in table_fields:
if field in TABLE_DATES[table]:
expected = FIX_NULL_OR_INCORRECT_DATETIME_QUERY.format(
field=field, date_field=TABLE_DATES[table][field])
else:
expected = field
expected_list.append(expected)
expected_cols = ', '.join(expected_list)
self.assertEqual(result_list, expected_cols)
def get_cols(table_id):
"""
Generates the fields to choose along with case statements to generate datetime
:param table_id: table for which the fields
:return:
"""
table_fields = field_mapping.get_domain_fields(table_id)
col_exprs = []
for field in table_fields:
if field in TABLE_DATES[table_id]:
if field_mapping.is_field_required(table_id, field):
col_expr = (
' CASE'
' WHEN EXTRACT(DATE FROM {field}) = {date_field}'
' THEN {field}'
' ELSE CAST(DATETIME({date_field}, EXTRACT(TIME FROM {field})) AS TIMESTAMP)'
' END AS {field}').format(
field=field, date_field=TABLE_DATES[table_id][field])
else:
col_expr = (' CASE'
' WHEN EXTRACT(DATE FROM {field}) = {date_field}'
' THEN {field}'
' ELSE NULL'
' END AS {field}').format(
field=field,
date_field=TABLE_DATES[table_id][field])
else:
col_expr = field
col_exprs.append(col_expr)
cols = ', '.join(col_exprs)
return cols
def get_date_fields(table_id):
"""
The function retrieves a list of date related fields for the given table
:param table_id:
:return:
"""
return [
field for field in field_mapping.get_domain_fields(table_id)
if DATE_FIELD_KEY_WORD in field
]
def get_cols(self, table):
"""
Generates the fields to choose along with case statements to generate datetime
And ensures no null datetime values
:param self: instance of EnsureDateDatetimeConsistency class
:param table: table for which the fields are pulled
:return: cols
"""
table_fields = field_mapping.get_domain_fields(table)
col_exprs = []
for field in table_fields:
if field in TABLE_DATES[table]:
col_expr = FIX_NULL_OR_INCORRECT_DATETIME_QUERY.format(
field=field, date_field=TABLE_DATES[table][field])
else:
col_expr = field
col_exprs.append(col_expr)
cols = ', '.join(col_exprs)
return cols