def deduplicate_other_data(data_import_output):
try:
if data_import_output is not None:
maternal_script = dedup_maternal
vitals_script = dedup_vitals
baseline_script = dedup_baseline
neolab_script = dedup_neolab
mat_completeness_script = dedup_mat_completeness
inject_sql(maternal_script, "deduplicate-maternal")
inject_sql(vitals_script, "deduplicate-vitals")
inject_sql(baseline_script, "deduplicate-baseline")
inject_sql(neolab_script, "deduplicate-neolabs")
inject_sql(mat_completeness_script, "deduplicate-mat-completeness")
#Add Return Value For Kedro Not To Throw Data Error And To Be Used As Input For Step 2
return dict(status='Success', message="Deduplication Complete")
else:
logging.error("Data Deduplication Did Not Execute To Completion")
return None
except Exception as e:
logging.error("!!! An error occured deduplicating data: ")
cron_log = open(cron_log_file, "a+")
#cron_log = open("C:\/Users\/morris\/Documents\/BRTI\/logs\/data_pipeline_cron.log","a+")
cron_log.write(
"StartTime: {0} Instance: {1} Status: Failed Stage: Deduplicating Data "
.format(cron_time, mode))
cron_log.close()
logging.error(e)
sys.exit(1)
def create_summary_discharges():
discharges_count = 0
tble_exists = False
try:
tble_exists = (table_exists('derived', 'discharges'))
if tble_exists:
discharges_count = table_data_count('derived', 'discharges')
if (discharges_count > 0):
summary_discharges_script = summary_discharges_query()
# Run Summary Admissions Query
inject_sql(summary_discharges_script, "create-summary-discharges")
else:
pass
except Exception as e:
logging.error("!!! An error occured creating summary discharges: ")
cron_log = open(cron_log_file, "a+")
cron_log.write(
"StartTime: {0} Instance: {1} Status: Failed Stage: Creating Summary Discharges "
.format(cron_time, mode))
cron_log.close()
logging.error(e)
sys.exit(1)
def create_summary_diagnosis(join_tables_output):
try:
#Test If Previous Node Has Completed Successfully
if join_tables_output is not None:
sql_script = summary_discharge_diagnosis_query()
inject_sql(sql_script, "create-summary-diagnosis")
#Add Return Value For Kedro Not To Throw Data Error
return dict(
status='Success',
message = "Creating Summary Diagnosis Complete"
)
else:
logging.error(
"Creating Summary Diagnosis Did Not Execute To Completion")
return None
except Exception as e:
logging.error("!!! An error occured creating summary diagnosis: ")
cron_log = open(cron_log_file,"a+")
#cron_log = open("C:\/Users\/morris\/Documents\/BRTI\/logs\/data_pipeline_cron.log","a+")
cron_log.write("StartTime: {0} Instance: {1} Status: Failed Stage: Creating Summary Diagnosis ".format(cron_time,mode))
cron_log.close()
logging.error(formatError(e))
sys.exit(1)
def create_maternal_completeness_summary():
try:
maternal_completeness_count = 0
mat_completeness_exists = False
mat_completeness_exists = table_exists('derived',
'maternity_completeness')
if mat_completeness_exists:
maternal_completeness_count = table_data_count(
'derived', 'maternity_completeness')
if (maternal_completeness_count > 0):
sql_script = summary_maternal_completeness_query()
inject_sql(sql_script, "create-summary-maternal-completeness")
else:
pass
except Exception as e:
logging.error("!!! An error occured creating Vital Signs Summaries: ")
cron_log = open(cron_log_file, "a+")
cron_log.write(
"StartTime: {0} Instance: {1} Status: Failed Stage: Creating Summary Maternal Completeness "
.format(cron_time, mode))
cron_log.close()
logging.error(formatError(e))
sys.exit(1)
def create_summary_baseline(join_tables_output):
tble_exists = False
try:
# Test if table exist before executing query
tble_exists = table_exists('derived', 'baseline')
#Test If Previous Node Has Completed Successfully
if tble_exists:
if join_tables_output is not None:
sql_script = summary_baseline_query()
inject_sql(sql_script, "create-summary-baseline")
#Add Return Value For Kedro Not To Throw Data Error
return dict(status='Success',
message="Creating Summary Baseline Complete")
else:
logging.error(
"Creating Summary Baseline Did Not Execute To Completion")
return None
else:
return dict(status='Skipped')
except Exception as e:
logging.error("!!! An error occured creating summary baseline: ")
cron_log = open(cron_log_file, "a+")
#cron_log = open("C:\/Users\/morris\/Documents\/BRTI\/logs\/data_pipeline_cron.log","a+")
cron_log.write(
"StartTime: {0} Instance: {1} Status: Failed Stage: Creating Summary Baseline "
.format(cron_time, mode))
cron_log.close()
raise e
logging.error(formatError(e))
sys.exit(1)
def create_summary_neolabs():
neolab_count = 0
neolab_exists = False
try:
neolab_exists = table_exists('derived', 'neolab')
if neolab_exists:
neolab_count = table_data_count('derived', 'neolab')
if (neolab_count > 0):
sql_script = summary_neolab_query()
inject_sql(sql_script, "create-summary-neolabs")
else:
pass
except Exception as e:
logging.error("!!! An error occured creating summary Neolab")
logging.error(e)
cron_log = open(cron_log_file, "a+")
cron_log.write(
"StartTime: {0} Instance: {1} Status: Failed Stage: Creating Summary Neolab"
.format(cron_time, mode))
cron_log.close()
sys.exit(1)
def create_summary_joined_vitalsigns():
vital_signs_count = 0
tble_exists = False
try:
tble_exists = (table_exists('derived', 'vitalsigns')
and table_exists('derived', 'summary_day1_vitals')
and table_exists('derived', 'summary_day2_vitals')
and table_exists('derived', 'summary_day3_vitals'))
if tble_exists:
vital_signs_count = table_data_count('derived', 'vitalsigns')
if (vital_signs_count > 0):
summary_joined_vitals_script = summary_joined_vitals_query()
# Run Summary Joined Vital Signs Query
inject_sql(summary_joined_vitals_script,
"create-summary-joined-vital-signs")
else:
pass
except Exception as e:
logging.error("!!! An error occured creating joined vital signs: ")
cron_log = open(cron_log_file, "a+")
cron_log.write(
"StartTime: {0} Instance: {1} Status: Failed Stage: Creating Summary Maternal Outcomes "
.format(cron_time, mode))
cron_log.close()
logging.error(e)
sys.exit(1)
def create_summary_vitalsigns():
vital_signs_count = 0
tble_exists = False
try:
tble_exists = table_exists('derived','vitalsigns');
if tble_exists:
vital_signs_count = table_data_count('derived','vitalsigns')
if (vital_signs_count> 0):
summary_vitals_script = summary_vital_signs_query()
summary_vitals_day1_script = summary_day_one_vitals_query()
summary_vitals_day2_script = summary_day_two_vitals_query()
summary_vitals_day3_script = summary_day_three_vitals_query()
# Run Summary Vital Signs Query
inject_sql(summary_vitals_script, "create-summary-vital-signs")
# Run Day1 Summary Vital Signs Query
inject_sql(summary_vitals_day1_script, "create-summary-day1-vital-signs")
# Run Day2 Summary Vital Signs Query
inject_sql(summary_vitals_day2_script, "create-summary-day2-vital-signs")
# Run Day3 Summary Vital Signs Query
inject_sql(summary_vitals_day3_script, "create-summary-day3-vital-signs")
else:
pass;
except Exception as e:
logging.error("!!! An error occured creating Vital Signs Summaries: ")
cron_log = open(cron_log_file,"a+")
cron_log.write("StartTime: {0} Instance: {1} Status: Failed Stage: Creating Summary Maternal Outcomes ".format(cron_time,mode))
cron_log.close()
logging.error(formatError(e))
sys.exit(1)
def deduplicate_discharges(data_import_output):
try:
if data_import_output is not None:
sql_script = dedup_discharges
inject_sql(sql_script, "deduplicate-discharges")
#Add Return Value For Kedro Not To Throw Data Error And To Be Used As Input For Step 2
return dict(status='Success',
message="Discharges Deduplication Complete")
else:
logging.error("Data Importation Did Not Execute To Completion")
return None
except Exception as e:
logging.error("!!! An error occured deduplicating discharges: ")
cron_log = open(cron_log_file, "a+")
#cron_log = open("C:\/Users\/morris\/Documents\/BRTI\/logs\/data_pipeline_cron.log","a+")
cron_log.write(
"StartTime: {0} Instance: {1} Status: Failed Stage: Deduplicating Discharges "
.format(cron_time, mode))
cron_log.close()
logging.error(e.with_traceback())
sys.exit(1)
def deduplicate_admissions(data_import_output):
try:
if data_import_output is not None:
sql_script = dedup_admissions
inject_sql(sql_script, "deduplicate-admissions")
#Add Return Value For Kedro Not To Throw Data Error
return dict(status='Success',
message="Admissions Deduplication Complete")
else:
logging.error("Data Importation Did Not Execute To Completion")
return None
except Exception as e:
logging.error("!!! An error occured deduplicating discharges: ")
logging.error(e)
#Only Open This File When Need Be To Write To It
cron_log = open(cron_log_file, "a+")
cron_log.write(
"StartTime: {0} Instance: {1} Status: Failed Stage: Deduplicating Admissions "
.format(cron_time, mode))
cron_log.close()
sys.exit(1)
def manually_fix_admissions(tidy_data_output):
try:
#Test If Previous Node Has Completed Successfully
if tidy_data_output is not None:
sql_script = manually_fix_admissions_query()
inject_sql(sql_script, "manually-fix-admissions")
#Add Return Value For Kedro Not To Throw Data Error
return dict(status='Success',
message="Manual Fixing Of Admissions Complete")
else:
logging.error(
"Manual Fixing Of Admissions Did Not Execute To Completion")
return None
except Exception as e:
logging.error("!!! An error occured manually fixing admissions: ")
cron_log = open(cron_log_file, "a+")
cron_log.write(
"StartTime: {0} Instance: {1} Status: Failed Stage: Manually Fixing Admissions "
.format(cron_time, mode))
cron_log.close()
logging.error(formatError(e))
sys.exit(1)
def union_views():
if ('country' in params and str(params['country']).lower()) == 'zimbabwe':
try:
adm_cols = pd.DataFrame(get_table_columns('admissions', 'derived'),
columns=["column_name", "data_type"])
old_adm_cols = pd.DataFrame(get_table_columns(
'old_smch_admissions', 'derived'),
columns=["column_name", "data_type"])
old_disc_cols = pd.DataFrame(get_table_columns(
'old_smch_discharges', 'derived'),
columns=["column_name", "data_type"])
disc_cols = pd.DataFrame(get_table_columns('discharges',
'derived'),
columns=["column_name", "data_type"])
old_matched_cols = pd.DataFrame(
get_table_columns('old_smch_matched_admissions_discharges',
'derived'),
columns=["column_name", "data_type"])
matched_cols = pd.DataFrame(get_table_columns(
'joined_admissions_discharges', 'derived'),
columns=["column_name", "data_type"])
# Match Data Types For Admissions
for index, row in adm_cols.iterrows():
col_name = str(row['column_name']).strip()
data_type = row['data_type']
using = ''
for index2, row2 in old_adm_cols.iterrows():
if col_name == row2['column_name']:
try:
if str(data_type) != str(row2['data_type']):
using = f'''USING "{col_name}"::{data_type}'''
query = f'''ALTER table derived.old_smch_admissions ALTER column "{col_name}" TYPE {data_type} {using};'''
inject_sql(query, "OLD ADMISSIONS")
except Exception as ex:
query = f'''ALTER table derived.old_smch_admissions DROP column "{col_name}" '''
inject_sql(query,
f'''DROPPING ADMISSIONS {col_name}''')
# Match Data Types For Discharges
for index, row in disc_cols.iterrows():
col_name = str(row['column_name']).strip()
data_type = row['data_type']
using = ''
for index2, row2 in old_disc_cols.iterrows():
if col_name == row2['column_name']:
try:
if str(data_type) != str(row2['data_type']):
using = f'''USING "{col_name}"::{data_type}'''
query = f''' ALTER table derived.old_smch_discharges ALTER column "{col_name}" TYPE {data_type} {using};'''
inject_sql(query, "OLD DISCHARGES")
except Exception as ex:
query = f'''ALTER table derived.old_smch_discharges DROP column "{col_name}" '''
inject_sql(
query,
f'''DROPPING DISCHARGE COLL {col_name}''')
# Match Data Types For Matched Data
for index, row in matched_cols.iterrows():
col_name = str(row['column_name']).strip()
data_type = row['data_type']
using = ''
for index2, row2 in old_matched_cols.iterrows():
if col_name == row2['column_name']:
try:
if str(data_type) != str(row2['data_type']):
using = f'''USING "{col_name}"::{data_type}'''
query = f''' ALTER table derived.old_smch_matched_admissions_discharges ALTER column "{col_name}" TYPE {data_type} {using};'''
inject_sql(query, "Union Views")
except Exception as ex:
query = f'''ALTER table derived.old_smch_matched_admissions_discharges DROP column "{col_name}" '''
inject_sql(
query,
f'''DROPPING MATCHED COLL {col_name} ''')
old_smch_admissions = None
old_smch_discharges = None
old_matched_smch_data = None
new_smch_admissions = None
new_smch_discharges = None
new_smch_matched_data = None
if table_exists('derived', 'old_smch_admissions'):
old_smch_admissions = catalog.load('read_old_smch_admissions')
if table_exists('derived', 'old_smch_discharges'):
old_smch_discharges = catalog.load('read_old_smch_discharges')
if table_exists('derived',
'old_smch_matched_admissions_discharges'):
old_matched_smch_data = catalog.load(
'read_old_smch_matched_data')
if table_exists('derived', 'admissions'):
new_smch_admissions = catalog.load('read_new_smch_admissions')
if table_exists('derived', 'discharges'):
new_smch_discharges = catalog.load('read_new_smch_discharges')
if table_exists('derived', 'joined_admissions_discharges'):
new_smch_matched_data = catalog.load('read_new_smch_matched')
if old_smch_admissions is not None and not old_smch_admissions.empty:
for position, admission in old_smch_admissions.iterrows():
age_list = []
if 'AgeB.value' in admission and str(
admission['AgeB.value']) != 'nan':
age_list = str(admission['AgeB.value']).split(",")
# Initialise Hours
hours = 0
period = 0
# If size of List is 1 it either means its days only or hours only
if len(age_list) == 1:
age = age_list[0]
# Check if hours or Days
if 'hour' in age:
hours = [
int(s) for s in age.replace("-", "").split()
if s.isdigit()
]
# Check if value contains figures
if len(hours) > 0:
period = hours[0]
else:
if "an" in age:
# IF AN HOUR
period = 1
elif 'day' in age:
hours = [
int(s) for s in age.replace("-", "").split()
if s.isdigit()
]
if len(hours) > 0:
period = hours[0] * 24
elif 'second' in age:
# FEW SECONDS CAN BE ROUNDED OFF 1 HOUR
period = 1
elif 'minute' in age:
# MINUTES CAN BE ROUNDED OFF 1 HOUR
period = 1
pass
# Contains Both Hours and Days
elif len(age_list) == 2:
age_days = age_list[0]
age_hours = age_list[1]
if 'day' in age_days and 'hour' in age_hours:
number_hours_days = [
int(s) for s in age_days.split()
if s.isdigit()
]
number_hours = [
int(s) for s in age_hours.split()
if s.isdigit()
]
if (len(number_hours) > 0
and len(number_hours_days) > 0):
period = (number_hours_days[0]) * 24 + (
number_hours[0])
else:
pass
if period > 0:
old_smch_admissions.loc[position, 'Age.value'] = period
if period < 2:
old_smch_admissions.loc[
position,
'AgeCategory'] = 'Fresh Newborn (< 2 hours old)'
elif period > 2 and period <= 23:
old_smch_admissions.loc[
position,
'AgeCategory'] = 'Newborn (2 - 23 hrs old)'
elif period > 23 and period <= 47:
old_smch_admissions.loc[
position,
'AgeCategory'] = 'Newborn (1 day - 1 day 23 hrs old)'
elif period > 47 and period <= 71:
old_smch_admissions.loc[
position,
'AgeCategory'] = 'Infant (2 days - 2 days 23 hrs old)'
else:
old_smch_admissions.loc[
position,
'AgeCategory'] = 'Infant (> 3 days old)'
########################## UPDATE ADMISSION SCRIPT WITH NEW KEYS ########################
key_change(old_smch_admissions, admission, position,
'BW.value', 'BirthWeight.value')
key_change(old_smch_admissions, admission, position,
'Conv.value', 'Convulsions.value')
key_change(old_smch_admissions, admission, position,
'SRNeuroOther.value',
'SymptomReviewNeurology.value')
key_change(old_smch_admissions, admission, position,
'LBW.value', 'LowBirthWeight.value')
key_change(old_smch_admissions, admission, position,
'AW.value', 'AdmissionWeight.value')
key_change(old_smch_admissions, admission, position,
'BSmgdL.value', 'BSUnitmg.value')
key_change(old_smch_admissions, admission, position,
'BSmmol.value', 'BloodSugarmmol.value')
key_change(old_smch_admissions, admission, position,
'BSmg.value', 'BloodSugarmg.value')
if "Age.value" in old_smch_admissions:
old_smch_admissions['Age.value'] = pd.to_numeric(
old_smch_admissions['Age.value'], errors='coerce')
if 'AdmissionWeight.value' in old_smch_admissions:
old_smch_admissions[
'AdmissionWeight.value'] = pd.to_numeric(
old_smch_admissions['AdmissionWeight.value'],
errors='coerce')
if 'BirthWeight.value' in old_smch_admissions:
old_smch_admissions['BirthWeight.value'] = pd.to_numeric(
old_smch_admissions['BirthWeight.value'],
errors='coerce')
format_date(old_smch_admissions, 'DateTimeAdmission.value')
format_date(old_smch_admissions, 'EndScriptDatetime.value')
format_date(old_smch_admissions, 'DateHIVtest.value')
format_date(old_smch_admissions, 'ANVDRLDate.value')
if old_smch_discharges is not None and not old_smch_discharges.empty:
for position, discharge in old_smch_discharges.iterrows():
key_change(old_smch_discharges, discharge, position,
'BWTDis.value', 'BirthWeight.value')
key_change(old_smch_discharges, discharge, position,
'BirthDateDis.value', 'DOBTOB.value')
key_change(old_smch_discharges, discharge, position,
'Delivery.value', 'ModeDelivery.value')
key_change(old_smch_discharges, discharge, position,
'NNUAdmTemp.value', 'Temperature.value')
key_change(old_smch_discharges, discharge, position,
'GestBirth.value', 'Gestation.value')
key_change(old_smch_discharges, discharge, position,
'PresComp.value', 'AdmReason.value')
#Format Dates Discharge Table
format_date(old_smch_discharges, 'DateAdmissionDC.value')
format_date(old_smch_discharges, 'DateDischVitals.value')
format_date(old_smch_discharges, 'DateDischWeight.value')
format_date(old_smch_discharges, 'DateTimeDischarge.value')
format_date(old_smch_discharges, 'EndScriptDatetime.value')
format_date(old_smch_discharges, 'DateWeaned.value')
format_date(old_smch_discharges, 'DateTimeDeath.value')
format_date(old_smch_discharges, 'DateAdmission.value')
format_date(old_smch_discharges, 'BirthDateDis.value')
if old_matched_smch_data is not None and not old_matched_smch_data.empty:
for position, matched_admission in old_matched_smch_data.iterrows(
):
age_list = []
if 'AgeB.value' in matched_admission and str(
matched_admission['AgeB.value']) != 'nan':
age_list = str(
matched_admission['AgeB.value']).split(",")
# Initialise Hours
hours = 0
period = 0
# If size of List is 1 it either means its days only or hours only
if len(age_list) == 1:
age = age_list[0]
# Check if hours or Days
if 'hour' in age:
hours = [
int(s) for s in age.replace("-", "").split()
if s.isdigit()
]
# Check if value contains figures
if len(hours) > 0:
period = hours[0]
else:
if "an" in age:
# IF AN HOUR
period = 1
elif 'day' in age:
hours = [
int(s) for s in age.replace("-", "").split()
if s.isdigit()
]
if len(hours) > 0:
period = hours[0] * 24
elif 'second' in age:
# FEW SECONDS CAN BE ROUNDED OFF 1 HOUR
period = 1
elif 'minute' in age:
# MINUTES CAN BE ROUNDED OFF 1 HOUR
period = 1
pass
# Contains Both Hours and Days
elif len(age_list) == 2:
age_days = age_list[0]
age_hours = age_list[1]
if 'day' in age_days and 'hour' in age_hours:
number_hours_days = [
int(s) for s in age_days.split()
if s.isdigit()
]
number_hours = [
int(s) for s in age_hours.split()
if s.isdigit()
]
if (len(number_hours) > 0
and len(number_hours_days) > 0):
period = (number_hours_days[0]) * 24 + (
number_hours[0])
else:
pass
if period > 0:
old_matched_smch_data.loc[position,
'Age.value'] = period
if period < 2:
old_matched_smch_data.loc[
position,
'AgeCategory'] = 'Fresh Newborn (< 2 hours old)'
elif period > 2 and period <= 23:
old_matched_smch_data.loc[
position,
'AgeCategory'] = 'Newborn (2 - 23 hrs old)'
elif period > 23 and period <= 47:
old_matched_smch_data.loc[
position,
'AgeCategory'] = 'Newborn (1 day - 1 day 23 hrs old)'
elif period > 47 and period <= 71:
old_matched_smch_data.loc[
position,
'AgeCategory'] = 'Infant (2 days - 2 days 23 hrs old)'
else:
old_matched_smch_data.loc[
position,
'AgeCategory'] = 'Infant (> 3 days old)'
########################## UPDATE ADMISSION SCRIPT WITH NEW KEYS ########################
key_change(old_matched_smch_data, matched_admission,
position, 'BW.value', 'BirthWeight.value')
key_change(old_matched_smch_data, matched_admission,
position, 'Conv.value', 'Convulsions.value')
key_change(old_matched_smch_data, matched_admission,
position, 'SRNeuroOther.value',
'SymptomReviewNeurology.value')
key_change(old_matched_smch_data, matched_admission,
position, 'LBW.value', 'LowBirthWeight.value')
key_change(old_matched_smch_data, matched_admission,
position, 'AW.value', 'AdmissionWeight.value')
key_change(old_matched_smch_data, matched_admission,
position, 'BSmgdL.value', 'BSUnitmg.value')
key_change(old_matched_smch_data, matched_admission,
position, 'BSmmol.value',
'BloodSugarmmol.value')
key_change(old_matched_smch_data, matched_admission,
position, 'BSmg.value', 'BloodSugarmg.value')
key_change(old_matched_smch_data, matched_admission,
position, 'BWTDis.value', 'BirthWeight.value')
key_change(old_matched_smch_data, matched_admission,
position, 'BirthDateDis.value', 'DOBTOB.value')
key_change(old_matched_smch_data, matched_admission,
position, 'Delivery.value',
'ModeDelivery.value')
key_change(old_matched_smch_data, matched_admission,
position, 'NNUAdmTemp.value',
'Temperature.value')
key_change(old_matched_smch_data, matched_admission,
position, 'GestBirth.value', 'Gestation.value')
key_change(old_matched_smch_data, matched_admission,
position, 'PresComp.value', 'AdmReason.value')
if "Age.value" in old_matched_smch_data:
old_matched_smch_data['Age.value'] = pd.to_numeric(
old_matched_smch_data['Age.value'], errors='coerce')
if 'AdmissionWeight.value' in old_matched_smch_data:
old_matched_smch_data[
'AdmissionWeight.value'] = pd.to_numeric(
old_matched_smch_data['AdmissionWeight.value'],
errors='coerce')
if 'BirthWeight.value' in old_matched_smch_data:
old_matched_smch_data['BirthWeight.value'] = pd.to_numeric(
old_matched_smch_data['BirthWeight.value'],
errors='coerce')
if 'BirthWeight.value_discharge' in old_matched_smch_data:
old_matched_smch_data['BirthWeight.value'] = pd.to_numeric(
old_matched_smch_data['BirthWeight.value_value'],
errors='coerce')
format_date(old_matched_smch_data, 'DateTimeAdmission.value')
format_date(old_matched_smch_data, 'EndScriptDatetime.value')
format_date(old_matched_smch_data, 'DateHIVtest.value')
format_date(old_matched_smch_data, 'ANVDRLDate.value')
#Format Dates Discharge Table
format_date(old_matched_smch_data, 'DateAdmissionDC.value')
format_date(old_matched_smch_data, 'DateDischVitals.value')
format_date(old_matched_smch_data, 'DateDischWeight.value')
format_date(old_matched_smch_data, 'DateTimeDischarge.value')
format_date(old_matched_smch_data, 'EndScriptDatetime.value')
format_date(old_matched_smch_data, 'DateWeaned.value')
format_date(old_matched_smch_data, 'DateTimeDeath.value')
format_date(old_matched_smch_data, 'DateAdmission.value')
format_date(old_matched_smch_data, 'BirthDateDis.value')
# SAVE OLD NEW ADMISSIONS
if new_smch_admissions is not None and old_smch_admissions is not None:
combined_adm_df = pd.concat(
[new_smch_admissions, old_smch_admissions],
ignore_index=True)
if not combined_adm_df.empty:
catalog.save('create_derived_old_new_admissions_view',
combined_adm_df)
# SAVE OLD NEW DISCHARGES
if new_smch_discharges is not None and old_smch_discharges is not None:
combined_dis_df = pd.concat(
[new_smch_discharges, old_smch_discharges],
ignore_index=True)
if not combined_dis_df.empty:
catalog.save('create_derived_old_new_discharges_view',
combined_dis_df)
# SAVE MATCHED DATA
if new_smch_matched_data is not None and old_matched_smch_data is not None:
#Correct UID column to suit the lower case uid in new_smch_matched_data
if 'UID' in old_matched_smch_data.columns:
old_matched_smch_data = old_matched_smch_data.rename(
columns={'UID': 'uid'}, inplace=False)
combined_matched_df = pd.concat(
[new_smch_matched_data, old_matched_smch_data],
ignore_index=True)
if not combined_matched_df.empty:
catalog.save('create_derived_old_new_matched_view',
combined_matched_df)
except Exception as ex:
logging.error("!!! An error occured creating union views: ")
logging.error(ex)
exit()