コード例 #1
0
def deduplicate_other_data(data_import_output):
    try:
        if data_import_output is not None:
            maternal_script = dedup_maternal
            vitals_script = dedup_vitals
            baseline_script = dedup_baseline
            neolab_script = dedup_neolab
            mat_completeness_script = dedup_mat_completeness
            inject_sql(maternal_script, "deduplicate-maternal")
            inject_sql(vitals_script, "deduplicate-vitals")
            inject_sql(baseline_script, "deduplicate-baseline")
            inject_sql(neolab_script, "deduplicate-neolabs")
            inject_sql(mat_completeness_script, "deduplicate-mat-completeness")
            #Add Return Value For Kedro Not To Throw Data Error And To Be Used As Input For Step 2
            return dict(status='Success', message="Deduplication Complete")
        else:
            logging.error("Data Deduplication Did Not Execute To Completion")
            return None

    except Exception as e:
        logging.error("!!! An error occured deduplicating data: ")
        cron_log = open(cron_log_file, "a+")
        #cron_log = open("C:\/Users\/morris\/Documents\/BRTI\/logs\/data_pipeline_cron.log","a+")
        cron_log.write(
            "StartTime: {0}   Instance: {1}   Status: Failed Stage: Deduplicating Data "
            .format(cron_time, mode))
        cron_log.close()
        logging.error(e)
        sys.exit(1)
コード例 #2
0
def create_summary_discharges():
    discharges_count = 0
    tble_exists = False
    try:
        tble_exists = (table_exists('derived', 'discharges'))

        if tble_exists:
            discharges_count = table_data_count('derived', 'discharges')

        if (discharges_count > 0):
            summary_discharges_script = summary_discharges_query()

            # Run  Summary Admissions Query
            inject_sql(summary_discharges_script, "create-summary-discharges")
        else:
            pass

    except Exception as e:
        logging.error("!!! An error occured creating summary discharges: ")
        cron_log = open(cron_log_file, "a+")
        cron_log.write(
            "StartTime: {0}   Instance: {1}   Status: Failed   Stage: Creating Summary Discharges "
            .format(cron_time, mode))
        cron_log.close()
        logging.error(e)
        sys.exit(1)
コード例 #3
0
def create_summary_diagnosis(join_tables_output):    
    try:
        #Test If Previous Node Has Completed Successfully
        if join_tables_output is not None:
           
            sql_script = summary_discharge_diagnosis_query()
            inject_sql(sql_script, "create-summary-diagnosis")
            #Add Return Value For Kedro Not To Throw Data Error
            return dict(
            status='Success',
            message = "Creating Summary Diagnosis Complete"
            )
        else:
            logging.error(
                "Creating Summary Diagnosis Did Not Execute To Completion")
            return None

    except Exception as e:
        logging.error("!!! An error occured creating summary diagnosis: ")
        cron_log = open(cron_log_file,"a+")
        #cron_log = open("C:\/Users\/morris\/Documents\/BRTI\/logs\/data_pipeline_cron.log","a+")
        cron_log.write("StartTime: {0}   Instance: {1}   Status: Failed   Stage: Creating Summary Diagnosis ".format(cron_time,mode))
        cron_log.close()
        logging.error(formatError(e))
        sys.exit(1)
コード例 #4
0
def create_maternal_completeness_summary():
    try:
        maternal_completeness_count = 0
        mat_completeness_exists = False
        mat_completeness_exists = table_exists('derived',
                                               'maternity_completeness')

        if mat_completeness_exists:
            maternal_completeness_count = table_data_count(
                'derived', 'maternity_completeness')

        if (maternal_completeness_count > 0):

            sql_script = summary_maternal_completeness_query()
            inject_sql(sql_script, "create-summary-maternal-completeness")
        else:
            pass

    except Exception as e:
        logging.error("!!! An error occured creating Vital Signs Summaries: ")
        cron_log = open(cron_log_file, "a+")
        cron_log.write(
            "StartTime: {0}   Instance: {1}   Status: Failed   Stage: Creating Summary Maternal Completeness "
            .format(cron_time, mode))
        cron_log.close()
        logging.error(formatError(e))
        sys.exit(1)
コード例 #5
0
def create_summary_baseline(join_tables_output):
    tble_exists = False
    try:
        # Test if table exist before executing query
        tble_exists = table_exists('derived', 'baseline')
        #Test If Previous Node Has Completed Successfully
        if tble_exists:
            if join_tables_output is not None:

                sql_script = summary_baseline_query()
                inject_sql(sql_script, "create-summary-baseline")
                #Add Return Value For Kedro Not To Throw Data Error
                return dict(status='Success',
                            message="Creating Summary Baseline Complete")
            else:
                logging.error(
                    "Creating Summary Baseline Did Not Execute To Completion")
                return None
        else:
            return dict(status='Skipped')

    except Exception as e:
        logging.error("!!! An error occured creating summary baseline: ")
        cron_log = open(cron_log_file, "a+")
        #cron_log = open("C:\/Users\/morris\/Documents\/BRTI\/logs\/data_pipeline_cron.log","a+")
        cron_log.write(
            "StartTime: {0}   Instance: {1}   Status: Failed   Stage: Creating Summary Baseline "
            .format(cron_time, mode))
        cron_log.close()
        raise e
        logging.error(formatError(e))
        sys.exit(1)
コード例 #6
0
def create_summary_neolabs():
    neolab_count = 0
    neolab_exists = False
    try:
        neolab_exists = table_exists('derived', 'neolab')

        if neolab_exists:
            neolab_count = table_data_count('derived', 'neolab')

        if (neolab_count > 0):

            sql_script = summary_neolab_query()
            inject_sql(sql_script, "create-summary-neolabs")
        else:
            pass

    except Exception as e:
        logging.error("!!! An error occured creating summary Neolab")
        logging.error(e)
        cron_log = open(cron_log_file, "a+")
        cron_log.write(
            "StartTime: {0}   Instance: {1}   Status: Failed   Stage: Creating Summary Neolab"
            .format(cron_time, mode))
        cron_log.close()
        sys.exit(1)
コード例 #7
0
def create_summary_joined_vitalsigns():
    vital_signs_count = 0
    tble_exists = False
    try:
        tble_exists = (table_exists('derived', 'vitalsigns')
                       and table_exists('derived', 'summary_day1_vitals')
                       and table_exists('derived', 'summary_day2_vitals')
                       and table_exists('derived', 'summary_day3_vitals'))

        if tble_exists:
            vital_signs_count = table_data_count('derived', 'vitalsigns')

        if (vital_signs_count > 0):
            summary_joined_vitals_script = summary_joined_vitals_query()

            # Run  Summary Joined Vital Signs Query
            inject_sql(summary_joined_vitals_script,
                       "create-summary-joined-vital-signs")
        else:
            pass

    except Exception as e:
        logging.error("!!! An error occured creating joined vital signs: ")
        cron_log = open(cron_log_file, "a+")
        cron_log.write(
            "StartTime: {0}   Instance: {1}   Status: Failed   Stage: Creating Summary Maternal Outcomes "
            .format(cron_time, mode))
        cron_log.close()
        logging.error(e)
        sys.exit(1)
コード例 #8
0
def create_summary_vitalsigns():
    vital_signs_count = 0
    tble_exists = False
    try:
        tble_exists = table_exists('derived','vitalsigns');
        if tble_exists:
                vital_signs_count = table_data_count('derived','vitalsigns')

        if (vital_signs_count> 0):
           
            summary_vitals_script = summary_vital_signs_query()
            summary_vitals_day1_script = summary_day_one_vitals_query()
            summary_vitals_day2_script = summary_day_two_vitals_query()
            summary_vitals_day3_script = summary_day_three_vitals_query()
               
                # Run Summary Vital Signs Query
            inject_sql(summary_vitals_script, "create-summary-vital-signs")
                # Run Day1 Summary Vital Signs Query
            inject_sql(summary_vitals_day1_script, "create-summary-day1-vital-signs")

                    # Run Day2 Summary Vital Signs Query
            inject_sql(summary_vitals_day2_script, "create-summary-day2-vital-signs")

                    # Run Day3 Summary Vital Signs Query
            inject_sql(summary_vitals_day3_script, "create-summary-day3-vital-signs")
        else:
            pass;

    except Exception as e:
        logging.error("!!! An error occured creating Vital Signs Summaries: ")
        cron_log = open(cron_log_file,"a+")
        cron_log.write("StartTime: {0}   Instance: {1}   Status: Failed   Stage: Creating Summary Maternal Outcomes ".format(cron_time,mode))
        cron_log.close()
        logging.error(formatError(e))
        sys.exit(1)
コード例 #9
0
def deduplicate_discharges(data_import_output):
    try:
        if data_import_output is not None:
            sql_script = dedup_discharges
            inject_sql(sql_script, "deduplicate-discharges")
            #Add Return Value For Kedro Not To Throw Data Error And To Be Used As Input For Step 2
            return dict(status='Success',
                        message="Discharges Deduplication Complete")
        else:
            logging.error("Data Importation Did Not Execute To Completion")
            return None

    except Exception as e:
        logging.error("!!! An error occured deduplicating discharges: ")
        cron_log = open(cron_log_file, "a+")
        #cron_log = open("C:\/Users\/morris\/Documents\/BRTI\/logs\/data_pipeline_cron.log","a+")
        cron_log.write(
            "StartTime: {0}   Instance: {1}   Status: Failed Stage: Deduplicating Discharges "
            .format(cron_time, mode))
        cron_log.close()
        logging.error(e.with_traceback())
        sys.exit(1)
コード例 #10
0
def deduplicate_admissions(data_import_output):
    try:
        if data_import_output is not None:
            sql_script = dedup_admissions
            inject_sql(sql_script, "deduplicate-admissions")
            #Add Return Value For Kedro Not To Throw Data Error
            return dict(status='Success',
                        message="Admissions Deduplication Complete")
        else:
            logging.error("Data Importation Did Not Execute To Completion")
            return None

    except Exception as e:
        logging.error("!!! An error occured deduplicating discharges: ")
        logging.error(e)
        #Only Open This File When Need Be To Write To It
        cron_log = open(cron_log_file, "a+")
        cron_log.write(
            "StartTime: {0}   Instance: {1}   Status: Failed Stage: Deduplicating Admissions "
            .format(cron_time, mode))
        cron_log.close()
        sys.exit(1)
def manually_fix_admissions(tidy_data_output):
    try:
        #Test If Previous Node Has Completed Successfully
        if tidy_data_output is not None:
            sql_script = manually_fix_admissions_query()
            inject_sql(sql_script, "manually-fix-admissions")
            #Add Return Value For Kedro Not To Throw Data Error
            return dict(status='Success',
                        message="Manual Fixing Of Admissions Complete")
        else:
            logging.error(
                "Manual Fixing Of Admissions Did Not Execute To Completion")
            return None

    except Exception as e:
        logging.error("!!! An error occured manually fixing admissions: ")
        cron_log = open(cron_log_file, "a+")
        cron_log.write(
            "StartTime: {0}   Instance: {1}   Status: Failed Stage: Manually Fixing Admissions "
            .format(cron_time, mode))
        cron_log.close()
        logging.error(formatError(e))
        sys.exit(1)
コード例 #12
0
def union_views():
    if ('country' in params and str(params['country']).lower()) == 'zimbabwe':
        try:
            adm_cols = pd.DataFrame(get_table_columns('admissions', 'derived'),
                                    columns=["column_name", "data_type"])
            old_adm_cols = pd.DataFrame(get_table_columns(
                'old_smch_admissions', 'derived'),
                                        columns=["column_name", "data_type"])
            old_disc_cols = pd.DataFrame(get_table_columns(
                'old_smch_discharges', 'derived'),
                                         columns=["column_name", "data_type"])
            disc_cols = pd.DataFrame(get_table_columns('discharges',
                                                       'derived'),
                                     columns=["column_name", "data_type"])
            old_matched_cols = pd.DataFrame(
                get_table_columns('old_smch_matched_admissions_discharges',
                                  'derived'),
                columns=["column_name", "data_type"])
            matched_cols = pd.DataFrame(get_table_columns(
                'joined_admissions_discharges', 'derived'),
                                        columns=["column_name", "data_type"])

            # Match Data Types For Admissions
            for index, row in adm_cols.iterrows():
                col_name = str(row['column_name']).strip()
                data_type = row['data_type']
                using = ''
                for index2, row2 in old_adm_cols.iterrows():
                    if col_name == row2['column_name']:
                        try:
                            if str(data_type) != str(row2['data_type']):
                                using = f'''USING "{col_name}"::{data_type}'''
                                query = f'''ALTER table derived.old_smch_admissions ALTER column "{col_name}" TYPE {data_type}  {using};'''
                                inject_sql(query, "OLD ADMISSIONS")
                        except Exception as ex:
                            query = f'''ALTER table derived.old_smch_admissions DROP column "{col_name}" '''
                            inject_sql(query,
                                       f'''DROPPING ADMISSIONS {col_name}''')

            # Match Data Types For Discharges
            for index, row in disc_cols.iterrows():
                col_name = str(row['column_name']).strip()
                data_type = row['data_type']
                using = ''
                for index2, row2 in old_disc_cols.iterrows():
                    if col_name == row2['column_name']:
                        try:
                            if str(data_type) != str(row2['data_type']):
                                using = f'''USING "{col_name}"::{data_type}'''
                                query = f''' ALTER table derived.old_smch_discharges ALTER column "{col_name}" TYPE {data_type} {using};'''
                                inject_sql(query, "OLD DISCHARGES")
                        except Exception as ex:
                            query = f'''ALTER table derived.old_smch_discharges DROP column "{col_name}" '''
                            inject_sql(
                                query,
                                f'''DROPPING DISCHARGE COLL {col_name}''')

            # Match Data Types For Matched Data
            for index, row in matched_cols.iterrows():
                col_name = str(row['column_name']).strip()
                data_type = row['data_type']
                using = ''
                for index2, row2 in old_matched_cols.iterrows():
                    if col_name == row2['column_name']:
                        try:
                            if str(data_type) != str(row2['data_type']):
                                using = f'''USING "{col_name}"::{data_type}'''
                                query = f''' ALTER table derived.old_smch_matched_admissions_discharges ALTER column "{col_name}" TYPE {data_type} {using};'''
                                inject_sql(query, "Union Views")
                        except Exception as ex:
                            query = f'''ALTER table derived.old_smch_matched_admissions_discharges DROP column "{col_name}" '''
                            inject_sql(
                                query,
                                f'''DROPPING MATCHED COLL {col_name} ''')

            old_smch_admissions = None
            old_smch_discharges = None
            old_matched_smch_data = None
            new_smch_admissions = None
            new_smch_discharges = None
            new_smch_matched_data = None
            if table_exists('derived', 'old_smch_admissions'):
                old_smch_admissions = catalog.load('read_old_smch_admissions')
            if table_exists('derived', 'old_smch_discharges'):
                old_smch_discharges = catalog.load('read_old_smch_discharges')
            if table_exists('derived',
                            'old_smch_matched_admissions_discharges'):
                old_matched_smch_data = catalog.load(
                    'read_old_smch_matched_data')
            if table_exists('derived', 'admissions'):
                new_smch_admissions = catalog.load('read_new_smch_admissions')
            if table_exists('derived', 'discharges'):
                new_smch_discharges = catalog.load('read_new_smch_discharges')
            if table_exists('derived', 'joined_admissions_discharges'):
                new_smch_matched_data = catalog.load('read_new_smch_matched')

            if old_smch_admissions is not None and not old_smch_admissions.empty:

                for position, admission in old_smch_admissions.iterrows():

                    age_list = []

                    if 'AgeB.value' in admission and str(
                            admission['AgeB.value']) != 'nan':
                        age_list = str(admission['AgeB.value']).split(",")
                    # Initialise Hours
                    hours = 0
                    period = 0
                    # If size of List is 1 it either means its days only or hours only

                    if len(age_list) == 1:
                        age = age_list[0]
                        # Check if hours or Days
                        if 'hour' in age:

                            hours = [
                                int(s) for s in age.replace("-", "").split()
                                if s.isdigit()
                            ]
                            # Check if value contains figures
                            if len(hours) > 0:
                                period = hours[0]
                            else:
                                if "an" in age:
                                    # IF AN HOUR
                                    period = 1

                        elif 'day' in age:
                            hours = [
                                int(s) for s in age.replace("-", "").split()
                                if s.isdigit()
                            ]
                            if len(hours) > 0:
                                period = hours[0] * 24
                        elif 'second' in age:
                            # FEW SECONDS CAN BE ROUNDED OFF 1 HOUR
                            period = 1
                        elif 'minute' in age:
                            # MINUTES CAN BE ROUNDED OFF 1 HOUR
                            period = 1
                            pass
                    # Contains Both Hours and Days
                    elif len(age_list) == 2:
                        age_days = age_list[0]
                        age_hours = age_list[1]
                        if 'day' in age_days and 'hour' in age_hours:
                            number_hours_days = [
                                int(s) for s in age_days.split()
                                if s.isdigit()
                            ]
                            number_hours = [
                                int(s) for s in age_hours.split()
                                if s.isdigit()
                            ]
                            if (len(number_hours) > 0
                                    and len(number_hours_days) > 0):
                                period = (number_hours_days[0]) * 24 + (
                                    number_hours[0])

                    else:
                        pass

                    if period > 0:
                        old_smch_admissions.loc[position, 'Age.value'] = period
                        if period < 2:
                            old_smch_admissions.loc[
                                position,
                                'AgeCategory'] = 'Fresh Newborn (< 2 hours old)'
                        elif period > 2 and period <= 23:
                            old_smch_admissions.loc[
                                position,
                                'AgeCategory'] = 'Newborn (2 - 23 hrs old)'
                        elif period > 23 and period <= 47:
                            old_smch_admissions.loc[
                                position,
                                'AgeCategory'] = 'Newborn (1 day - 1 day 23 hrs old)'
                        elif period > 47 and period <= 71:
                            old_smch_admissions.loc[
                                position,
                                'AgeCategory'] = 'Infant (2 days - 2 days 23 hrs old)'
                        else:
                            old_smch_admissions.loc[
                                position,
                                'AgeCategory'] = 'Infant (> 3 days old)'
                    ########################## UPDATE ADMISSION SCRIPT WITH NEW KEYS ########################

                    key_change(old_smch_admissions, admission, position,
                               'BW.value', 'BirthWeight.value')
                    key_change(old_smch_admissions, admission, position,
                               'Conv.value', 'Convulsions.value')
                    key_change(old_smch_admissions, admission, position,
                               'SRNeuroOther.value',
                               'SymptomReviewNeurology.value')
                    key_change(old_smch_admissions, admission, position,
                               'LBW.value', 'LowBirthWeight.value')
                    key_change(old_smch_admissions, admission, position,
                               'AW.value', 'AdmissionWeight.value')
                    key_change(old_smch_admissions, admission, position,
                               'BSmgdL.value', 'BSUnitmg.value')
                    key_change(old_smch_admissions, admission, position,
                               'BSmmol.value', 'BloodSugarmmol.value')
                    key_change(old_smch_admissions, admission, position,
                               'BSmg.value', 'BloodSugarmg.value')
                if "Age.value" in old_smch_admissions:
                    old_smch_admissions['Age.value'] = pd.to_numeric(
                        old_smch_admissions['Age.value'], errors='coerce')
                if 'AdmissionWeight.value' in old_smch_admissions:
                    old_smch_admissions[
                        'AdmissionWeight.value'] = pd.to_numeric(
                            old_smch_admissions['AdmissionWeight.value'],
                            errors='coerce')
                if 'BirthWeight.value' in old_smch_admissions:
                    old_smch_admissions['BirthWeight.value'] = pd.to_numeric(
                        old_smch_admissions['BirthWeight.value'],
                        errors='coerce')

                format_date(old_smch_admissions, 'DateTimeAdmission.value')
                format_date(old_smch_admissions, 'EndScriptDatetime.value')
                format_date(old_smch_admissions, 'DateHIVtest.value')
                format_date(old_smch_admissions, 'ANVDRLDate.value')

            if old_smch_discharges is not None and not old_smch_discharges.empty:
                for position, discharge in old_smch_discharges.iterrows():
                    key_change(old_smch_discharges, discharge, position,
                               'BWTDis.value', 'BirthWeight.value')
                    key_change(old_smch_discharges, discharge, position,
                               'BirthDateDis.value', 'DOBTOB.value')
                    key_change(old_smch_discharges, discharge, position,
                               'Delivery.value', 'ModeDelivery.value')
                    key_change(old_smch_discharges, discharge, position,
                               'NNUAdmTemp.value', 'Temperature.value')
                    key_change(old_smch_discharges, discharge, position,
                               'GestBirth.value', 'Gestation.value')
                    key_change(old_smch_discharges, discharge, position,
                               'PresComp.value', 'AdmReason.value')
                #Format Dates Discharge Table
                format_date(old_smch_discharges, 'DateAdmissionDC.value')
                format_date(old_smch_discharges, 'DateDischVitals.value')
                format_date(old_smch_discharges, 'DateDischWeight.value')
                format_date(old_smch_discharges, 'DateTimeDischarge.value')
                format_date(old_smch_discharges, 'EndScriptDatetime.value')
                format_date(old_smch_discharges, 'DateWeaned.value')
                format_date(old_smch_discharges, 'DateTimeDeath.value')
                format_date(old_smch_discharges, 'DateAdmission.value')
                format_date(old_smch_discharges, 'BirthDateDis.value')

            if old_matched_smch_data is not None and not old_matched_smch_data.empty:
                for position, matched_admission in old_matched_smch_data.iterrows(
                ):

                    age_list = []

                    if 'AgeB.value' in matched_admission and str(
                            matched_admission['AgeB.value']) != 'nan':
                        age_list = str(
                            matched_admission['AgeB.value']).split(",")
                    # Initialise Hours
                    hours = 0
                    period = 0
                    # If size of List is 1 it either means its days only or hours only

                    if len(age_list) == 1:
                        age = age_list[0]
                        # Check if hours or Days
                        if 'hour' in age:

                            hours = [
                                int(s) for s in age.replace("-", "").split()
                                if s.isdigit()
                            ]
                            # Check if value contains figures
                            if len(hours) > 0:
                                period = hours[0]
                            else:
                                if "an" in age:
                                    # IF AN HOUR
                                    period = 1

                        elif 'day' in age:
                            hours = [
                                int(s) for s in age.replace("-", "").split()
                                if s.isdigit()
                            ]
                            if len(hours) > 0:
                                period = hours[0] * 24
                        elif 'second' in age:
                            # FEW SECONDS CAN BE ROUNDED OFF 1 HOUR
                            period = 1
                        elif 'minute' in age:
                            # MINUTES CAN BE ROUNDED OFF 1 HOUR
                            period = 1
                            pass
                    # Contains Both Hours and Days
                    elif len(age_list) == 2:
                        age_days = age_list[0]
                        age_hours = age_list[1]
                        if 'day' in age_days and 'hour' in age_hours:
                            number_hours_days = [
                                int(s) for s in age_days.split()
                                if s.isdigit()
                            ]
                            number_hours = [
                                int(s) for s in age_hours.split()
                                if s.isdigit()
                            ]
                            if (len(number_hours) > 0
                                    and len(number_hours_days) > 0):
                                period = (number_hours_days[0]) * 24 + (
                                    number_hours[0])

                    else:
                        pass

                    if period > 0:
                        old_matched_smch_data.loc[position,
                                                  'Age.value'] = period
                        if period < 2:
                            old_matched_smch_data.loc[
                                position,
                                'AgeCategory'] = 'Fresh Newborn (< 2 hours old)'
                        elif period > 2 and period <= 23:
                            old_matched_smch_data.loc[
                                position,
                                'AgeCategory'] = 'Newborn (2 - 23 hrs old)'
                        elif period > 23 and period <= 47:
                            old_matched_smch_data.loc[
                                position,
                                'AgeCategory'] = 'Newborn (1 day - 1 day 23 hrs old)'
                        elif period > 47 and period <= 71:
                            old_matched_smch_data.loc[
                                position,
                                'AgeCategory'] = 'Infant (2 days - 2 days 23 hrs old)'
                        else:
                            old_matched_smch_data.loc[
                                position,
                                'AgeCategory'] = 'Infant (> 3 days old)'
                    ########################## UPDATE ADMISSION SCRIPT WITH NEW KEYS ########################
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'BW.value', 'BirthWeight.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'Conv.value', 'Convulsions.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'SRNeuroOther.value',
                               'SymptomReviewNeurology.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'LBW.value', 'LowBirthWeight.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'AW.value', 'AdmissionWeight.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'BSmgdL.value', 'BSUnitmg.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'BSmmol.value',
                               'BloodSugarmmol.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'BSmg.value', 'BloodSugarmg.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'BWTDis.value', 'BirthWeight.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'BirthDateDis.value', 'DOBTOB.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'Delivery.value',
                               'ModeDelivery.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'NNUAdmTemp.value',
                               'Temperature.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'GestBirth.value', 'Gestation.value')
                    key_change(old_matched_smch_data, matched_admission,
                               position, 'PresComp.value', 'AdmReason.value')
                if "Age.value" in old_matched_smch_data:
                    old_matched_smch_data['Age.value'] = pd.to_numeric(
                        old_matched_smch_data['Age.value'], errors='coerce')
                if 'AdmissionWeight.value' in old_matched_smch_data:
                    old_matched_smch_data[
                        'AdmissionWeight.value'] = pd.to_numeric(
                            old_matched_smch_data['AdmissionWeight.value'],
                            errors='coerce')
                if 'BirthWeight.value' in old_matched_smch_data:
                    old_matched_smch_data['BirthWeight.value'] = pd.to_numeric(
                        old_matched_smch_data['BirthWeight.value'],
                        errors='coerce')
                if 'BirthWeight.value_discharge' in old_matched_smch_data:
                    old_matched_smch_data['BirthWeight.value'] = pd.to_numeric(
                        old_matched_smch_data['BirthWeight.value_value'],
                        errors='coerce')
                format_date(old_matched_smch_data, 'DateTimeAdmission.value')
                format_date(old_matched_smch_data, 'EndScriptDatetime.value')
                format_date(old_matched_smch_data, 'DateHIVtest.value')
                format_date(old_matched_smch_data, 'ANVDRLDate.value')
                #Format Dates Discharge Table
                format_date(old_matched_smch_data, 'DateAdmissionDC.value')
                format_date(old_matched_smch_data, 'DateDischVitals.value')
                format_date(old_matched_smch_data, 'DateDischWeight.value')
                format_date(old_matched_smch_data, 'DateTimeDischarge.value')
                format_date(old_matched_smch_data, 'EndScriptDatetime.value')
                format_date(old_matched_smch_data, 'DateWeaned.value')
                format_date(old_matched_smch_data, 'DateTimeDeath.value')
                format_date(old_matched_smch_data, 'DateAdmission.value')
                format_date(old_matched_smch_data, 'BirthDateDis.value')
            # SAVE OLD NEW ADMISSIONS
            if new_smch_admissions is not None and old_smch_admissions is not None:
                combined_adm_df = pd.concat(
                    [new_smch_admissions, old_smch_admissions],
                    ignore_index=True)
                if not combined_adm_df.empty:
                    catalog.save('create_derived_old_new_admissions_view',
                                 combined_adm_df)
            # SAVE OLD NEW DISCHARGES
            if new_smch_discharges is not None and old_smch_discharges is not None:
                combined_dis_df = pd.concat(
                    [new_smch_discharges, old_smch_discharges],
                    ignore_index=True)
                if not combined_dis_df.empty:
                    catalog.save('create_derived_old_new_discharges_view',
                                 combined_dis_df)

            # SAVE MATCHED DATA
            if new_smch_matched_data is not None and old_matched_smch_data is not None:
                #Correct UID column to suit the lower case uid in new_smch_matched_data
                if 'UID' in old_matched_smch_data.columns:
                    old_matched_smch_data = old_matched_smch_data.rename(
                        columns={'UID': 'uid'}, inplace=False)
                combined_matched_df = pd.concat(
                    [new_smch_matched_data, old_matched_smch_data],
                    ignore_index=True)
                if not combined_matched_df.empty:
                    catalog.save('create_derived_old_new_matched_view',
                                 combined_matched_df)

        except Exception as ex:
            logging.error("!!! An error occured creating union views: ")
            logging.error(ex)
            exit()