def d210(source_output_path, source_name, STG_tables): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") INS_DTTM = ",CURRENT_TIMESTAMP AS INS_DTTM \n" stg_tables_df = funcs.get_stg_tables(STG_tables, source_name) for stg_tables_df_index, stg_tables_df_row in stg_tables_df.iterrows(): Table_name = stg_tables_df_row['Table name'] create_stg_view = "REPLACE VIEW " + pm.gdev1v_stg + "." + Table_name + " AS LOCK ROW FOR ACCESS \n" create_stg_view = create_stg_view + "SELECT\n" STG_table_columns = funcs.get_stg_table_columns( STG_tables, source_name, Table_name) for STG_table_columns_index, STG_table_columns_row in STG_table_columns.iterrows( ): Column_name = STG_table_columns_row['Column name'] comma = ',' if STG_table_columns_index > 0 else ' ' comma_Column_name = comma + Column_name create_stg_view = create_stg_view + comma_Column_name + "\n" create_stg_view = create_stg_view + INS_DTTM create_stg_view = create_stg_view + "from " + pm.gdev1t_stg + "." + Table_name + ";\n\n" f.write(create_stg_view) f.close()
def d415(source_output_path, source_name, STG_tables): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") stg_tables_df = funcs.get_stg_tables(STG_tables, source_name) for stg_tables_df_index, stg_tables_df_row in stg_tables_df.iterrows(): stg_table_name = stg_tables_df_row['Table name'] del_script = "DEL FROM " + pm.GCFR_V + ".GCFR_Transform_KeyCol " del_script = del_script + " WHERE OUT_OBJECT_NAME = '" + stg_table_name + "';\n" STG_table_columns = funcs.get_stg_table_columns( STG_tables, source_name, stg_table_name, True) exe_ = "EXEC " + pm.MACRO_DB + ".GCFR_Register_Tfm_KeyCol('" + pm.SI_VIEW + "'" _p = ",'" + stg_table_name + "'" _p = _p + ",'SEQ_NO' );\n\n" exe_p = exe_ + _p exe_p_ = "" for STG_table_columns_index, STG_table_columns_row in STG_table_columns.iterrows( ): if STG_table_columns_row['PK'].upper() == 'Y': Column_name = STG_table_columns_row['Column name'] _p = ",'" + stg_table_name + "'" _p = _p + ",'" + Column_name + "' );\n" exe_p_ = exe_p_ + exe_ + _p exe_p = exe_p_ + "\n" if exe_p_ != "" else exe_p f.write(del_script) f.write(exe_p) f.close()
def d340(source_output_path, source_name, STG_tables, BKEY): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") stg_tables_df = STG_tables.loc[ (STG_tables['Source system name'] == source_name) & ~(STG_tables['Key domain name'].isnull()) & ~(STG_tables['Natural key'].isnull())] for stg_tables_df_index, stg_tables_df_row in stg_tables_df.replace( np.nan, "", regex=True).iterrows(): key_domain_name = stg_tables_df_row['Key domain name'] stg_table_name = stg_tables_df_row['Table name'] stg_Column_name = stg_tables_df_row['Column name'] bkey_df = BKEY.loc[(BKEY['Key domain name'] == key_domain_name)] key_set_name = bkey_df['Key set name'].values[0] Key_set_ID = str(int(bkey_df['Key set ID'].values[0])) Key_domain_ID = str(int(bkey_df['Key domain ID'].values[0])) Physical_table = bkey_df['Physical table'].values[0] script = "CALL " + pm.APPLY_DB + ".GCFR_PP_BKEY(" script = script + "'BK_" + Key_set_ID + "_" + stg_table_name + "_" + stg_Column_name + "_" + Key_domain_ID + "'" script = script + ",6, oRC, oRM);" f.write(script + '\n') f.close()
def d400(source_output_path, source_name, STG_tables): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") stg_tables_df = funcs.get_stg_tables(STG_tables, source_name) for stg_tables_df_index, stg_tables_df_row in stg_tables_df.iterrows(): Table_name = stg_tables_df_row['Table name'] Fallback = ', Fallback' if stg_tables_df_row['Fallback'].upper( ) == 'Y' else '' create_stg_table = "create multiset table " + pm.gdev1t_stg + "." + Table_name + Fallback + "\n" + "(\n" STG_table_columns = funcs.get_stg_table_columns( STG_tables, source_name, Table_name, True) pi_columns = "" for STG_table_columns_index, STG_table_columns_row in STG_table_columns.iterrows( ): Column_name = STG_table_columns_row['Column name'] comma = ',' if STG_table_columns_index > 0 else ' ' comma_Column_name = comma + Column_name Data_type = str(STG_table_columns_row['Data type']) character_set = " CHARACTER SET UNICODE NOT CASESPECIFIC " if "CHAR" in Data_type.upper( ) or "VARCHAR" in Data_type.upper() else "" not_null = " not null " if STG_table_columns_row['Mandatory'].upper( ) == 'Y' or STG_table_columns_row['PK'].upper() == 'Y' else " " create_stg_table = create_stg_table + comma_Column_name + " " + Data_type + character_set + not_null + "\n" if STG_table_columns_row['PK'].upper() == 'Y': pi_columns = pi_columns + ',' + Column_name if pi_columns != "" else Column_name extra_columns = ",Start_Ts\tTIMESTAMP(6) WITH TIME ZONE \n" \ + ",End_Ts\tTIMESTAMP(6) WITH TIME ZONE \n"\ + ",Start_Date\tDATE FORMAT 'YYYY-MM-DD' \n" \ + ",End_Date\tDATE FORMAT 'YYYY-MM-DD' \n" \ + ",Record_Deleted_Flag\tBYTEINT\n" \ + ",Ctl_Id\tSMALLINT COMPRESS(997)\n" \ + ",File_Id\tSMALLINT\n" \ + ",Process_Name\tVARCHAR(128)\n" \ + ",Process_Id\tINTEGER\n" \ + ",Update_Process_Name\tVARCHAR(128) \n" \ + ",Update_Process_Id\tINTEGER\n " if pi_columns == "": pi_columns = "SEQ_NO" seq_column = ",SEQ_NO DECIMAL(10,0) NOT NULL GENERATED ALWAYS AS IDENTITY\n\t (START WITH 1 INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 NO CYCLE)\n" else: seq_column = "" Primary_Index = ")Primary Index (" + pi_columns + ")" create_stg_table = create_stg_table + extra_columns + seq_column + Primary_Index create_stg_table = create_stg_table + ";\n\n" f.write(create_stg_table) f.close()
def d200(source_output_path, source_name, STG_tables): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") INS_DTTM = ",INS_DTTM TIMESTAMP(6) NOT NULL \n" stg_tables_df = funcs.get_stg_tables(STG_tables, source_name) for stg_tables_df_index, stg_tables_df_row in stg_tables_df.iterrows(): Table_name = stg_tables_df_row['Table name'] Fallback = ', Fallback' if stg_tables_df_row['Fallback'].upper( ) == 'Y' else '' create_stg_table = "create multiset table " + pm.gdev1t_stg + "." + Table_name + Fallback + "\n" + "(\n" create_wrk_table = "create multiset table " + pm.gdev1t_WRK + "." + Table_name + Fallback + "\n" + "(\n" STG_table_columns = funcs.get_stg_table_columns( STG_tables, source_name, Table_name) pi_columns = "" for STG_table_columns_index, STG_table_columns_row in STG_table_columns.iterrows( ): Column_name = STG_table_columns_row['Column name'] comma = ',' if STG_table_columns_index > 0 else ' ' comma_Column_name = comma + Column_name Data_type = str(STG_table_columns_row['Data type']) character_set = " CHARACTER SET UNICODE NOT CASESPECIFIC " if "CHAR" in Data_type.upper( ) or "VARCHAR" in Data_type.upper() else "" not_null = " not null " if STG_table_columns_row['Mandatory'].upper( ) == 'Y' or STG_table_columns_row['PK'].upper() == 'Y' else " " create_stg_table = create_stg_table + comma_Column_name + " " + Data_type + character_set + not_null + "\n" create_wrk_table = create_wrk_table + comma_Column_name + " " + Data_type + character_set + not_null + "\n" if STG_table_columns_row['PK'].upper() == 'Y': pi_columns = pi_columns + ',' + Column_name if pi_columns != "" else Column_name wrk_extra_columns = ",REJECTED INTEGER\n" + ",BATCH_LOADED INTEGER\n" + ",NEW_ROW INTEGER\n" if pi_columns == "": pi_columns = "SEQ_NO" seq_column = ",SEQ_NO DECIMAL(10,0) NOT NULL GENERATED ALWAYS AS IDENTITY\n\t (START WITH 1 INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 NO CYCLE)\n" else: seq_column = "" Primary_Index = ")Primary Index (" + pi_columns + ")" create_stg_table = create_stg_table + INS_DTTM + seq_column + Primary_Index create_wrk_table = create_wrk_table + INS_DTTM + wrk_extra_columns + seq_column + Primary_Index create_stg_table = create_stg_table + ";\n\n" create_wrk_table = create_wrk_table + ";\n\n" f.write(create_stg_table) f.write(create_wrk_table) f.close()
def d320(source_output_path, source_name, STG_tables, BKEY): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") separator = pm.separator stg_tables_df = STG_tables.loc[ (STG_tables['Source system name'] == source_name) & ~(STG_tables['Key domain name'].isnull()) & ~(STG_tables['Natural key'].isnull())] for stg_tables_df_index, stg_tables_df_row in stg_tables_df.replace( np.nan, "", regex=True).iterrows(): key_domain_name = stg_tables_df_row['Key domain name'] stg_table_name = stg_tables_df_row['Table name'] stg_Column_name = stg_tables_df_row['Column name'] Bkey_filter = str(stg_tables_df_row['Bkey filter']).upper() Bkey_filter = "WHERE " + Bkey_filter if Bkey_filter != "" and "JOIN" not in Bkey_filter else Bkey_filter Bkey_filter = Bkey_filter + "\n" if Bkey_filter != "" else Bkey_filter Natural_key_list = stg_tables_df_row['Natural key'].split(separator) trim_Trailing_Natural_key_list = [] for i in Natural_key_list: trim_Trailing_Natural_key_list.append( "TRIM(Trailing '.' from TRIM(" + i.strip() + "))") Source_Key = funcs.list_to_string(trim_Trailing_Natural_key_list, separator) coalesce_count = Source_Key.upper().count("COALESCE") separator_count = Source_Key.count(separator) compare_string = "_" * separator_count if coalesce_count > separator_count else "''" Source_Key_cond = "WHERE " if "WHERE" not in Bkey_filter else " AND " Source_Key_cond = Source_Key_cond + "COALESCE(Source_Key," + compare_string + ") <> " + compare_string + " " bkey_df = BKEY.loc[(BKEY['Key domain name'] == key_domain_name)] Key_set_ID = str(int(bkey_df['Key set ID'].values[0])) Key_domain_ID = str(int(bkey_df['Key domain ID'].values[0])) script = "REPLACE VIEW " + pm.INPUT_VIEW_DB + ".BK_" + Key_set_ID + "_" + stg_table_name + "_" + stg_Column_name + "_" + Key_domain_ID + "_IN AS LOCK ROW FOR ACCESS\n" script = script + "SELECT " + Source_Key + " AS Source_Key\n" script = script + "FROM " + pm.gdev1v_stg + "." + stg_table_name + "\n" script = script + Bkey_filter + Source_Key_cond + "\n" script = script + "GROUP BY 1;" + "\n" f.write(script) f.write('\n') f.close()
def d410(source_output_path, source_name, STG_tables): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") stg_tables_df = funcs.get_stg_tables(STG_tables, source_name) for stg_tables_df_index, stg_tables_df_row in stg_tables_df.iterrows(): stg_table_name = stg_tables_df_row['Table name'] script = "REPLACE VIEW " + pm.SI_VIEW + "." + stg_table_name + " AS\n" script = script + "SELECT * FROM " + pm.SI_DB + "." + stg_table_name + ";\n\n" f.write(script) f.close()
def d300(source_output_path, source_name, STG_tables, BKEY): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") Key_domain_names_df = STG_tables.loc[ (STG_tables['Source system name'] == source_name) & ~(STG_tables['Key domain name'].isnull())][['Key domain name' ]].drop_duplicates() for Key_domain_names_df_index, Key_domain_names_df_row in Key_domain_names_df.iterrows( ): key_domain_name = Key_domain_names_df_row['Key domain name'] bkey_df = BKEY.loc[(BKEY['Key domain name'] == key_domain_name)] key_set_name = bkey_df['Key set name'].values[0] Key_set_ID = str(int(bkey_df['Key set ID'].values[0])) Key_domain_ID = str(int(bkey_df['Key domain ID'].values[0])) Physical_table = bkey_df['Physical table'].values[0] script1 = "EXEC " + pm.MACRO_DB + ".GCFR_Register_Bkey_Key_Set(" + Key_set_ID + ", '" + key_set_name + "', '" + Physical_table + "', '" + pm.G_BKEY_V + "');" script2 = "CALL " + pm.UT_DB + ".GCFR_UT_BKEY_St_Key_CT('" + pm.G_BKEY_T + "', '" + Physical_table + "', '1', :OMessage);" script3 = "CALL " + pm.UT_DB + ".GCFR_UT_BKEY_St_Key_CV('" + pm.G_BKEY_T + "', '" + Physical_table + "', '" + pm.G_BKEY_V + "', :OMessage);" script4 = "CALL " + pm.UT_DB + ".GCFR_UT_BKEY_Key_Set_RI_Check(" + Key_set_ID + ", :OMessage);" script5 = "CALL " + pm.UT_DB + ".GCFR_UT_BKEY_St_Key_NextId_CT('" + Physical_table + "', '1', :OMessage);" script6 = "CALL " + pm.UT_DB + ".GCFR_UT_BKEY_St_Key_NextId_CV('" + Physical_table + "', :OMessage);" script7 = "CALL " + pm.UT_DB + ".GCFR_UT_BKEY_S_K_NextId_Log_CT('" + Physical_table + "', '1', :OMessage);" script8 = "CALL " + pm.UT_DB + ".GCFR_UT_BKEY_S_K_NextId_Log_CV('" + Physical_table + "', :OMessage);" script9 = "CALL " + pm.UT_DB + ".GCFR_UT_BKEYStandKeyNextId_Gen('" + pm.G_BKEY_T + "', '" + Physical_table + "', " + Key_set_ID + ", :OMessage);" script10 = "EXEC " + pm.MACRO_DB + ".GCFR_Register_Bkey_Domain(" + Key_set_ID + ", " + Key_domain_ID + ", '" + key_domain_name + "');" f.write(script1 + '\n') f.write(script2 + '\n') f.write(script3 + '\n') f.write(script4 + '\n') f.write(script5 + '\n') f.write(script6 + '\n') f.write(script7 + '\n') f.write(script8 + '\n') f.write(script9 + '\n') f.write(script10 + '\n') f.write('\n') f.close()
def d001(source_output_path, source_name, STG_tables): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") stg_tables_df = STG_tables.loc[STG_tables['Source system name'] == source_name][['Table name' ]].drop_duplicates() for STG_tables_index, STG_tables_row in stg_tables_df.iterrows(): Table_name = STG_tables_row['Table name'] # print(Table_name) f.write("delete from " + pm.automation_db + "." + pm.SOURCE_TABLES_LKP_table + " where SOURCE_NAME = '" + source_name + "';\n") f.write("insert into " + pm.automation_db + "." + pm.SOURCE_TABLES_LKP_table + "(SOURCE_NAME, TABLE_NAME)\n") f.write("VALUES ('" + source_name + "', '" + Table_name + "')" + ";\n") f.write("\n") f.close()
def d330(source_output_path, source_name, STG_tables, BKEY): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") stg_tables_df = STG_tables.loc[ (STG_tables['Source system name'] == source_name) & ~(STG_tables['Key domain name'].isnull()) & ~(STG_tables['Natural key'].isnull())] for stg_tables_df_index, stg_tables_df_row in stg_tables_df.replace( np.nan, "", regex=True).iterrows(): key_domain_name = stg_tables_df_row['Key domain name'] stg_table_name = stg_tables_df_row['Table name'] stg_Column_name = stg_tables_df_row['Column name'] bkey_df = BKEY.loc[(BKEY['Key domain name'] == key_domain_name)] key_set_name = bkey_df['Key set name'].values[0] Key_set_ID = str(int(bkey_df['Key set ID'].values[0])) Key_domain_ID = str(int(bkey_df['Key domain ID'].values[0])) Physical_table = bkey_df['Physical table'].values[0] script = "EXEC " + pm.MACRO_DB + ".GCFR_Register_Process(" script = script + "'BK_" + Key_set_ID + "_" + stg_table_name + "_" + stg_Column_name + "_" + Key_domain_ID + "'," script = script + "'define bkey for the table " + key_set_name + " and the domain " + key_domain_name + "'," script = script + "21, 99, 1" script = script + "'" + pm.G_BKEY_V + "'," script = script + "'" + Physical_table + "'," script = script + "'" + pm.G_BKEY_T + "'," script = script + "'" + Physical_table + "'," script = script + "'" + pm.TMP_DB + "'," script = script + "'" + Key_set_ID + "'," script = script + "'" + Key_domain_ID + "'," script = script + "'',0,0,0,0);" f.write(script + '\n') f.close()
def d000(source_output_path, source_name, Table_mapping, STG_tables, BKEY): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") for table_maping_index, table_maping_row in Table_mapping[ Table_mapping['Source'] == source_name].iterrows(): prcess_type = "TXF" layer = str(table_maping_row['Layer']) process_name = prcess_type + "_" + layer + "_" + str( table_maping_row['Mapping name']) target_table = str(table_maping_row['Target table name']) Historization_algorithm = str( table_maping_row['Historization algorithm']) f.write("delete from " + pm.automation_db + "." + pm.etl_process_table + " where process_name = '" + process_name + "';\n") f.write( "insert into " + pm.automation_db + "." + pm.etl_process_table + "(SOURCE_NAME, PROCESS_TYPE, PROCESS_NAME, BASE_TABLE, APPLY_TYPE, RECORD_ID)\n" ) f.write("VALUES ('" + source_name + "', '" + prcess_type + "', '" + process_name + "', '" + target_table + "', '" + Historization_algorithm + "', NULL)" + ";\n") f.write("\n") for STG_tables_index, STG_tables_row in STG_tables.loc[ (STG_tables['Source system name'] == source_name) & (STG_tables['Key set name'].notnull())].iterrows(): Key_set_name = STG_tables_row['Key set name'] Key_domain_name = STG_tables_row['Key domain name'] Table_name = STG_tables_row['Table name'] Column_name = STG_tables_row['Column name'] prcess_type = "BKEY" target_table = "" Historization_algorithm = "INSERT" for BKEY_index, BKEY_row in BKEY.loc[ (BKEY['Key set name'] == Key_set_name) & (BKEY['Key domain name'] == Key_domain_name)].iterrows(): Key_set_id = int(BKEY_row['Key set ID']) Key_domain_ID = int(BKEY_row['Key domain ID']) Physical_table = BKEY_row['Physical table'] # print(Table_name, Column_name, Key_set_id,Key_domain_ID,Physical_table) process_name = "BK_" + str( Key_set_id) + "_" + Table_name + "_" + Column_name + "_" + str( Key_domain_ID) f.write("delete from " + pm.automation_db + "." + pm.etl_process_table + " where process_name = '" + process_name + "';\n") f.write( "insert into " + pm.automation_db + "." + pm.etl_process_table + "(SOURCE_NAME, PROCESS_TYPE, PROCESS_NAME, BASE_TABLE, APPLY_TYPE, RECORD_ID)\n" ) f.write("VALUES ('" + source_name + "', '" + prcess_type + "', '" + process_name + "', '" + target_table + "', '" + Historization_algorithm + "', NULL)" + ";\n") f.write("\n") f.close()
def d420(source_output_path, source_name, STG_tables, BKEY): file_name = funcs.get_file_name(__file__) f = open(source_output_path + "/" + file_name + ".sql", "w+") separator = pm.separator stg_tables_df = funcs.get_stg_tables(STG_tables, source_name) for stg_tables_df_index, stg_tables_df_row in stg_tables_df.iterrows(): stg_table_name = stg_tables_df_row['Table name'] stg_Natural_key_df = STG_tables.loc[ (STG_tables['Source system name'] == source_name) & (STG_tables['Table name'] == stg_table_name) & ~(STG_tables['Natural key'].isnull())] bkey_Natural_key_list = [] for stg_Natural_key_df_index, stg_Natural_key_df_row in stg_Natural_key_df.iterrows( ): # bkey_Natural_key_list = stg_Natural_key_df_row['Natural key'].split(separator) bkey_Natural_key_list.append(stg_Natural_key_df_row['Natural key']) bkey_Natural_key_list_str = funcs.list_to_string( bkey_Natural_key_list, ',').upper() stg_table_has_pk = True if len(STG_tables.loc[ (STG_tables['Table name'] == stg_table_name) & (STG_tables['PK'].str.upper() == 'Y')].index) > 0 else False if not stg_table_has_pk: seq_pk_col = " SEQ_NO\n," else: seq_pk_col = " " create_view_script = "REPLACE VIEW " + pm.SI_VIEW + "." + stg_table_name + " AS\nSELECT \n" from_clause = "FROM " + pm.gdev1v_stg + "." + stg_table_name + " t" STG_table_columns = funcs.get_stg_table_columns( STG_tables, source_name, stg_table_name, True) bkeys_left_join = "" bkeys_left_join_count = 0 for STG_table_columns_index, STG_table_columns_row in STG_table_columns.iterrows( ): Column_name = STG_table_columns_row['Column name'].upper() alias = Column_name if Column_name in bkey_Natural_key_list_str: if "COALESCE" in bkey_Natural_key_list_str: Column_name = "COALESCE( " + Column_name + ",'')" Column_name = "TRIM(Trailing '.' from TRIM(" + Column_name + ")) " + alias comma = ',' if STG_table_columns_index > 0 else seq_pk_col comma_Column_name = comma + Column_name create_view_script = create_view_script + comma_Column_name + "\n" try: Key_domain_name = STG_table_columns_row['Key domain name'] bkey_physical_table = BKEY.loc[( BKEY['Key domain name'].str.upper() == Key_domain_name )]['Physical table'].values[0] bkeys_left_join_count = bkeys_left_join_count + 1 bk_alias = " bk" + str(bkeys_left_join_count) bkeys_left_join = bkeys_left_join + "LEFT JOIN " + pm.G_BKEY_V + "." + bkey_physical_table + bk_alias + "\n" Natural_key = STG_table_columns_row['Natural key'] split_Natural_key = Natural_key.replace(" ", "").split(separator) trim_Natural_key = [] for i in split_Natural_key: trim_Natural_key.append("TRIM(Trailing '.' from TRIM(" + i.strip() + "))") Natural_key = funcs.list_to_string(trim_Natural_key, separator) bkeys_left_join = bkeys_left_join + "\tON " + bk_alias + ".Source_Key = " + Natural_key + "\n" except: pass create_view_script = create_view_script + from_clause + "\n" + bkeys_left_join f.write(create_view_script + "\n") # f.write(Natural_key_list_str + "\n") f.close()