def cyto_ai_balance_training_pipeline(self, data_df): data_processing_obj = DataProcessing() sampling_obj = DataSampling() sys_obj = SysConfig() train_obj = DataTrainning() test_set_ratio = sys_obj.get_test_set_ratio() y_key = sys_obj.get_y_key() model_output_dir = sys_obj.get_model_output_dir() log_file = sys_obj.get_log_file() model_threshold = 200 model_count = 1 print(data_df) #### data preprocessing print("start cast_all_to_numeric.") data_df = data_processing_obj.cast_all_to_numeric(data_df) print("end cast_all_to_numeric.") # print("########################") # print(data_df) ### convert to category data_df[y_key] = data_df[y_key].astype('category') ### sampling disease_df = data_df.loc[data_df[y_key] == 1, :] normal_df = data_df.loc[data_df[y_key] == 0, :] print("### disease_df") print(disease_df) print("### normal_df") print(normal_df) print(len(disease_df), len(normal_df)) # # log_file = '/app/data/model/RF_3000_log.txt' fh_writer = open(log_file, 'w') while model_count < model_threshold: # (train_set, train_label, test_set, test_label) = sampling_obj.category2_sampling_pipeline(normal_df, disease_df,y_key, test_set_ratio) (train_set, test_set) = sampling_obj.category2_simple_sampling_pipeline( normal_df, disease_df, y_key, test_set_ratio) print("###############################") print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") print(model_count) # print("######################## train_set") # print(train_set) # print("######################## test_set") # print(test_set) # #### temp data test_set_bak = test_set.copy() # # test_set_bak[y_key] = test_set_bak[y_key].astype('int') ### get x,y (train_set_x, train_set_y) = sampling_obj.get_x_y_from_dataframe( train_set, y_key) (test_set_x, test_set_y) = sampling_obj.get_x_y_from_dataframe( test_set, y_key) # print("######################## train_set_x") # print(train_set_x) # print("######################## train_set_y") # print(train_set_y) # print("######################## test_set_x") # print(test_set_x) # print("######################## test_set_y") # print(test_set_y) ### seperate by disease test_set_disease = test_set_bak.loc[test_set_bak[y_key] == 1, :] test_set_normal = test_set_bak.loc[test_set_bak[y_key] == 0, :] (test_set_disease_x, test_set_disease_y) = sampling_obj.get_x_y_from_dataframe( test_set_disease, y_key) (test_set_normal_x, test_set_normal_y) = sampling_obj.get_x_y_from_dataframe( test_set_normal, y_key) # print("######################## test_set_disease") # print(test_set_disease) # print("######################## test_set_disease_x") # print(test_set_disease_x) # print("######################## test_set_disease_y") # print(test_set_disease_y) # print("######################## test_set_normal") # print(test_set_normal) # print("######################## test_set_normal_x") # print(test_set_normal_x) # print("######################## test_set_normal_y") # print(test_set_normal_y) test_set_disease_len = len(test_set_disease_y) test_set_normal_len = len(test_set_normal_y) print( "######################## test_set_disease_len, test_set_normal_len" ) print("{}, {}".format(test_set_disease_len, test_set_normal_len)) # ### feature list # # feature_dict = {} # # feature_dict['numeric'] = list(train_set_x.columns.values) ### training the model my_model = train_obj.SKRandomForest_Category( train_set_x, train_set_y) test_score = my_model.score(test_set_x, test_set_y) test_set_diseas_score = my_model.score(test_set_disease_x, test_set_disease_y) test_set_normal_score = my_model.score(test_set_normal_x, test_set_normal_y) print("Test score = {}".format(test_score)) print("test_set_diseas_score score = {}".format( test_set_diseas_score)) print("test_set_normal_score score = {}".format( test_set_normal_score)) ### log file fh_writer.write("{}\t{}\t{}\n".format(test_score, test_set_diseas_score, test_set_normal_score)) ### model model_file = model_output_dir + str(model_count) + '.pkl' joblib.dump(my_model, model_file) model_count += 1 fh_writer.close() return data_df
def cyto_xgboost_balance_training_pipeline(self, data_df): data_processing_obj = DataProcessing() sampling_obj = DataSampling() sys_obj = SysConfig() train_obj = DataTrainning() test_set_ratio = sys_obj.get_test_set_ratio() y_key = sys_obj.get_y_key() model_output_dir = sys_obj.get_model_output_dir() log_file = sys_obj.get_log_file() model_threshold = 200 model_count = 1 print(data_df) #### data preprocessing print("start cast_all_to_numeric.") data_df = data_processing_obj.cast_all_to_numeric(data_df) print("end cast_all_to_numeric.") # print("########################") # print(data_df) ### convert to category # data_df[y_key] = data_df[y_key].astype('category') ### sampling disease_df = data_df.loc[data_df[y_key] == 1, :] normal_df = data_df.loc[data_df[y_key] == 0, :] # # log_file = '/app/data/model/RF_3000_log.txt' # fh_writer = open(log_file, 'w') # while model_count < model_threshold: # (train_set, train_label, test_set, test_label) = sampling_obj.category2_sampling_pipeline(normal_df, disease_df,y_key, test_set_ratio) (train_set, test_set) = sampling_obj.category2_simple_sampling_pipeline( normal_df, disease_df, y_key, test_set_ratio) print("###############################") print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") print(model_count) # print("######################## train_set") # print(train_set) # print("######################## test_set") # print(test_set) # #### temp data test_set_bak = test_set.copy() # # test_set_bak[y_key] = test_set_bak[y_key].astype('int') ### get x,y (train_set_x, train_set_y) = sampling_obj.get_x_y_from_dataframe(train_set, y_key) (test_set_x, test_set_y) = sampling_obj.get_x_y_from_dataframe(test_set, y_key) # print("######################## train_set_x") # print(train_set_x) # print("######################## train_set_y") # print(train_set_y) # print("######################## test_set_x") # print(test_set_x) # print("######################## test_set_y") # print(test_set_y) ### seperate by disease test_set_disease = test_set_bak.loc[test_set_bak[y_key] == 1, :] test_set_normal = test_set_bak.loc[test_set_bak[y_key] == 0, :] (test_set_disease_x, test_set_disease_y) = sampling_obj.get_x_y_from_dataframe( test_set_disease, y_key) (test_set_normal_x, test_set_normal_y) = sampling_obj.get_x_y_from_dataframe( test_set_normal, y_key) # print("######################## test_set_disease") # print(test_set_disease) # print("######################## test_set_disease_x") # print(test_set_disease_x) # print("######################## test_set_disease_y") # print(test_set_disease_y) # print("######################## test_set_normal") # print(test_set_normal) # print("######################## test_set_normal_x") # print(test_set_normal_x) # print("######################## test_set_normal_y") # print(test_set_normal_y) test_set_disease_len = len(test_set_disease_y) test_set_normal_len = len(test_set_normal_y) print( "######################## test_set_disease_len, test_set_normal_len" ) print("{}, {}".format(test_set_disease_len, test_set_normal_len)) # ### feature list # # feature_dict = {} # # feature_dict['numeric'] = list(train_set_x.columns.values) ### training the model my_model = train_obj.xgboot_training(train_set_x, train_set_y, test_set_x, test_set_y) # # test_set_x = preprocessing.scale(test_set_x) # # test_set_x = scaler.scale(test_set_x) # test_set_x = scaler.transform(test_set_x) # # test_set_y = to_categorical(test_set_y) # test_score = my_model.evaluate(test_set_x, test_set_y) # # test_set_diseas_score = my_model.score(test_set_disease_x, test_set_disease_y) # # test_set_normal_score = my_model.score(test_set_normal_x, test_set_normal_y) # print("\ntest data set, %s: %.2f%%" % (my_model.metrics_names[1], test_score[1]*100)) # print("Test score:", test_score[0]) # print('Test accuracy:', test_score[1]) return data_df