def __do_one_hot_encodings(self): df_train, cv = self.res_data_dict[ g_singletonDataFilePath.getTrainDir()] df_testset1 = self.res_data_dict[g_singletonDataFilePath.getTest1Dir()] df_testset2 = self.res_data_dict[g_singletonDataFilePath.getTest2Dir()] enc = OneHotEncoder(sparse=False) cross_feature_dict = self.__get_label_encode_dict() to_be_encoded = [] for _, new_feature_name in cross_feature_dict.iteritems(): to_be_encoded.append(new_feature_name) #fix all data source to_be_stacked_df = pd.concat([ df_train[to_be_encoded], df_testset1[to_be_encoded], df_testset2[to_be_encoded] ], axis=0) enc.fit(to_be_stacked_df) enc, to_be_encoded = self.__filter_too_big_onehot_encoding( enc, to_be_encoded, df_train, df_testset1, df_testset2) # transform on seprate data source self.res_data_dict[g_singletonDataFilePath.getTrainDir( )] = self.__do_one_hot_encoding(df_train, enc, to_be_encoded), cv self.res_data_dict[g_singletonDataFilePath.getTest1Dir( )] = self.__do_one_hot_encoding(df_testset1, enc, to_be_encoded) self.res_data_dict[g_singletonDataFilePath.getTest2Dir( )] = self.__do_one_hot_encoding(df_testset2, enc, to_be_encoded) return
def get_history_data_dict(self): """ indexes for quick search key = 'start_district_id','time_id' value = 'gap its data includes those from train, test1, test2. """ t0 = time() filename = "../data_preprocessed/" + 'traintest_history_data.dict.pickle' dumpload = DumpLoad(filename) if dumpload.isExisiting(): return dumpload.load() test1data_df = ExploreOrder().load_gapdf( g_singletonDataFilePath.getTest1Dir()) test2data_df = ExploreOrder().load_gapdf( g_singletonDataFilePath.getTest2Dir()) traindata_df = ExploreOrder().load_gapdf( g_singletonDataFilePath.getTrainDir()) df = pd.concat([traindata_df, test1data_df, test2data_df], axis=0) self.__fileter_earlier_date(df) res_dict = self.__generate_dict(df) dumpload.dump(res_dict) print "dump weather dict:", round(time() - t0, 3), "s" return res_dict
def run(self): self.__unittest() data_dir = g_singletonDataFilePath.getTest2Dir() # self.save_all_csv( data_dir+ 'weather_data/') # self.combine_all_csv(data_dir + 'weather_data/temp/', 'weather_', 'weather.csv') self.get_weather_dict(data_dir) return
def get_history_data_dict(self): """ indexes for quick search key = 'start_district_id','time_id' value = 'gap its data includes those from train, test1, test2. """ t0 = time() filename = "../data_preprocessed/" + 'traintest_history_data.dict.pickle' dumpload = DumpLoad( filename) if dumpload.isExisiting(): return dumpload.load() test1data_df = ExploreOrder().load_gapdf(g_singletonDataFilePath.getTest1Dir()) test2data_df = ExploreOrder().load_gapdf(g_singletonDataFilePath.getTest2Dir()) traindata_df = ExploreOrder().load_gapdf(g_singletonDataFilePath.getTrainDir()) df = pd.concat([traindata_df, test1data_df,test2data_df], axis=0) self.__fileter_earlier_date(df) res_dict = self.__generate_dict(df) dumpload.dump(res_dict) print "dump weather dict:", round(time()-t0, 3), "s" return res_dict
def run(self): self.__unittest() data_dir = g_singletonDataFilePath.getTest2Dir() # self.save_all_csv( data_dir+ 'weather_data/') # self.combine_all_csv(data_dir + 'weather_data/temp/', 'weather_', 'weather.csv') self.get_weather_dict(data_dir) return
def run(self): # self.__unittest() data_dir = g_singletonDataFilePath.getTest2Dir() # self.save_all_csv(data_dir+ 'traffic_data/') # self.combine_all_csv(data_dir + 'traffic_data/temp/', 'traffic_', 'traffic.csv') self.get_traffic_dict(data_dir) return
def save_model(self): if not self.save_final_model: return dumpload = DumpLoad('logs/' + self.application_start_time + '_estimator.pickle') dumpload.dump(self) self.predictTestSet(g_singletonDataFilePath.getTest2Dir()) return
def save_model(self): if not self.save_final_model: return dumpload = DumpLoad('logs/' + self.application_start_time + '_estimator.pickle') dumpload.dump(self) self.predictTestSet(g_singletonDataFilePath.getTest2Dir()) return
def __save_final_data(self): df_train, _ = self.res_data_dict[g_singletonDataFilePath.getTrainDir()] df_testset1 = self.res_data_dict[g_singletonDataFilePath.getTest1Dir()] df_testset2 = self.res_data_dict[g_singletonDataFilePath.getTest2Dir()] df_train.to_csv('temp/df_train_final.csv') df_testset1.to_csv('temp/df_testset1_final.csv') df_testset2.to_csv('temp/df_testset2_final.csv') return
def run(self): self.getFeaturesLabel() self.getFeaturesforTestSet(g_singletonDataFilePath.getTest2Dir()) self.getFeaturesforTestSet(g_singletonDataFilePath.getTest1Dir()) self.get_train_validationset() self.__save_final_data() return
def __save_final_data(self): df_train, _ = self.res_data_dict[g_singletonDataFilePath.getTrainDir()] df_testset1 = self.res_data_dict[g_singletonDataFilePath.getTest1Dir()] df_testset2 = self.res_data_dict[g_singletonDataFilePath.getTest2Dir()] df_train.to_csv('temp/df_train_final.csv') df_testset1.to_csv('temp/df_testset1_final.csv') df_testset2.to_csv('temp/df_testset2_final.csv') return
def run(self): self.getFeaturesLabel() self.getFeaturesforTestSet(g_singletonDataFilePath.getTest2Dir()) self.getFeaturesforTestSet(g_singletonDataFilePath.getTest1Dir()) self.get_train_validationset() self.__save_final_data() return
def __do_one_hot_encodings(self): df_train, cv = self.res_data_dict[g_singletonDataFilePath.getTrainDir()] df_testset1 = self.res_data_dict[g_singletonDataFilePath.getTest1Dir()] df_testset2 = self.res_data_dict[g_singletonDataFilePath.getTest2Dir()] enc = OneHotEncoder(sparse=False) cross_feature_dict = self.__get_label_encode_dict() to_be_encoded = [] for _, new_feature_name in cross_feature_dict.iteritems(): to_be_encoded.append(new_feature_name) #fix all data source to_be_stacked_df = pd.concat([df_train[to_be_encoded], df_testset1[to_be_encoded], df_testset2[to_be_encoded]], axis = 0) enc.fit(to_be_stacked_df) enc, to_be_encoded = self.__filter_too_big_onehot_encoding(enc, to_be_encoded, df_train, df_testset1, df_testset2) # transform on seprate data source self.res_data_dict[g_singletonDataFilePath.getTrainDir()] = self.__do_one_hot_encoding(df_train, enc, to_be_encoded),cv self.res_data_dict[g_singletonDataFilePath.getTest1Dir()] = self.__do_one_hot_encoding(df_testset1,enc, to_be_encoded) self.res_data_dict[g_singletonDataFilePath.getTest2Dir()] = self.__do_one_hot_encoding(df_testset2, enc, to_be_encoded) return
def run(self): self.unitTest() data_dir = g_singletonDataFilePath.getTest2Dir() # self.get_gap_meanmedian_dict() # self.saveAllGapCsv(data_dir) # self.combine_gap_csv(data_dir) df = self.load_gapdf(data_dir) res = self.get_outlier_threshold(df['gap']) # self.get_gap_dict(data_dir) return
def __do_prepare_data(self): if len(self.res_data_dict) != 0: # the data has already been preprocessed return self.__get_feature_label() self.__get_feature_for_test_set(g_singletonDataFilePath.getTest2Dir()) self.__get_feature_for_test_set(g_singletonDataFilePath.getTest1Dir()) self.__do_label_encoding() self.__do_one_hot_encodings() return
def run(self): self.unitTest() data_dir = g_singletonDataFilePath.getTest2Dir() # self.get_gap_meanmedian_dict() # self.saveAllGapCsv(data_dir) # self.combine_gap_csv(data_dir) df = self.load_gapdf(data_dir) res = self.get_outlier_threshold(df['gap']) # self.get_gap_dict(data_dir) return
def __do_prepare_data(self): if len(self.res_data_dict) != 0: # the data has already been preprocessed return self.__get_feature_label() self.__get_feature_for_test_set(g_singletonDataFilePath.getTest2Dir()) self.__get_feature_for_test_set(g_singletonDataFilePath.getTest1Dir()) self.__do_label_encoding() self.__do_one_hot_encodings() return
def __do_label_encoding(self): df_train, _ = self.res_data_dict[g_singletonDataFilePath.getTrainDir()] df_testset1 = self.res_data_dict[g_singletonDataFilePath.getTest1Dir()] df_testset2 = self.res_data_dict[g_singletonDataFilePath.getTest2Dir()] le = LabelEncoder() cross_feature_dict = self.__get_label_encode_dict() for _, new_feature_name in cross_feature_dict.iteritems(): to_be_stacked = [df_train[new_feature_name], df_testset1[new_feature_name], df_testset2[new_feature_name]] le.fit(pd.concat(to_be_stacked, axis=0)) df_train[new_feature_name] = le.transform(df_train[new_feature_name]) df_testset1[new_feature_name] = le.transform(df_testset1[new_feature_name]) df_testset2[new_feature_name] = le.transform(df_testset2[new_feature_name]) return
def __do_label_encoding(self): df_train, _ = self.res_data_dict[g_singletonDataFilePath.getTrainDir()] df_testset1 = self.res_data_dict[g_singletonDataFilePath.getTest1Dir()] df_testset2 = self.res_data_dict[g_singletonDataFilePath.getTest2Dir()] le = LabelEncoder() cross_feature_dict = self.__get_label_encode_dict() for _, new_feature_name in cross_feature_dict.iteritems(): to_be_stacked = [ df_train[new_feature_name], df_testset1[new_feature_name], df_testset2[new_feature_name] ] le.fit(pd.concat(to_be_stacked, axis=0)) df_train[new_feature_name] = le.transform( df_train[new_feature_name]) df_testset1[new_feature_name] = le.transform( df_testset1[new_feature_name]) df_testset2[new_feature_name] = le.transform( df_testset2[new_feature_name]) return