def process_train_woe(cfg=None, feature_name=None, target=None): print 'run into process_train_woe: \n', feature_name, time.asctime( time.localtime(time.time())) feature_path = 'E:\\ScoreCard\\cs_model\\cs_m1_pos_model_daily\\raw_data\\dataset_split_by_cols\\' feature_path = feature_path + feature_name + '.csv' feature = pd.read_csv(feature_path) rst = [] if feature.columns[0] in list(cfg.bin_var_list): feature.loc[feature[feature.columns[0]].isnull()] = -1 fp.change_feature_dtype(feature, cfg.variable_type) dataset = pd.merge(feature.reset_index(), target.reset_index()).drop('index', axis=1) var = feature.columns[0] del feature del target riv = fp.proc_woe_continuous(dataset, var, cfg.global_bt, cfg.global_gt, cfg.min_sample, alpha=0.05) else: # process woe transformation of discrete variables print 'process woe transformation of discrete variables: \n', time.asctime( time.localtime(time.time())) feature.loc[feature[feature.columns[0]].isnull()] = 'missing' fp.change_feature_dtype(feature, cfg.variable_type) dataset = pd.merge(feature.reset_index(), target.reset_index()).drop('index', axis=1) var = feature.columns[0] del feature del target riv = fp.proc_woe_discrete(dataset, var, cfg.global_bt, cfg.global_gt, cfg.min_sample, alpha=0.05) rst.append(riv) feature_detail = eval.eval_feature_detail(rst) rst_path = 'E:\\ScoreCard\\cs_model\\cs_m1_pos_model_daily\\gendata\\WOE_Rule\\' rst_path = rst_path + feature_name + '.pkl' result = (riv, feature_detail) output = open(rst_path, 'wb') pickle.dump(result, output) output.close() return result
def get_cut_result(self): df = self.X.copy() df['target'] = self.y self.civ_dict = {} woe_fp = {} for column in list(df.columns): if column == 'target': continue self.log.info("------ Now dealing the var={}".format(column)) tmp_df = df[~df[column].isin(self.special_value[column])] if column in self.classVars: civ = fp.proc_woe_discrete(tmp_df, column, self.global_bt, self.global_gt, 0.05 * len(tmp_df), alpha=0.05) else: civ = fp.proc_woe_continuous(tmp_df, column, self.global_bt, self.global_gt, 0.05 * len(tmp_df), alpha=0.05) self.civ_dict[column] = civ woe_fp[column] = fp if self.file_name is not None: feature_detail = eval.eval_feature_detail([v for k, v in self.civ_dict.items()], self.file_name) else: feature_detail = None if self.is_changed: changed_df = df.copy() changed_df = changed_df.drop(['target'], axis=1) for column in list(df.columns): if column == 'target': continue changed_df[column] = woe_fp[column].woe_trans(df[column], self.civ_dict[column]) if self.is_tag: for column in list(df.columns): woe_list = [str(i) for i in self.civ_dict[column].woe_list] split_list = self.civ_dict[column].split_list if len(split_list) == 0: continue changed_df[column] = changed_df[column].map( lambda x: '(' + str(split_list[-1]) + ',' + 'inf]' if x == woe_list[-1] else '(' + str(split_list[woe_list.index(x) - 1]) + ',' + str( split_list[woe_list.index(x)]) + ']') else: changed_df = None return woe_fp, changed_df, feature_detail
def get_iv(df, cols, target, outputfile='./data/feature_detail_iv_list.csv'): import woe.feature_process as fp import woe.eval as eval # 分别用于计算连续变量与离散变量的woe。它们的输入形式相同: # proc_woe_discrete(df,var,global_bt,global_gt,min_sample,alpha=0.01) # # proc_woe_continuous(df,var,global_bt,global_gt,min_sample,alpha=0.01) # 输入: # df: DataFrame,要计算woe的数据,必须包含'target'变量,且变量取值为{0,1} # var:要计算woe的变量名 # global_bt:全局变量bad total。df的正样本数量 # global_gt:全局变量good total。df的负样本数量 # min_sample:指定每个bin中最小样本量,一般设为样本总量的5%。 # alpha:用于自动计算分箱时的一个标准,默认0.01.如果iv_划分>iv_不划分*(1+alpha)则划分。 data = df.copy() data_woe = data data_woe.rename(columns={target: 'target'}, inplace=True) #用于存储所有数据的woe值 civ_list = [] n_positive = sum(data['target']) n_negtive = len(data) - n_positive for column in list(cols): if data[column].dtypes == 'object' or 'category': civ = fp.proc_woe_discrete(data, column, n_positive, n_negtive, 0.05 * len(data), alpha=0.05) else: civ = fp.proc_woe_continuous(data, column, n_positive, n_negtive, 0.05 * len(data), alpha=0.05) civ_list.append(civ) data_woe[column] = fp.woe_trans(data[column], civ) civ_df = eval.eval_feature_detail(civ_list, outputfile) #删除iv值过小的变量 # iv_thre = 0.001 # iv = civ_df[['var_name','iv']].drop_duplicates() # x_columns = iv.var_name[iv.iv > iv_thre] return civ_df
def cal_iv(df, cate_vars, cont_vars, target): #%% woe分箱, iv and transform df_woe = df civ_list = [] n_positive = sum(df[target]) n_negtive = len(df) - n_positive for var in cate_vars: civ = fp.proc_woe_discrete(df, var, n_positive, n_negtive, 0.05*len(df), alpha=0.05) civ_list.append(civ) df_woe[var] = fp.woe_trans(df[var], civ) for var in cont_vars: civ = fp.proc_woe_continuous(df, var, n_positive, n_negtive, 0.05*len(df), alpha=0.05) civ_list.append(civ) df_woe[var] = fp.woe_trans(df[var], civ) civ_df = eval.eval_feature_detail(civ_list,'output_feature_detail_0927.csv') df_iv = civ_df[['var_name','iv']].drop_duplicates() return df_iv.sort_values('iv',ascending=False)
def process_train_woe(infile_path=None,outfile_path=None,rst_path=None): print 'run into process_train_woe: \n',time.asctime(time.localtime(time.time())) config_path = 'E:\\Code\\Python_ML_Code\\cs_model\\config\\config_cs_model_pos_m2.csv' data_path = infile_path cfg = config.config() cfg.load_file(config_path,data_path) bin_var_list = [tmp for tmp in cfg.bin_var_list if tmp in list(cfg.dataset_train.columns)] for var in bin_var_list: # fill null cfg.dataset_train.loc[cfg.dataset_train[var].isnull(), (var)] = -1 # change feature dtypes fp.change_feature_dtype(cfg.dataset_train, cfg.variable_type) rst = [] # process woe transformation of continuous variables print 'process woe transformation of continuous variables: \n',time.asctime(time.localtime(time.time())) print 'cfg.global_bt',cfg.global_bt print 'cfg.global_gt', cfg.global_gt for var in bin_var_list: rst.append(fp.proc_woe_continuous(cfg.dataset_train,var,cfg.global_bt,cfg.global_gt,cfg.min_sample,alpha=0.05)) # process woe transformation of discrete variables print 'process woe transformation of discrete variables: \n',time.asctime(time.localtime(time.time())) for var in [tmp for tmp in cfg.discrete_var_list if tmp in list(cfg.dataset_train.columns)]: # fill null cfg.dataset_train.loc[cfg.dataset_train[var].isnull(), (var)] = 'missing' rst.append(fp.proc_woe_discrete(cfg.dataset_train,var,cfg.global_bt,cfg.global_gt,cfg.min_sample,alpha=0.05)) feature_detail = eval.eval_feature_detail(rst, outfile_path) print 'save woe transformation rule into pickle: \n',time.asctime(time.localtime(time.time())) output = open(rst_path, 'wb') pickle.dump(rst,output) output.close() return feature_detail,rst
criterion.cuda() data = pd.read_csv('./data/cs-training.csv').iloc[:, 1:] data.rename(columns={'SeriousDlqin2yrs': 'target'}, inplace=True) data = data.dropna() ''' woe分箱, iv and transform ''' print("woe....") data_woe = data # 用于存储所有数据的woe值 info_value_list = [] n_positive = sum(data['target']) n_negtive = len(data) - n_positive for column in list(data.columns[1:]): if data[column].dtypes == 'object': info_value = fp.proc_woe_discrete(data, column, n_positive, n_negtive, 0.05 * len(data), alpha=0.05) else: info_value = fp.proc_woe_continuous(data, column, n_positive, n_negtive, 0.05 * len(data), alpha=0.05) info_value_list.append(info_value) data_woe[column] = fp.woe_trans(data[column], info_value) info_df = eval.eval_feature_detail(info_value_list, './dataDump/woe_info.csv') # 删除iv值过小的变量
# fill null cfg.dataset_train.loc[cfg.dataset_train[var].isnull(), (var)] = 0 # change feature dtypes fp.change_feature_dtype(cfg.dataset_train, cfg.variable_type) rst = [] # process woe transformation of continuous variables for var in cfg.bin_var_list: rst.append( fp.proc_woe_continuous(cfg.dataset_train, var, cfg.global_bt, cfg.global_gt, cfg.min_sample, alpha=0.05)) # process woe transformation of discrete variables for var in cfg.discrete_var_list: # fill null cfg.dataset_train.loc[cfg.dataset_train[var].isnull(), (var)] = 'missing' rst.append( fp.proc_woe_discrete(cfg.dataset_train, var, cfg.global_bt, cfg.global_gt, cfg.min_sample, alpha=0.05)) feature_detail = eval.eval_feature_detail(rst, 'output_feature_detail.csv')
import woe.feature_process as fp import woe.eval as eval civ_list = [] # woe_index = ['贷款总额','首付金额','duringDay'] woe_index = [ '贷款总额', '首付金额', 'duringDay', '停车超时报警平均时间', '离线超时报警平均时间', '风险点报警平均时间' ] # 如果将分箱放入到交叉验证流程中,结果如何? n_positive = sum(alldata['target']) n_negtive = len(alldata) - n_positive for i in woe_index: if alldata[i].dtypes == 'object': civ = fp.proc_woe_discrete(alldata, i, n_positive, n_negtive, 0.05 * len(alldata), alpha=0.05) else: civ = fp.proc_woe_continuous(alldata, i, n_positive, n_negtive, 0.05 * len(alldata), alpha=0.05) civ_list.append(civ) alldata[i] = fp.woe_trans(alldata[i], civ) civ_df = eval.eval_feature_detail(civ_list) iv_thre = 0.001 iv = civ_df[['var_name', 'iv']].drop_duplicates() # 计算特征的iv值,查看特征的重要性 '''
def process_train_woe(infile_path=None, outfile_path=None, rst_path=None): print 'run into process_train_woe: \n', time.asctime( time.localtime(time.time())) config_path = r'E:\Code\Python_ML_Code\cs_model\config\config_cs_daily_model_lr.csv' data_path = infile_path cfg = config.config() cfg.load_file(config_path, data_path) # rst = [] output = open(rst_path, 'rb') rst = pickle.load(output) output.close() exists_var_list = [rst[i].var_name for i in range(rst.__len__())] bin_var_list = [ tmp for tmp in cfg.bin_var_list if tmp in list(cfg.dataset_train.columns) and tmp not in exists_var_list ] for var in bin_var_list: # fill null cfg.dataset_train.loc[cfg.dataset_train[var].isnull(), (var)] = -1 # change feature dtypes fp.change_feature_dtype(cfg.dataset_train, cfg.variable_type) # process woe transformation of continuous variables print 'process woe transformation of continuous variables: \n', time.asctime( time.localtime(time.time())) print 'cfg.global_bt', cfg.global_bt print 'cfg.global_gt', cfg.global_gt for var in bin_var_list: print var if rst.__len__() == 0: pass else: output = open(rst_path, 'rb') rst = pickle.load(output) output.close() print 'load' rst.append( fp.proc_woe_continuous(cfg.dataset_train, var, cfg.global_bt, cfg.global_gt, cfg.min_sample, alpha=0.05)) output = open(rst_path, 'wb') pickle.dump(rst, output) output.close() print 'dump' # process woe transformation of discrete variables print 'process woe transformation of discrete variables: \n', time.asctime( time.localtime(time.time())) for var in [ tmp for tmp in cfg.discrete_var_list if tmp in list(cfg.dataset_train.columns) and tmp not in exists_var_list ]: print var # fill null cfg.dataset_train.loc[cfg.dataset_train[var].isnull(), (var)] = 'missing' if rst.__len__() == 0: pass else: output = open(rst_path, 'rb') rst = pickle.load(output) output.close() print 'load' rst.append( fp.proc_woe_discrete(cfg.dataset_train, var, cfg.global_bt, cfg.global_gt, cfg.min_sample, alpha=0.05)) output = open(rst_path, 'wb') pickle.dump(rst, output) output.close() print 'dump' feature_detail = eval.eval_feature_detail(rst, outfile_path) return feature_detail, rst