def f_getAucKs(pretrpath): ''' 计算出AUC和 KS :param pretrpath: 使用最优模型预测出的结果路径 :return: ''' data = pd.read_csv(pretrpath) # 读入数据 index_name ,target_name, P_value ks = ModelUtil.ks(data.P_value, data.iloc[:, 1]) Auc = ModelUtil.AUC(data.P_value, data.iloc[:, 1]) return Auc, ks
def f_getmodelen(model_path): ''' 依据模型路径 给出需要输入模型的变量个数 :param model_path: 模型路径 :param implen: 重要变量长度 :return: ''' x = ModelUtil.load_bstmodel(model_path) try: # xgboost 获取变量的方法 x = x.feature_names modellen = len(x) except: try: # 随机森林的获取方法 modellen = x[0].n_features_ except: try: # gbm 获取入模变量的方法 modellen = len(x.feature_name()) except: # 评分卡 获取入模变量的方法 modellen = len(x.params) - 1 return modellen
def f_recursionrfModel(train_path, test_path, index_name, target_name, modelconf, i,optimizationType): ''' 递归的将上一轮的重要变量重新作为输入 从而达到筛选变量的作用 :param train_path: 训练集 :param test_path: 测试集 :param index_name: :param target_name: :param modelconf: 模型配置文件路径 :param i: 记录递归的次数 :return: 最终递归完成的模型输出 结果的路径列表 ''' train_path = ModelUtil.load_data(train_path) test_path = ModelUtil.load_data(test_path) initmodelmain = ModelMain(train_path, test_path, index_name, target_name) initpathlist = initmodelmain.ccxrf_main(modelconf,optimizationType) # 1.计算出重要变量的个数 implen, impvar = f_getImplen(initpathlist[2]) # 2.计算出模型的AUC和KS train_auc, train_ks = f_getAucKs(initpathlist[3]) test_auc, test_ks = f_getAucKs(initpathlist[4]) # 3.判断出模型重要变量占总变量的百分比情况 imppct = f_getVarpctrf(initpathlist[1], implen) # 入模变量 == 重要变量 flag = f_flag(train_auc, train_ks, test_auc, test_ks, imppct) i = i + 1 if i < 5: if flag: print('递归调用 ' * 20) newselectcol = impvar + [index_name, target_name] print('---入选模型的变量个数%s' % len(newselectcol)) train_path = ModelUtil.load_data(train_path)[newselectcol] test_path = ModelUtil.load_data(test_path)[newselectcol] print('##' * 20, i, '##' * 20) # 后续优化 递归的同时修改配置文件modelconf return f_recursionrfModel(train_path, test_path, index_name, target_name, modelconf, i,optimizationType) else: print('满足条件结束递归 ' * 10) return initpathlist else: print('递归次数达到要求结束递归' * 10) return initpathlist
def f_splitdata(dummyAfterdf, target_name): ''' 切分数据集为训练集和测试集 :param rawdata: :param target_name: :return: ''' x_columns = [x for x in dummyAfterdf.columns if x not in [target_name]] y_columns = target_name tr, te = ModelUtil.splitdata(dummyAfterdf, x_columns, y_columns) return tr, te
def f_getVarpctrf(model_path, implen): ''' 依据模型路径 给出需要输入模型的变量个数 :param model_path: 模型路径 :param implen: 重要变量长度 :return: ''' x = ModelUtil.load_bstmodel(model_path) try: # 随机森林的获取方法 modellen = x.n_features_ except: modellen = np.nan return implen == modellen
def f_getVarpctgbm(model_path, implen): ''' 依据模型路径 给出需要输入模型的变量个数 :param model_path: 模型路径 :param implen: 重要变量长度 :return: ''' x = ModelUtil.load_bstmodel(model_path) try: # gbm 获取入模变量的方法 modellen = len(x.feature_name()) except: modellen = np.nan return implen == modellen
def f_getVarpctboost(model_path, implen): ''' 依据模型路径 给出需要输入模型的变量个数 :param model_path: 模型路径 :param implen: 重要变量长度 :return: ''' x = ModelUtil.load_bstmodel(model_path) try: # xgboost 获取变量的方法 x = x.feature_names modellen = len(x) except: modellen = np.nan return implen == modellen
def __init__(self, modelname, dummyList, Allcol, bstmodelpath): self.modelname = modelname self.dummyList = dummyList # 这个也会获取到 self.Allcol = Allcol # f_genAllcol(dummyAfterdf) 可以由这个函数获取到 self.bstmodelpath = bstmodelpath self.bstmodel = ModelUtil.load_bstmodel(bstmodelpath)
for x in ls: fill_dict[x] = 0 # print(fill_dict) var = dummy2df.fillna(fill_dict) return var # dummyList = list(set(res[4]) - set(res[5])) # dummyAfterdf = f_dummyOld(tr, dummyList) # f_dummyNew(te.head(3), dummyList, f_genAllcol(dummyAfterdf)) if __name__ == '__main__': # 测试一下 保存模型处理数据的类 modelpath = r'C:\Users\liyin\Desktop\CcxMLOGE\TestUnit\ccxboost\model20171211190055\modeltxt\model_ccxboost_2017-12-11.txt' bst = ModelUtil.load_bstmodel(modelpath) Allcol = bst.feature_names import numpy as np dd = pd.DataFrame(np.ones(len(Allcol))).T dd.columns = Allcol dd.index = ['All'] dummyList = list(set([i.split('_')[0] for i in Allcol if '_' in i])) psd = processData('ccxboost', dummyList, dd, modelpath) psd.modelname psd.save('111', r'C:\Users\liyin\Desktop\CcxMLOGE\TestUnit') path = 'C:/Users/liyin/Desktop/CcxMLOGE/TestUnit/predict/ccxboost111.model' with open(path, 'rb') as f: psd_1 = pickle.load(f)