def n2v_classification(): global train_x, train_y n2v_res, rcnt = [0, 0, 0, 0], 5 for i in tqdm(range(rcnt)): node_feas, labels = dcopy(train_x.values), dcopy(train_y.values) embe_feas = read_embeds(N2VPathName_new + '/embeds_n2v_%d.dat' % i) np_fealab = np.hstack((node_feas, embe_feas, labels)) columns_name = ['f%02d' % i for i in range(np_fealab.shape[1] - 1)] + ['label'] df_fealab = pd.DataFrame(data=np_fealab, columns=columns_name, dtype=float) df_fealab['label'] = df_fealab['label'].astype(int) y_cols_name = ['label'] x_cols_name = [x for x in df_fealab.columns if x not in y_cols_name] n2vdf_x = df_fealab[x_cols_name] n2vdf_y = df_fealab[y_cols_name] print(n2vdf_x.shape, n2vdf_y.shape) lgb_res = lgb_train_model_with_split(n2vdf_x, n2vdf_y, RANDOM_SEED) for i in range(len(n2v_res)): n2v_res[i] += lgb_res[i] n2v_res = [i / rcnt for i in n2v_res] gc.collect() return n2v_res
def GCN_classfication(emsize): gcn_res, rcnt = [0, 0, 0, 0], 5 rs = 7 for i in range(rcnt): for idx in range(subSum - 1, subSum): #print(idx) #print(tolFeaturePathName+"/tolFeature_%d.pkl" % idx) tolFeature = load_pickle(tolFeaturePathName, "/tolFeature_%d.pkl" % idx) tolFeature_df = pd.DataFrame(tolFeature, columns=[ 'label', 'AF1', 'AF2', 'AF3', 'AF4', 'AF5', 'AF6', 'AF7', 'AF8' ]) sp_muldG = load_pickle(muldigPathName, "/G_%d.pkl" % idx) #sp_mulG = load_pickle(mulgPathName+"/G_%d.pkl" % idx) #print(mulgPathName+"/G_%d.pkl" % idx) y_cols_name = ['label'] x_cols_name = [ x for x in tolFeature_df.columns if x not in y_cols_name ] global scipy_adj_matrix, train_x, train_y train_x = dcopy(tolFeature_df[x_cols_name]) train_y = dcopy(tolFeature_df[y_cols_name]) pos_cnt, neg_cnt = int( train_y.sum()), int(len(train_y) - train_y.sum()) scipy_adj_matrix = get_scipy_adj_matrix(sp_muldG) #print('pos node cnts:', pos_cnt) #print('neg node cnts:', neg_cnt, 'pos/all ratio:', # pos_cnt / (pos_cnt + neg_cnt)) fGCNembedding = get_GCN_embedding(epoch=6, lr=0.005, weight_decay=1e-6, esize=emsize, random_seed=rs + i) print("finish calculate embedding data!") save_pickle(fGCNembedding, GCNPathName + "%d" % testNum, "/fGCNembedding_%d.pkl" % idx) trainX, trainY, testX, testY = get_input_data( GCNPathName + "%d" % testNum, "/fGCNembedding", esize, True, False, True) #ftrainX, ftrainY, ftestX, ftestY = get_pure_feature(False) #print(ftrainX[:,subSum-1,:].size) #print(trainX[:, subSum-1, :]) inputX = trainX[:, subSum - 1, :] #inputX=np.concatenate((ftrainX[:,subSum-1,:], trainX[:, subSum-1, :]), axis=1) #inputX=trainX trainX_2D, trainY_1D = inputX, trainY[:, subSum - 1, 0] lgb_res = lgb_train_model_with_split(pd.DataFrame(trainX_2D), pd.DataFrame(trainY_1D), 2011) #print_res(lgb_res) for j in range(len(gcn_res)): gcn_res[j] += lgb_res[j] gcn_res = [i / rcnt for i in gcn_res] print(gcn_res) gc.collect() return gcn_res
def pure_feature_classification(): trainX, trainY, testX, testY = get_pure_feature(False) trainX_2D, trainY_1D = trainX[:, subSum - 1, :], trainY[:, subSum - 1, 0] result = lgb_train_model_with_split(pd.DataFrame(trainX_2D), pd.DataFrame(trainY_1D), 2011) print_res(result) return result
def GCN_classfication(): trainX, trainY, testX, testY = get_input_data( GCNPathName+"%d_%d" %(embSize,testNum), "/fGCNembedding",True, False) trainX_2D, trainY_1D = trainX[:, subSum-1, :], trainY[:, subSum-1, 0] ftrainX, ftrainY, ftestX, ftestY = get_pure_feature(False) inputX=np.concatenate((ftrainX[:,subSum-1,:], trainX[:, subSum-1, :]), axis=1) result = lgb_train_model_with_split( pd.DataFrame(inputX), pd.DataFrame(trainY_1D), 2011) print_res(result)
def LSTM_classification(modelFileName): # 不切分训练测试集 new_model = load_RNNmodel(modelFileName) trainX, trainY, testX, testY = get_pure_feature(False) print(trainX.shape,trainY.shape) trainX_emb, testX_emb = get_autoEncoder_Embedding_Layer( trainX, trainX, new_model) #实际上传入的test也是trainX 因为在这里不切分数据集 trainX_2D = trainX_emb[:, subSum-1, :] trainY_1D = trainY[:, subSum-1, 0] result = lgb_train_model_with_split( pd.DataFrame(trainX_2D), pd.DataFrame(trainY_1D), 2011) print_res(result)
def feature_classification(): global train_x, train_y res, rcnt = [0, 0, 0, 0], 5 for i in tqdm(range((rcnt))): trainX, trainY, testX, testY = get_pure_feature(False) trainX_2D, trainY_1D = trainX[:, subSum - 1, :], trainY[:, subSum - 1, :] print(trainX_2D.shape, trainY_1D.shape) f_res = lgb_train_model_with_split(pd.DataFrame(trainX_2D), pd.DataFrame(trainY_1D), RANDOM_SEED + i) print(f_res) for j in range(len(f_res)): res[j] += f_res[j] res = [j / rcnt for j in res] gc.collect() return res
def GCN_LSTM_classification(modelFileName): new_model = load_RNNmodel(modelFileName) trainX, trainY, testX, testY = get_input_data( GCNPathName+"%d_%d" %(embSize,testNum), "/fGCNembedding", True, False) #trainX= trainX[:, :, :8] ftrainX, ftrainY, ftestX, ftestY = get_pure_feature(False) print(trainX.shape,trainY.shape) trainX_emb, testX_emb = get_autoEncoder_Embedding_Layer( trainX, trainX, new_model) print(trainX_emb.shape) for i in range(subSum-1,subSum): #print(str(i)+":------") inputX=np.concatenate((ftrainX[:,i,:], trainX_emb[:, i, :]), axis=1) #inputX=trainX_emb[:, i, :] print(inputX.shape) trainX_2D, trainY_1D =inputX, trainY[:,subSum-1, 0] result = lgb_train_model_with_split( pd.DataFrame(trainX_2D), pd.DataFrame(trainY_1D), 2011) print_res(result)