def pretrain_ae(model, dataset, y): train_loader = DataLoader(dataset, batch_size=256, shuffle=True) print(model) optimizer = Adam(model.parameters(), lr=1e-3) for epoch in range(30): # adjust_learning_rate(optimizer, epoch) for batch_idx, (x, _) in enumerate(train_loader): x = x.cuda() x_bar, _ = model(x) loss = F.mse_loss(x_bar, x) optimizer.zero_grad() loss.backward() optimizer.step() with torch.no_grad(): x = torch.Tensor(dataset.x).cuda().float() x_bar, z = model(x) loss = F.mse_loss(x_bar, x) print('{} loss: {}'.format(epoch, loss)) kmeans = KMeans(n_clusters=4, n_init=20).fit(z.data.cpu().numpy()) eva(y, kmeans.labels_, epoch) torch.save(model.state_dict(), 'dblp1.pkl')
def trainer(dataset): model = DAEGC(num_features=args.input_dim, hidden_size=args.hidden_size, embedding_size=args.embedding_size, alpha=args.alpha, num_clusters=args.n_clusters).to(device) print(model) optimizer = Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # data process dataset = utils.data_preprocessing(dataset) adj = dataset.adj.to(device) adj_label = dataset.adj_label.to(device) M = utils.get_M(adj).to(device) # data and label data = torch.Tensor(dataset.x).to(device) y = dataset.y.cpu().numpy() with torch.no_grad(): _, z = model.gat(data, adj, M) # get kmeans and pretrain cluster result kmeans = KMeans(n_clusters=args.n_clusters, n_init=20) y_pred = kmeans.fit_predict(z.data.cpu().numpy()) model.cluster_layer.data = torch.tensor(kmeans.cluster_centers_).to(device) eva(y, y_pred, 'pretrain') for epoch in range(args.max_epoch): model.train() if epoch % args.update_interval == 0: # update_interval A_pred, z, Q = model(data, adj, M) q = Q.detach().data.cpu().numpy().argmax(1) # Q eva(y, q, epoch) A_pred, z, q = model(data, adj, M) p = target_distribution(Q.detach()) kl_loss = F.kl_div(q.log(), p, reduction='batchmean') re_loss = F.binary_cross_entropy(A_pred.view(-1), adj_label.view(-1)) loss = 10 * kl_loss + re_loss optimizer.zero_grad() loss.backward() optimizer.step()
def train_idec(dataset): model = SDCN(500, 500, 2000, 2000, 500, 500, n_input=args.n_input, n_z=args.n_z, n_clusters=args.n_clusters, v=1.0).to(device) print(model) optimizer = Adam(model.parameters(), lr=args.lr) # KNN Graph adj = load_graph(args.name, args.k) adj = adj.cuda() # cluster parameter initiate data = torch.Tensor(dataset.x).to(device) y = dataset.y with torch.no_grad(): _, _, _, _, z = model.ae(data) kmeans = KMeans(n_clusters=args.n_clusters, n_init=20) y_pred = kmeans.fit_predict(z.data.cpu().numpy()) y_pred_last = y_pred model.cluster_layer.data = torch.tensor(kmeans.cluster_centers_).to(device) eva(y, y_pred, 'pae') for epoch in range(200): if epoch % 1 == 0: # update_interval _, tmp_q, pred, _ = model(data, adj) tmp_q = tmp_q.data p = target_distribution(tmp_q) res1 = tmp_q.cpu().numpy().argmax(1) #Q res2 = pred.data.cpu().numpy().argmax(1) #Z res3 = p.data.cpu().numpy().argmax(1) #P eva(y, res1, str(epoch) + 'Q') eva(y, res2, str(epoch) + 'Z') eva(y, res3, str(epoch) + 'P') x_bar, q, pred, _ = model(data, adj) kl_loss = F.kl_div(q.log(), p, reduction='batchmean') ce_loss = F.kl_div(pred.log(), p, reduction='batchmean') re_loss = F.mse_loss(x_bar, data) loss = 0.1 * kl_loss + 0.01 * ce_loss + re_loss optimizer.zero_grad() loss.backward() optimizer.step()
def pretrain_daegc(dataset): model = DAEGC(num_features=args.input_dim, hidden_size=args.hidden1_dim, embedding_size=args.hidden2_dim, alpha=args.alpha).to(device) print(model) optimizer = Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) #Some porcess adj, adj_label = load_graph(args.name, args.k) adj_dense = adj.to_dense() adj_numpy = adj_dense.data.cpu().numpy() t = 2 tran_prob = normalize(adj_numpy, norm="l1", axis=0) M_numpy = sum( [np.linalg.matrix_power(tran_prob, i) for i in range(1, t + 1)]) / t M = torch.Tensor(M_numpy).cuda() adj = adj_dense.cuda() adj_label = adj_label.cuda() # cluster parameter initiate data = torch.Tensor(dataset.x).cuda() y = dataset.y for epoch in range(30): model.train() A_pred, z = model(data, adj, M) loss = F.binary_cross_entropy(A_pred.view(-1), adj_label.to_dense().view(-1)) optimizer.zero_grad() loss.backward() optimizer.step() with torch.no_grad(): _, z = model(data, adj, M) kmeans = KMeans(n_clusters=args.n_clusters, n_init=20).fit(z.data.cpu().numpy()) eva(y, kmeans.labels_, epoch) torch.save(model.state_dict(), 'predaegc_uai.pkl')
def construct_env(mname='base'): rdata = raw_data(proj_root) evaluator = eva(rdata) DATA_DIR = "../data/" TEST_DIR = "../test/" model = None if mname == 'base': model = base_model(rdata, DATA_DIR) elif mname == 'emfa': model = cem(rdata, DATA_DIR) elif mname == 'dummy': model = dummy(rdata, DATA_DIR) else: print "invalid model!" exit() model.train() sim_vec = evaluator.nec_eva(TEST_DIR, model) cdf_plot(sim_vec) norm_dist_plot(sim_vec) return model #if len(sys.argv) > 0: # rdata = raw_data(proj_root) # evaluator = eva(rdata) # # DATA_DIR = sys.argv[2] # TEST_DIR = sys.argv[3] # # model = None # if sys.argv[1]=='base': # model = base_model(rdata, DATA_DIR) # elif sys.argv[1]=='emfa': # model = cem(rdata, DATA_DIR) # elif sys.argv[1]=='dummy': # model = dummy(rdata, DATA_DIR) # else: # print "invalid model!" # exit() # # model.train() # sim_vec = evaluator.nec_eva(TEST_DIR, model) # cdf_plot(sim_vec) # norm_dist_plot(sim_vec) # suf_res = evaluator.suf_eva(TEST_DIR, model) # evaluator.suf_histo(suf_res, bin=20) #else: # test()
def pretrain(dataset): model = GAT( num_features=args.input_dim, hidden_size=args.hidden_size, embedding_size=args.embedding_size, alpha=args.alpha, ).to(device) print(model) optimizer = Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # data process dataset = utils.data_preprocessing(dataset) adj = dataset.adj.to(device) adj_label = dataset.adj_label.to(device) M = utils.get_M(adj).to(device) # data and label x = torch.Tensor(dataset.x).to(device) y = dataset.y.cpu().numpy() for epoch in range(args.max_epoch): model.train() A_pred, z = model(x, adj, M) loss = F.binary_cross_entropy(A_pred.view(-1), adj_label.view(-1)) optimizer.zero_grad() loss.backward() optimizer.step() with torch.no_grad(): _, z = model(x, adj, M) kmeans = KMeans(n_clusters=args.n_clusters, n_init=20).fit(z.data.cpu().numpy()) acc, nmi, ari, f1 = eva(y, kmeans.labels_, epoch) if epoch % 5 == 0: torch.save(model.state_dict(), f"./pretrain/predaegc_{args.name}_{epoch}.pkl")
def train_dtfu(dataset): model = DTFU(500, 500, 2000, 2000, 500, 500, n_input=opt.args.n_input, n_z=opt.args.n_z, n_clusters=opt.args.n_clusters, v=1.0).to(device) print(model) optimizer = Adam(model.parameters(), lr=opt.args.lr) # KNN Graph adj = load_graph(opt.args.name, opt.args.k) adj = adj.to(device) data = torch.Tensor(dataset.x).to(device) # cluster parameter initiate y = dataset.y with torch.no_grad(): _, _, _, _, z, _, _, _ = model.ael(data) kmeans = KMeans(n_clusters=opt.args.n_clusters, n_init=20) y_pred = kmeans.fit_predict(z.data.cpu().numpy()) y_pred_last = y_pred model.cluster_layer.data = torch.tensor(kmeans.cluster_centers_).to(device) eva(y, y_pred, 'pae') M = np.zeros((700, 4)) for epoch in range(700): if epoch % 1 == 0: # update_interval _, tmp_q, pred, _, _ = model(data, adj) tmp_q = tmp_q.data p = target_distribution(tmp_q) res1 = tmp_q.cpu().numpy().argmax(1) # Q res2 = pred.data.cpu().numpy().argmax(1) # Z res3 = p.data.cpu().numpy().argmax(1) # P eva(y, res1, str(epoch) + 'Q') eva(y, res2, str(epoch) + 'Z') M[epoch, 0], M[epoch, 1], M[epoch, 2], M[epoch, 3] = eva(y, res2, str(epoch) + 'Z') # eva(y, res3, str(epoch) + 'P') x_bar, q, pred, _, A_pred = model(data, adj) kl_loss = F.kl_div(q.log(), p, reduction='batchmean') ce_loss = F.kl_div(pred.log(), p, reduction='batchmean') re_loss = F.mse_loss(x_bar, data) re_graphloss = F.mse_loss(A_pred, adj.to_dense()) loss = opt.args.lambda_v1 * kl_loss + opt.args.lambda_v2 * ce_loss + re_loss + opt.args.lambda_v3 * re_graphloss print('{} loss: {}'.format(epoch, loss)) optimizer.zero_grad() loss.backward() optimizer.step() print('acc:', np.max(M[:, 0])) print('nmi:', np.max(M[:, 1])) print('ari:', np.max(M[:, 2])) print('f1:', np.max(M[:, 3]))
def daegc(dataset): model = Self_DAEGC(num_features=args.input_dim, hidden_size=args.hidden1_dim, embedding_size=args.hidden2_dim, alpha=args.alpha, num_clusters=args.n_clusters).to(device) print(model) optimizer = Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # Some porcess adj, adj_label = load_graph(args.name, args.k) adj_dense = adj.to_dense() adj_numpy = adj_dense.data.cpu().numpy() t = 2 tran_prob = normalize(adj_numpy, norm="l1", axis=0) M_numpy = sum( [np.linalg.matrix_power(tran_prob, i) for i in range(1, t + 1)]) / t M = torch.Tensor(M_numpy).cuda() adj = adj_dense.cuda() adj_label = adj_label.cuda() # cluster parameter initiate data = torch.Tensor(dataset.x).cuda() y = dataset.y with torch.no_grad(): _, z = model.pre_daegc(data, adj, M) kmeans = KMeans(n_clusters=args.n_clusters, n_init=20) y_pred = kmeans.fit_predict(z.data.cpu().numpy()) #y_pred_last = y_pred #kmeans.cluster_centers_ 从z中获得初始聚类中心 model.cluster_layer.data = torch.tensor(kmeans.cluster_centers_).to(device) eva(y, y_pred, 'pae') for epoch in range(30): model.train() if epoch % args.update_interval == 0: #[1,3,5] # update_interval A_pred, z, tmp_q = model(data, adj, M) tmp_q = tmp_q.data p = target_distribution(tmp_q) res1 = tmp_q.cpu().numpy().argmax(1) # Q res3 = p.data.cpu().numpy().argmax(1) # P eva(y, res1, str(epoch) + 'Q') # eva(y, res3, str(epoch) + 'P') A_pred, z, q = model(data, adj, M) kl_loss = F.kl_div(q.log(), p, reduction='batchmean') re_loss = F.binary_cross_entropy(A_pred.view(-1), adj_label.to_dense().view(-1)) loss = 10 * kl_loss + re_loss optimizer.zero_grad() loss.backward() optimizer.step()
def train_graphscc(dataset): model = GraphSCC(512, 256, 64, 64, 256, 512, n_input=args.n_input, n_z=args.n_z, n_clusters=args.n_clusters, v=1.0).to(device) print(model) model.pretrain_ae(LoadDataset(dataset.x)) optimizer = Adam(model.parameters(), lr=args.lr) data = torch.Tensor(dataset.x).to(device) y = dataset.y with torch.no_grad(): xbar, _, _, _, z = model.ae(data) kmeans = KMeans(n_clusters=args.n_clusters, n_init=20, random_state=666) y_pred = kmeans.fit_predict(z.data.cpu().numpy()) model.cluster_layer.data = torch.tensor(kmeans.cluster_centers_).to(device) y_pred_last = y_pred pae_acc, pae_nmi, pae_ari = eva(y, y_pred, 'pae', pp=False) print(':pae_acc {:.4f}'.format(pae_acc), ', pae_nmi {:.4f}'.format(pae_nmi), ', pae_ari {:.4f}'.format(pae_ari)) features = z.data.cpu().numpy() error_rate = construct_graph_kmean(args.name, features.copy(), y, y, load_type='csv', topk=args.k, method='ncos') adj = load_graph(args.name, k=args.k, n=dataset.x.shape[0]) adj = adj.cuda() patient = 0 series = False for epoch in range(args.train_epoch): if epoch % 1 == 0: # update_interval xbar, tmp_q, pred, z = model(data, adj) tmp_q = tmp_q.data p = target_distribution(tmp_q) res1 = tmp_q.cpu().numpy().argmax(1) #Q res2 = pred.data.cpu().numpy().argmax(1) #Z res3 = p.data.cpu().numpy().argmax(1) #P Q_acc, Q_nmi, Q_ari = eva(y, res1, str(epoch) + 'Q', pp=False) Z_acc, Z_nmi, Z_ari = eva(y, res2, str(epoch) + 'Z', pp=False) P_acc, P_nmi, p_ari = eva(y, res3, str(epoch) + 'P', pp=False) print(epoch, ':Q_acc {:.5f}'.format(Q_acc), ', Q_nmi {:.5f}'.format(Q_nmi), ', Q_ari {:.5f}'.format(Q_ari)) print(epoch, ':Z_acc {:.5f}'.format(Z_acc), ', Z_nmi {:.5f}'.format(Z_nmi), ', Z_ari {:.5f}'.format(Z_ari)) print(epoch, ':P_acc {:.5f}'.format(P_acc), ', P_nmi {:.5f}'.format(P_nmi), ', p_ari {:.5f}'.format(p_ari)) delta_label = np.sum(res2 != y_pred_last).astype( np.float32) / res2.shape[0] y_pred_last = res2 if epoch > 0 and delta_label < 0.0001: if series: patient += 1 else: patient = 0 series = True if patient == 300: print('Reached tolerance threshold. Stopping training.') print("Z_acc: {}".format(Z_acc), "Z_nmi: {}".format(Z_nmi), "Z_ari: {}".format(Z_ari)) break else: series = False x_bar, q, pred, _ = model(data, adj) kl_loss = F.kl_div(q.log(), p, reduction='batchmean') ce_loss = F.kl_div(pred.log(), p, reduction='batchmean') re_loss = F.mse_loss(x_bar, data) loss = args.kl_loss * kl_loss + args.ce_loss * ce_loss + re_loss print(epoch, ':kl_loss {:.5f}'.format(kl_loss), ', ce_loss {:.5f}'.format(ce_loss), ', re_loss {:.5f}'.format(re_loss), ', total_loss {:.5f}'.format(loss)) optimizer.zero_grad() loss.backward() optimizer.step() Q_acc, Q_nmi, Q_ari = eva(y, res1, str(epoch) + 'Q', pp=False) Z_acc, Z_nmi, Z_ari = eva(y, res2, str(epoch) + 'Z', pp=False) P_acc, P_nmi, p_ari = eva(y, res3, str(epoch) + 'P', pp=False) print(epoch, ':Q_acc {:.4f}'.format(Q_acc), ', Q_nmi {:.4f}'.format(Q_nmi), ', Q_ari {:.4f}'.format(Q_ari)) print(epoch, ':Z_acc {:.4f}'.format(Z_acc), ', Z_nmi {:.4f}'.format(Z_nmi), ', Z_ari {:.4f}'.format(Z_ari)) print(epoch, ':P_acc {:.4f}'.format(P_acc), ', P_nmi {:.4f}'.format(P_nmi), ', p_ari {:.4f}'.format(p_ari)) print(args)
def train_sdcn(dataset): model = SDCN(256, 128, 64, 64, 128, 256, n_input=args.n_input, n_z=args.n_z, n_clusters=args.n_clusters, v=1.0).to(device) print(model) # instantiate the object net of the class #model.apply(weight_init) optimizer = Adam(model.parameters(), lr=args.lr) # KNN Graph # adj = load_graph(args.name, args.k) # adj = adj.cuda() adj = buildGraphNN(features.T) adj = normalize(adj) adj = sparse_mx_to_torch_sparse_tensor(adj) # cluster parameter initiate data = torch.Tensor(dataset.x).to(device) y = dataset.y # with torch.no_grad(): # _, _, _, _, z = model.ae(data) # kmeans = KMeans(n_clusters=args.n_clusters, n_init=20) # y_pred = kmeans.fit_predict(z.data.cpu().numpy()) kmeans = KMeans(n_clusters=args.n_clusters, n_init=20) y_pred = kmeans.fit_predict(features.T) y_pred_last = y_pred ######model.cluster_layer.data = torch.tensor(kmeans.cluster_centers_).to(device) eva(y, y_pred, 'kmeans') for epoch in range(100): if epoch % 20 == 0: # update_interval _, tmp_q, pred, _, _ = model(data, adj) tmp_q = tmp_q.data p = target_distribution(tmp_q) res1 = tmp_q.cpu().numpy().argmax(1) #Q res2 = pred.data.cpu().numpy().argmax(1) #Z res3 = p.data.cpu().numpy().argmax(1) #P eva(y, res1, str(epoch) + 'Q') eva(y, res2, str(epoch) + 'Z') eva(y, res3, str(epoch) + 'P') x_bar, q, pred, _, adj_bar = model(data, adj) kl_loss = F.kl_div(q.log(), p, reduction='batchmean') ce_loss = F.kl_div(pred.log(), p, reduction='batchmean') re_loss = F.mse_loss(x_bar, data) #reAJD_loss = F.mse_loss(adj_bar, adj) #print(kl_loss, ce_loss, re_loss) loss = 0.01 * kl_loss + 0.1 * ce_loss + re_loss optimizer.zero_grad() loss.backward() optimizer.step()
Yl = np.append(Yl, np.array(Y_list_sure), axis=0) # Evaluate classification performance if seperate_train % 1 == 0: # Prediction labels cls_label_Ft_1, cls_label_Ft_2, cls_label_Ft_3 = sess.run( [ classified_label_X_prob_1, classified_label_X_prob_2, classified_label_X_graph ], feed_dict={X: _Xt}) # Show performance print('====================') print("Iter = ", seperate_train) # Performance of C1 prec, rec, f1, retrieved, f1Ind, precInd, recInd = evaluation.eva( Yt.T, cls_label_Ft_1.T, 5) print('C1: Pre=%.4f ' % prec, 'Rec=%.4f ' % rec, 'F1=%.4f ' % f1, 'N-R=%d ' % retrieved) # Performance of C2 prec, rec, f1, retrieved, f1Ind, precInd, recInd = evaluation.eva( Yt.T, cls_label_Ft_2.T, 5) print('C2: Pre=%.4f ' % prec, 'Rec=%.4f ' % rec, 'F1=%.4f ' % f1, 'N-R=%d ' % retrieved) # Final performance prec, rec, f1, retrieved, f1Ind, precInd, recInd, _, mAP = evaluation_mAP.eva( Yt.T, cls_label_Ft_3.T, 5) print('Final: Pre=%.4f ' % prec, 'Rec=%.4f ' % rec, 'F1=%.4f ' % f1, 'N-R=%d ' % retrieved, 'mAP=%.4f ' % mAP)
def train_sdcn(dataset): # KNN Graph adj = load_graph(args.name, args.k) adj = adj.cuda() model = SDCN(dataset=dataset, n_enc_1=500, n_enc_2=500, n_enc_3=2000, n_dec_1=2000, n_dec_2=500, n_dec_3=500, n_input=args.n_input, n_z=args.n_z, n_clusters=args.n_clusters, v=1.0, adj=adj).to(device) # print(model) # model.pretrain(args.pretrain_path) if args.name == 'cite': optimizer = Adam(model.parameters(), lr=args.lr, weight_decay=5e-4) if args.name == 'dblp': optimizer = Adam(model.parameters(), lr=args.lr) # cluster parameter initiate data = torch.Tensor(dataset.x).to(device) y = dataset.y with torch.no_grad(): _, _, _, _, z = model.ae(data) kmeans = KMeans(n_clusters=args.n_clusters, n_init=20) y_pred = kmeans.fit_predict(z.data.cpu().numpy()) y_pred_last = y_pred model.cluster_layer.data = torch.tensor(kmeans.cluster_centers_).to(device) eva(y, y_pred, 'pae') # ## Visualization # with SummaryWriter(comment='model') as w: # testdata = torch.rand(3327,3703).cuda() # testadj = torch.rand(3327,3327).cuda() # w.add_graph(model,(testdata,testadj)) # ## for epoch in range(200): if epoch % 1 == 0: # update_interval # x_bar为decoder的结果 # q为t分布 # predict为softmax后预测的种类 # z为encoder的潜在特征表示 _, tmp_q, pred, _, adj_re = model(data, adj) tmp_q = tmp_q.data p = target_distribution(tmp_q) res1 = tmp_q.cpu().numpy().argmax(1) #Q res2 = pred.data.cpu().numpy().argmax(1) #Z res3 = p.data.cpu().numpy().argmax(1) #P # eva(y, res1, str(epoch) + 'Q') acc, nmi, ari, f1 = eva(y, res2, str(epoch) + 'Z') ## Visualization # writer.add_scalar('checkpoints/scalar', acc, epoch) # writer.add_scalar('checkpoints/scalar', nmi, epoch) # writer.add_scalar('checkpoints/scalar', ari, epoch) # writer.add_scalar('checkpoints/scalar', f1, epoch) ## # eva(y, res3, str(epoch) + 'P') # x_bar为decoder的结果 # q为t分布 # predict为softmax后预测的种类 # z为encoder的潜在特征表示 x_bar, q, pred, _, adj_re = model(data, adj) kl_loss = F.kl_div(q.log(), p, reduction='batchmean') ce_loss = F.kl_div(pred.log(), p, reduction='batchmean') re_loss = F.mse_loss(x_bar, data) # print('kl_loss:{} ce_loss:{}'.format(kl_loss, ce_loss)) # print(p.shape) # torch.Size([3327, 6]) # print(q.log().shape) # torch.Size([3327, 6]) # print(pred.log().shape) # torch.Size([3327, 6]) loss = 0.1 * kl_loss + 0.01 * ce_loss + re_loss # adj_loss = norm*F.binary_cross_entropy(adj_re.view(-1).cpu(), adj_label.to_dense().view(-1).cpu(), weight = weight_tensor) # loss = 0.1 * kl_loss + 0.01 * ce_loss + re_loss + 0.1*adj_loss optimizer.zero_grad() loss.backward() optimizer.step()