def main(): # Load data start = time.time() N, _adj, _feats, _labels, train_adj, train_feats, train_nodes, val_nodes, test_nodes, y_train, y_val, y_test, val_mask, test_mask = utils.load_data(args.dataset) print('Loaded data in {:.2f} seconds!'.format(time.time() - start)) # Prepare Train Data start = time.time() _, parts = utils.partition_graph(train_adj, train_nodes, args.num_clusters_train) parts = [np.array(pt) for pt in parts] train_features, train_support, y_train = utils.preprocess_multicluster(train_adj, parts, train_feats, y_train, args.num_clusters_train, args.batch_size) print('Train Data pre-processed in {:.2f} seconds!'.format(time.time() - start)) # Prepare Test Data if args.test == 1: y_test, test_mask = y_val, val_mask start = time.time() _, test_features, test_support, y_test, test_mask = utils.preprocess(_adj, _feats, y_test, np.arange(N), args.num_clusters_test, test_mask) print('Test Data pre-processed in {:.2f} seconds!'.format(time.time() - start)) # Shuffle Batches batch_idxs = list(range(len(train_features))) # model model = GCN(fan_in=_in, fan_out=_out, layers=args.layers, dropout=args.dropout, normalize=True, bias=False).float() model.cuda() # Loss Function criterion = torch.nn.CrossEntropyLoss() # Optimization Algorithm optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) # Learning Rate Schedule scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=args.lr, steps_per_epoch=int(args.num_clusters_train/args.batch_size), epochs=args.epochs+1, anneal_strategy='linear') model.train() # Train for epoch in range(args.epochs + 1): np.random.shuffle(batch_idxs) avg_loss = 0 start = time.time() for batch in batch_idxs: loss = train(model.cuda(), criterion, optimizer, train_features[batch], train_support[batch], y_train[batch], dataset=args.dataset) if args.lr_scheduler == 1: scheduler.step() avg_loss += loss.item() # Write Train stats to tensorboard writer.add_scalar('time/train', time.time() - start, epoch) writer.add_scalar('loss/train', avg_loss/len(train_features), epoch) if args.test == 1: # Test on cpu f1 = test(model.cpu(), test_features, test_support, y_test, test_mask, device='cpu') print('f1: {:.4f}'.format(f1))
def test_memorize_minibatch(self): for db_name in self.db_names: db_info = get_db_info(db_name) train_data, val_data, _ = get_train_val_test_datasets( dataset_name=db_name, train_test_split='use_full_train', encoders=dict(CATEGORICAL='CategoricalOrdinalEnc', SCALAR='ScalarRobustScalerEnc', DATETIME='DatetimeScalarEnc', LATLONG='LatLongScalarEnc', TEXT='TextSummaryScalarEnc'), ) train_loader = get_dataloader( dataset=train_data, batch_size=256, sampler_class_name='SequentialSampler', num_workers=0, max_nodes_per_graph=False) writer = DummyWriter() model = GCN(writer, db_info=db_info, hidden_dim=256, n_init_layers=3, activation_class_name='SELU', activation_class_kwargs={}, loss_class_kwargs={}, loss_class_name='CrossEntropyLoss', p_dropout=0.0, drop_whole_embeddings=True, n_layers=3, readout_class_name='AvgPooling', readout_kwargs={}) if torch.cuda.is_available(): model.cuda() model.device = torch.device('cuda:0') else: model.device = torch.device('cpu') model.train() optimizer = AdamW(model.parameters(), lr=0.001, weight_decay=0.0) bdgl, features, label = next(iter(train_loader)) recursive_to((bdgl, features, label), model.device) for _ in tqdm(range(200)): optimizer.zero_grad() output = model(bdgl, features) loss = model.loss_fxn(output, label) if loss < 1e-4: break loss.backward() optimizer.step() else: tqdm.write(f'Loss: {loss}') self.fail("Didn't memorize minibatch")
mlp = MLP(dropout=args.dropout) optimizer = optim.Adam(itertools.chain(model.parameters(), mlp.parameters()), lr=args.lr, weight_decay=args.weight_decay) creterion = torch.nn.TripletMarginLoss(margin=args.margin, p=2) CE = torch.nn.CrossEntropyLoss() loss_list = RunningAvg(window_size=200) loss_list_CE = RunningAvg(window_size=200) acc_list = RunningAvg(window_size=200) loss_by_iter = [] if args.cuda: device = torch.device(args.cuda_device) model.cuda(device) mlp.cuda(device) def train_step(epoch, loss_save): for _, data_train_group in tqdm(enumerate(dataloader_train), desc='Training', total=len(dataloader_train)): # pass three times first, then back propagate the loss model.train() mlp.train() for data_train in data_train_group: vector3 = [] regularization = 0 for data_train_item in data_train: # and/or children: [[1,2],[3,4]]
def execute(params, budget=None, max_epoch=243, device='cpu', seed=42): np.random.seed(seed) torch.manual_seed(seed) if device == "cuda": torch.cuda.manual_seed(seed) # Load data if params['dataset'] == "cora": adj, features, labels, idx_train, idx_val, idx_test = load_data( dataset=params['dataset'], train_percent=0.052) if params['dataset'] == "citeseer": adj, features, labels, idx_train, idx_val, idx_test = load_citeseer( train_percent=0.036) # Model and optimizer model = GCN(nfeat=features.shape[1], nhid=params['hidden'], nclass=labels.max().item() + 1, dropout=params['dropout']) optimizer = optim.Adam(model.parameters(), lr=params['lr'], weight_decay=params['weight_decay']) if device == "cuda": model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() # train model if device == "cuda": start = torch.cuda.Event(enable_timing=True) end = torch.cuda.Event(enable_timing=True) start.record() else: t1 = time.time_ns() model.train() num_epoch = int(budget) if budget != None else max_epoch for epoch in range(num_epoch): optimizer.zero_grad() output = model(features, adj) loss_train = F.nll_loss(output[idx_train], labels[idx_train]) acc_train = accuracy(output[idx_train], labels[idx_train]) loss_train.backward() optimizer.step() # evaluation model.eval() output = model(features, adj) loss_val = F.nll_loss(output[idx_val], labels[idx_val]) acc_val = accuracy(output[idx_val], labels[idx_val]) if device == "cuda": end.record() torch.cuda.synchronize() total_time = start.elapsed_time(end) / 1e3 sys.stdout.flush() acc_val = acc_val.item() else: t2 = time.time_ns() total_time = (t2 - t1) / 1e9 print() print( f"dataset={params['dataset']}, num_epoch={num_epoch}, device={next(model.parameters()).device}" ) print("Validation results:", "loss= {:.4f}".format(loss_val.item()), "accuracy= {:.4f}".format(acc_val)) print("Total training time: {:.4f} sec".format(total_time)) return 1 - acc_val
dropout=args.dropout, nnz=adj._nnz()) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) gamma = 0 eta = 10**-6 #should be decreasing with epochs print('!!!!!!!!!!!!CHECK!!!!!!!!!!!!') print('save (location) of plots') print('Gamma:', gamma) print('Eta:', eta) print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') if args.cuda: model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() #parameters ew_mean = nn.Parameter(torch.randn(adj._nnz())) ew_std = nn.Parameter(torch.ones(adj._nnz())) adj_indices = adj._indices() adj_size = adj.size()
# Load data #adj, features, labels, idx_train, idx_val, idx_test = load_data() adj, A_tilde, adj_sct1, adj_sct2, adj_sct4, features, labels, idx_train, idx_val, idx_test = load_citation( args.dataset, args.normalization, args.cuda) # Model and optimizer model = GCN(nfeat=features.shape[1], para3=args.hid1, para4=args.hid2, nclass=labels.max().item() + 1, dropout=args.dropout, smoo=args.smoo) PATH = "state_dict_model.pt" model.load_state_dict(torch.load(PATH)) if args.cuda: model = model.cuda() features = features.cuda() A_tilde = A_tilde.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) scheduler = StepLR(optimizer, step_size=50, gamma=0.9) def test():
def train(): acc_test_list = [] for ii in range(args.run_time): #seed = gen_seeds() seed = args.seed print("dataset:{}, epochs:{}, weight_decay:{},lr:{},dropout:{},seed:{}, alpha:{}, features_perturbation: rate1:{},lambda1:{}; adj_pertubation: rate2:{},lambda2:{}".format( args.dataset, args.epochs, args.weight_decay, args.lr, args.dropout, seed, args.alpha, args.rate1, args.lambda1, args.rate2,args.lambda2)) args.cuda = not args.no_cuda and torch.cuda.is_available() np.random.seed(seed) torch.manual_seed(seed) if args.cuda: torch.cuda.manual_seed(seed) #torch.cuda.manual_seed(seed) if args.dataset == "R8": adj, features, labels, idx_train, idx_val, idx_test = load_corpus(args.dataset, args.normalization,args.cuda) else: adj, features, labels, idx_train, idx_val, idx_test, indices = load_citation(args.dataset, args.normalization, args.cuda) model = GCN(nfeat=features.shape[1], nhid=args.hidden, nclass=labels.max().item() + 1, dropout=args.dropout) if args.cuda: model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) global x,y,z x,y,z=0,0,0 win = vis.line(X=np.array([x]), Y=np.array([y]), opts=dict(title='loss_CE')) global b_1,b_2 b_0,b_1,b_2=0,0,0 win_b0 = vis.line(X=np.array([x]), Y=np.array([b_0]), opts=dict(title='b')) def train(epoch): t = time.time() model.train() optimizer.zero_grad() output = model(features, adj) output = F.log_softmax(output, dim=1) loss_CE = F.nll_loss(output[idx_train], labels[idx_train]) #l2_reg = sum(torch.sum(param ** 2) for param in model.reg_params) acc_train = accuracy(output[idx_train], labels[idx_train]) x = epoch y = loss_CE.detach().cpu().numpy() vis.line(X=np.array([x]), Y=np.array([y]), win=win, update='append') #loss_train = loss_CE + args.weight_decay /2 *l2_reg loss_train = loss_CE loss_train.backward() optimizer.step() if not args.fastmode: model.eval() output = model(features, adj) output = F.log_softmax(output, dim=1) loss_val = F.nll_loss(output[idx_val], labels[idx_val]) acc_val = accuracy(output[idx_val], labels[idx_val]) if ii == 0: if epoch%10==0: print('Epoch: {:04d}'.format(epoch + 1), 'loss_train: {:.4f}'.format(loss_train.item()), 'loss_CE: {:.4f}'.format(loss_CE.item()), 'acc_train: {:.4f}'.format(acc_train.item()), #'loss_0: {:.4f}'.format(loss_0.item()), # 'loss_fx: {:.4f}'.format(loss_fx.item()), # 'loss_logfx: {:.4f}'.format(loss_logfx.item()), # 'acc_train: {:.4f}'.format(acc_train.item()), 'loss_val: {:.4f}'.format(loss_val.item()), 'acc_val: {:.4f}'.format(acc_val.item()), 'time: {:.4f}s'.format(time.time() - t)) return loss_val.item() def test(): model.eval() output = model(features,adj) output = F.log_softmax(output, dim=1) loss_test = F.nll_loss(output[idx_test], labels[idx_test]) acc_test = accuracy(output[idx_test], labels[idx_test]) print("Test set results:", "loss_test= {:.4f}".format(loss_test.item()), "accuracy= {:.4f}".format(acc_test.item())) return acc_test t_total = time.time() loss_values = [] # bad_counter = 0 # best = args.epochs+1 # best_epoch=0 # for epoch in range(args.epochs): # # loss_values.append(train(epoch)) # torch.save(model.state_dict(), './checkpoints/{}/{}.pkl'.format(args.dataset, epoch)) # if loss_values[-1] < best: # best = loss_values[-1] # best_epoch = epoch # bad_counter = 0 # else: # bad_counter += 1 # if bad_counter == args.patience: # break # files = glob.glob('./checkpoints/{}/*.pkl'.format(args.dataset)) # =[/checkpoints/{}/{}/0.pkl] # for file in files: # epoch_nb = int(file.split('.')[-2].split('/')[-1]) # if epoch_nb < best_epoch: # os.remove(file) # files = glob.glob('./checkpoints/{}/*.pkl'.format(args.dataset)) # for file in files: # epoch_nb = int(file.split('.')[-2].split('/')[-1]) # if epoch_nb > best_epoch: # os.remove(file) # print("Optimization Finished!") # print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) # # # Testing # print('Loading {}th epoch'.format(best_epoch)) # model.load_state_dict(torch.load('./checkpoints/{}/{}.pkl'.format(args.dataset, best_epoch))) for epoch in range(args.epochs): loss_values.append(train(epoch)) if epoch>args.early_stop and loss_values[-1] > np.mean(loss_values[-(args.early_stop+1):-1]): print("Early stopping...") break #train(epoch) print("Optimization Finished!") print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) acc_test = test() acc_test_list.append(acc_test) acc_test = acc_test.view(1, 1) acc_test = acc_test.cpu().numpy() with open("./results/{}/{}.txt".format(args.dataset, args.dataset), 'a') as f: # f.write("不丢弃dropout,加上b= {:.07f}×(output-output_1),epoch : {:04d},weight_decacy={},系数lam ={:.07f}".format f.write("dataset{} epoch : {:04d},weight_decacy={}, lr{},seed{},dropout{},features_perturbation: rate1:{},lambda1:{}; adj_pertubation: rate2:{},lambda2:{}".format (args.dataset, args.epochs, args.weight_decay, args.lr, seed, args.dropout,args.rate1,args.lambda1,args.rate2,args.lambda2)) np.savetxt(f, acc_test, fmt="%.6f") acc_test_list = torch.FloatTensor(acc_test_list) acc_test_std = torch.std(acc_test_list) avg_test = torch.mean(acc_test_list) avg_test = avg_test.view(1, 1) avg_test = avg_test.cpu().numpy() acc_test_std = acc_test_std.view(1, 1) acc_test_std = acc_test_std.cpu().numpy() print("总共做类{}次实验,平均值为:{:.04f}".format(args.run_time, avg_test.item())) print("总共做类{}次实验,误差值为:{:.04f}".format(args.run_time,acc_test_std.item())) with open("./results/{}/{}.txt".format(args.dataset, args.dataset), 'a') as f: f.write("总共做类{}次实验,平均值为:{:.04f}\n".format(args.run_time, avg_test.item())) f.write("总共做类{}次实验,误差值为:{:.04f}\n".format(args.run_time, acc_test_std.item()))
nclass=labels.max().item() + 1, dropout=0) student_params = list(model_S.parameters()) teacher_params = list(model_T.parameters()) for param in teacher_params: param.requires_grad = False optimizer_S = optim.Adam(model_S.parameters(), lr=args.lr, weight_decay=args.weight_decay) optimizer_T = WeightEMA(teacher_params, student_params, alpha=args.alpha) if args.cuda: model_S.cuda(args.gpu) model_T.cuda(args.gpu) features = features.cuda(args.gpu) adj_T = adj_T.cuda(args.gpu) adj_S = adj_S.cuda(args.gpu) labels = labels.cuda(args.gpu) pseudo_labels = pseudo_labels.cuda(args.gpu) idx_train = idx_train.cuda(args.gpu) idx_val = idx_val.cuda(args.gpu) idx_test = idx_test.cuda(args.gpu) idx_self = idx_self.cuda(args.gpu) features, labels, pseudo_labels = Variable(features), Variable( labels), Variable(pseudo_labels)
def get_validation_loss(no_cuda=True, fastmode=True, seed=42, epochs=50, lr=0.01, weight_decay=5e-4, hidden_dim_gcn=32, output_dim=16, dropout_prop=0.5, hidden_dim_fc=32, verbose=False ): # Training settings # help='Weight decay (L2 loss on parameters).') cuda = not no_cuda and torch.cuda.is_available() np.random.seed(seed) torch.manual_seed(seed) if cuda: torch.cuda.manual_seed(seed) # Load data (training_features, validation_features, training_adj, validation_adj, training_labels, validation_labels) = load_all_ligands() # Model and optimizer model = GCN(init_dim=11, hidden_dim_gcn=hidden_dim_gcn, output_dim=output_dim, dropout_prop=dropout_prop, hidden_dim_fc=hidden_dim_fc) optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) loss = nn.BCELoss() if cuda: model.cuda() training_features = [features.cuda() for features in training_features] validation_features = [features.cuda() for features in validation_features] training_adj = [adj.cuda() for adj in training_adj] validation_adj = [adj.cuda() for adj in validation_adj] training_adj = [adj.cuda() for adj in training_adj] validation_adj = [adj.cuda() for adj in validation_adj] training_labels = [labels.cuda() for labels in training_labels] validation_labels = [labels.cuda() for labels in validation_labels] def train(epoch): model.train() for i in range(len(training_labels)): optimizer.zero_grad() output = model(training_features[i], training_adj[i]) loss_train = loss(output, training_labels[i]) #acc_train = accuracy(output, labels) if i % 32 == 0: loss_train.backward() optimizer.step() if verbose: print('Epoch: {:04d}'.format(epoch+1), 'loss_train: {:.4f}'.format(loss_train.item())) def validate(): model.eval() for i in range(len(validation_labels)): output = model(validation_features[i], validation_adj[i]) loss_validation = loss(output, validation_labels[i]) #acc_validation = accuracy(output, labels) if verbose: print('Epoch: {:04d}'.format(epoch+1), 'loss_validation: {:.4f}'.format(loss_validation.item())) return loss_validation.item() # Train model t_total = time.time() for epoch in range(epochs): train(epoch) if not fastmode: # Evaluate validation set performance separately, # deactivates dropout during validation run. validate() if verbose: print("Optimization Finished!") print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) # Testing return validate()
def experimenter(data_name='cora', train_ratio=0.03, cuda=True, random_seed=42, hidden=16, dropout_ratio=0.5, learning_rate=0.01, weight_decay=5e-4, num_epochs=65, early_stopping=30, task='classification', public_splitting=False): # helper function to run epxeriment if data_name in ['cora', 'citeseer', 'pubmed']: print("Loading Classification Datasets") Tmat, eadj, edge_name, edge_feature_dict, adj, features, edge_features, labels, idx_train, idx_val, idx_test = tqdm( load_data(data_name=data_name, train_ratio=train_ratio, public_splitting=public_splitting)) model = GCN(nfeat_v=features.shape[1], nfeat_e=edge_features.shape[1], nhid=hidden, nclass=labels.max().item() + 1, dropout=dropout_ratio) ssl_agent_n = PairwiseDistance(adj, features, nhid=args.hidden, cuda=cuda, node=True) ssl_agent_e = PairwiseDistance(eadj, edge_features, nhid=args.hidden, cuda=cuda, node=False) else: ValueError("The input data is not supported! ") print(">" * 100) print("Loaded and preprocessed the graph data! ") print(">" * 100) optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) np.random.seed(random_seed) torch.manual_seed(random_seed) if cuda: torch.cuda.manual_seed(random_seed) torch.set_default_tensor_type('torch.cuda.FloatTensor') if cuda: model.cuda() Tmat = Tmat.cuda() eadj = eadj.cuda() adj = adj.cuda() features = features.cuda() edge_features = edge_features.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() # pooling = pooling.cuda() # node_count = node_count.cuda() if task == "classification": criteria = F.nll_loss acc_measure = accuracy elif task == "regression": criteria = torch.nn.L1Loss acc_measure = RMSELoss # --------------------------------------- # training function # --------------------------------------- # count_time = 0 def train(epoch): t = time.time() model.train() optimizer.zero_grad() output, n_feature, e_feature = model(features, edge_features, eadj, adj, Tmat, task) loss_n = ssl_agent_n.classification_loss(n_feature) loss_e = ssl_agent_e.classification_loss(e_feature) loss_train = criteria(output[idx_train], labels[idx_train]) loss_train = loss_train + 0.1 * loss_n + 0.1 * loss_e acc_train = acc_measure(output[idx_train], labels[idx_train]) loss_train.backward() optimizer.step() if not args.fastmode: # Evaluate validation set performance separately, # deactivates dropout during validation run. model.eval() output, _, _ = model(features, edge_features, eadj, adj, Tmat, task) loss_val = criteria(output[idx_val], labels[idx_val]) acc_val = acc_measure(output[idx_val], labels[idx_val]) print('Epoch: {:04d}'.format(epoch + 1), 'loss_train: {:.4f}'.format(loss_train.item()), 'acc_train: {:.4f}'.format(acc_train.item()), 'loss_val: {:.4f}'.format(loss_val.item()), 'acc_val: {:.4f}'.format(acc_val.item()), 'time: {:.4f}s'.format(time.time() - t)) return loss_val.item() # ------------------------------------------- # testing function # ------------------------------------------- def test(): model.eval() output, _, _ = model(features, edge_features, eadj, adj, Tmat, task) loss_test = criteria(output[idx_test], labels[idx_test]) acc_test = acc_measure(output[idx_test], labels[idx_test]) print("Test set results:", "loss= {:.4f}".format(loss_test.item()), "accuracy= {:.4f}".format(acc_test.item())) return acc_test.item() def pretrain(epoch): t = time.time() model.train() optimizer.zero_grad() output, node_f, edge_f = model(features, edge_features, eadj, adj, Tmat, task) loss_n = ssl_agent_n.classification_loss(node_f) loss_e = ssl_agent_e.classification_loss(edge_f) # loss_train = criteria(output[idx_train], labels[idx_train]) loss_train = loss_n + loss_e acc_train = acc_measure(output[idx_train], labels[idx_train]) loss_train.backward() optimizer.step() if not args.fastmode: # Evaluate validation set performance separately, # deactivates dropout during validation run. model.eval() output, _, _ = model(features, edge_features, eadj, adj, Tmat, task) loss_val = criteria(output[idx_val], labels[idx_val]) acc_val = acc_measure(output[idx_val], labels[idx_val]) print('Epoch: {:04d}'.format(epoch + 1), 'loss_train: {:.4f}'.format(loss_train.item()), 'acc_train: {:.4f}'.format(acc_train.item()), 'loss_val: {:.4f}'.format(loss_val.item()), 'acc_val: {:.4f}'.format(acc_val.item()), 'time: {:.4f}s'.format(time.time() - t)) return loss_val.item() # Train model t_total = time.time() val_watch = [] pre_val_watch = [] input_idx_train = idx_train # for epoch in range(50): # pre_val_watch.append(pretrain(epoch)) # test() # if epoch > early_stopping and pre_val_watch[-1] > np.mean(pre_val_watch[-(early_stopping + 1):-1]): # print("Early stopping...") # break for epoch in range(num_epochs): # val_adj, val_fea = ssl_agent.transform_data() val_watch.append(train(epoch)) test() if epoch > early_stopping and val_watch[-1] > np.mean( val_watch[-(early_stopping + 1):-1]): print("Early stopping...") break print("Optimization Finished!") print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) print("Printing the weights : ") return test()
args = parser.parse_args() n_anchor = 1000 n_bits = args.n_bits n_class = 21 n_epoch = 10 topk = 15 # dataset: 'cifar10', 'nuswide', 'ImageNet', 'sun' dataset = 'nuswide' dset = load_data(dataset) meta = torch.load('nuswide_2000_32_0.4454_0.5912') anchor = meta['anchor'] gcn = GCN(500, n_bits, n_class, meta['anchor_affnty'], 40) gcn.load_state_dict(meta['state_dict']) gcn.cuda() test_loader = data.DataLoader(dataset=db(dset.testdata, dset.testlabel), batch_size=100, shuffle=False, num_workers=4) tH = [] gcn.eval() for images, _ in test_loader: in_aff, out_aff = rbf_affnty(images, anchor, topk=topk) images = Variable(images).cuda() in_aff = Variable(in_aff).cuda() out_aff = Variable(out_aff).cuda() out, _ = gcn(images, in_aff, out_aff) tH.append(out.data.cpu().numpy())
# print('idx_train的维度:',idx_train.shape) # 140*1 训练节点标签 # print('idx_val的维度:',idx_val.shape) # 300*1 验证节点标签 # print('idx_test的维度:',idx_test.shape) # 1000*1 测试节点标签 # Model and optimizer model = GCN(nfeat=features.shape[1], nhid=args.hidden, nclass=labels.max().item() + 1, dropout=args.dropout) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # 数据写入cuda,便于后续加速 if args.cuda: model.cuda() # . cuda()会分配到显存里(如果gpu可用) features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() def train(epoch): t = time.time() # 返回当前时间 model.train() optimizer.zero_grad() # optimizer.zero_grad()意思是把梯度置零,也就是把loss关于weight的导数变成0. # pytorch中每一轮batch需要设置optimizer.zero_gra output = model(features, adj)
features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() features, adj, labels = Variable(features), Variable(adj), Variable(labels) #net = nn.DataParallel(net, device_ids=range(1)) #net.load_state_dict(torch.load(opt.modelIn)) optimizer = optim.Adam(net.parameters(), lr=0.01, weight_decay=5e-4) net.cuda() for epoch_n in range(200): train(epoch_n) loss_f = nn.CrossEntropyLoss() net.eval() output = net(features, adj) output = F.log_softmax(output, dim =1) loss_test = F.nll_loss(output[idx_test], labels[idx_test]) acc_test = accuracy(output[idx_test], labels[idx_test]) print("Test set results:", "loss= {:.4f}".format(loss_test.data[0]), "accuracy= {:.4f}".format(acc_test.data[0])) for c in opt.c: #print (type(adj), type(features)) acc, avg_distort = acc_under_attack(features,adj,labels, net, c, attack_cw)
def train_gcn(dataset, test_ratio=0.5, val_ratio=0.2, seed=1, n_hidden=64, n_epochs=200, lr=1e-2, weight_decay=5e-4, dropout=0.5, use_embs=False, verbose=True, cuda=False): data = dataset.get_data() # train text embs if use_embs: pad_ix, n_tokens, matrix, pretrained_embs = data['features'] if pretrained_embs is not None: pretrained_embs = torch.FloatTensor(pretrained_embs) features = torch.LongTensor(matrix) else: pad_ix = None n_tokens = None pretrained_embs = None features = torch.FloatTensor(data['features']) labels = torch.LongTensor(data['labels']) n = len(data['ids']) train_mask, val_mask, test_mask = get_masks(n, data['main_ids'], data['main_labels'], test_ratio=test_ratio, val_ratio=val_ratio, seed=seed) train_mask = torch.BoolTensor(train_mask) val_mask = torch.BoolTensor(val_mask) test_mask = torch.BoolTensor(test_mask) if cuda: torch.cuda.set_device("cuda:0") features = features.cuda() labels = labels.cuda() train_mask = train_mask.cuda() val_mask = val_mask.cuda() test_mask = test_mask.cuda() g = DGLGraph(data['graph']) g = dgl.transform.add_self_loop(g) n_edges = g.number_of_edges() degs = g.in_degrees().float() norm = torch.pow(degs, -0.5) norm[torch.isinf(norm)] = 0 if cuda: norm = norm.cuda() g.ndata['norm'] = norm.unsqueeze(1) if use_embs: if pretrained_embs is not None: in_feats = 100 else: in_feats = 64 else: in_feats = features.shape[1] # + 1 for unknown class n_classes = data['n_classes'] + 1 model = GCN(g, in_feats=in_feats, n_hidden=n_hidden, n_classes=n_classes, activation=F.relu, dropout=dropout, use_embs=use_embs, pretrained_embs=pretrained_embs, pad_ix=pad_ix, n_tokens=n_tokens) if cuda: model.cuda() loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.9, patience=20, min_lr=1e-10) best_f1 = -100 # initialize graph dur = [] for epoch in range(n_epochs): model.train() if epoch >= 3: t0 = time.time() # forward mask_probs = torch.empty(features.shape).uniform_(0, 1) if cuda: mask_probs = mask_probs.cuda() mask_features = torch.where(mask_probs > 0.2, features, torch.zeros_like(features)) logits = model(mask_features) loss = loss_fcn(logits[train_mask], labels[train_mask]) optimizer.zero_grad() loss.backward() optimizer.step() if epoch >= 3: dur.append(time.time() - t0) f1 = evaluate(model, features, labels, val_mask) scheduler.step(1 - f1) if f1 > best_f1: best_f1 = f1 torch.save(model.state_dict(), 'best_model.pt') if verbose: print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | F1 {:.4f} | " "ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.item(), f1, n_edges / np.mean(dur) / 1000)) model.load_state_dict(torch.load('best_model.pt')) f1 = evaluate(model, features, labels, test_mask) if verbose: print() print("Test F1 {:.2}".format(f1)) return f1
torch.manual_seed(args.seed) # 为CPU设置随机种子用于生成随机数,以使得结果是确定的 if args.cuda: torch.cuda.manual_seed(args.seed) # 为GPU设置随机种子 # 加载数据 adj, features, labels, idx_train, idx_val, idx_test = load_data() # 模型和优化器 model = GCN(nfeat=features.shape[1], nhid=args.hidden, nclass=labels.max().item() + 1, dropout=args.dropout, meta_size=args.meta_size) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.cuda: # 如果使用GUP则执行这里,数据写入cuda,便于后续加速 model.cuda() # 模型放到GPU上跑 features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() def train(epoch): """ 定义训练函数 :param epoch: 训练轮数 """ t = time.time() model.train()
def train_run(img_name): adj, features, labels, idx_train, idx_val, idx_test, rgbxy = load_data( dataset=img_name) # Model and optimizer model = GCN(nfeat=features.shape[1], nhid=80, nclass=labels.max().item() + 1, dropout=0.5) optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) print("a\n") if args.cuda: model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() def train(epoch): t = time.time() model.train() optimizer.zero_grad() output = model(features, adj) loss_train1 = F.nll_loss(output[idx_train], labels[idx_train]) ''' print(labels.shape) #print(output.shape) c = torch.where(labels.cpu() == 0,torch.ones(labels.shape),torch.zeros(labels.shape)) print(c.shape) background_points = torch.sum( c ) total_points = labels.shape[0] foreground_points = total_points - background_points l = labels[idx_train].cpu() ba = l.shape[0] deep = output.shape[1] labels_one_hot = torch.zeros((ba,deep)) #labels_one_hot[:,0] = torch.where(l == 0, torch.ones(ba)/background_points, torch.zeros(ba)) #labels_one_hot[:,0] = torch.where(l == 0, torch.ones(ba)*total_points/background_points, torch.zeros(ba)) labels_one_hot[:,0] = torch.where(l == 0, torch.ones(ba), torch.zeros(ba)) for i in range(1,deep): #labels_one_hot[:,i] = torch.where(l == i, torch.ones(ba)/foreground_points*foreground_inputs, torch.zeros(ba)) #labels_one_hot[:,i] = torch.where(l == i, torch.ones(ba)*total_points/foreground_points, torch.zeros(ba)) labels_one_hot[:,i] = torch.where(l == i, torch.ones(ba), torch.zeros(ba)) loss_func = torch.nn.BCELoss() #print(output[idx_train].shape) #print(labels_one_hot.shape) soft = torch.nn.Softmax(dim=1) loss_train1 = loss_func( soft(output[idx_train]).cpu(), labels_one_hot) ''' ''' rowsum = torch.sum(output[idx_train],1) r_inv = torch.pow(rowsum, -1) r_inv[torch.isinf(r_inv)] = 0 r_mat_inv = torch.diag(r_inv) mx = torch.mm(r_mat_inv, output[idx_train]) loss_train1 = loss_func( mx.cpu(), labels_one_hot) ''' #print("b= {}, f = {}".format(background_points,foreground_points)) l = output[idx_test].shape[0] W = torch.zeros((l, l, rgbxy.shape[1])).cuda() ones = torch.unsqueeze(torch.ones((l, 5)), 0) rgbxy_2d = rgbxy[idx_test] rgbxy_3d = [] for i in range(l): rgbxy_3d.append(rgbxy_2d) rgbxy_3d = tuple(rgbxy_3d) rgbxy_3d = torch.stack(rgbxy_3d, dim=1).cuda() # ones_t = ones.transpose(ones, 1,2) ''' for i in range(l): for j in range(l): W[i][j] = rgbxy[idx_test[i]]-rgbxy[idx_test[j]] ''' W = rgbxy_3d.transpose(0, 1) - rgbxy_3d #''' sigma = args.sigma W = torch.exp(-torch.norm(W, dim=2) / 2 / sigma) d = torch.sum(W, dim=1) n_cut = 1 for k in range(output.shape[1]): s = output[idx_test, k] n_cut = n_cut + torch.mm(torch.mm(torch.unsqueeze( s, 0), W), torch.unsqueeze(1 - s, 1)) / (torch.dot(d, s)) lamda = args.lamda print(n_cut) loss_train = loss_train1.cuda() + lamda * n_cut acc_train = accuracy(output[idx_train], labels[idx_train]) loss_train.backward() optimizer.step() if not False: # Evaluate validation set performance separately, # deactivates dropout during validation run. model.eval() output = model(features, adj) #loss_val = F.nll_loss(output[idx_val], labels[idx_val]) loss_val = loss_train1 acc_val = accuracy(output[idx_val], labels[idx_val]) print('Epoch: {:04d}'.format(epoch + 1), 'loss_train: {:.4f}'.format(loss_train.item()), 'acc_train: {:.4f}'.format(acc_train.item()), 'loss_val: {:.4f}'.format(loss_val.item()), 'acc_val: {:.4f}'.format(acc_val.item()), 'time: {:.4f}s'.format(time.time() - t)) def test(): model.eval() output = model(features, adj) loss_test = F.nll_loss(output[idx_test], labels[idx_test]) acc_test = accuracy(output[idx_test], labels[idx_test]) print("Test set results:", "loss= {:.4f}".format(loss_test.item()), "accuracy= {:.4f}".format(acc_test.item())) #np.save(img_name+'predict',output.detach().cpu().numpy()) print("save prediction!---" + img_name + 'predict') with open("retport.txt", 'a') as f: f.write(img_name + "\t") f.write("test acc:" + str(acc_test) + "\n") # Train model t_total = time.time() for epoch in range(200): train(epoch) print("Optimization Finished!") print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) # Testing test() from postprocess import run run(img_name)