def main(args): # save_dir = args.save_dir log_dir = args.log_dir train_dir = args.data_dir if not os.path.exists(save_dir): os.makedirs(save_dir) if not os.path.exists(log_dir): os.makedirs(log_dir) adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = utils.load_data( args.data_type) features = utils.preprocess_features(features) support = [utils.preprocess_adj(adj)] args.num_supports = 1 args.input_size, args.features_size = features[2][1], features[2] args.output_size = y_train.shape[1] config_proto = utils.get_config_proto() sess = tf.Session(config=config_proto) model = GCN(args, sess, name="gcn") summary_writer = tf.summary.FileWriter(log_dir) for epoch in range(1, args.nb_epoch + 1): epoch_start_time = time.time() feed_dict = utils.construct_feed_dict(model, features, support, y_train, train_mask) _, train_loss, train_acc, summaries = model.train(feed_dict) if epoch % args.summary_epoch == 0: summary_writer.add_summary(summaries, epoch) if epoch % args.print_epoch == 0: feed_dict_val = utils.construct_feed_dict(model, features, support, y_val, val_mask) val_loss, val_acc = model.evaluate(feed_dict_val) print "epoch %d, train_loss %f, train_acc %f, val_loss %f, val_acc %f, time %.5fs" % \ (epoch, train_loss, train_acc, val_loss, val_acc, time.time()-epoch_start_time) if args.anneal and epoch >= args.anneal_start: sess.run(model.lr_decay_op) model.saver.save(sess, os.path.join(save_dir, "model.ckpt")) print "Model stored...."
def build_classifier(gnn, input_dim, hidden_dim, num_labels, args): if gnn == 'GCN': return GCN(input_dim, hidden_dim, num_labels, args.num_layers) elif gnn == 'GAT': return GAT(input_dim, hidden_dim, num_labels, args.num_layers, args.num_heads, args.merge, args.dropout) else: raise NotImplementedError( '%d is not implemented yet or doesn\'t exist.' % gnn)
def __init__(self, params, logger, data_logger=None, epochs_logger=None): self._logger = logger self._epoch_logger = epochs_logger self._data_logger = EmptyLogger() if data_logger is None else data_logger self._parameters = params self._lr = params["lr"] self._is_nni = params['is_nni'] self._device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') self._mse_loss = self.weighted_mse_loss self._temporal_loss = self.weighted_mse_loss self.model = GCN(num_of_features=self._parameters["feature_matrices"][0].shape[1], hid_size=self._parameters["hid_size"], num_of_classes=self._parameters["number_of_classes"], activation=self._parameters["activation"], dropout=self._parameters["dropout"]) self.model = self.model.to(self._device) self.opt = self._parameters["optimizer"](self.model.parameters(), lr=self._parameters['lr'], weight_decay=self._parameters['weight_decay'])
def train_GCN(args, features, labels, adj, train_mask, val_mask, test_mask, batch_test=False): #features, labels, adj, train_mask, val_mask, test_mask = [data[i] for i in range(6)] model = GCN(args, features, labels, adj) #model = GCN_LPA(args, features, labels, adj) def get_feed_dict(mask, dropout): feed_dict = {model.label_mask: mask, model.dropout: dropout} return feed_dict with tf.Session() as sess: sess.run(tf.global_variables_initializer()) best_val_acc = 0 final_test_acc = 0 for epoch in range(args.epochs): # train _, train_loss, train_acc, train_weight = sess.run( [model.optimizer, model.loss, model.accuracy, model.adj], feed_dict=get_feed_dict(train_mask, args.dropout)) #print(train_weight) # validation val_loss, val_acc, val_weight = sess.run( [model.loss, model.accuracy, model.adj], feed_dict=get_feed_dict(val_mask, 0.0)) #print(val_weight) #exit() # test test_loss, test_acc = sess.run([model.loss, model.accuracy], feed_dict=get_feed_dict( test_mask, 0.0)) if val_acc >= best_val_acc: best_val_acc = val_acc final_test_acc = test_acc if not batch_test: print( 'epoch %d train loss: %.4f acc: %.4f val loss: %.4f acc: %.4f test loss: %.4f acc: %.4f' % (epoch, train_loss, train_acc, val_loss, val_acc, test_loss, test_acc)) if not batch_test: print('final test acc: %.4f' % final_test_acc) else: return final_test_acc
def __init__(self, in_features=66, kernel_size=10, d_model=512, device='cuda', h=4, dropout=0.1, dct_n=10, num_stage=12, d_ff=256, N=6, d_att_model=128): super(AttModelModVel, self).__init__() self.kernel_size = kernel_size self.dct_n = dct_n self.h = h self.device = device self.in_features = in_features c = copy.deepcopy self.pos_encoding = PositionalEncoding(d_model=in_features, dropout=dropout) attn = MultiHeadedAttention(h=h, d_model=d_att_model) ff = PositionwiseFeedForward(d_model=d_att_model, d_ff=d_ff, dropout=dropout) self.encoder = Encoder( EncoderLayer(d_att_model, c(attn), c(ff), dropout), N) self.gcn = GCN.GCN(input_feature=dct_n * 2, hidden_feature=d_model, p_dropout=0.3, num_stage=num_stage, node_n=in_features) self.init_lin = nn.Linear(in_features, d_att_model) #self.inter_lin = nn.Linear(d_att_model, in_features) self.inter_conv = nn.Sequential( nn.Conv1d(in_channels=d_att_model, out_channels=in_features, kernel_size=11, bias=False), nn.LeakyReLU(0.1), nn.Conv1d(in_channels=in_features, out_channels=in_features, kernel_size=11, bias=False), nn.LeakyReLU(0.1), nn.Conv1d(in_channels=in_features, out_channels=in_features, kernel_size=11, bias=False), nn.LeakyReLU(0.1)) for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform(p)
def create_model(params, input_dim): model_type = params['model'] hidden1 = int(params['hidden']) depth = int(params['depth']) dropout = float(params['dropout']) degree = int(params['max_degree']) if model_type == 'dense': return MLP(input_dim=input_dim, hidden1=hidden1, dropout=dropout), False elif depth != 0: return Deep_GCN(input_dim=input_dim, hidden1=hidden1, depth=depth, dropout=dropout, degree=degree), True else: return GCN(input_dim=input_dim, hidden1=hidden1, dropout=dropout, degree=degree), True
def save_output(root_dir, output_dir): # dataset dataset = save_predict_dataset(root_dir) #model model = GCN(4, 512) if use_gpu: model = model.cuda() #model = torch.nn.DataParallel(model).cuda() model.load_state_dict(torch.load(os.path.join(save_dir, model_name))) model.train(False) for n in range(int(len(dataset) / (6 * 9))): #test full_output = np.zeros((4, 2848, 4288), dtype='float32') #(C, H, W) title = '' for idx in range(6 * 9 * n, 6 * 9 * (n + 1)): image, name = dataset[idx] r = int((idx % (6 * 9)) / 9) #row c = (idx % (6 * 9)) % 9 #column title = name if use_gpu: image = image.cuda() image = Variable(image, volatile=True) #forward output = model(image.unsqueeze(0)) output = F.sigmoid(output) output = output[0] if c < 8: if r == 5: full_output[:, r * 512:r * 512 + 512 - 224, c * 512:c * 512 + 512] = output.cpu().data.numpy()[:, :-224, :] else: full_output[:, r * 512:r * 512 + 512, c * 512:c * 512 + 512] = output.cpu().data.numpy() for i, d in enumerate(['MA', 'EX', 'HE', 'SE']): if not os.path.exists(os.path.join(output_dir, d)): os.makedirs(os.path.join(output_dir, d)) im = np.expand_dims(full_output[i], axis=0).transpose(1, 2, 0) im = full_output[i] * 255 im = np.uint8(im) im = Image.fromarray(im) im.save(os.path.join(output_dir, d, title + '.jpg'))
def __init__(self, in_features=66, kernel_size=10, d_model=512, device='cuda', h=4, dropout=0.1, dct_n=10, num_stage=12, d_ff=128, N=4, d_att_model=128): super(AttModelResNetVel, self).__init__() self.kernel_size = kernel_size self.dct_n = dct_n self.h = h self.device = device c = copy.deepcopy self.pos_encoding = PositionalEncoding(d_model=d_att_model, dropout=dropout) attn = MultiHeadedAttention(h=h, d_model=d_att_model) ff = PositionwiseFeedForward(d_model=d_att_model, d_ff=d_ff, dropout=dropout) self.pos_encoder = Encoder( EncoderLayer(d_att_model, c(attn), c(ff), dropout), N) self.vel_encoder = Encoder( EncoderLayer(d_att_model, c(attn), c(ff), dropout), N) self.gcn = GCN.GCN(input_feature=dct_n * 2, hidden_feature=d_model, p_dropout=0.3, num_stage=num_stage, node_n=in_features) self.init_lin = nn.Linear(in_features, d_att_model) self.inter_lin = nn.Linear(d_att_model, in_features) self.res_conv_block = ResConvBlock(in_features) self.init_lin_vel = nn.Linear(in_features, d_att_model) self.inter_lin_vel = nn.Linear(d_att_model, in_features) self.res_conv_block_vel = ResConvBlock(in_features)
def load_trained_vector(epoch, number, n2i_f, file_homes): global node2index node2index = cPickle.load(n2i_f) node_count = len(node2index) node_dim = 128 n_repr = 128 gcn = GCN(node_count, node_dim, n_repr) gcn.load_state_dict( torch.load(file_homes + '/networks/GCN_%d_%d.pth' % (number, epoch), map_location='cpu')) f = open(files_home + '/networks/adj_matrix_%d_full' % (number), 'rb') full_adj_matrix = cPickle.load(f) full_adj_matrix = sparse_mx_to_torch_sparse_tensor(full_adj_matrix) init_input = torch.LongTensor([j for j in range(0, node_count)]) gcn.eval() rp_matrix = gcn(init_input, full_adj_matrix) #gcn.to(device) return rp_matrix.double()
def init_model(): if args.model == "GCN": model = GCN(**model_args) elif args.model == "GAT": model_args["num_heads"] = 8 model_args["n_units"] = 8 model_args["dropout"] = 0.6 model_args["activation"] = "elu" model = GAT(**model_args) else: model_args["n_layers"] = args.hidden_layers if args.model == "JKNetConCat": model = JKNetConCat(**model_args) elif args.model == "JKNetMaxpool": model = JKNetMaxpool(**model_args) else: print("Model should be GCN, GAT, JKNetConCat or JKNetMaxpool.") assert False optimizer = th.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) return model, optimizer
amide_train = PandasTools.LoadSDF(dir_train,smilesName='SMILES',molColName='Molecule',\ includeFingerprints=False) amide_test = PandasTools.LoadSDF(dir_test, smilesName='SMILES',molColName='Molecule',\ includeFingerprints=False) '''If test atoms not in train --> problem ''' _, atom_dict = unique_atom_list(amide_train) train_loader = DataLoader(data_frame_to_list(amide_train, atom_dict), batch_size=batch_size, shuffle=True) test_loader = DataLoader(data_frame_to_list(amide_test, atom_dict), batch_size=batch_size, shuffle=True) model = GCN(num_node_features=13, hidden_channels=64, num_classes=2) optimizer = torch.optim.Adam(model.parameters(), lr=0.01) criterion = torch.nn.CrossEntropyLoss() def train(loader): model.train() for data in loader: # Iterate in batches over the training dataset. out = model(data.x, data.edge_index, data.batch) # Perform a single forward pass. loss = criterion(out, data.y) # Compute the loss. loss.backward() # Derive gradients. optimizer.step() # Update parameters based on gradients. optimizer.zero_grad() # Clear gradients.
def main(): logger = logging.getLogger('stdout') logger.setLevel(logging.DEBUG) hdlr = logging.StreamHandler(sys.stdout) logger.addHandler(hdlr) logger.debug('starting training') config_path = "../../../config/gcn.conf" conf = configparser.ConfigParser() conf.read(config_path) logger.debug(f'running on {device}') data_dir = conf['path']['gcn_data'] + '/data.pkl' if not os.path.exists(data_dir): pdb_data.main() try: with open(data_dir, 'rb') as f: data_dict = pickle.load(f) logger.debug('data successfully found') except: logger.error(f'can not find data at {data_dir}') vocab_path = conf['path']['vocab'] with open(vocab_path, 'r') as of: aa_vocab = json.load(of) train_data, val_data, test_data, target_dim = split_data( data_dict, aa_vocab, 'mf') #change to variable GO type later logger.debug(f'size of training set = {len(train_data)}') logger.debug(f'size of validation set = {len(val_data)}') logger.debug(f'# of go terms = {target_dim}') plt.figure(figsize=(10, 6)) lm_model_paths = conf['path']['lm'] if type(lm_model_paths) != list: lm_model_paths = [lm_model_paths] for lm_model_path in lm_model_paths: language_model = torch.load(lm_model_path, map_location=device) lm_embedding = language_model['model_state_dict'][ 'module.word_embeddings.weight'] num_trials = int(conf['model']['num_trials']) for i in range(num_trials): model_name = lm_model_path.split('/')[-1].split( '.')[0] + '_GCN' + '_iter_{}'.format(i) if conf['model']['model'] == 'GCN': model = GCN(target_dim, lm_embedding) elif conf['model']['model'] == 'Toy': model = Toy(target_dim, lm_embedding) model.to(device) logger.debug(f'traning on {model_name}') train(logger, model, train_data, val_data, target_dim) model_dir = conf['path']['models'] torch.save(model, model_dir + '/' + model_name + '.model') scores, labels = evaluate(model, val_data, target_dim) precision, recall, ap_score = compute_pr(scores, labels, target_dim) logger.debug('Average precision score for {}: {}'.format( model_name, ap_score)) plt.step(recall, precision, where='post', label='AP score for {}: {}'.format(model_name, ap_score)) plt.xlabel('Recall') plt.ylabel('Precision') plt.ylim([0.0, 1.05]) plt.xlim([0.0, 1.0]) plt.title('micro-averaged PR curve over all classes') plt.legend() figure_dir = conf['path']['figures'] plt.savefig(figure_dir + '/temp.png') logger.debug('plot saved to: ' + figure_dir + '/temp.png')
test_mask = torch.from_numpy(test_mask.astype(np.int)).to(device) i = torch.from_numpy(features[0]).long().to(device) v = torch.from_numpy(features[1]).to(device) feature = torch.sparse.FloatTensor(i.t(), v, features[2]).to(device) i = torch.from_numpy(supports[0]).long().to(device) v = torch.from_numpy(supports[1]).to(device) support = torch.sparse.FloatTensor(i.t(), v, supports[2]).float().to(device) print('x :', feature) print('sp:', support) num_features_nonzero = feature._nnz() feat_dim = feature.shape[1] net = GCN(feat_dim, num_classes, num_features_nonzero) net.to(device) optimizer = optim.Adam(net.parameters(), lr=args.learning_rate) net.train() for epoch in range(args.epochs): out = net((feature, support)) out = out[0] loss = masked_loss(out, train_label, train_mask) loss += args.weight_decay * net.l2_loss() acc = masked_acc(out, train_label, train_mask) optimizer.zero_grad() loss.backward()
target = torch.tensor(target) true_labels = [ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ] # Using One-Hot Encoding of the nodes for initializing feature matrix x feat = torch.eye(adj.size(0), dtype=torch.float32) n_feat = feat.size(0) # n_feat = 34 n_hid = args.hid # n_hid = 10 n_out = args.out # out = 2 # Initialize the GCN model model = GCN(adj, n_feat, n_hid, n_out) #### Training #### criterion = nn.CrossEntropyLoss(ignore_index=-1) optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) if not args.no_vis: #### Plot animation using celluloid fig = plt.figure() camera = Camera(fig) model.train() for i in range(args.epochs): optimizer.zero_grad() pred = model(feat)
def sgcn(feat_data, labels, lap_matrix, train_nodes, valid_nodes, test_nodes, args, device, calculate_grad_vars=False, full_batch=False): # use multiprocess sample data process_ids = np.arange(args.batch_num) pool = mp.Pool(args.pool_num) lap_matrix_sq = lap_matrix.multiply(lap_matrix) jobs = prepare_data(pool, sampler, process_ids, train_nodes, samp_num_list, len(feat_data), lap_matrix, lap_matrix_sq, args.n_layers) susage = GCN(nfeat=feat_data.shape[1], nhid=args.nhid, num_classes=num_classes, layers=args.n_layers, dropout=args.dropout).to(device) susage.to(device) print(susage) adjs_full, input_nodes_full, sampled_nodes_full = full_batch_sampler( train_nodes, len(feat_data), lap_matrix, args.n_layers) adjs_full = package_mxl(adjs_full, device) optimizer = optim.SGD(filter(lambda p: p.requires_grad, susage.parameters()), lr=args.lr) loss_train = [] loss_test = [] grad_variance_all = [] loss_train_all = [] best_model = copy.deepcopy(susage) best_val_loss = 10 # randomly pick a large number is fine best_val_index = 0 best_val_cnt = 0 for epoch in np.arange(args.epoch_num): # fetch train data train_data = [job.get() for job in jobs] pool.close() pool.join() # prepare next epoch train data pool = mp.Pool(args.pool_num) jobs = prepare_data(pool, sampler, process_ids, train_nodes, samp_num_list, len(feat_data), lap_matrix, lap_matrix_sq, args.n_layers) # it can also run full-batch GD by ignoring all the samplings if full_batch: inner_loop_num = args.batch_num cur_train_loss, cur_train_loss_all, grad_variance = full_step( susage, optimizer, feat_data, labels, train_nodes, valid_nodes, adjs_full, train_data, inner_loop_num, device, calculate_grad_vars=calculate_grad_vars) else: inner_loop_num = args.batch_num cur_train_loss, cur_train_loss_all, grad_variance = sgd_step( susage, optimizer, feat_data, labels, train_nodes, valid_nodes, adjs_full, train_data, inner_loop_num, device, calculate_grad_vars=calculate_grad_vars) loss_train_all.extend(cur_train_loss_all) grad_variance_all.extend(grad_variance) # calculate test loss susage.eval() susage.zero_grad() val_loss, _ = susage.calculate_loss_grad(feat_data, adjs_full, labels, valid_nodes) if val_loss + 5e-4 < best_val_loss: best_val_loss = val_loss del best_model best_model = copy.deepcopy(susage) best_val_index = epoch best_val_cnt = 0 cur_test_loss = val_loss best_val_cnt += 1 loss_train.append(cur_train_loss) loss_test.append(cur_test_loss) # print progress print('Epoch: ', epoch, '| train loss: %.8f' % cur_train_loss, '| test loss: %.8f' % cur_test_loss) if best_val_cnt > 10: break f1_score_test = best_model.calculate_f1(feat_data, adjs_full, labels, test_nodes) return best_model, loss_train, loss_test, loss_train_all, f1_score_test, grad_variance_all, best_val_index
v = torch.FloatTensor(values) shape = coo.shape return torch.sparse.FloatTensor(i, v, torch.Size(shape)) if args.model == 'GCN': #x = preprocess_features(features) x = features.todense() x = torch.FloatTensor(x).to(device) labels = mat_contents['Label'] labels = torch.LongTensor(labels-1).squeeze().to(device) #adj_pt, _, _ = normalize_trans(adj+sp.eye(adj.shape[0])) adj_pt = normalize_trans(adj)[0] + sp.eye(adj.shape[0]) adj_pt = np_sparse_to_pt_sparse(adj_pt).to(device) model = GCN(x.size()[-1], 128, nb_classes).to(device) optimizer = optim.Adam(model.parameters(), lr=0.01) loss_func = nn.CrossEntropyLoss() train(model, optimizer, loss_func, adj_pt, x, labels, train_mask_real, val_mask, test_mask, batch_size=128, epochs = 200, iters_per_epoch=int(nb_train_real/128)+1, patience=10) elif args.model == 'GFNN' or args.model == 'SGC': x = preprocess_features(features) x = torch.FloatTensor(x).to(device) labels = mat_contents['Label'] labels = torch.LongTensor(labels-1).squeeze().to(device) adj_pt, _, _ = normalize_trans(adj+sp.eye(adj.shape[0])) adj_pt = np_sparse_to_pt_sparse(adj_pt).to(device) x = torch.spmm(adj_pt, x) x = torch.spmm(adj_pt, x)
parser.add_argument('--dropout', type=float, default=0.5, help='Dropout rate (1 - keep probability).') args = parser.parse_args() np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) # Load data adj, features, labels, idx_train, idx_val, idx_test = load_citeseer() # Model and optimizer model = GCN(input_size=features.shape[1], hidden_size=args.hidden, n_classes=labels.max().item() + 1) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() criterion = nn.CrossEntropyLoss()
def graphsaint(feat_data, labels, adj_matrix, train_nodes, valid_nodes, test_nodes, args, device, concat=False): samp_num_list = np.array([args.samp_num for _ in range(args.n_layers)]) # use multiprocess sample data process_ids = np.arange(args.batch_num) graphsaint_sampler_ = graphsaint_sampler(adj_matrix, train_nodes, node_budget=args.batch_size) if concat: susage = GraphSageGCN(nfeat=feat_data.shape[1], nhid=args.nhid, num_classes=args.num_classes, layers=args.n_layers, dropout=args.dropout, multi_class=args.multi_class).to(device) else: susage = GCN(nfeat=feat_data.shape[1], nhid=args.nhid, num_classes=args.num_classes, layers=args.n_layers, dropout=args.dropout, multi_class=args.multi_class).to(device) susage.to(device) print(susage) optimizer = optim.Adam(susage.parameters()) adjs_full, input_nodes_full, sampled_nodes_full = graphsaint_sampler_.full_batch( train_nodes, len(feat_data), args.n_layers) adjs_full = package_mxl(adjs_full, device) best_model = copy.deepcopy(susage) susage.zero_grad() cur_test_loss = susage.calculate_loss_grad( feat_data, adjs_full, labels, valid_nodes) best_val, cnt = 0, 0 loss_train = [cur_test_loss] loss_test = [cur_test_loss] grad_variance_all = [] loss_train_all = [cur_test_loss] times = [] data_prepare_times = [] for epoch in np.arange(args.epoch_num): train_nodes_p = args.batch_size * \ np.ones_like(train_nodes)/len(train_nodes) # prepare train data tp0 = time.time() pool = mp.Pool(args.pool_num) jobs = prepare_data(pool, graphsaint_sampler_.mini_batch, process_ids, train_nodes, train_nodes_p, samp_num_list, len(feat_data), adj_matrix, args.n_layers) # fetch train data train_data = [job.get() for job in jobs] pool.close() pool.join() tp1 = time.time() data_prepare_times += [tp1-tp0] inner_loop_num = args.batch_num t2 = time.time() cur_train_loss, cur_train_loss_all, grad_variance = boost_step(susage, optimizer, feat_data, labels, train_nodes, valid_nodes, adjs_full, train_data, inner_loop_num, device, calculate_grad_vars=bool(args.show_grad_norm) if epoch<int(200/args.batch_num) else False) t3 = time.time() times += [t3-t2] print('mvs_gcn_plus run time per epoch is %0.3f' % (t3-t2)) loss_train_all.extend(cur_train_loss_all) grad_variance_all.extend(grad_variance) # calculate test loss susage.eval() susage.zero_grad() cur_test_loss = susage.calculate_loss_grad( feat_data, adjs_full, labels, valid_nodes) val_f1 = susage.calculate_f1(feat_data, adjs_full, labels, valid_nodes) if val_f1 > best_val: best_model = copy.deepcopy(susage) if val_f1 > best_val + 1e-2: best_val = val_f1 cnt = 0 else: cnt += 1 if cnt == args.n_stops // args.batch_num: break loss_train.append(cur_train_loss) loss_test.append(cur_test_loss) # print progress print('Epoch: ', epoch, '| train loss: %.8f' % cur_train_loss, '| val loss: %.8f' % cur_test_loss, '| val f1: %.8f' % val_f1) if bool(args.show_grad_norm): times, data_prepare_times = times[int(200/args.batch_num):], data_prepare_times[int(200/args.batch_num):] print('Average training time is %0.3f' % np.mean(times)) print('Average data prepare time is %0.3f'%np.mean(data_prepare_times)) f1_score_test = best_model.calculate_f1( feat_data, adjs_full, labels, test_nodes) return best_model, loss_train, loss_test, loss_train_all, f1_score_test, grad_variance_all
def __init__(self, nfeat, nhid, nclass, dropout): super(GCN_Net, self).__init__() self.gc1 = GCN(nfeat, nhid) self.gc2 = GCN(nhid, nclass) self.dropout = dropout
# X_train = torch.LongTensor(X_train.values) # X_test = (torch.LongTensor(X_test.values)) # Y_train = Variable(torch.LongTensor(Y_train.values), requires_grad=False) # Y_test = torch.LongTensor(Y_test.values) # print Y_train ## Neural Model # Loss and Optimizer nComm = 2 nHid1 = 50 nHid2 = 20 nHid3 = 20 nHid4 = 10 nFeat = X_Tags_Feature.shape[1] #len(Tags_Features) gcn_model = GCN(nFeat, nHid1, nComm) #gcn_model = GCN(nFeat, nHid1, nHid2, nHid3, nHid4, nComm) #gcn_model.load_state_dict(torch.load('gcn_complete2.pkl')) criterion = nn.CrossEntropyLoss() criterion2 = nn.MSELoss() gcn_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,gcn_model.parameters()), lr=args.learning_rate, weight_decay=args.weight_decay) epoch_loss = [] epoch_accuracy = [] train_accuracy = [] epochs = [] correct = 0 try: for epoch in range(1, args.num_epochs + 1):
def main(args, print_fn=print): print_fn("Experiment arguments: {}".format(args)) if args.random_seed: torch.manual_seed(args.random_seed) else: torch.manual_seed(123) # Load dataset if args.dataset.startswith('ogbl'): graph, split_edge = load_ogb_dataset(args.dataset) else: raise NotImplementedError num_nodes = graph.num_nodes() # set gpu if args.gpu_id >= 0 and torch.cuda.is_available(): device = 'cuda:{}'.format(args.gpu_id) else: device = 'cpu' if args.dataset == 'ogbl-collab': # ogbl-collab dataset is multi-edge graph use_coalesce = True else: use_coalesce = False # Generate positive and negative edges and corresponding labels # Sampling subgraphs and generate node labeling features seal_data = SEALData(g=graph, split_edge=split_edge, hop=args.hop, neg_samples=args.neg_samples, subsample_ratio=args.subsample_ratio, use_coalesce=use_coalesce, prefix=args.dataset, save_dir=args.save_dir, num_workers=args.num_workers, print_fn=print_fn) node_attribute = seal_data.ndata['feat'] edge_weight = seal_data.edata['weight'].float() train_data = seal_data('train') val_data = seal_data('valid') test_data = seal_data('test') train_graphs = len(train_data.graph_list) # Set data loader train_loader = GraphDataLoader(train_data, batch_size=args.batch_size, num_workers=args.num_workers) val_loader = GraphDataLoader(val_data, batch_size=args.batch_size, num_workers=args.num_workers) test_loader = GraphDataLoader(test_data, batch_size=args.batch_size, num_workers=args.num_workers) # set model if args.model == 'gcn': model = GCN(num_layers=args.num_layers, hidden_units=args.hidden_units, gcn_type=args.gcn_type, pooling_type=args.pooling, node_attributes=node_attribute, edge_weights=edge_weight, node_embedding=None, use_embedding=True, num_nodes=num_nodes, dropout=args.dropout) elif args.model == 'dgcnn': model = DGCNN(num_layers=args.num_layers, hidden_units=args.hidden_units, k=args.sort_k, gcn_type=args.gcn_type, node_attributes=node_attribute, edge_weights=edge_weight, node_embedding=None, use_embedding=True, num_nodes=num_nodes, dropout=args.dropout) else: raise ValueError('Model error') model = model.to(device) parameters = model.parameters() optimizer = torch.optim.Adam(parameters, lr=args.lr) loss_fn = BCEWithLogitsLoss() print_fn("Total parameters: {}".format(sum([p.numel() for p in model.parameters()]))) # train and evaluate loop summary_val = [] summary_test = [] for epoch in range(args.epochs): start_time = time.time() loss = train(model=model, dataloader=train_loader, loss_fn=loss_fn, optimizer=optimizer, device=device, num_graphs=args.batch_size, total_graphs=train_graphs) train_time = time.time() if epoch % args.eval_steps == 0: val_pos_pred, val_neg_pred = evaluate(model=model, dataloader=val_loader, device=device) test_pos_pred, test_neg_pred = evaluate(model=model, dataloader=test_loader, device=device) val_metric = evaluate_hits(args.dataset, val_pos_pred, val_neg_pred, args.hits_k) test_metric = evaluate_hits(args.dataset, test_pos_pred, test_neg_pred, args.hits_k) evaluate_time = time.time() print_fn("Epoch-{}, train loss: {:.4f}, hits@{}: val-{:.4f}, test-{:.4f}, " "cost time: train-{:.1f}s, total-{:.1f}s".format(epoch, loss, args.hits_k, val_metric, test_metric, train_time - start_time, evaluate_time - start_time)) summary_val.append(val_metric) summary_test.append(test_metric) summary_test = np.array(summary_test) print_fn("Experiment Results:") print_fn("Best hits@{}: {:.4f}, epoch: {}".format(args.hits_k, np.max(summary_test), np.argmax(summary_test)))
help='Dropout rate (1 - keep probability).') args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() # print(args.no_cuda, args.epochs) np.random.RandomState(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) # Load data adj, features, labels, idx_train, idx_val, idx_test = load_dataset() # Model and optimizer model = GCN(n_feat=features.shape[1], n_hid=args.hidden, n_class=labels.max().item() + 1, dropout=args.dropout) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.cuda: model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda()
def main(): p = Preprocess(config, 'Cora') model = GCN(config, p.num_node_features, p.num_classes) train(model, p.data, config) test(model, p.data, config)
print() time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) print('Best F1 score: {:.4f}'.format(best_f1)) # load best model weights model.load_state_dict(best_model_wts) return model if __name__ == '__main__': # dataset dataloaders = make_dataloaders(batch_size=batch_size) #model model = GCN(4, 512) if use_gpu: model = model.cuda() #model = torch.nn.DataParallel(model).cuda() model.load_state_dict(torch.load(os.path.join(save_dir, 'gcn_v5.pth'))) #training optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = ReduceLROnPlateau(optimizer, 'min', verbose=True) model = train_model(model, num_epochs, dataloaders, optimizer, scheduler) #save save_model(model, save_dir, model_name)
def main(opt): train_dataset = BADataset(opt.dataroot, opt.L, True, False, False) train_dataloader = BADataloader(train_dataset, batch_size=opt.batchSize, \ shuffle=True, num_workers=opt.workers, drop_last=True) valid_dataset = BADataset(opt.dataroot, opt.L, False, True, False) valid_dataloader = BADataloader(valid_dataset, batch_size=opt.batchSize, \ shuffle=True, num_workers=opt.workers, drop_last=True) test_dataset = BADataset(opt.dataroot, opt.L, False, False, True) test_dataloader = BADataloader(test_dataset, batch_size=opt.batchSize, \ shuffle=True, num_workers=opt.workers, drop_last=True) all_dataset = BADataset(opt.dataroot, opt.L, False, False, False) all_dataloader = BADataloader(all_dataset, batch_size=opt.batchSize, \ shuffle=False, num_workers=opt.workers, drop_last=False) opt.n_edge_types = train_dataset.n_edge_types opt.n_node = train_dataset.n_node net = GCN(opt, kernel_size=2, n_blocks=1, state_dim_bottleneck=opt.state_dim, annotation_dim_bottleneck=opt.annotation_dim) net.double() print(net) criterion = nn.BCELoss() if opt.cuda: net.cuda() criterion.cuda() optimizer = optim.Adam(net.parameters(), lr=opt.lr) early_stopping = EarlyStopping(patience=opt.patience, verbose=True) os.makedirs(OutputDir, exist_ok=True) train_loss_ls = [] valid_loss_ls = [] test_loss_ls = [] for epoch in range(0, opt.niter): train_loss = train(epoch, train_dataloader, net, criterion, optimizer, opt) valid_loss = valid(valid_dataloader, net, criterion, opt) test_loss = test(test_dataloader, net, criterion, opt) train_loss_ls.append(train_loss) valid_loss_ls.append(valid_loss) test_loss_ls.append(test_loss) early_stopping(valid_loss, net, OutputDir) if early_stopping.early_stop: print("Early stopping") break df = pd.DataFrame({ 'epoch': [i for i in range(1, len(train_loss_ls) + 1)], 'train_loss': train_loss_ls, 'valid_loss': valid_loss_ls, 'test_loss': test_loss_ls }) df.to_csv(OutputDir + '/loss.csv', index=False) net.load_state_dict(torch.load(OutputDir + '/checkpoint.pt')) inference(all_dataloader, net, criterion, opt, OutputDir)
def train_and_val_1Fold(batch_size, num_epochs, num_layers, weight_decay, num_input_features, hidden, device, lr, step_size, lr_decay, m, folder, fold, augment=False, opt=False, testing=False, it=None): """ the data of the pt1 dataset is split into train val and test in 4 different ways this function trains and validates using the train and val split of one of these 4 possible splits :param batch_size: (int) :param num_epochs: (int) number of epochs :param num_layers: (int) number of graph convolutional layers :param num_input_features: (int) number of node features :param hidden: (int) number of hidden representations per node :param device: (str) either "cpu" or "cuda" :param lr: learning rate :param step_size: indicates after how many epochs the learning rate is decreased by a factor of lr_decay :param lr_decay: factor by which the learning rate is decreased :param m: str, the model that should be trained :param folder: which dataset to use (33 or 4 node features) :param augment: boolean, determines whether the dataset should be augmented or not :param fold: int, determines which of the 4 possible splits is considered :param opt: (bool), determine whether the function is called during the hyperparameter optimization or not :return: """ if folder == "pT1_dataset/graphs/paper-graphs/distance-based_10_13_14_35/": folder_short = "paper/" if folder == "pT1_dataset/graphs/base-dataset/": folder_short = "base/" all_lists = load_obj(folder, augment=0, sd=0) all_train_lists = all_lists[0] all_val_lists = all_lists[1] all_test_lists = all_lists[2] all_train_aug_lists = all_lists[ 3] # contains train and augmented train graphs val_res = [] # will contain the best validation accuracy train_accs = [] # will contain the training accuracy of every epoch val_accs = [] # will contain the validation accuracy of every epoch losses = [] # will contain the training loss of every epoch val_losses = [] # will contain the validation loss of every epoch # get the training and validation data lists # augment data by adding/subtracting small random values from node features if augment: num_train = len(all_train_lists[fold]) # num_train_aug = len(all_train_aug_lists[k]) indices = list(range(0, num_train)) # get all original graphs # randomly select augmented graphs n_aug = 5 # n_aug determines by which factor the dataset should be augmented choice = random.sample(range(1, n_aug), n_aug - 1) for j in choice: indices.extend( random.sample(range(num_train * j, num_train * (j + 1)), num_train)) # create the train_data_list and val_data_list used for the DataLoader train_data_list = [ all_train_aug_lists[fold][i] for i in indices ] # contains all original graphs plus num_aug augmented graphs val_data_list = all_val_lists[fold] print("augm. train size: " + str(len(train_data_list)) + " val size: " + str(len(val_data_list))) else: # create the train_data_list and val_data_list used for the DataLoader train_data_list = all_train_lists[fold] val_data_list = all_val_lists[fold] test_data_list = all_test_lists[fold] # print("train size: " + str(len(train_data_list)) + " val size: " + str(len(val_data_list))) # if testing: # for entry in val_data_list: # train_data_list.append(entry) # add val to train data # val_data_list = test_data_list # use test_data_list for measuring the performance print("train size: " + str(len(train_data_list)) + " val size: " + str(len(val_data_list))) # initialize train loader train_loader = DataLoader(train_data_list, batch_size=batch_size, shuffle=True, drop_last=True) # initialize val loader val_loader = DataLoader(val_data_list, batch_size=batch_size, shuffle=True) # initialize the model if m == "GCN": model = GCN(num_layers=num_layers, num_input_features=num_input_features, hidden=hidden).to(device) # initialize the model elif m == "GCNWithJK": model = GCNWithJK(num_layers=num_layers, num_input_features=num_input_features, hidden=hidden, mode="cat").to(device) # initialize the model elif m == "GraphSAGE": model = GraphSAGE(num_layers=num_layers, num_input_features=num_input_features, hidden=hidden).to(device) elif m == "GraphSAGEWithJK": model = GraphSAGEWithJK(num_layers=num_layers, num_input_features=num_input_features, hidden=hidden, mode="cat").to(device) elif m == "GIN": model = GIN(num_layers=num_layers, num_input_features=num_input_features, hidden=hidden).to(device) elif m == "GATNet": model = GATNet(num_layers=num_layers, num_input_features=num_input_features, hidden=hidden).to(device) elif m == "GraphNN": model = GraphNN(num_layers=num_layers, num_input_features=num_input_features, hidden=hidden).to(device) elif m == "NMP": model = NMP(num_layers=num_layers, num_input_features=num_input_features, hidden=hidden, nn=MLP).to(device) optimizer = torch.optim.Adam( model.parameters(), lr=lr, weight_decay=weight_decay ) # define the optimizer, weight_decay corresponds to L2 regularization scheduler = StepLR(optimizer, step_size=step_size, gamma=lr_decay) # learning rate decay crit = torch.nn.NLLLoss() # compute training and validation accuracy for every epoch for epoch in range(num_epochs): if epoch == 0: train_acc, loss, _, _ = evaluate( model, train_loader, crit, device) # compute the accuracy for the training data train_accs.append(train_acc) losses.append(loss) val_acc, val_loss, img_name, TP_TN_FP_FN = evaluate( model, val_loader, crit, device) # compute the accuracy for the test data running_val_acc = np.asarray([0, 0, val_acc]) val_accs.append(val_acc) val_losses.append(val_loss) TP_TN_FP_FN_res = np.copy(TP_TN_FP_FN) val_res = np.copy(running_val_acc) img_name_res = img_name if testing: torch.save( model, "Parameters/" + folder_short + m + "_fold" + str(fold) + "it" + str(it) + ".pt") # train the model train(model, train_loader, optimizer, crit, device) scheduler.step() # ge train acc and loss train_acc, loss, _, _ = evaluate( model, train_loader, crit, device) # compute the accuracy for the training data train_accs.append(train_acc) losses.append(loss) # get validation acc and loss val_acc, val_loss, img_name, TP_TN_FP_FN = evaluate( model, val_loader, crit, device) # compute the accuracy for the validation data running_val_acc[0] = running_val_acc[1] running_val_acc[1] = running_val_acc[2] running_val_acc[2] = val_acc if np.mean(running_val_acc) > np.mean( val_res ) and not testing: # if this is current best save the list of predictions and corresponding labels img_name_res = img_name TP_TN_FP_FN_res = np.copy(TP_TN_FP_FN) val_res = np.copy(running_val_acc) if running_val_acc[2] > val_res[ 2] and testing: # if this is current best save the list of predictions and corresponding labels img_name_res = img_name TP_TN_FP_FN_res = np.copy(TP_TN_FP_FN) val_res = np.copy(running_val_acc) torch.save( model, "Parameters/" + folder_short + m + "_fold" + str(fold) + "it" + str(it) + ".pt") val_accs.append(val_acc) val_losses.append(val_loss) if stdev(losses[-20:]) < 0.05 and mean(train_accs[-20:]) < 0.55: boolean = True else: boolean = False #################################################################### ################################################################### # plot the training and test accuracies #################################################################### if not opt: plt.rc("font", size=5) x = range(num_epochs + 1) ltype = ["--", "-"] plt.subplot(2, 1, 1) plt.plot(x, train_accs, color=(1, 0, 0), linestyle=ltype[0], label="train {}".format(fold)) plt.plot(x, val_accs, color=(0, 1, 0), linestyle=ltype[1], label="val {}".format(fold)) plt.ylim(0.5, 1) # plt.plot(x, np.mean(np.asarray(train_accs), axis=0), color=(0,0,0), label="train avg") # plt.plot(x, np.mean(np.asarray(val_accs), axis=0), color=(0,0,1), label="val avg") plt.vlines(val_accs.index(val_res[1]), 0.5, 1) plt.xticks(np.arange(min(x), max(x) + 1, 1.0)) plt.legend() if folder == "pT1_dataset/graphs/paper-graphs/distance-based_10_13_14_35/": title = "paper-graphs, " + m + " Validation accuracy: " + str( round(100 * val_res[1], 2)) + "%" + " Fold:" + str(fold) plt.title(title) if folder == "pT1_dataset/graphs/base-dataset/": title = "base-dataset, " + m + " Validation accuracy: " + str( round(100 * val_res[1], 2)) + "%" + " Fold:" + str(fold) plt.title(title) # plot_loss = plt.subplot(2, 1, 2) plot_loss.plot(x, losses, color=(1, 0, 0), linestyle=ltype[0], label="train {}".format(fold)) plot_loss.plot(x, val_losses, color=(0, 1, 0), linestyle=ltype[1], label="val {}".format(fold)) # plt.plot(x, np.mean(np.asarray(losses), axis=0), color=(0,0,0), label="train avg") # plt.plot(x, np.mean(np.asarray(val_losses), axis=0), color=(0,0,1), label="val avg") plot_loss.set_title("train and val loss") plot_loss.legend() plt.xticks(np.arange(min(x), max(x) + 1, 1.0)) plt.show() ####################################################################### # compute number of false positives, false negatives, true positives and true negatives ###################################################################### if not testing: print("true positives: ", TP_TN_FP_FN_res[0]) print("true negatives: ", TP_TN_FP_FN_res[1]) print("false positives: ", TP_TN_FP_FN_res[2]) print("false_negatives: ", TP_TN_FP_FN_res[3]) print("FP images:", img_name_res[2]) print("FN images:", img_name_res[3]) # print("best val accuracy:", val_res) return ( val_res, boolean, np.asarray(train_accs), np.asarray(val_accs), np.asarray(losses), np.asarray(val_losses), img_name_res ) # the boolean tells that train_and_val was completed (good param combination)
def sgcn_plus_v2(feat_data, labels, lap_matrix, train_nodes, valid_nodes, test_nodes, args, device, calculate_grad_vars=False): # use multiprocess sample data process_ids = np.arange(args.batch_num) pool = mp.Pool(args.pool_num) lap_matrix_sq = lap_matrix.multiply(lap_matrix) jobs = prepare_data(pool, sampler, process_ids, train_nodes, samp_num_list, len(feat_data), lap_matrix, lap_matrix_sq, args.n_layers) susage = GCN(nfeat=feat_data.shape[1], nhid=args.nhid, num_classes=num_classes, layers=args.n_layers, dropout=args.dropout).to(device) susage.to(device) print(susage) adjs_full, input_nodes_full, sampled_nodes_full = full_batch_sampler( train_nodes, len(feat_data), lap_matrix, args.n_layers) adjs_full = package_mxl(adjs_full, device) # this stupid wrapper is only used for sgcn++ forward_wrapper = ForwardWrapperMomentum(len(feat_data), args.nhid, args.n_layers, num_classes) optimizer = optim.SGD(filter(lambda p: p.requires_grad, susage.parameters()), lr=0.1) loss_train = [] loss_test = [] grad_variance_all = [] loss_train_all = [] best_model = copy.deepcopy(susage) best_val_loss = 10 # randomly pick a large number is fine best_val_index = 0 best_val_cnt = 0 for epoch in np.arange(args.epoch_num): # fetch train data train_data = [job.get() for job in jobs] pool.close() pool.join() # prepare next epoch train data pool = mp.Pool(args.pool_num) jobs = prepare_data(pool, sampler, process_ids, train_nodes, samp_num_list, len(feat_data), lap_matrix, lap_matrix_sq, args.n_layers) inner_loop_num = args.batch_num # compare with sgcn_plus, the only difference is we use multi_level_spider_step_v1 here cur_train_loss, cur_train_loss_all, grad_variance = multi_level_momentum_step( susage, optimizer, feat_data, labels, train_nodes, valid_nodes, adjs_full, sampled_nodes_full, train_data, inner_loop_num, forward_wrapper, device, calculate_grad_vars=calculate_grad_vars) loss_train_all.extend(cur_train_loss_all) grad_variance_all.extend(grad_variance) # calculate validate loss susage.eval() susage.zero_grad() val_loss, _ = susage.calculate_loss_grad(feat_data, adjs_full, labels, valid_nodes) if val_loss + 0.01 < best_val_loss: best_val_loss = val_loss del best_model best_model = copy.deepcopy(susage) best_val_index = epoch best_val_cnt = 0 cur_test_loss = val_loss best_val_cnt += 1 loss_train.append(cur_train_loss) loss_test.append(cur_test_loss) # print progress print('Epoch: ', epoch, '| train loss: %.8f' % cur_train_loss, '| test loss: %.8f' % cur_test_loss) if best_val_cnt > 10: break f1_score_test = best_model.calculate_f1(feat_data, adjs_full, labels, test_nodes) return best_model, loss_train[: best_val_index], loss_test[: best_val_index], loss_train_all, f1_score_test, grad_variance_all
tensor_y = torch.from_numpy(dataset.y).to(DEVICE) tensor_train_mask = torch.from_numpy(dataset.train_mask).to(DEVICE) tensor_val_mask = torch.from_numpy(dataset.val_mask).to(DEVICE) tensor_test_mask = torch.from_numpy(dataset.test_mask).to(DEVICE) normalize_adjacency = CoraData.normalization(dataset.adjacency) # 规范化邻接矩阵 num_nodes, input_dim = node_feature.shape indices = torch.from_numpy( np.asarray([normalize_adjacency.row, normalize_adjacency.col]).astype('int64')).long() values = torch.from_numpy(normalize_adjacency.data.astype(np.float32)) tensor_adjacency = torch.sparse.FloatTensor(indices, values, (num_nodes, num_nodes)).to(DEVICE) # 模型定义:Model, Loss, Optimizer model = GCN(input_dim).to(DEVICE) criterion = nn.CrossEntropyLoss().to(DEVICE) optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DACAY) def train(): model.train() train_loss_history = [] train_acc_history = [] val_acc_history = [] val_loss_history = [] train_y = tensor_y[tensor_train_mask] for epoch in range(EPOCHS): logits = model(tensor_adjacency, tensor_x) # 前向传播
# data loading method 2 adj, features, labels, y_train, y_val, y_test, train_mask, val_mask, test_mask = load_data( 'cora') support = [preprocess_adj(adj)] adj = tf.cast(tf.SparseTensor(*support[0]), tf.float32) features = preprocess_features(features).tocoo() nf_shape = features.data.shape features = tf.SparseTensor(indices=np.array(list( zip(features.row, features.col)), dtype=np.int64), values=tf.cast(features.data, tf.float32), dense_shape=features.shape) graph = [features, adj] model = GCN(16, labels.shape[-1], 0.5, nf_shape) criterion = tf.keras.losses.CategoricalCrossentropy(from_logits=True) def loss(model, x, y, train_mask, training=True): y_ = model(x[0], x[1], training=training) test_mask_logits = tf.gather_nd(y_, tf.where(train_mask)) masked_labels = tf.gather_nd(y, tf.where(train_mask)) return criterion(y_true=masked_labels, y_pred=test_mask_logits) # return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=test_mask_logits, labels=masked_labels)) # return masked_softmax_cross_entropy(y_, y, train_mask)
""" import torch import torch.nn.functional as F import networkx as nx import matplotlib.animation as animation import matplotlib.pyplot as plt from model import GCN from build_graph import build_karate_club_graph import warnings warnings.filterwarnings('ignore') net = GCN(34, 5, 2) print(net) G = build_karate_club_graph() inputs = torch.eye(34) labeled_nodes = torch.tensor( [0, 33]) # only the instructor and the president nodes are labeled labels = torch.tensor([0, 1]) optimizer = torch.optim.Adam(net.parameters(), lr=0.01) all_logits = [] for epoch in range(20): logits = net(G, inputs) all_logits.append(logits.detach()) logp = F.log_softmax(logits, 1)