idx_train = torch.LongTensor(idx_train) idx_val = torch.LongTensor(idx_val) idx_test = torch.LongTensor(idx_test) # Train model t_total = time.time() loss_values = [] bad_counter = 0 best = args.epochs + 1 best_epoch = 0 if __name__ == '__main__': adj, features, labels = load_prepared_data(dataset) model = GAT(nfeat=features.shape[1], nhid=args.hidden, nclass=int(labels.max()) + 1, dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.cuda: model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() for epoch in range(args.epochs): loss_values.append(train(epoch))
if __name__ == '__main__': adj, features, labels, idx_train, idx_val, idx_test = load_data_gat( path='data/', dataset_str='citeseer') features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() features, adj, labels = Variable(features), Variable(adj), Variable(labels) model = GAT(nfeat=features.shape[1], nhid=12, nclass=int(labels.max()) + 1, dropout=0.6, nheads=8, alpha=0.2) model.load_state_dict(torch.load('693.pkl')) compute_test() # x = torch.cat([att(features, adj) for att in model.attentions], dim=1) # embeddings = x.cpu().data.numpy() # time_start = time.time() # # pca_50 = PCA(n_components=50) # pca_result_50 = pca_50.fit_transform(embeddings) # # tsne = TSNE(n_components=2, verbose=1, perplexity=50, n_iter=1000) # tsne_results = tsne.fit_transform(pca_result_50)
def __call__(self, trial): print('TRIAL N0 - %d'%trial.number) ################################################################################################### # Hyperparameters ################################################################################################### self.configs = {} self.configs['seed'] = self.seed ################################################################ # Initialize Hyperparameters for search # Comment This block when specifying hyperparameters self.configs['learning_rate'] = trial.suggest_uniform('learning_rate',0,1) self.configs['weight_decay'] = trial.suggest_uniform('weight_decay',0,1) self.configs['model_dropout'] = trial.suggest_uniform('model_dropout',0,1) self.configs['attention_dropout'] = trial.suggest_uniform('attention_dropout',0,1) ################################################################ ''' ################################################################ # Specifying hyperparamters # Comment this block when searching for hyperparameters # Set n_trials to 1 self.configs['learning_rate'] = 0.005 self.configs['weight_decay'] = 5e-4 self.configs['model_dropout'] = 0.6 self.configs['attention_dropout'] = 0.6 ################################################################ ''' self.configs['hidden_dims'] = 8 self.configs['num_heads'] = 8 self.configs['num_heads_output'] = 1 ################################################################################################### graph = tf.Graph() with tf.Session(graph=graph) as sess: ################################################################################################### # Set random seed ################################################################################################### tf.set_random_seed(self.seed) random.seed(self.seed) np.random.seed(self.seed) ################################################################################################### # Define placeholders self.placeholders = { 'adj': tf.sparse_placeholder(tf.float32, name='adj'), 'features': tf.sparse_placeholder(tf.float32, shape=tf.constant(self.features[2], dtype=tf.int64), name='features'), 'labels': tf.placeholder(tf.float32, shape=(None, self.y_train.shape[1]), name='labels'), 'labels_mask': tf.placeholder(tf.int32,name='labels_mask'), 'model_dropout': tf.placeholder_with_default(0., shape=(), name='model_dropout'), 'attention_dropout': tf.placeholder_with_default(0., shape=(), name='attention_dropout') } # Create model self.model = GAT(self.configs, self.placeholders, input_dim=self.features[2][1], logging=True) # Init variables sess.run(tf.global_variables_initializer()) patience_count = 0 best_val_accuracy = 0.0 # Train model for epoch in range(self.epochs): # Construct feed dictionary feed_dict = construct_feed_dict(self.features, self.adj_t, self.y_train, self.train_mask, self.placeholders) feed_dict.update({self.placeholders['model_dropout']: self.configs['model_dropout']}) feed_dict.update({self.placeholders['attention_dropout']: self.configs['attention_dropout']}) # Training step outs = sess.run([self.model.opt_op, self.model.loss, self.model.accuracy], feed_dict=feed_dict) # Evaluate train_loss, train_acc, _ = evaluate(self.model, sess, self.features, self.adj_t, self.y_train, self.train_mask, self.placeholders) val_loss, val_acc, _ = evaluate(self.model, sess, self.features, self.adj_t, self.y_val, self.val_mask, self.placeholders) test_loss, test_acc, _ = evaluate(self.model, sess, self.features, self.adj_t, self.y_test, self.test_mask, self.placeholders) unlabeled_loss, unlabeled_acc, _ = evaluate(self.model, sess, self.features, self.adj_t, self.y_unlabeled, self.unlabeled_mask, self.placeholders) if self.verbose: # Print results print('[Epoch %03d] Loss - %0.04f\t Train Accuracy - %0.04f\t Val Accuracy - %0.04f'%(epoch+1, outs[1], train_acc, val_acc)) if val_acc > best_val_accuracy: best_val_accuracy = val_acc patience_count = 0 trial.set_user_attr('Train Accuracy', train_acc) trial.set_user_attr('Train Loss', train_loss) trial.set_user_attr('Val Accuracy', val_acc) trial.set_user_attr('Val Loss', val_loss) trial.set_user_attr('Test Accuracy', test_acc) trial.set_user_attr('Test Loss', test_loss) trial.set_user_attr('Unlabeled Accuracy', unlabeled_acc) trial.set_user_attr('Unlabeled Loss', unlabeled_loss) ''' ############################################################################################################# # Fetch Attention Weights to analyze ############################################################################################################# feed_dict.update({self.placeholders['model_dropout']: 0.}) feed_dict.update({self.placeholders['attention_dropout']: 0.}) layer_1_attention_matrices = [sess.run(self.model.layers[0].attention_matrices[i], feed_dict=feed_dict) for i in range(self.configs['num_heads'])] layer_1_attention_matrices = [sparse_tensor_to_coo(sp_mat) for sp_mat in layer_1_attention_matrices] trial.set_user_attr('layer_1_attention_matrices', layer_1_attention_matrices) layer_2_attention_matrices = [sess.run(self.model.layers[1].attention_matrices[i], feed_dict=feed_dict) for i in range(self.configs['num_heads_output'])] layer_2_attention_matrices = [sparse_tensor_to_coo(sp_mat) for sp_mat in layer_2_attention_matrices] trial.set_user_attr('layer_2_attention_matrices', layer_2_attention_matrices) ############################################################################################################# ''' else: patience_count = patience_count + 1 if patience_count >= self.early_stopping: #Early stopping break print() print('Train Accuracy - %0.04f'%(trial.user_attrs['Train Accuracy'])) print('Val Accuracy - %0.04f'%(trial.user_attrs['Val Accuracy'])) print('Test Accuracy - %0.04f'%(trial.user_attrs['Test Accuracy'])) print('Unlabeled Accuracy - %0.04f'%(trial.user_attrs['Unlabeled Accuracy'])) print() return trial.user_attrs['Val Accuracy']
verbose=args.verbose).to(args.device) model_args = { "num_features": data.num_features, "num_classes": data.num_classes, "hidden_size": args.hidden, "dropout": args.dropout, "activation": args.activation } if args.model == "gcn": model = GCN(**model_args) elif args.model == "gat": model_args["num_heads"] = args.num_heads model_args["hidden_size"] = int(args.hidden / args.num_heads) model = GAT(**model_args) elif args.model == "mlp": model = MLP(**model_args) else: gen_type, post_type = args.model.split("_") gen_config = copy.deepcopy(model_args) gen_config["type"] = gen_type gen_config["neg_ratio"] = args.neg_ratio if gen_type == "lsm": gen_config["hidden_x"] = args.hidden_x if gen_type == "sbm": gen_config["p0"] = args.p0 gen_config["p1"] = args.p1 post_config = copy.deepcopy(model_args)
parser.add_argument('--patience', type=int, default=100, help='Patience') args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) # Load data adj, features, labels, idx_train, idx_val, idx_test = load_data() # Model and optimizer model = GAT(nfeat=features.shape[1], nhid=args.hidden, nclass=int(labels.max()) + 1, dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.cuda: model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() features, adj, labels = Variable(features), Variable(adj), Variable(labels) def train(epoch):
def main(dataset_name, yang_splits, nheads, hidden_units, feat_drop_rate, coefs_drop_rate, epochs, learning_rate, l2_weight, patience, data_seed, net_seed, checkpoint_path, verbose): # reproducibility np.random.seed(data_seed) tf.random.set_seed(net_seed) if yang_splits: features, o_h_labels, graph, mask_train, mask_val, mask_test = read_dataset(dataset_name, yang_splits=True) labels = np.array([np.argmax(l) for l in o_h_labels], dtype=np.int32) else: if verbose > 0: print("reading dataset") features, neighbors, labels, o_h_labels, keys = read_dataset(dataset_name) num_classes = len(set(labels)) if verbose > 0: print("shuffling dataset") features, neighbors, labels, o_h_labels, keys = permute(features, neighbors, labels, o_h_labels, keys) if verbose > 0: print("obtaining masks") mask_train, mask_val, mask_test = split(dataset_name, labels) if verbose > 0: print("calculating adjacency matrix") graph = adjacency_matrix(neighbors) # add self loops to adj matrix graph = graph + sp.eye(graph.shape[0]) num_classes = get_num_classes(dataset_name) features = normalize_features(features) y_train = np.multiply(o_h_labels, np.broadcast_to(mask_train.T, o_h_labels.T.shape).T ) y_val = np.multiply(o_h_labels, np.broadcast_to(mask_val.T, o_h_labels.T.shape).T ) y_test = np.multiply(o_h_labels, np.broadcast_to(mask_test.T, o_h_labels.T.shape).T ) optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate) if verbose > 0: print("defining model") model = GAT(graph, num_classes, hidden_units, nheads, feat_drop_rate, coefs_drop_rate) model.compile(loss=lambda y_true, y_pred: masked_loss(y_true, y_pred)+l2_weight*tf.reduce_sum([tf.nn.l2_loss(w) for w in model.weights if not 'bias' in w.name]), optimizer=optimizer, metrics=[masked_accuracy]) if verbose > 0: print("begin training") tb = TensorBoard(log_dir='logs') if dataset_name == 'cora': monitor = 'acc_loss' elif dataset_name == 'pubmed': monitor = 'loss' else: monitor = 'acc' es = EarlyStoppingAccLoss(patience, monitor, checkpoint_path) model.fit(features, y_train, epochs=epochs, batch_size=len(features), shuffle=False, validation_data=(features, y_val), callbacks=[tb, es], verbose=verbose) file_writer = tf.summary.create_file_writer("./logs/results/") file_writer.set_as_default() # log best performances on train and val set loss, accuracy = model.evaluate(features, y_train, batch_size=len(features), verbose=0) print("accuracy on training: " + str(accuracy)) tf.summary.scalar('bw_loss', data=loss, step=1) tf.summary.scalar('bw_accuracy', data=accuracy, step=1) v_loss, v_accuracy = model.evaluate(features, y_val, batch_size=len(features), verbose=0) print("accuracy on validation: " + str(v_accuracy)) tf.summary.scalar('bw_val_loss', data=v_loss, step=1) tf.summary.scalar('bw_val_accuracy', data=v_accuracy, step=1) tf.summary.scalar('bw_epoch', data=es.stopped_epoch, step=1) if verbose > 0: print("test the model on test set") t_loss, t_accuracy = model.evaluate(features, y_test, batch_size=len(features), verbose=0) print("accuracy on test: " + str(t_accuracy))
def __init__(self,graph,sparse = False,epochs = 200,learning_rate = 0.005, weight_decay = 5e-4,hidden = 8,nb_heads = 8,drop_out = 0.6, alpha = 0.2 ,patience = 100,train = 1500,val = 2000,test = 3100): self.graph = graph self.sparse = sparse self.epochs = epochs self.learning_rate = learning_rate self.weight_decay = weight_decay self.hidden = hidden self.nb_heads = nb_heads self.drop_out = drop_out self.alpha = alpha self.patience = patience self.train = train self.val = val self.test = test idx_train,idx_val , idx_test = self.load_data() random.seed(random_seed) np.random.seed(random_seed) torch.manual_seed(random_seed) if self.sparse: model = SpGAT(nfeat=self.features.shape[1], nhid=self.hidden, nclass=int(self.labels.max()) + 1, dropout=self.drop_out, nheads=self.nb_heads, alpha=self.alpha) else: model = GAT(nfeat=self.features.shape[1], nhid=self.hidden, nclass=int(self.labels.max()) + 1, dropout=self.drop_out, nheads=self.nb_heads, alpha=self.alpha) optimizer = optim.Adam(model.parameters(), lr=self.learning_rate, weight_decay=self.weight_decay) #利用GPU # device = torch.device("cuda:0") # torch.cuda.empty_cache() # model.to(device) # self.features = self.features.to(device) # self.adj = self.adj.to(device) # self.labels = self.labels.to(device) # idx_train = idx_train.to(device) # idx_val = idx_val.to(device) # idx_test = idx_test.to(device) features, adj, labels = Variable(self.features), Variable(self.adj), Variable(self.labels) t_total = time.time() loss_values = [] bad_counter = 0 best = self.epochs + 1 best_epoch = 0 for epoch in range(self.epochs): t = time.time() model.train() optimizer.zero_grad() output = model(features, adj) loss_train = F.nll_loss(output[idx_train], labels[idx_train]) acc_train = accuracy(output[idx_train], labels[idx_train]) loss_train.backward() optimizer.step() model.eval() output = model(features, adj) loss_val = F.nll_loss(output[idx_val], labels[idx_val]) acc_val = accuracy(output[idx_val], labels[idx_val]) print('Epoch: {:04d}'.format(epoch + 1), 'loss_train: {:.4f}'.format(loss_train.data), 'acc_train: {:.4f}'.format(acc_train.data), 'loss_val: {:.4f}'.format(loss_val.data), 'acc_val: {:.4f}'.format(acc_val.data), 'time: {:.4f}s'.format(time.time() - t)) loss_values.append(loss_val.data) torch.save(model.state_dict(), '{}.pkl'.format(epoch)) if loss_values[-1] < best: best = loss_values[-1] best_epoch = epoch bad_counter = 0 else: bad_counter += 1 if bad_counter == self.patience: break files = glob.glob('*.pkl') for file in files: epoch_nb = int(file.split('.')[0]) if epoch_nb < best_epoch: os.remove(file) print("Optimization Finished!") print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) print('Loading {}th epoch'.format(best_epoch)) model.load_state_dict(torch.load('{}.pkl'.format(best_epoch))) model.eval() output = model(features, adj) loss_test = F.nll_loss(output[idx_test], labels[idx_test]) acc_test = accuracy(output[idx_test], labels[idx_test]) print("Test set results:", "loss= {:.4f}".format(loss_test.data), "accuracy= {:.4f}".format(acc_test.data))
np.random.seed(42) torch.manual_seed(42) if use_gpu: torch.cuda.manual_seed(42) model, optimizer = None, None best_acc = 0 early_stop = 0 # Define the model and optimizer if (opt.model == 'attention'): print("| Constructing Graph Attention Network model...") model = GAT(nfeat=features.shape[1], nhid=opt.num_hidden, nclass=int(labels.max().item()) + 1, dropout=opt.dropout, nheads=opt.nb_heads, nouts=opt.nb_outs, alpha=opt.alpha) else: raise NotImplementedError if (opt.optimizer == 'sgd'): optimizer = optim.SGD(model.parameters(), lr=opt.lr, weight_decay=opt.weight_decay, momentum=0.9) elif (opt.optimizer == 'adam'): optimizer = optim.Adam(model.parameters(), lr=opt.lr, weight_decay=opt.weight_decay)
def main(dataset_name, yang_splits, nheads, hidden_units, feat_drop_rate, coefs_drop_rate, l2_weight, data_seed, checkpoint_path, verbose, tsne): # reproducibility np.random.seed(data_seed) if yang_splits: features, o_h_labels, graph, mask_train, mask_val, mask_test = read_dataset( dataset_name, yang_splits=True) labels = np.array([np.argmax(l) for l in o_h_labels], dtype=np.int32) else: if verbose > 0: print("reading dataset") features, neighbors, labels, o_h_labels, keys = read_dataset( dataset_name) num_classes = len(set(labels)) if verbose > 0: print("shuffling dataset") features, neighbors, labels, o_h_labels, keys = permute( features, neighbors, labels, o_h_labels, keys) if verbose > 0: print("obtaining masks") mask_train, mask_val, mask_test = split(dataset_name, labels) if verbose > 0: print("calculating adjacency matrix") graph = adjacency_matrix(neighbors) # add self loops to adj matrix graph = graph + sp.eye(graph.shape[0]) num_classes = get_num_classes(dataset_name) features = normalize_features(features) y_test = np.multiply(o_h_labels, np.broadcast_to(mask_test.T, o_h_labels.T.shape).T) if verbose > 0: print("defining model") model = GAT(graph, num_classes, hidden_units, nheads, feat_drop_rate, coefs_drop_rate) model.compile(loss=lambda y_true, y_pred: masked_loss( y_true, y_pred) + l2_weight * tf.reduce_sum( [tf.nn.l2_loss(w) for w in model.weights if not 'bias' in w.name]), metrics=[masked_accuracy]) if verbose > 0: print("load model from checkpoint") wpath = os.path.join(checkpoint_path, 'cp.ckpt') model.load_weights(wpath).expect_partial() if verbose > 0: print("test the model on test set") t_loss, t_accuracy = model.evaluate(features, y_test, batch_size=len(features), verbose=0) print("accuracy on test: " + str(t_accuracy)) if tsne: if verbose > 0: print("calculating t-SNE plot") intermediate_output = model.call(features, training=False, intermediate=True) plot_tsne(intermediate_output[mask_test], labels[mask_test], len(o_h_labels[0]), 'GAT')
def build_model(args, in_feats, n_hidden, n_classes, device, n_layers=1, backend='geometric', edge_index=None, num_nodes=None): if args.model == 'graphsaint': assert backend == 'geometric' model_spec = args.variant else: model_spec = args.model if backend == 'geometric': print("Using Geometric Backend") if model_spec == 'gs-mean': model = geo.GraphSAGE(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif model_spec == "gcn": model = geo.GCN(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif model_spec == "gat": print("Warning, GAT doesn't respect n_layers") heads = [8, args.gat_out_heads] # Fixed head config n_hidden_per_head = int(n_hidden / heads[0]) model = geo.GAT(in_feats, n_hidden_per_head, n_classes, F.relu, args.dropout, 0.6, heads).to(device) elif model_spec == "mlp": model = geo.MLP(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif model_spec == 'jknet-sageconv': # Geometric JKNEt with SAGECOnv model = JKNet(tg.nn.SAGEConv, in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout, mode="cat", conv_kwargs={ "normalize": False }, backend="geometric").to(device) elif model_spec == 'jknet-graphconv': model = JKNet(tg.nn.GraphConv, in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout, mode="cat", conv_kwargs={ "aggr": "mean" }, backend="geometric").to(device) elif model_spec == "sgnet": model = geo.SGNet(in_channels=in_feats, out_channels=n_classes, K=n_layers, cached=True).to(device) else: raise NotImplementedError( f"Unknown model spec 'f{model_spec} for backend {backend}") elif backend == 'dgl': # DGL models if model_spec == 'gs-mean': model = GraphSAGE(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout, 'mean').to(device) elif model_spec == 'mlp': model = MLP(in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout).to(device) elif model_spec == 'mostfrequent': model = MostFrequentClass() elif model_spec == 'gat': print("Warning, GAT doesn't respect n_layers") heads = [8, args.gat_out_heads] # Fixed head config # Div num_hidden by heads for same capacity n_hidden_per_head = int(n_hidden / heads[0]) assert n_hidden_per_head * heads[ 0] == n_hidden, f"{n_hidden} not divisible by {heads[0]}" model = GAT(1, in_feats, n_hidden_per_head, n_classes, heads, F.elu, 0.6, 0.6, 0.2, False).to(device) elif model_spec == 'node2vec': assert edge_index is not None model = tg.nn.Node2Vec( edge_index, n_hidden, args.n2v_walk_length, args.n2v_context_size, walks_per_node=args.n2v_walks_per_node, p=args.n2v_p, q=args.n2v_q, num_negative_samples=args.n2v_num_negative_samples, num_nodes=num_nodes, sparse=True) elif model_spec == 'jknet-sageconv': # DGL JKNet model = JKNet(dgl.nn.pytorch.SAGEConv, in_feats, n_hidden, n_classes, n_layers, F.relu, args.dropout, mode="cat", conv_args=["mean"], backend='dgl').to(device) elif model_spec == 'sgnet': model = SGNet(in_feats, n_classes, k=n_layers, cached=True, bias=True, norm=None).to(device) else: raise NotImplementedError( f"Unknown model spec 'f{model_spec} for backend {backend}") else: raise NotImplementedError(f"Unknown backend: {backend}") return model
if args.sparse: model = SpGAT(nfeat=feat_dim, # nfeat=features.shape[1], nhid=args.hidden, # nclass=int(labels.max()) + 1, <<<<<<< HEAD nclass=label_dim, ======= nclass=151, >>>>>>> origin/master dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha) else: model = GAT(nfeat=feat_dim, # nfeat=features.shape[1], nhid=args.hidden, # nclass=int(labels.max()) + 1, <<<<<<< HEAD nclass=label_dim, ======= nclass=151, >>>>>>> origin/master dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.cuda:
torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') print('loading data') x, y, adj = load_data() y_train = y[:60000] x_train = x[:60000] y_test = y[60000:] x_test = x[60000:] adj = torch.Tensor(adj) print('loading on cuda') model = GAT(nfeat=1, nhid=5, nclass=1, dropout=0.8, nheads=2, alpha=0.2) optimizer = optim.Adam(model.parameters(), lr=0.005, weight_decay=5e-4) model.cuda() adj = adj.cuda() def train(epoch): # LR = [1.00e-02, 8.89e-03, 7.78e-03, 6.67e-03, 5.56e-03, 4.45e-03, 3.34e-03, 2.23e-03, 1.12e-03, 1.00e-05] print('Epoch: ' + str(epoch)) n_iter = 500 batch_size = int(60000 / n_iter)
rng_key = random.PRNGKey(args.seed) step_size = args.lr num_epochs = args.epochs n_nodes = adj.shape[0] n_feats = features.shape[1] # GAT params nheads = [8, 1] nhid = [8] dropout = args.dropout # probability of keeping residual = False init_fun, predict_fun = GAT(nheads=nheads, nhid=nhid, nclass=labels.shape[1], dropout=dropout, residual=residual) input_shape = (-1, n_nodes, n_feats) rng_key, init_key = random.split(rng_key) _, init_params = init_fun(init_key, input_shape) opt_init, opt_update, get_params = optimizers.adam(step_size) @jit def update(i, opt_state, batch): params = get_params(opt_state) return opt_update(i, grad(loss)(params, batch), opt_state) opt_state = opt_init(init_params)
import torch from models import GAT, ConvDecoder from main import parse_args args = parse_args() gat = GAT( seed=args.seed, nn_args=dict(args=GAT.adapt_args(True, args)), optim_args=dict(), ) gat.set_save_valid_conditions('save', 'every', 600, 'epochs') gat.train_n_epochs(args.epochs_gat) gat.load_checkpoint(max_epochs=args.epochs_gat ) # line should be unnecessary once using latest ptutils conv = ConvDecoder( seed=args.seed, nn_args=dict( args=ConvDecoder.adapt_args(False, args), entity_embeddings=gat.final_entity_embeddings, relation_embeddings=gat.final_relation_embeddings, ), optim_args=dict(), extra_things_to_use_in_hash=gat.get_path(gat.epochs), ) conv.set_save_valid_conditions('save', 'every', 10, 'epochs') conv.train_n_epochs(args.epochs_conv) # f**k it conv.conv.n_samples = 5
# Define the model and optimizer if (opt.model == 'basic'): print("| Constructing basic GCN model...") model = GCN(nfeat=features.shape[1], nhid=opt.num_hidden, nclass=labels.max().item() + 1, dropout=opt.dropout, init=opt.init_type, nheads=opt.nb_heads, alpha=opt.alpha) elif (opt.model == 'attention'): print("| Constructing Attention GCN model...") model = GAT(nfeat=features.shape[1], nhid=opt.num_hidden, nclass=int(labels.max().item()) + 1, dropout=opt.dropout, nheads=opt.nb_heads, alpha=opt.alpha) elif (opt.model == 'semi-attention'): print("| Constructing Semi-Attention GCN model ...") model_gx = GCN_cont(nfeat=features.shape[1], nembed=opt.embedding_size, nx=adj.shape[0], dropout=opt.dropout_gcn, init=opt.init_type, nheads=opt.nb_heads, alpha=opt.alpha, neg_samp=opt.neg_samp) model_x = GAT_cont(nfeat=features.shape[1], nhid=opt.num_hidden, nclass=int(labels.max().item()) + 1,
# Normalize adjacency matrices individually for i in range(len(A)): A[i] = A[i] + unit d = np.array(A[i].sum(1)).flatten() # print (np.array(A[i])) d_inv = 1. / d d_inv[np.isinf(d_inv)] = 0. D_inv = sp.diags(d_inv) A[i] = D_inv.dot(A[i]).tocsr() A[i] = utils.convert_sparse_matrix_to_sparse_tensor(A[i]) # A = list(map(lambda x: utils.convert_sparse_matrix_to_sparse_tensor(x),A)) # x = tf.sparse_placeholder(tf.float32) model1 = GAT() model2 = GAT() # for i in A: # print (i) # add self attention loops to every tensor matrix checkpt_file = './a.ckpt' nb_epochs = 180 # Sparse tensor dropout here with tf.Graph().as_default(): A_in = [tf.sparse_placeholder(dtype=tf.float32) for _ in range(len(A))] # print(A_in[89].get_shape()) X_in = tf.sparse_placeholder(dtype=tf.float32) attn_drop = tf.placeholder(dtype=tf.float32) ffd_drop = tf.placeholder(dtype=tf.float32)
model = None if args.model == 'GAGAN': model = GAGAN(n_out=dataset.__nlabels__(), n_feat=dataset.__nfeats__(), n_attns=args.n_attns, n_dense=args.n_dense, dim_attn=args.dim_attn, dim_dense=args.dim_dense, dropout=args.dropout) elif args.model == 'GAT': model = GAT(nclass=dataset.__nlabels__(), nfeat=dataset.__nfeats__(), nhid=args.dim_attn, nheads=args.n_attns, nhid_linear=args.dim_dense, nlinear=args.n_dense, alpha=args.alpha, dropout=args.dropout, use_distance_aware_adj=args.use_dist_aware_adj) else: raise ValueError('args.model should be one of GAGAN, GAT') criterion = nn.CrossEntropyLoss() def compute_test(): model.eval() losses_batch = [] acc_batch = []
random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) # Load data adj, features, labels, idx_train, idx_val, idx_test = gcn_utils.load_data( args.dataset) # Model and optimizer if args.model == 'GAT': model = GAT(nfeat=features.shape[1], nhid=args.hidden, nclass=int(labels.max()) + 1, dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha) elif args.model == 'GCN': model = GCN(nfeat=features.shape[1], nhid=args.hidden, nclass=int(labels.max()) + 1, dropout=args.dropout) else: raise ValueError("Model {} not registered".format(args.model)) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.cuda:
# Model and optimizer if args.sparse: model = SpGAT( nfeat=features.shape[1], nhid=args.hidden, nclass=1, # regression dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha) else: model = GAT( ncells=features.shape[0], #features is cells x gene x gene features ngenes=features.shape[1], nfeat=features.shape[2], nhid=args.hidden, nclass=1, # regression dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha) optimizer = optim.Adam(model.parameters(), lr=args.lr) #, #weight_decay=args.weight_decay) if args.cuda: model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda()
# Load data adj, features, labels, idx_train, idx_val, idx_test = load_data() # Model and optimizer if args.sparse: model = SpGAT(nfeat=features.shape[1], nhid=args.hidden, nclass=int(labels.max()) + 1, dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha) else: model = GAT(nfeat=features.shape[1], nhid=args.hidden, nclass=int(labels.max()) + 1, dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.cuda: model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda()
label = data.label.to(device) train = Dataset(data.idx_train) val = Dataset(data.idx_val) test = Dataset(data.idx_test) collate = Collate(data.adj) train_loader = DataLoader(dataset=train, batch_size=args.batch_size, collate_fn=collate) val_loader = DataLoader(dataset=val, batch_size=args.batch_size, collate_fn=collate) test_loader = DataLoader(dataset=test, batch_size=args.batch_size, collate_fn=collate) """ =========================================================================== Training =========================================================================== """ # Model and optimizer model = GAT(n_feature=data.n_feature, n_hidden=args.hidden, n_class=data.n_class, dropout=args.dropout, n_head=args.head).to(device) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) metric = torchmetrics.Accuracy().to(device) for epoch in range(1, args.epoch+1): t = time.time() # Training model.train() loss_train = 0 metric.reset() for edge_batch, idx_batch in train_loader: optimizer.zero_grad() label_batch = label[idx_batch] output = model(feature, edge_batch.to(device))[idx_batch] loss_batch = F.nll_loss(output, label_batch) # mean loss to backward loss_batch.backward()