idx_train = torch.LongTensor(idx_train)
idx_val = torch.LongTensor(idx_val)
idx_test = torch.LongTensor(idx_test)
# Train model
t_total = time.time()
loss_values = []
bad_counter = 0
best = args.epochs + 1
best_epoch = 0
if __name__ == '__main__':
adj, features, labels = load_prepared_data(dataset)
model = GAT(nfeat=features.shape[1],
nhid=args.hidden,
nclass=int(labels.max()) + 1,
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
for epoch in range(args.epochs):
loss_values.append(train(epoch))
if __name__ == '__main__':
adj, features, labels, idx_train, idx_val, idx_test = load_data_gat(
path='data/', dataset_str='citeseer')
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
features, adj, labels = Variable(features), Variable(adj), Variable(labels)
model = GAT(nfeat=features.shape[1],
nhid=12,
nclass=int(labels.max()) + 1,
dropout=0.6,
nheads=8,
alpha=0.2)
model.load_state_dict(torch.load('693.pkl'))
compute_test()
# x = torch.cat([att(features, adj) for att in model.attentions], dim=1)
# embeddings = x.cpu().data.numpy()
# time_start = time.time()
#
# pca_50 = PCA(n_components=50)
# pca_result_50 = pca_50.fit_transform(embeddings)
#
# tsne = TSNE(n_components=2, verbose=1, perplexity=50, n_iter=1000)
# tsne_results = tsne.fit_transform(pca_result_50)
def __call__(self, trial):
print('TRIAL N0 - %d'%trial.number)
###################################################################################################
# Hyperparameters
###################################################################################################
self.configs = {}
self.configs['seed'] = self.seed
################################################################
# Initialize Hyperparameters for search
# Comment This block when specifying hyperparameters
self.configs['learning_rate'] = trial.suggest_uniform('learning_rate',0,1)
self.configs['weight_decay'] = trial.suggest_uniform('weight_decay',0,1)
self.configs['model_dropout'] = trial.suggest_uniform('model_dropout',0,1)
self.configs['attention_dropout'] = trial.suggest_uniform('attention_dropout',0,1)
################################################################
'''
################################################################
# Specifying hyperparamters
# Comment this block when searching for hyperparameters
# Set n_trials to 1
self.configs['learning_rate'] = 0.005
self.configs['weight_decay'] = 5e-4
self.configs['model_dropout'] = 0.6
self.configs['attention_dropout'] = 0.6
################################################################
'''
self.configs['hidden_dims'] = 8
self.configs['num_heads'] = 8
self.configs['num_heads_output'] = 1
###################################################################################################
graph = tf.Graph()
with tf.Session(graph=graph) as sess:
###################################################################################################
# Set random seed
###################################################################################################
tf.set_random_seed(self.seed)
random.seed(self.seed)
np.random.seed(self.seed)
###################################################################################################
# Define placeholders
self.placeholders = {
'adj': tf.sparse_placeholder(tf.float32, name='adj'),
'features': tf.sparse_placeholder(tf.float32, shape=tf.constant(self.features[2], dtype=tf.int64), name='features'),
'labels': tf.placeholder(tf.float32, shape=(None, self.y_train.shape[1]), name='labels'),
'labels_mask': tf.placeholder(tf.int32,name='labels_mask'),
'model_dropout': tf.placeholder_with_default(0., shape=(), name='model_dropout'),
'attention_dropout': tf.placeholder_with_default(0., shape=(), name='attention_dropout')
}
# Create model
self.model = GAT(self.configs, self.placeholders, input_dim=self.features[2][1], logging=True)
# Init variables
sess.run(tf.global_variables_initializer())
patience_count = 0
best_val_accuracy = 0.0
# Train model
for epoch in range(self.epochs):
# Construct feed dictionary
feed_dict = construct_feed_dict(self.features, self.adj_t, self.y_train, self.train_mask, self.placeholders)
feed_dict.update({self.placeholders['model_dropout']: self.configs['model_dropout']})
feed_dict.update({self.placeholders['attention_dropout']: self.configs['attention_dropout']})
# Training step
outs = sess.run([self.model.opt_op, self.model.loss, self.model.accuracy], feed_dict=feed_dict)
# Evaluate
train_loss, train_acc, _ = evaluate(self.model, sess, self.features, self.adj_t, self.y_train, self.train_mask, self.placeholders)
val_loss, val_acc, _ = evaluate(self.model, sess, self.features, self.adj_t, self.y_val, self.val_mask, self.placeholders)
test_loss, test_acc, _ = evaluate(self.model, sess, self.features, self.adj_t, self.y_test, self.test_mask, self.placeholders)
unlabeled_loss, unlabeled_acc, _ = evaluate(self.model, sess, self.features, self.adj_t, self.y_unlabeled, self.unlabeled_mask, self.placeholders)
if self.verbose:
# Print results
print('[Epoch %03d] Loss - %0.04f\t Train Accuracy - %0.04f\t Val Accuracy - %0.04f'%(epoch+1, outs[1], train_acc, val_acc))
if val_acc > best_val_accuracy:
best_val_accuracy = val_acc
patience_count = 0
trial.set_user_attr('Train Accuracy', train_acc)
trial.set_user_attr('Train Loss', train_loss)
trial.set_user_attr('Val Accuracy', val_acc)
trial.set_user_attr('Val Loss', val_loss)
trial.set_user_attr('Test Accuracy', test_acc)
trial.set_user_attr('Test Loss', test_loss)
trial.set_user_attr('Unlabeled Accuracy', unlabeled_acc)
trial.set_user_attr('Unlabeled Loss', unlabeled_loss)
'''
#############################################################################################################
# Fetch Attention Weights to analyze
#############################################################################################################
feed_dict.update({self.placeholders['model_dropout']: 0.})
feed_dict.update({self.placeholders['attention_dropout']: 0.})
layer_1_attention_matrices = [sess.run(self.model.layers[0].attention_matrices[i], feed_dict=feed_dict) for i in range(self.configs['num_heads'])]
layer_1_attention_matrices = [sparse_tensor_to_coo(sp_mat) for sp_mat in layer_1_attention_matrices]
trial.set_user_attr('layer_1_attention_matrices', layer_1_attention_matrices)
layer_2_attention_matrices = [sess.run(self.model.layers[1].attention_matrices[i], feed_dict=feed_dict) for i in range(self.configs['num_heads_output'])]
layer_2_attention_matrices = [sparse_tensor_to_coo(sp_mat) for sp_mat in layer_2_attention_matrices]
trial.set_user_attr('layer_2_attention_matrices', layer_2_attention_matrices)
#############################################################################################################
'''
else:
patience_count = patience_count + 1
if patience_count >= self.early_stopping: #Early stopping
break
print()
print('Train Accuracy - %0.04f'%(trial.user_attrs['Train Accuracy']))
print('Val Accuracy - %0.04f'%(trial.user_attrs['Val Accuracy']))
print('Test Accuracy - %0.04f'%(trial.user_attrs['Test Accuracy']))
print('Unlabeled Accuracy - %0.04f'%(trial.user_attrs['Unlabeled Accuracy']))
print()
return trial.user_attrs['Val Accuracy']
verbose=args.verbose).to(args.device)
model_args = {
"num_features": data.num_features,
"num_classes": data.num_classes,
"hidden_size": args.hidden,
"dropout": args.dropout,
"activation": args.activation
}
if args.model == "gcn":
model = GCN(**model_args)
elif args.model == "gat":
model_args["num_heads"] = args.num_heads
model_args["hidden_size"] = int(args.hidden / args.num_heads)
model = GAT(**model_args)
elif args.model == "mlp":
model = MLP(**model_args)
else:
gen_type, post_type = args.model.split("_")
gen_config = copy.deepcopy(model_args)
gen_config["type"] = gen_type
gen_config["neg_ratio"] = args.neg_ratio
if gen_type == "lsm":
gen_config["hidden_x"] = args.hidden_x
if gen_type == "sbm":
gen_config["p0"] = args.p0
gen_config["p1"] = args.p1
post_config = copy.deepcopy(model_args)
parser.add_argument('--patience', type=int, default=100, help='Patience')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
# Load data
adj, features, labels, idx_train, idx_val, idx_test = load_data()
# Model and optimizer
model = GAT(nfeat=features.shape[1], nhid=args.hidden, nclass=int(labels.max()) + 1, dropout=args.dropout, nheads=args.nb_heads, alpha=args.alpha)
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
features, adj, labels = Variable(features), Variable(adj), Variable(labels)
def train(epoch):
def main(dataset_name, yang_splits,
nheads, hidden_units, feat_drop_rate, coefs_drop_rate,
epochs, learning_rate, l2_weight, patience,
data_seed, net_seed, checkpoint_path, verbose):
# reproducibility
np.random.seed(data_seed)
tf.random.set_seed(net_seed)
if yang_splits:
features, o_h_labels, graph, mask_train, mask_val, mask_test = read_dataset(dataset_name, yang_splits=True)
labels = np.array([np.argmax(l) for l in o_h_labels], dtype=np.int32)
else:
if verbose > 0: print("reading dataset")
features, neighbors, labels, o_h_labels, keys = read_dataset(dataset_name)
num_classes = len(set(labels))
if verbose > 0: print("shuffling dataset")
features, neighbors, labels, o_h_labels, keys = permute(features, neighbors, labels, o_h_labels, keys)
if verbose > 0: print("obtaining masks")
mask_train, mask_val, mask_test = split(dataset_name, labels)
if verbose > 0: print("calculating adjacency matrix")
graph = adjacency_matrix(neighbors)
# add self loops to adj matrix
graph = graph + sp.eye(graph.shape[0])
num_classes = get_num_classes(dataset_name)
features = normalize_features(features)
y_train = np.multiply(o_h_labels, np.broadcast_to(mask_train.T, o_h_labels.T.shape).T )
y_val = np.multiply(o_h_labels, np.broadcast_to(mask_val.T, o_h_labels.T.shape).T )
y_test = np.multiply(o_h_labels, np.broadcast_to(mask_test.T, o_h_labels.T.shape).T )
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
if verbose > 0: print("defining model")
model = GAT(graph, num_classes, hidden_units, nheads, feat_drop_rate, coefs_drop_rate)
model.compile(loss=lambda y_true, y_pred: masked_loss(y_true, y_pred)+l2_weight*tf.reduce_sum([tf.nn.l2_loss(w) for w in model.weights if not 'bias' in w.name]),
optimizer=optimizer, metrics=[masked_accuracy])
if verbose > 0: print("begin training")
tb = TensorBoard(log_dir='logs')
if dataset_name == 'cora':
monitor = 'acc_loss'
elif dataset_name == 'pubmed':
monitor = 'loss'
else:
monitor = 'acc'
es = EarlyStoppingAccLoss(patience, monitor, checkpoint_path)
model.fit(features, y_train, epochs=epochs, batch_size=len(features), shuffle=False, validation_data=(features, y_val), callbacks=[tb, es], verbose=verbose)
file_writer = tf.summary.create_file_writer("./logs/results/")
file_writer.set_as_default()
# log best performances on train and val set
loss, accuracy = model.evaluate(features, y_train, batch_size=len(features), verbose=0)
print("accuracy on training: " + str(accuracy))
tf.summary.scalar('bw_loss', data=loss, step=1)
tf.summary.scalar('bw_accuracy', data=accuracy, step=1)
v_loss, v_accuracy = model.evaluate(features, y_val, batch_size=len(features), verbose=0)
print("accuracy on validation: " + str(v_accuracy))
tf.summary.scalar('bw_val_loss', data=v_loss, step=1)
tf.summary.scalar('bw_val_accuracy', data=v_accuracy, step=1)
tf.summary.scalar('bw_epoch', data=es.stopped_epoch, step=1)
if verbose > 0: print("test the model on test set")
t_loss, t_accuracy = model.evaluate(features, y_test, batch_size=len(features), verbose=0)
print("accuracy on test: " + str(t_accuracy))
def __init__(self,graph,sparse = False,epochs = 200,learning_rate = 0.005,
weight_decay = 5e-4,hidden = 8,nb_heads = 8,drop_out = 0.6,
alpha = 0.2 ,patience = 100,train = 1500,val = 2000,test = 3100):
self.graph = graph
self.sparse = sparse
self.epochs = epochs
self.learning_rate = learning_rate
self.weight_decay = weight_decay
self.hidden = hidden
self.nb_heads = nb_heads
self.drop_out = drop_out
self.alpha = alpha
self.patience = patience
self.train = train
self.val = val
self.test = test
idx_train,idx_val , idx_test = self.load_data()
random.seed(random_seed)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
if self.sparse:
model = SpGAT(nfeat=self.features.shape[1],
nhid=self.hidden,
nclass=int(self.labels.max()) + 1,
dropout=self.drop_out,
nheads=self.nb_heads,
alpha=self.alpha)
else:
model = GAT(nfeat=self.features.shape[1],
nhid=self.hidden,
nclass=int(self.labels.max()) + 1,
dropout=self.drop_out,
nheads=self.nb_heads,
alpha=self.alpha)
optimizer = optim.Adam(model.parameters(),
lr=self.learning_rate,
weight_decay=self.weight_decay)
#利用GPU
# device = torch.device("cuda:0")
# torch.cuda.empty_cache()
# model.to(device)
# self.features = self.features.to(device)
# self.adj = self.adj.to(device)
# self.labels = self.labels.to(device)
# idx_train = idx_train.to(device)
# idx_val = idx_val.to(device)
# idx_test = idx_test.to(device)
features, adj, labels = Variable(self.features), Variable(self.adj), Variable(self.labels)
t_total = time.time()
loss_values = []
bad_counter = 0
best = self.epochs + 1
best_epoch = 0
for epoch in range(self.epochs):
t = time.time()
model.train()
optimizer.zero_grad()
output = model(features, adj)
loss_train = F.nll_loss(output[idx_train], labels[idx_train])
acc_train = accuracy(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
model.eval()
output = model(features, adj)
loss_val = F.nll_loss(output[idx_val], labels[idx_val])
acc_val = accuracy(output[idx_val], labels[idx_val])
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(loss_train.data),
'acc_train: {:.4f}'.format(acc_train.data),
'loss_val: {:.4f}'.format(loss_val.data),
'acc_val: {:.4f}'.format(acc_val.data),
'time: {:.4f}s'.format(time.time() - t))
loss_values.append(loss_val.data)
torch.save(model.state_dict(), '{}.pkl'.format(epoch))
if loss_values[-1] < best:
best = loss_values[-1]
best_epoch = epoch
bad_counter = 0
else:
bad_counter += 1
if bad_counter == self.patience:
break
files = glob.glob('*.pkl')
for file in files:
epoch_nb = int(file.split('.')[0])
if epoch_nb < best_epoch:
os.remove(file)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
print('Loading {}th epoch'.format(best_epoch))
model.load_state_dict(torch.load('{}.pkl'.format(best_epoch)))
model.eval()
output = model(features, adj)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:",
"loss= {:.4f}".format(loss_test.data),
"accuracy= {:.4f}".format(acc_test.data))
np.random.seed(42)
torch.manual_seed(42)
if use_gpu:
torch.cuda.manual_seed(42)
model, optimizer = None, None
best_acc = 0
early_stop = 0
# Define the model and optimizer
if (opt.model == 'attention'):
print("| Constructing Graph Attention Network model...")
model = GAT(nfeat=features.shape[1],
nhid=opt.num_hidden,
nclass=int(labels.max().item()) + 1,
dropout=opt.dropout,
nheads=opt.nb_heads,
nouts=opt.nb_outs,
alpha=opt.alpha)
else:
raise NotImplementedError
if (opt.optimizer == 'sgd'):
optimizer = optim.SGD(model.parameters(),
lr=opt.lr,
weight_decay=opt.weight_decay,
momentum=0.9)
elif (opt.optimizer == 'adam'):
optimizer = optim.Adam(model.parameters(),
lr=opt.lr,
weight_decay=opt.weight_decay)
def main(dataset_name, yang_splits, nheads, hidden_units, feat_drop_rate,
coefs_drop_rate, l2_weight, data_seed, checkpoint_path, verbose,
tsne):
# reproducibility
np.random.seed(data_seed)
if yang_splits:
features, o_h_labels, graph, mask_train, mask_val, mask_test = read_dataset(
dataset_name, yang_splits=True)
labels = np.array([np.argmax(l) for l in o_h_labels], dtype=np.int32)
else:
if verbose > 0: print("reading dataset")
features, neighbors, labels, o_h_labels, keys = read_dataset(
dataset_name)
num_classes = len(set(labels))
if verbose > 0: print("shuffling dataset")
features, neighbors, labels, o_h_labels, keys = permute(
features, neighbors, labels, o_h_labels, keys)
if verbose > 0: print("obtaining masks")
mask_train, mask_val, mask_test = split(dataset_name, labels)
if verbose > 0: print("calculating adjacency matrix")
graph = adjacency_matrix(neighbors)
# add self loops to adj matrix
graph = graph + sp.eye(graph.shape[0])
num_classes = get_num_classes(dataset_name)
features = normalize_features(features)
y_test = np.multiply(o_h_labels,
np.broadcast_to(mask_test.T, o_h_labels.T.shape).T)
if verbose > 0: print("defining model")
model = GAT(graph, num_classes, hidden_units, nheads, feat_drop_rate,
coefs_drop_rate)
model.compile(loss=lambda y_true, y_pred: masked_loss(
y_true, y_pred) + l2_weight * tf.reduce_sum(
[tf.nn.l2_loss(w) for w in model.weights if not 'bias' in w.name]),
metrics=[masked_accuracy])
if verbose > 0: print("load model from checkpoint")
wpath = os.path.join(checkpoint_path, 'cp.ckpt')
model.load_weights(wpath).expect_partial()
if verbose > 0: print("test the model on test set")
t_loss, t_accuracy = model.evaluate(features,
y_test,
batch_size=len(features),
verbose=0)
print("accuracy on test: " + str(t_accuracy))
if tsne:
if verbose > 0: print("calculating t-SNE plot")
intermediate_output = model.call(features,
training=False,
intermediate=True)
plot_tsne(intermediate_output[mask_test], labels[mask_test],
len(o_h_labels[0]), 'GAT')
def build_model(args,
in_feats,
n_hidden,
n_classes,
device,
n_layers=1,
backend='geometric',
edge_index=None,
num_nodes=None):
if args.model == 'graphsaint':
assert backend == 'geometric'
model_spec = args.variant
else:
model_spec = args.model
if backend == 'geometric':
print("Using Geometric Backend")
if model_spec == 'gs-mean':
model = geo.GraphSAGE(in_feats, n_hidden, n_classes, n_layers,
F.relu, args.dropout).to(device)
elif model_spec == "gcn":
model = geo.GCN(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout).to(device)
elif model_spec == "gat":
print("Warning, GAT doesn't respect n_layers")
heads = [8, args.gat_out_heads] # Fixed head config
n_hidden_per_head = int(n_hidden / heads[0])
model = geo.GAT(in_feats, n_hidden_per_head, n_classes, F.relu,
args.dropout, 0.6, heads).to(device)
elif model_spec == "mlp":
model = geo.MLP(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout).to(device)
elif model_spec == 'jknet-sageconv':
# Geometric JKNEt with SAGECOnv
model = JKNet(tg.nn.SAGEConv,
in_feats,
n_hidden,
n_classes,
n_layers,
F.relu,
args.dropout,
mode="cat",
conv_kwargs={
"normalize": False
},
backend="geometric").to(device)
elif model_spec == 'jknet-graphconv':
model = JKNet(tg.nn.GraphConv,
in_feats,
n_hidden,
n_classes,
n_layers,
F.relu,
args.dropout,
mode="cat",
conv_kwargs={
"aggr": "mean"
},
backend="geometric").to(device)
elif model_spec == "sgnet":
model = geo.SGNet(in_channels=in_feats,
out_channels=n_classes,
K=n_layers,
cached=True).to(device)
else:
raise NotImplementedError(
f"Unknown model spec 'f{model_spec} for backend {backend}")
elif backend == 'dgl': # DGL models
if model_spec == 'gs-mean':
model = GraphSAGE(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout, 'mean').to(device)
elif model_spec == 'mlp':
model = MLP(in_feats, n_hidden, n_classes, n_layers, F.relu,
args.dropout).to(device)
elif model_spec == 'mostfrequent':
model = MostFrequentClass()
elif model_spec == 'gat':
print("Warning, GAT doesn't respect n_layers")
heads = [8, args.gat_out_heads] # Fixed head config
# Div num_hidden by heads for same capacity
n_hidden_per_head = int(n_hidden / heads[0])
assert n_hidden_per_head * heads[
0] == n_hidden, f"{n_hidden} not divisible by {heads[0]}"
model = GAT(1, in_feats, n_hidden_per_head, n_classes, heads,
F.elu, 0.6, 0.6, 0.2, False).to(device)
elif model_spec == 'node2vec':
assert edge_index is not None
model = tg.nn.Node2Vec(
edge_index,
n_hidden,
args.n2v_walk_length,
args.n2v_context_size,
walks_per_node=args.n2v_walks_per_node,
p=args.n2v_p,
q=args.n2v_q,
num_negative_samples=args.n2v_num_negative_samples,
num_nodes=num_nodes,
sparse=True)
elif model_spec == 'jknet-sageconv':
# DGL JKNet
model = JKNet(dgl.nn.pytorch.SAGEConv,
in_feats,
n_hidden,
n_classes,
n_layers,
F.relu,
args.dropout,
mode="cat",
conv_args=["mean"],
backend='dgl').to(device)
elif model_spec == 'sgnet':
model = SGNet(in_feats,
n_classes,
k=n_layers,
cached=True,
bias=True,
norm=None).to(device)
else:
raise NotImplementedError(
f"Unknown model spec 'f{model_spec} for backend {backend}")
else:
raise NotImplementedError(f"Unknown backend: {backend}")
return model
if args.sparse:
model = SpGAT(nfeat=feat_dim,
# nfeat=features.shape[1],
nhid=args.hidden,
# nclass=int(labels.max()) + 1,
<<<<<<< HEAD
nclass=label_dim,
=======
nclass=151,
>>>>>>> origin/master
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
else:
model = GAT(nfeat=feat_dim,
# nfeat=features.shape[1],
nhid=args.hidden,
# nclass=int(labels.max()) + 1,
<<<<<<< HEAD
nclass=label_dim,
=======
nclass=151,
>>>>>>> origin/master
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.cuda:
torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print('loading data')
x, y, adj = load_data()
y_train = y[:60000]
x_train = x[:60000]
y_test = y[60000:]
x_test = x[60000:]
adj = torch.Tensor(adj)
print('loading on cuda')
model = GAT(nfeat=1, nhid=5, nclass=1, dropout=0.8, nheads=2, alpha=0.2)
optimizer = optim.Adam(model.parameters(), lr=0.005, weight_decay=5e-4)
model.cuda()
adj = adj.cuda()
def train(epoch):
# LR = [1.00e-02, 8.89e-03, 7.78e-03, 6.67e-03, 5.56e-03, 4.45e-03, 3.34e-03, 2.23e-03, 1.12e-03, 1.00e-05]
print('Epoch: ' + str(epoch))
n_iter = 500
batch_size = int(60000 / n_iter)
rng_key = random.PRNGKey(args.seed)
step_size = args.lr
num_epochs = args.epochs
n_nodes = adj.shape[0]
n_feats = features.shape[1]
# GAT params
nheads = [8, 1]
nhid = [8]
dropout = args.dropout # probability of keeping
residual = False
init_fun, predict_fun = GAT(nheads=nheads,
nhid=nhid,
nclass=labels.shape[1],
dropout=dropout,
residual=residual)
input_shape = (-1, n_nodes, n_feats)
rng_key, init_key = random.split(rng_key)
_, init_params = init_fun(init_key, input_shape)
opt_init, opt_update, get_params = optimizers.adam(step_size)
@jit
def update(i, opt_state, batch):
params = get_params(opt_state)
return opt_update(i, grad(loss)(params, batch), opt_state)
opt_state = opt_init(init_params)
import torch
from models import GAT, ConvDecoder
from main import parse_args
args = parse_args()
gat = GAT(
seed=args.seed,
nn_args=dict(args=GAT.adapt_args(True, args)),
optim_args=dict(),
)
gat.set_save_valid_conditions('save', 'every', 600, 'epochs')
gat.train_n_epochs(args.epochs_gat)
gat.load_checkpoint(max_epochs=args.epochs_gat
) # line should be unnecessary once using latest ptutils
conv = ConvDecoder(
seed=args.seed,
nn_args=dict(
args=ConvDecoder.adapt_args(False, args),
entity_embeddings=gat.final_entity_embeddings,
relation_embeddings=gat.final_relation_embeddings,
),
optim_args=dict(),
extra_things_to_use_in_hash=gat.get_path(gat.epochs),
)
conv.set_save_valid_conditions('save', 'every', 10, 'epochs')
conv.train_n_epochs(args.epochs_conv)
# f**k it
conv.conv.n_samples = 5
# Define the model and optimizer
if (opt.model == 'basic'):
print("| Constructing basic GCN model...")
model = GCN(nfeat=features.shape[1],
nhid=opt.num_hidden,
nclass=labels.max().item() + 1,
dropout=opt.dropout,
init=opt.init_type,
nheads=opt.nb_heads,
alpha=opt.alpha)
elif (opt.model == 'attention'):
print("| Constructing Attention GCN model...")
model = GAT(nfeat=features.shape[1],
nhid=opt.num_hidden,
nclass=int(labels.max().item()) + 1,
dropout=opt.dropout,
nheads=opt.nb_heads,
alpha=opt.alpha)
elif (opt.model == 'semi-attention'):
print("| Constructing Semi-Attention GCN model ...")
model_gx = GCN_cont(nfeat=features.shape[1],
nembed=opt.embedding_size,
nx=adj.shape[0],
dropout=opt.dropout_gcn,
init=opt.init_type,
nheads=opt.nb_heads,
alpha=opt.alpha,
neg_samp=opt.neg_samp)
model_x = GAT_cont(nfeat=features.shape[1],
nhid=opt.num_hidden,
nclass=int(labels.max().item()) + 1,
# Normalize adjacency matrices individually
for i in range(len(A)):
A[i] = A[i] + unit
d = np.array(A[i].sum(1)).flatten()
# print (np.array(A[i]))
d_inv = 1. / d
d_inv[np.isinf(d_inv)] = 0.
D_inv = sp.diags(d_inv)
A[i] = D_inv.dot(A[i]).tocsr()
A[i] = utils.convert_sparse_matrix_to_sparse_tensor(A[i])
# A = list(map(lambda x: utils.convert_sparse_matrix_to_sparse_tensor(x),A))
# x = tf.sparse_placeholder(tf.float32)
model1 = GAT()
model2 = GAT()
# for i in A:
# print (i)
# add self attention loops to every tensor matrix
checkpt_file = './a.ckpt'
nb_epochs = 180
# Sparse tensor dropout here
with tf.Graph().as_default():
A_in = [tf.sparse_placeholder(dtype=tf.float32) for _ in range(len(A))]
# print(A_in[89].get_shape())
X_in = tf.sparse_placeholder(dtype=tf.float32)
attn_drop = tf.placeholder(dtype=tf.float32)
ffd_drop = tf.placeholder(dtype=tf.float32)
model = None
if args.model == 'GAGAN':
model = GAGAN(n_out=dataset.__nlabels__(),
n_feat=dataset.__nfeats__(),
n_attns=args.n_attns,
n_dense=args.n_dense,
dim_attn=args.dim_attn,
dim_dense=args.dim_dense,
dropout=args.dropout)
elif args.model == 'GAT':
model = GAT(nclass=dataset.__nlabels__(),
nfeat=dataset.__nfeats__(),
nhid=args.dim_attn,
nheads=args.n_attns,
nhid_linear=args.dim_dense,
nlinear=args.n_dense,
alpha=args.alpha,
dropout=args.dropout,
use_distance_aware_adj=args.use_dist_aware_adj)
else:
raise ValueError('args.model should be one of GAGAN, GAT')
criterion = nn.CrossEntropyLoss()
def compute_test():
model.eval()
losses_batch = []
acc_batch = []
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
# Load data
adj, features, labels, idx_train, idx_val, idx_test = gcn_utils.load_data(
args.dataset)
# Model and optimizer
if args.model == 'GAT':
model = GAT(nfeat=features.shape[1],
nhid=args.hidden,
nclass=int(labels.max()) + 1,
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
elif args.model == 'GCN':
model = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=int(labels.max()) + 1,
dropout=args.dropout)
else:
raise ValueError("Model {} not registered".format(args.model))
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.cuda:
# Model and optimizer
if args.sparse:
model = SpGAT(
nfeat=features.shape[1],
nhid=args.hidden,
nclass=1, # regression
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
else:
model = GAT(
ncells=features.shape[0], #features is cells x gene x gene features
ngenes=features.shape[1],
nfeat=features.shape[2],
nhid=args.hidden,
nclass=1, # regression
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
optimizer = optim.Adam(model.parameters(), lr=args.lr) #,
#weight_decay=args.weight_decay)
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
# Load data
adj, features, labels, idx_train, idx_val, idx_test = load_data()
# Model and optimizer
if args.sparse:
model = SpGAT(nfeat=features.shape[1],
nhid=args.hidden,
nclass=int(labels.max()) + 1,
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
else:
model = GAT(nfeat=features.shape[1],
nhid=args.hidden,
nclass=int(labels.max()) + 1,
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
label = data.label.to(device)
train = Dataset(data.idx_train)
val = Dataset(data.idx_val)
test = Dataset(data.idx_test)
collate = Collate(data.adj)
train_loader = DataLoader(dataset=train, batch_size=args.batch_size, collate_fn=collate)
val_loader = DataLoader(dataset=val, batch_size=args.batch_size, collate_fn=collate)
test_loader = DataLoader(dataset=test, batch_size=args.batch_size, collate_fn=collate)
"""
===========================================================================
Training
===========================================================================
"""
# Model and optimizer
model = GAT(n_feature=data.n_feature, n_hidden=args.hidden, n_class=data.n_class, dropout=args.dropout, n_head=args.head).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
metric = torchmetrics.Accuracy().to(device)
for epoch in range(1, args.epoch+1):
t = time.time()
# Training
model.train()
loss_train = 0
metric.reset()
for edge_batch, idx_batch in train_loader:
optimizer.zero_grad()
label_batch = label[idx_batch]
output = model(feature, edge_batch.to(device))[idx_batch]
loss_batch = F.nll_loss(output, label_batch) # mean loss to backward
loss_batch.backward()
