epoch_nb = int(file.split('.')[0])
if epoch_nb < best_epoch:
os.remove(file)
files = glob.glob('*.pkl')
for file in files:
epoch_nb = int(file.split('.')[0])
if epoch_nb > best_epoch:
os.remove(file)
print('Optimization Finished!')
print('Total time elapsed: {:.4f}s'.format(time.time() - t_total))
#Restore best model
print('Loading {}th epoch'.format(best_epoch))
model.load_state_dict(torch.load('{}.pkl'.format(best_epoch)))
#Testing
acc_test = compute_test()
#Plot
output_dir = "results/random_seed_" + str(args.seed)
mkdir_p(output_dir)
fig, ax = plt.subplots()
ax.plot(train_losses, label = 'train loss')
ax.plot(val_losses, label = 'validation loss')
ax.set_xlabel('epochs')
ax.set_ylabel('cross entropy loss')
ax.legend()
for file in files:
filename = os.path.split(file)[-1]
epoch_nb = int(filename.split('.')[0])
if epoch_nb != best_epoch:
os.remove(file)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
print("Model arguments: ")
print(args)
print(model)
# Restore best model
print('Loading {}th epoch'.format(best_epoch))
model.load_state_dict(
torch.load(os.path.join(dump_dir, '{}.pkl'.format(best_epoch))))
# Testing
model.eval()
output_logits = model(features, adj)
output = F.log_softmax(output_logits, dim=-1)
loss_test = F.nll_loss(output[0, idx_test], labels[0, idx_test])
acc_test = accuracy(output[0, idx_test], labels[0, idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.data.item()),
"accuracy= {:.4f}".format(acc_test.data.item()))
with open(os.path.join('./result', "%s.txt" % (configStr, )), 'a') as f:
f.write("Test graph results: \tmean loss: %.4f \tmean acc: %4f \n" %
(loss_test, acc_test))
def main(opt):
data = get_dataset(opt)
g = data[0]
if opt['gpu'] < 0:
cuda = False
else:
cuda = True
g = g.int().to(opt['gpu'])
features = g.ndata['feat']
labels = g.ndata['label']
train_mask = g.ndata['train_mask']
val_mask = g.ndata['val_mask']
test_mask = g.ndata['test_mask']
num_feats = features.shape[1]
n_classes = data.num_labels
n_edges = data.graph.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes, train_mask.int().sum().item(),
val_mask.int().sum().item(), test_mask.int().sum().item()))
# add self loop
g = dgl.remove_self_loop(g)
g = dgl.add_self_loop(g)
n_edges = g.number_of_edges()
# create model
heads = ([opt['num_heads']] * opt['num_layers']) + [opt['num_out_heads']]
if opt['model'] == 'GAT':
model = GAT(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, heads, F.elu, opt['in_drop'], opt['attn_drop'],
opt['negative_slope'], opt['residual'], opt)
elif opt['model'] == 'AGNN':
model = AGNN(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, opt['in_drop'], opt)
print(model)
if opt['early_stop']:
stopper = EarlyStopping(patience=100)
if cuda:
model.cuda()
# use optimizer
optimizer = get_optimizer(opt['optimizer'],
parameters=model.parameters(),
lr=opt['lr'],
weight_decay=opt['weight_decay'])
# initialize graph
dur = []
for epoch in range(opt['epochs']):
# model.train()
if epoch >= 3:
t0 = time.time()
# forward
# logits = model(features)
# loss = loss_fcn(logits[train_mask], labels[train_mask])
# optimizer.zero_grad()
# loss.backward()
# optimizer.step()
loss, logits = train(model, optimizer, features, train_mask, labels)
if epoch >= 3:
dur.append(time.time() - t0)
train_acc = accuracy(logits[train_mask], labels[train_mask])
if opt['fastmode']:
val_acc = accuracy(logits[val_mask], labels[val_mask])
else:
val_acc = evaluate(model, features, labels, val_mask)
if opt['early_stop']:
if stopper.step(val_acc, model):
break
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainAcc {:.4f} |"
" ValAcc {:.4f} | ETputs(KTEPS) {:.2f}".format(
epoch, np.mean(dur), loss.item(), train_acc, val_acc,
n_edges / np.mean(dur) / 1000))
print()
if opt['early_stop']:
model.load_state_dict(torch.load('es_checkpoint.pt'))
acc = evaluate(model, features, labels, test_mask)
print("Test Accuracy {:.4f}".format(acc))
def train_ray(opt, checkpoint_dir=None, data_dir="../data"):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = get_dataset(opt)
g = data[0]
if opt['gpu'] < 0:
cuda = False
else:
cuda = True
g = g.int().to(opt['gpu'])
features = g.ndata['feat']
labels = g.ndata['label']
train_mask = g.ndata['train_mask']
val_mask = g.ndata['val_mask']
test_mask = g.ndata['test_mask']
num_feats = features.shape[1]
n_classes = data.num_classes
n_edges = data.graph.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes, train_mask.int().sum().item(),
val_mask.int().sum().item(), test_mask.int().sum().item()))
# add self loop
g = dgl.remove_self_loop(g)
g = dgl.add_self_loop(g)
n_edges = g.number_of_edges()
# create model
heads = ([opt['num_heads']] * opt['num_layers']) + [opt['num_out_heads']]
models = []
optimizers = []
datas = [g for i in range(opt['num_init'])]
for split in range(opt['num_init']):
if opt['model'] == 'GAT':
model = GAT(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, heads, F.elu, opt['in_drop'],
opt['attn_drop'], opt['negative_slope'],
opt['residual'], opt)
elif opt['model'] == 'AGNN':
model = AGNN(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, opt['in_drop'], opt)
train_this = train
model = model.to(device)
models.append(model)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
# model = model.to(device)
parameters = [p for p in model.parameters() if p.requires_grad]
optimizer = get_optimizer(opt['optimizer'],
parameters,
lr=opt['lr'],
weight_decay=opt['weight_decay'])
optimizers.append(optimizer)
# The `checkpoint_dir` parameter gets passed by Ray Tune when a checkpoint
# should be restored.
if checkpoint_dir:
checkpoint = os.path.join(checkpoint_dir, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint)
model.load_state_dict(model_state)
optimizer.load_state_dict(optimizer_state)
for epoch in range(1, opt['epochs']):
loss = np.mean([
train_this(model, optimizer, features, train_mask,
labels)[0].item()
for model, optimizer in zip(models, optimizers)
])
train_accs, val_accs, tmp_test_accs = average_test(models, datas)
with tune.checkpoint_dir(step=epoch) as checkpoint_dir:
best = np.argmax(val_accs)
path = os.path.join(checkpoint_dir, "checkpoint")
torch.save(
(models[best].state_dict(), optimizers[best].state_dict()),
path)
tune.report(loss=loss,
accuracy=np.mean(val_accs),
test_acc=np.mean(tmp_test_accs),
train_acc=np.mean(train_accs))
X, Y = load_dataset(args.seed)
split_rate = 0.8
x_train = X[0:int(split_rate * X.shape[0])]
x_test = X[int(split_rate * X.shape[0]):]
y_train = Y[0:int(split_rate * Y.shape[0])]
y_test = Y[int(split_rate * Y.shape[0]):]
# Restore best model
model = GAT(nfeat=120,
nhid=4,
nclass=int(y_train.max()) + 1,
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
model.load_state_dict(
torch.load('{}/{}.pkl'.format('output(Apr 12 20.08.47 2019)', '649')))
print('model loaded')
# Testing
mode = 'plot_chan_exp' # define which test to run: GADN, svm, run_record, cut_channel_GADN, cut_channel_svm
if mode == 'GADN':
compute_test(x_test, y_test, SpikeChans=True)
elif mode == 'cut_channel_GADN':
cut_chan_test(mode='GADN')
elif mode == 'svm':
clf = svm_run_and_test()
elif mode == 'run_record':
run_EEG_record(model)
elif mode == 'attents_box_chart':
attents_box(model)
elif mode == 'plot_chan_exp':
def train_ray_int(opt, checkpoint_dir=None, data_dir="../data"):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data = get_dataset(opt)
g = data[0]
if opt['gpu'] < 0:
cuda = False
else:
cuda = True
g = g.int().to(opt['gpu'])
# if opt["num_splits"] > 0:
# dataset.data = set_train_val_test_split(
# 23 * np.random.randint(0, opt["num_splits"]), # random prime 23 to make the splits 'more' random. Could remove
# dataset.data,
# num_development=5000 if opt["dataset"] == "CoauthorCS" else 1500)
features = g.ndata['feat']
labels = g.ndata['label']
train_mask = g.ndata['train_mask']
val_mask = g.ndata['val_mask']
test_mask = g.ndata['test_mask']
num_feats = features.shape[1]
n_classes = data.num_classes
n_edges = data.graph.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes, train_mask.int().sum().item(),
val_mask.int().sum().item(), test_mask.int().sum().item()))
# add self loop
g = dgl.remove_self_loop(g)
g = dgl.add_self_loop(g)
n_edges = g.number_of_edges()
# create model
heads = ([opt['num_heads']] * opt['num_layers']) + [opt['num_out_heads']]
if opt['model'] == 'GAT':
model = GAT(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, heads, F.elu, opt['in_drop'], opt['attn_drop'],
opt['negative_slope'], opt['residual'], opt)
elif opt['model'] == 'AGNN':
model = AGNN(g, opt['num_layers'], num_feats, opt['num_hidden'],
n_classes, opt['in_drop'], opt)
model = model.to(device)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
parameters = [p for p in model.parameters() if p.requires_grad]
optimizer = get_optimizer(opt["optimizer"],
parameters,
lr=opt["lr"],
weight_decay=opt["weight_decay"])
if checkpoint_dir:
checkpoint = os.path.join(checkpoint_dir, "checkpoint")
model_state, optimizer_state = torch.load(checkpoint)
model.load_state_dict(model_state)
optimizer.load_state_dict(optimizer_state)
train_this = train
this_test = test_OGB if opt['dataset'] == 'ogbn-arxiv' else test
best_time = best_epoch = train_acc = val_acc = test_acc = 0
for epoch in range(1, opt["epoch"]):
# loss = train(model, optimizer, data)
loss = train_this(model, optimizer, features, train_mask,
labels)[0].item()
if opt["no_early"]:
tmp_train_acc, tmp_val_acc, tmp_test_acc = this_test(model, g)
best_time = opt['time']
else:
tmp_train_acc, tmp_val_acc, tmp_test_acc = this_test(model, g)
if tmp_val_acc > val_acc:
best_epoch = epoch
train_acc = tmp_train_acc
val_acc = tmp_val_acc
test_acc = tmp_test_acc
with tune.checkpoint_dir(step=epoch) as checkpoint_dir:
path = os.path.join(checkpoint_dir, "checkpoint")
torch.save((model.state_dict(), optimizer.state_dict()), path)
tune.report(loss=loss,
accuracy=val_acc,
test_acc=test_acc,
train_acc=train_acc,
best_time=best_time,
best_epoch=best_epoch)
# break
# files = glob.glob('*.pkl')
# for file in files:
# epoch_nb = int(file.split('.')[0])
# if epoch_nb < best_epoch:
# os.remove(file)
files = glob.glob('*.pkl')
for file in files:
epoch_ = file.split('.')[0]
model = GAT(nfeat=features.shape[1],
nhid=args.hidden,
dropout=args.dropout,
nheads=args.nb_heads,
alpha=args.alpha)
model.load_state_dict(torch.load(file))
model.eval()
model = model.cuda()
embeddings = model.get_embedding(features, adj)
embeddings = embeddings.cpu().detach().numpy()
print(embeddings.shape, "Numpy array shape")
np.save(epoch_, embeddings)
os.remove(file)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# Restore best model
# print('Loading {}th epoch'.format(best_epoch))
# model.load_state_dict(torch.load('{}.pkl'.format(best_epoch)))
best = loss_values[-1]
best_epoch = epoch
bad_counter = 0
else:
bad_counter += 1
if bad_counter == args.patience:
break
files = glob.glob('*.pkl')
for file in files:
epoch_nb = int(file.split('.')[0])
if epoch_nb < best_epoch:
os.remove(file)
files = glob.glob('*.pkl')
for file in files:
epoch_nb = int(file.split('.')[0])
if epoch_nb > best_epoch:
os.remove(file)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# Restore best model
print('Loading {}th epoch'.format(best_epoch))
model.load_state_dict(torch.load('{}.pkl'.format(best_epoch)))
# Testing
compute_test()
idx_test = idx_test.cuda()
features, adj, labels = Variable(features), Variable(adj), Variable(labels)
def compute_test():
model.eval()
output = model(features, adj)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.data.item()),
"accuracy= {:.4f}".format(acc_test.data.item()))
# Load model
filename = glob.glob('{}_{}_*.pkl'.format(model._get_name(), args.dataset))[0]
print("Loading {}...".format(filename))
if args.cuda:
model_location = 'cuda'
else:
model_location = 'cpu'
state_dict = torch.load(filename, map_location=model_location)
model.load_state_dict(state_dict)
model.eval()
print('Loaded {}.'.format(filename))
# Testing
compute_test()
state_dict['gc1.weight']
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
features, adj, labels = Variable(features), Variable(adj), Variable(labels)
model = GAT(nfeat=features.shape[1],
nhid=12,
nclass=int(labels.max()) + 1,
dropout=0.6,
nheads=8,
alpha=0.2)
model.load_state_dict(torch.load('693.pkl'))
compute_test()
# x = torch.cat([att(features, adj) for att in model.attentions], dim=1)
# embeddings = x.cpu().data.numpy()
# time_start = time.time()
#
# pca_50 = PCA(n_components=50)
# pca_result_50 = pca_50.fit_transform(embeddings)
#
# tsne = TSNE(n_components=2, verbose=1, perplexity=50, n_iter=1000)
# tsne_results = tsne.fit_transform(pca_result_50)
#
# feat_cols = ['tsne-one', 'tsne-two']
# df = pd.DataFrame(tsne_results, columns=feat_cols)
# df['labels'] = labels.cpu().data.numpy()
