def run(n_trials=100):
start_time = time.time()
# Load data
adj, features, labels, idx_train, idx_test = load_data()
epochs = 200
lr = 0.01
weight_decay = 5e-4
hidden = 16
dropout = 0.5
for i in range(n_trials):
# Model and optimizer
model = GCN(nfeat=features.shape[1],
nhid=hidden,
nclass=labels.max().item() + 1,
dropout=dropout)
optimizer = optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
# Train model
for _ in range(epochs):
train(model, features, adj, idx_train, labels, optimizer)
done_training_time = time.time()
# Testing
metrics = test(model, features, adj, idx_test, labels)
params = model.state_dict()
for k in params:
params[k] = params[k].reshape(-1).data.tolist()
final_results = {**params, **metrics}
done_testing_time = time.time()
# Cache to file
records_filename = "records.csv"
write_results_to_file(records_filename, final_results)
# report_progress(i, start_time, done_training_time,
# done_testing_time, end_time)
end_time = time.time()
speed = n_trials / (end_time - start_time)
print(f"{n_trials} tasks completed in {end_time - start_time}.")
print(f"{round(speed, 3)} tasks/second for non-parallel implementation.")
bad_counter = 0
best = np.inf
best_epoch = 0
for epoch in range(args.epochs):
loss_val, acc_val, epoch_time = train(epoch)
epoch_time_list.append(epoch_time)
criteria = -acc_val if args.criteria == 'acc' else loss_val
criterias.append(criteria)
if args.early_stopping:
if criterias[-1] < best:
best = criterias[-1]
best_epoch = epoch
bad_counter = 0
if args.save:
torch.save(model.state_dict(), '{}.pkl'.format(run_id))
else:
bad_counter += 1
if bad_counter >= args.early_stopping and loss_val < args.loss_threshold:
print("Early stopping...")
break
print("Optimization Finished!")
total_time = time.time() - t_total
mean_time = np.mean(epoch_time_list)
print("Total time elapsed: {:.4f}s".format(total_time))
print("Time per epoch: {:.4f}s".format(mean_time))
if args.early_stopping and args.save:
print('Loading {}th epoch'.format(best_epoch))
iter += 1
index = content_g[subgraph]['index_subgraph']
idx_train = content_g[subgraph]['idx_train']
idx_val = content_g[subgraph]['idx_val']
adj = content_g[subgraph]['adj']
adj = torch.FloatTensor(np.array(adj.todense()))
val_loss, train_loss, train_acc, val_acc = train(
iter, features[index], adj, labels[index], idx_train, idx_val)
loss_values.append(val_loss)
train_loss_list.append(train_loss)
train_acc_list.append(train_acc)
val_loss_list.append(val_loss)
val_acc_list.append(val_acc)
torch.save(model.state_dict(), '{}.pkl'.format(iter))
if loss_values[-1] < best:
best = loss_values[-1]
best_epoch = iter
bad_counter = 0
else:
bad_counter += 1
if bad_counter == args.patience:
break
files = glob.glob('*.pkl')
for file in files:
epoch_nb = int(file.split('.')[0])
if epoch_nb < best_epoch:
os.remove(file)
'time: {:.4f}s'.format(time.time() - t))
def test():
print ('testing')
model.eval()
O=[]
for _ in range(len(x_test)):
x_s = torch.Tensor(x_train[_]).float()[None, :].permute(1,0).cuda()
output = model(x_s, adj)
O.append(output)
with open('31_365/pred_1_5_1_30_SL1.pkl','w') as f:
pickle.dump(O,f)
t_total = time.time()
for epoch in range(10):
train(epoch)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
test()
print ('saving')
# save model name - GCN_nfeat_nhid1..._nclass_batchsize
torch.save(model.state_dict(), '31_365/models/GCN_1_5_1_30_SL1.pt')
model.eval()
output, embeddings = model(features, adj)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
# print(output[idx_test],'output[idx_test]')
# print(labels_df.columns)
# print(labels[idx_test],'labels[idx_test]')
# writer.add_scalar('Loss/test', loss_test, epoch)
print(df_merged_labels.iloc[idx_test]['Object'])
print(df_merged_labels.iloc[:, 13:].columns)
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
# Train model
t_total = time.time()
from tqdm import trange
for epoch in trange(args.epochs):
train(epoch)
torch.save(model.state_dict(), '/home/arjun/pygcn/models_exp/test')
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# print(df_merged_labels.iloc[idx_test.detach().cpu(), 4], 'test')
# Testing
test()
def train_gcn(dataset,
test_ratio=0.5,
val_ratio=0.2,
seed=1,
n_hidden=64,
n_epochs=200,
lr=1e-2,
weight_decay=5e-4,
dropout=0.5,
use_embs=False,
verbose=True,
cuda=False):
data = dataset.get_data()
# train text embs
if use_embs:
pad_ix, n_tokens, matrix, pretrained_embs = data['features']
if pretrained_embs is not None:
pretrained_embs = torch.FloatTensor(pretrained_embs)
features = torch.LongTensor(matrix)
else:
pad_ix = None
n_tokens = None
pretrained_embs = None
features = torch.FloatTensor(data['features'])
labels = torch.LongTensor(data['labels'])
n = len(data['ids'])
train_mask, val_mask, test_mask = get_masks(n,
data['main_ids'],
data['main_labels'],
test_ratio=test_ratio,
val_ratio=val_ratio,
seed=seed)
train_mask = torch.BoolTensor(train_mask)
val_mask = torch.BoolTensor(val_mask)
test_mask = torch.BoolTensor(test_mask)
if cuda:
torch.cuda.set_device("cuda:0")
features = features.cuda()
labels = labels.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
g = DGLGraph(data['graph'])
g = dgl.transform.add_self_loop(g)
n_edges = g.number_of_edges()
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
if cuda:
norm = norm.cuda()
g.ndata['norm'] = norm.unsqueeze(1)
if use_embs:
if pretrained_embs is not None:
in_feats = 100
else:
in_feats = 64
else:
in_feats = features.shape[1]
# + 1 for unknown class
n_classes = data['n_classes'] + 1
model = GCN(g,
in_feats=in_feats,
n_hidden=n_hidden,
n_classes=n_classes,
activation=F.relu,
dropout=dropout,
use_embs=use_embs,
pretrained_embs=pretrained_embs,
pad_ix=pad_ix,
n_tokens=n_tokens)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.9,
patience=20,
min_lr=1e-10)
best_f1 = -100
# initialize graph
dur = []
for epoch in range(n_epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
mask_probs = torch.empty(features.shape).uniform_(0, 1)
if cuda:
mask_probs = mask_probs.cuda()
mask_features = torch.where(mask_probs > 0.2, features,
torch.zeros_like(features))
logits = model(mask_features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
f1 = evaluate(model, features, labels, val_mask)
scheduler.step(1 - f1)
if f1 > best_f1:
best_f1 = f1
torch.save(model.state_dict(), 'best_model.pt')
if verbose:
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | F1 {:.4f} | "
"ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur),
loss.item(), f1,
n_edges / np.mean(dur) / 1000))
model.load_state_dict(torch.load('best_model.pt'))
f1 = evaluate(model, features, labels, test_mask)
if verbose:
print()
print("Test F1 {:.2}".format(f1))
return f1
def main(args):
# convert boolean type for args
assert args.use_ist in ['True', 'False'], ["Only True or False for use_ist, get ",
args.use_ist]
assert args.split_input in ['True', 'False'], ["Only True or False for split_input, get ",
args.split_input]
assert args.split_output in ['True', 'False'], ["Only True or False for split_output, get ",
args.split_output]
assert args.self_loop in ['True', 'False'], ["Only True or False for self_loop, get ",
args.self_loop]
assert args.use_layernorm in ['True', 'False'], ["Only True or False for use_layernorm, get ",
args.use_layernorm]
assert args.use_random_proj in ['True', 'False'], ["Only True or False for use_random_proj, get ",
args.use_random_proj]
use_ist = (args.use_ist == 'True')
split_input = (args.split_input == 'True')
split_output = (args.split_output == 'True')
self_loop = (args.self_loop == 'True')
use_layernorm = (args.use_layernorm == 'True')
use_random_proj = (args.use_random_proj == 'True')
# make sure hidden layer is the correct shape
assert (args.n_hidden % args.num_subnet) == 0
# load and preprocess dataset
global t0
if args.dataset in {'cora', 'citeseer', 'pubmed'}:
data = load_data(args)
else:
raise NotImplementedError(f'{args.dataset} is not a valid dataset')
# randomly project the input to make it dense
if use_random_proj:
# densify input features with random projection
from sklearn import random_projection
# make sure input features are divisible by number of subnets
# otherwise some parameters of the last subnet will be handled improperly
n_components = int(data.features.shape[-1] / args.num_subnet) * args.num_subnet
transformer = random_projection.GaussianRandomProjection(n_components=n_components)
new_feature = transformer.fit_transform(data.features)
features = torch.FloatTensor(new_feature)
else:
assert (data.features.shape[-1] % args.num_subnet) == 0.
features = torch.FloatTensor(data.features)
labels = torch.LongTensor(data.labels)
train_mask = torch.ByteTensor(data.train_mask)
val_mask = torch.ByteTensor(data.val_mask)
test_mask = torch.ByteTensor(data.test_mask)
in_feats = features.shape[1]
n_classes = data.num_labels
n_edges = data.graph.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes,
train_mask.sum().item(),
val_mask.sum().item(),
test_mask.sum().item()))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
features = features.to(device)
labels = labels.to(device)
train_mask = train_mask.to(device)
val_mask = val_mask.to(device)
test_mask = test_mask.to(device)
# graph preprocess and calculate normalization factor
g = data.graph
# add self loop
if self_loop:
g.remove_edges_from(nx.selfloop_edges(g))
g.add_edges_from(zip(g.nodes(), g.nodes()))
g = DGLGraph(g)
g = g.to(device)
n_edges = g.number_of_edges()
# normalization
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
norm = norm.to(device)
g.ndata['norm'] = norm.unsqueeze(1)
# create GCN model
model = GCN(
g, in_feats, args.n_hidden, n_classes, args.n_layers, F.relu,
args.dropout, use_layernorm)
model = model.to(device)
loss_fcn = torch.nn.CrossEntropyLoss()
# initialize graph
dur = []
record = []
sub_models = []
opt_list = []
sub_dict_list = []
main_dict = None
for epoch in range(args.n_epochs):
if epoch >= 3:
t0 = time.time()
if use_ist:
model.eval()
# IST training:
# Distribute parameter to sub networks
num_subnet = args.num_subnet
if (epoch % args.iter_per_site) == 0.:
main_dict = model.state_dict()
feats_idx = [] # store all layer indices within a single list
# create input partition
if split_input:
feats_idx.append(torch.chunk(torch.randperm(in_feats), num_subnet))
else:
feats_idx.append(None)
# create hidden layer partitions
for i in range(1, args.n_layers):
feats_idx.append(torch.chunk(torch.randperm(args.n_hidden), num_subnet))
# create output layer partitions
if split_output:
feats_idx.append(torch.chunk(torch.randperm(args.n_hidden), num_subnet))
else:
feats_idx.append(None)
for subnet_id in range(args.num_subnet):
if (epoch % args.iter_per_site) == 0.:
# create the sub model to train
sub_model = GCN(
g, in_feats, args.n_hidden, n_classes,
args.n_layers, F.relu, args.dropout, use_layernorm,
split_input, split_output, args.num_subnet)
sub_model = sub_model.to(device)
sub_dict = main_dict.copy()
# split input params
if split_input:
idx = feats_idx[0][subnet_id]
sub_dict['layers.0.weight'] = main_dict['layers.0.weight'][idx, :]
# split hidden params (and output params)
for i in range(1, args.n_layers + 1):
if i == args.n_layers and not split_output:
pass # params stay the same
else:
idx = feats_idx[i][subnet_id]
sub_dict[f'layers.{i - 1}.weight'] = sub_dict[f'layers.{i -1}.weight'][:, idx]
sub_dict[f'layers.{i - 1}.bias'] = main_dict[f'layers.{i - 1}.bias'][idx]
sub_dict[f'layers.{i}.weight'] = main_dict[f'layers.{i}.weight'][idx, :]
# use a lr scheduler
curr_lr = args.lr
if epoch >= int(args.n_epochs*0.5):
curr_lr /= 10
if epoch >= int(args.n_epochs*0.75):
curr_lr /= 10
# import params into subnet for training
sub_model.load_state_dict(sub_dict)
sub_models.append(sub_model)
sub_models = sub_models[-num_subnet:]
optimizer = torch.optim.Adam(
sub_model.parameters(), lr=curr_lr,
weight_decay=args.weight_decay)
opt_list.append(optimizer)
opt_list = opt_list[-num_subnet:]
else:
sub_model = sub_models[subnet_id]
optimizer = opt_list[subnet_id]
# train a sub network
optimizer.zero_grad()
sub_model.train()
if split_input:
model_input = features[:, feats_idx[0][subnet_id]]
else:
model_input = features
logits = sub_model(model_input)
loss = loss_fcn(logits[train_mask], labels[train_mask])
# reset optimization for every sub training
loss.backward()
optimizer.step()
# save sub model parameter
if (
((epoch + 1) % args.iter_per_site == 0.)
or (epoch == args.n_epochs - 1)):
sub_dict = sub_model.state_dict()
sub_dict_list.append(sub_dict)
sub_dict_list = sub_dict_list[-num_subnet:]
# Merge parameter to main network:
# force aggregation if training about to end
if (
((epoch + 1) % args.iter_per_site == 0.)
or (epoch == args.n_epochs - 1)):
#keys = main_dict.keys()
update_dict = main_dict.copy()
# copy in the input parameters
if split_input:
if args.n_layers <= 1 and not split_output:
for idx, sub_dict in zip(feats_idx[0], sub_dict_list):
update_dict['layers.0.weight'][idx, :] = sub_dict['layers.0.weight']
else:
for i, sub_dict in enumerate(sub_dict_list):
curr_idx = feats_idx[0][i]
next_idx = feats_idx[1][i]
correct_rows = update_dict['layers.0.weight'][curr_idx, :]
correct_rows[:, next_idx] = sub_dict['layers.0.weight']
update_dict['layers.0.weight'][curr_idx, :] = correct_rows
else:
if args.n_layers <= 1 and not split_output:
update_dict['layers.0.weight'] = sum(sub_dict['layers.0.weight'] for sub_dict in sub_dict_list) / len(sub_dict_list)
else:
for i, sub_dict in enumerate(sub_dict_list):
next_idx = feats_idx[1][i]
update_dict['layers.0.weight'][:, next_idx] = sub_dict['layers.0.weight']
# copy the rest of the parameters
for i in range(1, args.n_layers + 1):
if i == args.n_layers:
if not split_output:
update_dict[f'layers.{i-1}.bias'] = sum(sub_dict[f'layers.{i-1}.bias'] for sub_dict in sub_dict_list) / len(sub_dict_list)
update_dict[f'layers.{i}.weight'] = sum(sub_dict[f'layers.{i}.weight'] for sub_dict in sub_dict_list) / len(sub_dict_list)
else:
for idx, sub_dict in zip(feats_idx[i], sub_dict_list):
update_dict[f'layers.{i-1}.bias'][idx] = sub_dict[f'layers.{i-1}.bias']
update_dict[f'layers.{i}.weight'][idx, :] = sub_dict[f'layers.{i}.weight']
else:
if i >= args.n_layers - 1 and not split_output:
for idx, sub_dict in zip(feats_idx[i], sub_dict_list):
update_dict[f'layers.{i-1}.bias'][idx] = sub_dict[f'layers.{i-1}.bias']
update_dict[f'layers.{i}.weight'][idx, :] = sub_dict[f'layers.{i}.weight']
else:
for idx, sub_dict in enumerate(sub_dict_list):
curr_idx = feats_idx[i][idx]
next_idx = feats_idx[i+1][idx]
update_dict[f'layers.{i-1}.bias'][curr_idx] = sub_dict[f'layers.{i-1}.bias']
correct_rows = update_dict[f'layers.{i}.weight'][curr_idx, :]
correct_rows[:, next_idx] = sub_dict[f'layers.{i}.weight']
update_dict[f'layers.{i}.weight'][curr_idx, :] = correct_rows
model.load_state_dict(update_dict)
else:
raise NotImplementedError('Should train with IST')
if epoch >= 3:
dur.append(time.time() - t0)
acc_val = evaluate(model, features, labels, val_mask)
acc_test = evaluate(model, features, labels, test_mask)
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Val Accuracy {:.4f} | Test Accuracy {:.4f} |"
"ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.item(),
acc_val, acc_test, n_edges / np.mean(dur) / 1000))
record.append([acc_val, acc_test])
all_test_acc = [v[1] for v in record]
all_val_acc = [v[0] for v in record]
acc = evaluate(model, features, labels, test_mask)
print(f"Final Test Accuracy: {acc:.4f}")
print(f"Best Val Accuracy: {max(all_val_acc):.4f}")
print(f"Best Test Accuracy: {max(all_test_acc):.4f}")
B = torch.sign(outputs).data.cpu().numpy()
binary_target = Variable(torch.Tensor(B)).cuda()
loss += quan_loss(outputs, binary_target) * 0.00001
# loss += torch.norm(torch.dot(outputs.t(), outputs), 2) * 0.00000001
# loss += torch.sum(outputs)/outputs.size(0) * 0.0000000001
loss.backward()
optimizer.step()
if (i + 1) % 100 == 0:
print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f' %
(epoch + 1, n_epoch, i + 1, len(unlabeled_loader),
loss.data[0]))
torch.save(
{
'state_dict': gcn.state_dict(),
'mean_val': mean_val,
'anchor': anchor,
'anchor_affnty': anchor_affnty
}, './ImageNet_%d_%d' % (n_labeled, n_bits))
'''
model = torch.load('./ImageNet_%d_%d' % (n_labeled, n_bits))
print model.keys()
anchor = model['anchor']
anchor_affnty = model['anchor_affnty']
gcn = GCN(4096, n_bits, 1000, anchor_affnty, 40)
gcn.load_state_dict(model['state_dict'])
gcn.cuda()
'''
traindata, testdata = load_ImageNet_full(mean_val)
print(
f'Epoch {epoch}, Loss: {loss:.4f}, Test Loss: {test_loss:.4f}, Test Accuracy: {test_acc:.4f}'
)
# scheduler.step(test_loss)
if epoch % 10 == 0:
plt.plot(train_losses)
plt.title('Loss vs Epoch')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.savefig('loss_vs_epoch')
plt.close()
plt.plot(train_losses, label='Train Loss')
plt.plot(test_losses, label='Test Loss')
plt.title('Loss vs Epoch')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend()
plt.savefig('train_test_loss_vs_epoch')
plt.close()
plt.plot(test_accs)
plt.title('Test Accuracy vs Epoch')
plt.ylabel('Test Accuracy')
plt.xlabel('Epoch')
plt.savefig('test_acc_vs_epoch')
plt.close()
torch.save(model.state_dict(), 'gcn_state_dict.pt')
# Train model
t_total = time.time()
loss_values = []
bad_counter = 0
best_loss = np.inf
best_epoch = 0
for epoch in range(args.epochs):
loss_values.append(train(epoch))
if loss_values[-1] < best_loss:
best_loss = loss_values[-1]
best_epoch = epoch
bad_counter = 0
best_state_dict = copy.deepcopy(model.state_dict())
else:
bad_counter += 1
if bad_counter == args.patience:
break
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# Restore best model
print('Loading {}th epoch'.format(best_epoch))
model.load_state_dict(best_state_dict)
acc_test = test()
acc_test = accuracy(output[idx_test],
labels[idx_test],
flag=False,
target_names=df_merged_labels.iloc[:, 13:].columns)
acc_train = accuracy(output[idx_train],
labels[idx_train],
flag=False,
target_names=df_merged_labels.iloc[:, 13:].columns)
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
# Train model
t_total = time.time()
for epoch in range(args.epochs):
train(epoch)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
#print(df_merged_labels.iloc[idx_test.detach().cpu(), 4], 'test')
#torch.save(model.state_dict(), '/home/arjun/same_set_best_72_robert_40_text_int_22.pt') best use 71
#torch.save(model.state_dict(), '/home/arjun/same_set_best_72_robert_40_text_int_22_symmetric_adj_array.pt') #68
torch.save(
model.state_dict(),
'/home/arjun/same_set_best_72_robert_40_text_int_22_symmetric_adj_array_stan_rand.pt'
)
#torch.save(model.state_dict(), '/home/arjun/same_set_best_72_robert_40_text_int_22_preprocess_no_preprocess_int.pt') worst porter 69
# Testing
test()
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
# Train model
t_total = time.time()
loss_values = []
bad_counter = 0
best = args.epochs + 1
best_epoch = 0
for epoch in range(args.epochs):
loss_values.append(train(epoch))
torch.save(model.state_dict(), '{}.pkl'.format(epoch))
if loss_values[-1] < best:
best = loss_values[-1]
best_epoch = epoch
bad_counter = 0
else:
bad_counter += 1
if bad_counter == args.patience:
break
files = glob.glob('*.pkl')
for file in files:
epoch_nb = int(file.split('.')[0])
if epoch_nb < best_epoch:
os.remove(file)
#criterion = nn.NLLLoss().to(device)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = optim.Adam(model.parameters(),lr=args.lr, weight_decay=args.weight_decay)
print("Total number of forward processes:" + str(args.epochs * args.batch_size))
if args.save_validation == True:
val_df = np.zeros([epochs_num * cv_fold,4]) #train_loss, val_loss, train_acc, val_acc
if args.save_test == True: # save test dataset
test_index = np.where(test_mask == True)[0]
pkl.dump(test_index, open(os.path.join(args.save,"ind." + args.dataset + ".index"),'wb'))
np.savetxt("ind.{}.test.index".format(args.dataset),test_index, fmt="%d")
# save initialize parameters
torch.save({'state_dict': model.state_dict()}, os.path.join(args.save, 'temp_hidden_' + str(args.hidden1) + '_linear_' + str(args.linear) + '_lr_'+str(args.lr)+'_wd_'+str(args.weight_decay)+'_bs_'+str(args.batch_size)+'_dt_' + str(args.dropout) + '.pth'))
for fold in range(cv_fold):
# load initialized parameters
checkpoint = torch.load(os.path.join(args.save, 'temp_hidden_' + str(args.hidden1) + '_linear_' + str(args.linear) + '_lr_'+str(args.lr)+'_wd_'+str(args.weight_decay)+'_bs_'+str(args.batch_size)+ '_dt_' + str(args.dropout) + '.pth'))
model.load_state_dict(checkpoint['state_dict'])
tmp_mask = train_mask.copy()
if args.cv == 0:
train_mask_tmp = tmp_mask[-1]
val_mask = tmp_mask[0]
else:
val_mask = tmp_mask.pop(fold)
train_mask_tmp = np.sum(tmp_mask, axis=0, dtype=bool)
best_acc = 0
batch_time_meter = RunningAverageMeter()
end = time.time()
print("\n\nRecall, V starts of with equal weights for all neighbours!")
V = normalise(adj_indices, model.edge_weights.detach(), adj_size, eps)
#values row 0
print('Row 0:\n', V[torch.where(adj_indices[0] == 0)[0]])
#values row 25
print('Row 25:\n', V[torch.where(adj_indices[0] == 25)[0]])
#values row 50
print('Row 50:\n', V[torch.where(adj_indices[0] == 50)[0]])
#values row 75
print('Row 75:\n', V[torch.where(adj_indices[0] == 75)[0]])
print('smallest value in V:', torch.min(V))
print('Average time per epoch:', sum(t_lst) / len(t_lst))
torch.save(
{
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'epoch': epoch
}, path + 'model-optimised.pt')
torch.save(
{
'projection': args.projection,
'learnable': args.learnable,
'nrm_mthd': name,
'indices': adj._indices(),
'values': adj._values(),
'size': adj.size(),
'features': features,
'labels': labels,
'idx_train': idx_train,
'idx_val': idx_val,
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
return acc_test
model_file = 'model_save/' + args.dataset + '.pkl'
# Train model
t_total = time.time()
max_acc = 0
acc_list = []
for epoch in range(args.epochs):
val_acc = train(epoch)
if val_acc > max_acc:
max_acc = val_acc
torch.save(model.state_dict(), model_file)
acc_list.append(val_acc)
if args.load_best:
model.load_state_dict(torch.load(model_file))
print(max(acc_list))
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# Testing
acc_test = test()
if len(args.save_file) > 0:
with open(args.save_file, 'a') as f:
f.write('GCN %.4f' % acc_test)
f.write('\n')
vecs = []
cnt += sum(res)
for j in res:
print(int(j), file=f)
print("")
print(cnt, "/", len(test_data))
#loss_test = F.nll_loss(output[idx_test], labels[idx_test])
#acc_test = accuracy(output[idx_test], labels[idx_test])
#print("Test set results:",
# "loss= {:.4f}".format(loss_test.item()),
# "accuracy= {:.4f}".format(acc_test.item()))
# Train model
t_total = time.time()
if args.train:
for epoch in range(args.epochs):
train(epoch)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
torch.save(model.state_dict(), 'model.mdl')
else:
model.load_state_dict(torch.load('model.mdl'))
# Extract Embedding
#emb = torch.nn.Sequential(*list(model.children())[:-1])
# Testing
test()
