class Runtime:
def __init__(self, dataset, hidden_channels: int) -> None:
self.data = dataset[0]
self.model = GCN(dataset=dataset, hidden_channels=hidden_channels)
self.optimizer = torch.optim.Adam(self.model.parameters(),
lr=0.01,
weight_decay=5e-4)
self.criterion = torch.nn.CrossEntropyLoss()
def train(self):
self.model.train()
self.optimizer.zero_grad() # Clear gradients.
out = self.model(
self.data.x,
self.data.edge_index) # Perform a single forward pass.
loss = self.criterion(
out[self.data.train_mask], self.data.y[self.data.train_mask]
) # Compute the loss solely based on the training nodes.
loss.backward() # Derive gradients.
self.optimizer.step() # Update parameters based on gradients.
return loss
def test(self):
self.model.eval()
out = self.model(self.data.x, self.data.edge_index)
pred = out.argmax(dim=1) # Use the class with highest probability.
test_correct = pred[self.data.test_mask] == self.data.y[
self.data.test_mask] # Check against ground-truth labels.
test_acc = int(test_correct.sum()) / int(
self.data.test_mask.sum()) # Derive ratio of correct predictions.
return test_acc
def load_trained_vector(epoch, number, n2i_f, file_homes):
global node2index
node2index = cPickle.load(n2i_f)
node_count = len(node2index)
node_dim = 128
n_repr = 128
gcn = GCN(node_count, node_dim, n_repr)
gcn.load_state_dict(
torch.load(file_homes + '/networks/GCN_%d_%d.pth' % (number, epoch),
map_location='cpu'))
f = open(files_home + '/networks/adj_matrix_%d_full' % (number), 'rb')
full_adj_matrix = cPickle.load(f)
full_adj_matrix = sparse_mx_to_torch_sparse_tensor(full_adj_matrix)
init_input = torch.LongTensor([j for j in range(0, node_count)])
gcn.eval()
rp_matrix = gcn(init_input, full_adj_matrix)
#gcn.to(device)
return rp_matrix.double()
def sgcn_plus_v2(feat_data,
labels,
lap_matrix,
train_nodes,
valid_nodes,
test_nodes,
args,
device,
calculate_grad_vars=False):
# use multiprocess sample data
process_ids = np.arange(args.batch_num)
pool = mp.Pool(args.pool_num)
lap_matrix_sq = lap_matrix.multiply(lap_matrix)
jobs = prepare_data(pool, sampler, process_ids, train_nodes, samp_num_list,
len(feat_data), lap_matrix, lap_matrix_sq,
args.n_layers)
susage = GCN(nfeat=feat_data.shape[1],
nhid=args.nhid,
num_classes=num_classes,
layers=args.n_layers,
dropout=args.dropout).to(device)
susage.to(device)
print(susage)
adjs_full, input_nodes_full, sampled_nodes_full = full_batch_sampler(
train_nodes, len(feat_data), lap_matrix, args.n_layers)
adjs_full = package_mxl(adjs_full, device)
# this stupid wrapper is only used for sgcn++
forward_wrapper = ForwardWrapperMomentum(len(feat_data), args.nhid,
args.n_layers, num_classes)
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
susage.parameters()),
lr=0.1)
loss_train = []
loss_test = []
grad_variance_all = []
loss_train_all = []
best_model = copy.deepcopy(susage)
best_val_loss = 10 # randomly pick a large number is fine
best_val_index = 0
best_val_cnt = 0
for epoch in np.arange(args.epoch_num):
# fetch train data
train_data = [job.get() for job in jobs]
pool.close()
pool.join()
# prepare next epoch train data
pool = mp.Pool(args.pool_num)
jobs = prepare_data(pool, sampler,
process_ids, train_nodes, samp_num_list,
len(feat_data), lap_matrix, lap_matrix_sq,
args.n_layers)
inner_loop_num = args.batch_num
# compare with sgcn_plus, the only difference is we use multi_level_spider_step_v1 here
cur_train_loss, cur_train_loss_all, grad_variance = multi_level_momentum_step(
susage,
optimizer,
feat_data,
labels,
train_nodes,
valid_nodes,
adjs_full,
sampled_nodes_full,
train_data,
inner_loop_num,
forward_wrapper,
device,
calculate_grad_vars=calculate_grad_vars)
loss_train_all.extend(cur_train_loss_all)
grad_variance_all.extend(grad_variance)
# calculate validate loss
susage.eval()
susage.zero_grad()
val_loss, _ = susage.calculate_loss_grad(feat_data, adjs_full, labels,
valid_nodes)
if val_loss + 0.01 < best_val_loss:
best_val_loss = val_loss
del best_model
best_model = copy.deepcopy(susage)
best_val_index = epoch
best_val_cnt = 0
cur_test_loss = val_loss
best_val_cnt += 1
loss_train.append(cur_train_loss)
loss_test.append(cur_test_loss)
# print progress
print('Epoch: ', epoch, '| train loss: %.8f' % cur_train_loss,
'| test loss: %.8f' % cur_test_loss)
if best_val_cnt > 10:
break
f1_score_test = best_model.calculate_f1(feat_data, adjs_full, labels,
test_nodes)
return best_model, loss_train[:
best_val_index], loss_test[:
best_val_index], loss_train_all, f1_score_test, grad_variance_all
def sgcn(feat_data,
labels,
lap_matrix,
train_nodes,
valid_nodes,
test_nodes,
args,
device,
calculate_grad_vars=False,
full_batch=False):
# use multiprocess sample data
process_ids = np.arange(args.batch_num)
pool = mp.Pool(args.pool_num)
lap_matrix_sq = lap_matrix.multiply(lap_matrix)
jobs = prepare_data(pool, sampler, process_ids, train_nodes, samp_num_list,
len(feat_data), lap_matrix, lap_matrix_sq,
args.n_layers)
susage = GCN(nfeat=feat_data.shape[1],
nhid=args.nhid,
num_classes=num_classes,
layers=args.n_layers,
dropout=args.dropout).to(device)
susage.to(device)
print(susage)
adjs_full, input_nodes_full, sampled_nodes_full = full_batch_sampler(
train_nodes, len(feat_data), lap_matrix, args.n_layers)
adjs_full = package_mxl(adjs_full, device)
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
susage.parameters()),
lr=args.lr)
loss_train = []
loss_test = []
grad_variance_all = []
loss_train_all = []
best_model = copy.deepcopy(susage)
best_val_loss = 10 # randomly pick a large number is fine
best_val_index = 0
best_val_cnt = 0
for epoch in np.arange(args.epoch_num):
# fetch train data
train_data = [job.get() for job in jobs]
pool.close()
pool.join()
# prepare next epoch train data
pool = mp.Pool(args.pool_num)
jobs = prepare_data(pool, sampler,
process_ids, train_nodes, samp_num_list,
len(feat_data), lap_matrix, lap_matrix_sq,
args.n_layers)
# it can also run full-batch GD by ignoring all the samplings
if full_batch:
inner_loop_num = args.batch_num
cur_train_loss, cur_train_loss_all, grad_variance = full_step(
susage,
optimizer,
feat_data,
labels,
train_nodes,
valid_nodes,
adjs_full,
train_data,
inner_loop_num,
device,
calculate_grad_vars=calculate_grad_vars)
else:
inner_loop_num = args.batch_num
cur_train_loss, cur_train_loss_all, grad_variance = sgd_step(
susage,
optimizer,
feat_data,
labels,
train_nodes,
valid_nodes,
adjs_full,
train_data,
inner_loop_num,
device,
calculate_grad_vars=calculate_grad_vars)
loss_train_all.extend(cur_train_loss_all)
grad_variance_all.extend(grad_variance)
# calculate test loss
susage.eval()
susage.zero_grad()
val_loss, _ = susage.calculate_loss_grad(feat_data, adjs_full, labels,
valid_nodes)
if val_loss + 5e-4 < best_val_loss:
best_val_loss = val_loss
del best_model
best_model = copy.deepcopy(susage)
best_val_index = epoch
best_val_cnt = 0
cur_test_loss = val_loss
best_val_cnt += 1
loss_train.append(cur_train_loss)
loss_test.append(cur_test_loss)
# print progress
print('Epoch: ', epoch, '| train loss: %.8f' % cur_train_loss,
'| test loss: %.8f' % cur_test_loss)
if best_val_cnt > 10:
break
f1_score_test = best_model.calculate_f1(feat_data, adjs_full, labels,
test_nodes)
return best_model, loss_train, loss_test, loss_train_all, f1_score_test, grad_variance_all, best_val_index
num_features_nonzero = feature._nnz()
feat_dim = feature.shape[1]
net = GCN(feat_dim, num_classes, num_features_nonzero)
net.to(device)
optimizer = optim.Adam(net.parameters(), lr=args.learning_rate)
net.train()
for epoch in range(args.epochs):
out = net((feature, support))
out = out[0]
loss = masked_loss(out, train_label, train_mask)
loss += args.weight_decay * net.l2_loss()
acc = masked_acc(out, train_label, train_mask)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch % 10 == 0:
print(epoch, loss.item(), acc.item())
net.eval()
out = net((feature, support))
out = out[0]
acc = masked_acc(out, test_label, test_mask)
print('test:', acc.item())
train_idx = train_idx.to(device)
val_idx = val_idx.to(device)
test_idx = test_idx.to(device)
print("Start training the model.............")
start = time.time()
for epoch in range(epochs):
model.train()
optimizer.zero_grad()
output = model(X, A)
train_loss = F.nll_loss(output[train_idx], torch.max(y[train_idx], 1)[1])
train_accuracy = accuracy(output[train_idx], torch.max(y[train_idx], 1)[1])
train_loss.backward()
optimizer.step()
model.eval()
output = model(X, A)
val_loss = F.nll_loss(output[val_idx], torch.max(y[val_idx], 1)[1])
val_accuracy = accuracy(output[val_idx], torch.max(y[val_idx], 1)[1])
print('Epoch: {:04d}'.format(epoch + 1),
'train loss: {:.4f}'.format(train_loss),
'train accuracy: {:.4f}%'.format(train_accuracy),
'validation loss: {:.4f}'.format(val_loss),
'validation accuracy: {:.4f}%'.format(val_accuracy))
if epoch == epochs - 1: continue
val_window.append(val_loss)
if len(val_window) >= window_size:
if stop_criterion(val_window, window_size):
print("*" * 30, end=" ")
# @Time : 2020/9/28 10:49 AM
# @Author : huangyajian
# @File : train.py
# @Software : PyCharm
# @Comment :
# Reference:**********************************************
from data_loader import gcn_load_data
from model import GCN
load_data_function = gcn_load_data
gcn_model = GCN(load_data_function=load_data_function,
hidden_unit=16,
learning_rate=0.01,
weight_decay=5e-4)
cost_val = []
epochs = 200
for i in range(epochs):
train_loss, train_acc = gcn_model.train()
# print("model loss: {}, model acc: {}".format(train_loss, train_acc))
gcn_model.update()
# val step
val_loss, val_acc = gcn_model.eval()
print(
"iteration: {}, train_loss: {}, train_acc: {}, val_loss: {}, val_acc: {}"
.format(i, train_loss, train_acc, val_loss, val_acc))
cost_val.append(val_loss)
test_loss, test_acc = gcn_model.test()
print("start test, the loss: {}, the acc: {}".format(test_loss, test_acc))
def main(files_home):
starttime = datetime.now()
print('start test model ', starttime)
number = args.number
f = open(os.path.join(files_home, files_name['node_index']), 'rb')
node2index = cPickle.load(f)
f_train = os.path.join(files_home, files_name['train_file'])
f_test = os.path.join(files_home, files_name['test_file']) ###
# testset = Sample_Set_Test(f_test,f_train, node2index)
# testloader = DataLoader(testset, batch_size=32, shuffle=False)
node_count = len(node2index)
node_dim = 128
n_repr = 128
gcn = GCN(node_count, node_dim, n_repr, dropout=args.dropout)
lp = Link_Prediction(n_repr, dropout=args.dropout)
if args.cuda == True:
gcn.cuda()
lp = nn.DataParallel(lp)
lp.cuda()
init_input = torch.LongTensor([j for j in range(0, node_count)]).cuda()
dis2gene_test_true, dis2gene_test_all = get_rank_test_samples(
f_train, f_test)
f = open(files_home + '/networks/adj_matrix_%d_full' % (number), 'rb')
full_adj_matrix = cPickle.load(f)
full_adj_matrix = sparse_mx_to_torch_sparse_tensor(full_adj_matrix).cuda()
for epoch in tqdm(range(0, args.epochs)):
if epoch % 9 != 0 and epoch < args.epochs - 5:
continue
gcn.load_state_dict(
torch.load(files_home + '/networks/GCN_%d_%d.pth' %
(number, epoch)))
lp.load_state_dict(
torch.load(files_home + '/networks/Link_Prediction_%d_%d.pth' %
(number, epoch)))
gcn.eval()
feature_matrix = gcn(init_input, full_adj_matrix)
# test_link(testloader, feature_matrix, lp)
if 0:
print('use gcn to prediction')
ap, prec, recall, f1score = test_priorization_gcn(
dis2gene_test_true, dis2gene_test_all, feature_matrix, lp)
else:
print('use gcn and word2ver to prediction')
ap, prec, recall, f1score = test_priorization_word_gcn(
dis2gene_test_true, dis2gene_test_all, feature_matrix, lp)
print('Performance for number=%d epoch=%d' % (number, epoch))
print('AP: ', ap)
print('Prec: ', prec)
print('Recall: ', recall)
print('F1score: ', f1score)
endtime = datetime.now()
print('finish test model! run spend ', endtime - starttime)
error_list = []
error_list_0to1 = []
error_list_1to0 = []
for i, (X_test, Y_test) in enumerate(test_loader):
#X_test_unique = X_test.unique()
X_test1 = torch.tensor(
X[X["QuestionId"].isin(X_test)]['PairId'].values)
Y_test1 = torch.tensor(
X[X["QuestionId"].isin(X_test)]['Credible'].values)
Q1 = torch.tensor(
X[X["QuestionId"].isin(X_test)]['QScore'].values)
A1 = torch.tensor(
X[X["QuestionId"].isin(X_test)]['AScore'].values)
QId1 = torch.tensor(
X[X["QuestionId"].isin(X_test)]['QuestionId'].values)
gcn_model.eval()
user_gcn_embed = gcn_model(X_Tags_Feature, Adj)
user_gcn_embed.squeeze_()
#predicted = np.rint(F.sigmoid(user_gcn_embed[X_test]).data.numpy())
predicted = torch.max(user_gcn_embed[X_test1], 1)[1].data
# clique size staff
predict_list += predicted.numpy().tolist()
Q_list += Q1.numpy().tolist()
A_list += A1.numpy().tolist()
QId_list += QId1.numpy().tolist()
y_list += Y_test1.numpy().tolist()
#X_list += X_test1.numpy().tolist()
data = pd.DataFrame()
data['QId'] = QId_list
data['QScore'] = Q_list
class ModelRunner:
def __init__(self, params, logger, data_logger=None, epochs_logger=None):
self._logger = logger
self._epoch_logger = epochs_logger
self._data_logger = EmptyLogger() if data_logger is None else data_logger
self._parameters = params
self._lr = params["lr"]
self._is_nni = params['is_nni']
self._device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
self._mse_loss = self.weighted_mse_loss
self._temporal_loss = self.weighted_mse_loss
self.model = GCN(num_of_features=self._parameters["feature_matrices"][0].shape[1],
hid_size=self._parameters["hid_size"],
num_of_classes=self._parameters["number_of_classes"],
activation=self._parameters["activation"],
dropout=self._parameters["dropout"])
self.model = self.model.to(self._device)
self.opt = self._parameters["optimizer"](self.model.parameters(),
lr=self._parameters['lr'],
weight_decay=self._parameters['weight_decay'])
@property
def logger(self):
return self._logger
@property
def data_logger(self):
return self._data_logger
def weighted_mse_loss(self, pred, target, weights=None):
if weights is None:
return ((pred - target) ** 2).sum(dim=1).sum().to(self._device)
elif self._parameters['loss_weights_type'] == 'sqrt(N/Nj)':
weights = torch.tensor(weights).to(device=self._device, dtype=torch.float)
b = (torch.sqrt((weights).sum() / weights) *(pred - target) ** 2).sum(dim=1).sum().to(self._device)
return b
elif self._parameters['loss_weights_type'] == '1/Njs':
weights = torch.tensor(weights).to(device=self._device, dtype=torch.float)
b = (torch.tensor(1. / weights) * (pred - target) ** 2).sum(dim=1).sum().to(self._device)
return b
def run(self):
train_results_l = []
test_results = []
# train
for epoch in range(self._parameters["epochs"]):
train_results = self.train(epoch)
train_results_l.append(train_results)
if epoch == self._parameters["epochs"]-1: # for grid
test_res = self.test(epoch)
test_results.append(test_res)
if self._is_nni:
nni.report_intermediate_result(test_res["f1_score_macro"][-1])
else:
print(self._parameters["it_num"],
self._parameters["iterations"],
epoch + 1,
self._parameters["epochs"],
self._parameters["lr"],
self._parameters["dropout"],
self._parameters["hid_size"],
self._parameters["weight_decay"],
self._parameters["temporal_pen"],
train_results['loss'].item(),
train_results['tempo_loss'].item(),
train_results['loss'].item() + train_results['tempo_loss'].item(),
train_results['f1_score_macro'][-1],
train_results['f1_score_micro'][-1],
test_res["loss"],
test_res["tempo_loss"],
test_res["loss"] + test_res["tempo_loss"],
test_res["f1_score_macro"][-1],
test_res["f1_score_micro"][-1])
self._logger.debug('Epoch: {:04d} '.format(epoch + 1) +
'lr: {:04f} '.format(self._parameters['lr']) +
'dropout: {:04f} '.format(self._parameters['dropout']) +
'hid_size: {:04f} '.format(self._parameters['hid_size']) +
'weight_decay: {:04f} '.format(self._parameters['weight_decay']) +
'temporal_pen: {:04f} '.format(self._parameters['temporal_pen']) +
'reg_loss_train: {:.4f} '.format(train_results['loss']) +
'temp_loss: {:.4f} '.format(train_results['tempo_loss']))
result = self.test('test', print_to_file=True)
if self._is_nni:
nni.report_final_result(result["f1_score_macro"])
return train_results_l, test_results, result, self._parameters
def train(self, epoch):
z_vals, outputs = [], []
labeled_indices = self._parameters['training_inds']
labels = self._parameters['training_labels']
tempo_loss = 0.
loss_train = 0.
self.model.train()
self.opt.zero_grad()
for idx, adj in enumerate(self._parameters["adj_matrices"]):
input_features = torch.from_numpy(self._parameters["feature_matrices"][idx]).to(dtype=torch.float,
device=self._device)
z, output = self.model(input_features, adj)
output = output[labeled_indices[idx], :]
# Njs are the weights of the loss using the adj mx.
# they should be either used or not.
Nj_s = [sum([labels[u][t][j] for u in range(len(self._parameters["adj_matrices"])) for t in range(len(labels[u]))]) for j in
range(self._parameters['number_of_classes'])]
loss_train += self._mse_loss(output, labels[idx], Nj_s)
# loss_train += self._mse_loss(output, labels[idx].float()) #without weights using the build-in mse
z_vals.append(z) # After 1 GCN layer
outputs.append(output) # Final predictions
# counts the number of cross_year_persons
z_appearances = 0.
for t in range(len(z_vals) - 1):
t_inds = self._parameters['training_inds'][t]
t_plus_one_inds = self._parameters['training_inds'][t + 1]
z_inds = [i for i in t_inds if i in t_plus_one_inds]
z_appearances += len(z_inds)
z_val_t = z_vals[t][z_inds, :]
z_val_t_plus_1 = z_vals[t+1][z_inds, :]
loss = self._temporal_loss(z_val_t_plus_1, z_val_t)
tempo_loss += self._parameters["temporal_pen"] * loss
tempo_loss /= z_appearances
loss_train /= sum([len(labeled_indices[u]) for u in range(len(outputs))])
total_loss = loss_train + tempo_loss
total_loss.backward()
self.opt.step()
f1_score_macro, f1_score_micro = [],[]
if epoch == self._parameters['epochs']-1:
for i in range(len(labels)):
f1_mac, f1_mic, list_real, list_pred = self.accuracy_f1_score(outputs[i], labels[i])
f1_score_macro.append(f1_mac)
f1_score_micro.append(f1_mic)
result = {"loss": loss_train,
"f1_score_macro": f1_score_macro,
"f1_score_micro": f1_score_micro,
"tempo_loss": tempo_loss}
return result
def test(self,epoch, print_to_file=False):
z_vals, outputs = [], []
labeled_indices = self._parameters['test_inds']
labels = self._parameters['test_labels']
tempo_loss = 0.
loss_test = 0.
test_z_appearances = 0.
self.model.eval()
for idx, adj in enumerate(self._parameters["adj_matrices"]):
test_mat = torch.from_numpy(self._parameters["feature_matrices"][idx]).to(self._device)
z, output = self.model(*[test_mat, adj])
output = output[labeled_indices[idx], :]
loss_test += self._mse_loss(output, labels[idx].float())
z_vals.append(z)
outputs.append(output)
if print_to_file:
grid_outputs_folder = str(self._parameters['name'])
self._logger.debug("\nprint to files")
for i in range(len(self._parameters["adj_matrices"])):
np_output = outputs[i].cpu().data.numpy()
products_path = os.path.join(os.getcwd(),'dataset',self._parameters["dataset_name"], "gcn_outputs",grid_outputs_folder)
if not os.path.exists(products_path):
os.makedirs(products_path)
with open(os.path.join("dataset",self._parameters["dataset_name"],"gcn_outputs",grid_outputs_folder, "gcn_" + str(i) + ".pkl"), "wb") as f:
pickle.dump(np_output, f, protocol=pickle.HIGHEST_PROTOCOL)
for t in range(len(z_vals) - 1):
t_inds = self._parameters['test_inds'][t]
t_plus_one_inds = self._parameters['test_inds'][t + 1]
z_inds = [i for i in t_inds if i in t_plus_one_inds]
test_z_appearances += len(z_inds)
z_val_t = z_vals[t][z_inds, :]
z_val_t_plus_1 = z_vals[t + 1][z_inds, :]
tempo_loss += self._parameters["temporal_pen"] * self._temporal_loss(z_val_t_plus_1, z_val_t)
tempo_loss /= test_z_appearances
loss_test /= sum([len(labeled_indices[u]) for u in range(len(outputs))])
f1_score_macro, f1_score_micro = [], []
real,pred = [],[]
if epoch == self._parameters['epochs'] - 1 or epoch == 'test':
for i in range(len(labels)): #running over the years
f1_mac, f1_mic, list_real, list_pred = self.accuracy_f1_score(outputs[i], labels[i])
f1_score_macro.append(f1_mac)
f1_score_micro.append(f1_mic)
real.extend(list_real)
pred.extend(list_pred)
self.confusion_matrix(real, pred) # of all years normalized to 1 for the last epoch test
result = {"loss": loss_test.data.item(),
"f1_score_macro": f1_score_macro,
"f1_score_micro": f1_score_micro,
"tempo_loss": tempo_loss.data.item()}
return result
def accuracy_f1_score(self,output, labels):
pred, real = [],[]
for person in range(labels.size(0)): #range of all persons
for label in range(labels.size(1)):
if labels[person,label]==0:
continue
else:
argmax = output[person].max(0)[1]
real.append(label)
pred.append(argmax.cpu().item())
f1_macro = f1_score(real, pred, average='macro')
f1_micro = f1_score(real, pred, average='micro')
return f1_macro, f1_micro, real,pred
def confusion_matrix(self, list_real, list_pred):
matrix = np.zeros((self._parameters["number_of_classes"], self._parameters["number_of_classes"])) # classes X classes
for i in range(len(list_pred)):
matrix[list_real[i], list_pred[i]] += 1
row_sums = matrix.sum(axis=1, dtype='float')
new_matrix = np.zeros((self._parameters["number_of_classes"], self._parameters["number_of_classes"])) # classes X classes
for i, (row, row_sum) in enumerate(zip(matrix, row_sums)):
if row_sum == 0:
new_matrix[i, :] = 0
else:
new_matrix[i, :] = row / row_sum
new_matrix = np.around(new_matrix, 3)
b = np.asarray(new_matrix)
self._parameters['diag_sum'] = np.trace(b)
self._parameters['diag_elements'] = np.diagonal(b)
print('Diagonal (sum): ', np.trace(b))
print('Diagonal (elements): ', np.diagonal(b))
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(new_matrix, interpolation='nearest')
fig.colorbar(cax)
ax.set_yticks(plt.np.arange(self._parameters["number_of_classes"]))
ax.set_yticklabels(i for i in range(self._parameters["number_of_classes"]))
ax.set_xticks(plt.np.arange(self._parameters["number_of_classes"]))
ax.set_xticklabels(i for i in range(self._parameters["number_of_classes"]))
ax.tick_params(axis='y', labelsize=7)
ax.tick_params(axis='x', labelsize=7, labelbottom=True, labeltop=False)
plt.title('Confusion matrix')
ax.axis('image')
plt.xlabel("Predicted label")
plt.ylabel("Real label")
mypath = "./dataset/"+self._parameters["dataset_name"]+"/figures"
if not os.path.exists(mypath):
os.makedirs(mypath)
plt.savefig("./dataset/"+self._parameters["dataset_name"]+"/figures/cofution_matrix"+str(self._parameters['name'])+time.strftime("%Y%m%d_%H%M%S")+".png")
plt.clf()
plt.close()
return
def train_gcn(training_features, training_adjs, training_labels, eval_features,
eval_adjs, eval_labels, params, class_weights, activations,
unary, coeffs, graph_params):
device = torch.device(
'cuda:1') if torch.cuda.is_available() else torch.device('cpu')
gcn = GCN(n_features=training_features[0].shape[1],
hidden_layers=params["hidden_layers"],
dropout=params["dropout"],
activations=activations,
p=graph_params["probability"],
normalization=params["edge_normalization"])
gcn.to(device)
opt = params["optimizer"](gcn.parameters(),
lr=params["lr"],
weight_decay=params["regularization"])
n_training_graphs = len(training_labels)
graph_size = graph_params["vertices"]
n_eval_graphs = len(eval_labels)
counter = 0 # For early stopping
min_loss = None
for epoch in range(params["epochs"]):
# -------------------------- TRAINING --------------------------
training_graphs_order = np.arange(n_training_graphs)
np.random.shuffle(training_graphs_order)
for i, idx in enumerate(training_graphs_order):
training_mat = torch.tensor(training_features[idx], device=device)
training_adj, training_lbs = map(
lambda x: torch.tensor(
data=x[idx], dtype=torch.double, device=device),
[training_adjs, training_labels])
gcn.train()
opt.zero_grad()
output_train = gcn(training_mat, training_adj)
output_matrix_flat = (
torch.mm(output_train, output_train.transpose(0, 1)) +
1 / 2).flatten()
training_criterion = gcn_build_weighted_loss(
unary, class_weights, training_lbs)
loss_train = coeffs[0] * training_criterion(output_train.view(output_train.shape[0]), training_lbs) + \
coeffs[1] * gcn_pairwise_loss(output_matrix_flat, training_adj.flatten()) + \
coeffs[2] * gcn_binomial_reg(output_train, graph_params)
loss_train.backward()
opt.step()
# -------------------------- EVALUATION --------------------------
graphs_order = np.arange(n_eval_graphs)
np.random.shuffle(graphs_order)
outputs = torch.zeros(graph_size * n_eval_graphs, dtype=torch.double)
output_xs = torch.zeros(graph_size**2 * n_eval_graphs,
dtype=torch.double)
adj_flattened = torch.tensor(
np.hstack([eval_adjs[idx].flatten() for idx in graphs_order]))
for i, idx in enumerate(graphs_order):
eval_mat = torch.tensor(eval_features[idx], device=device)
eval_adj, eval_lbs = map(
lambda x: torch.tensor(
data=x[idx], dtype=torch.double, device=device),
[eval_adjs, eval_labels])
gcn.eval()
output_eval = gcn(eval_mat, eval_adj)
output_matrix_flat = (
torch.mm(output_eval, output_eval.transpose(0, 1)) +
1 / 2).flatten()
output_xs[i * graph_size**2:(i + 1) *
graph_size**2] = output_matrix_flat.cpu()
outputs[i * graph_size:(i + 1) * graph_size] = output_eval.view(
output_eval.shape[0]).cpu()
all_eval_labels = torch.tensor(np.hstack(
[eval_labels[idx] for idx in graphs_order]),
dtype=torch.double)
eval_criterion = gcn_build_weighted_loss(unary, class_weights,
all_eval_labels)
loss_eval = (
coeffs[0] * eval_criterion(outputs, all_eval_labels) +
coeffs[1] * gcn_pairwise_loss(output_xs, adj_flattened) +
coeffs[2] * gcn_binomial_reg(outputs, graph_params)).item()
if min_loss is None:
current_min_loss = loss_eval
else:
current_min_loss = min(min_loss, loss_eval)
if epoch >= 10 and params[
"early_stop"]: # Check for early stopping during training.
if min_loss is None:
min_loss = current_min_loss
torch.save(gcn.state_dict(),
"tmp_time.pt") # Save the best state.
elif loss_eval < min_loss:
min_loss = current_min_loss
torch.save(gcn.state_dict(),
"tmp_time.pt") # Save the best state.
counter = 0
else:
counter += 1
if counter >= 40: # Patience for learning
break
# After stopping early, our model is the one with the best eval loss.
gcn.load_state_dict(torch.load("tmp_time.pt"))
os.remove("tmp_time.pt")
return gcn
def train_classification_gcn(self, Adj, features, nfeats, labels, nclasses,
train_mask, val_mask, test_mask, args):
model = GCN(in_channels=nfeats,
hidden_channels=args.hidden,
out_channels=nclasses,
num_layers=args.nlayers,
dropout=args.dropout2,
dropout_adj=args.dropout_adj2,
Adj=Adj,
sparse=args.sparse)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.w_decay)
bad_counter = 0
best_val = 0
best_model = None
best_loss = 0
best_train_loss = 0
if torch.cuda.is_available():
model = model.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
features = features.cuda()
labels = labels.cuda()
for epoch in range(1, args.epochs + 1):
model.train()
loss, accu = self.get_loss_fixed_adj(model, train_mask, features,
labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch % 10 == 0:
model.eval()
val_loss, accu = self.get_loss_fixed_adj(
model, val_mask, features, labels)
if accu > best_val:
bad_counter = 0
best_val = accu
best_model = copy.deepcopy(model)
best_loss = val_loss
best_train_loss = loss
else:
bad_counter += 1
if bad_counter >= args.patience:
break
print("Val Loss {:.4f}, Val Accuracy {:.4f}".format(
best_loss, best_val))
best_model.eval()
test_loss, test_accu = self.get_loss_fixed_adj(best_model, test_mask,
features, labels)
print("Test Loss {:.4f}, Test Accuracy {:.4f}".format(
test_loss, test_accu))
return best_val, test_accu, best_model
optimizer.zero_grad()
loss.backward()
optimizer.step()
# print(model.q)
# print(f'Ep.{ep+1} - loss: {avg_loss/100.:.6f} - mse loss: {avg_loss_mse/100.:.6f}')
data, label = datasets.load_digits(return_X_y=True)
data_max = np.max(data)
data = torch.from_numpy(data).type(torch.FloatTensor).cuda() / data_max
label = torch.from_numpy(label).type(torch.LongTensor).cuda()
train_id, valid_id = split_dataset(data, label)
clfr = GCN(64, 128, 128, 10, get_knn_graph(data, 8))
training_loop(clfr, data, label, train_id, valid_id, lr=2e-3, epoch=200, name='digits')
clfr.eval()
yy = torch.argmax(clfr(data)[0], dim=1)
train_acc = torch.sum(yy[train_id]==label[train_id]).cpu().numpy()*1./len(train_id)
valid_acc = torch.sum(yy[valid_id]==label[valid_id]).cpu().numpy()*1./len(valid_id)
print(f'training set accuracy: {train_acc:.6f}')
print(f'validation set accuracy: {valid_acc:.6f}')
init_matrix = get_knn_graph(data,8).cpu().numpy()
init_matrix = init_matrix / np.max(init_matrix)
adj_matrix = clfr.gc1.adj_matrix.detach()
adj_matrix = adj_matrix / torch.max(adj_matrix)
adj_matrix = adj_matrix.cpu().numpy()
plt.subplot(121)
sb.heatmap(init_matrix[:100,:100], cmap='Blues')
def main(args):
# 0. initial setting
# set environmet
cudnn.benchmark = True
if not os.path.isdir(os.path.join(args.path, './ckpt')):
os.mkdir(os.path.join(args.path, './ckpt'))
if not os.path.isdir(os.path.join(args.path, './results')):
os.mkdir(os.path.join(args.path, './results'))
if not os.path.isdir(os.path.join(args.path, './ckpt', args.name)):
os.mkdir(os.path.join(args.path, './ckpt', args.name))
if not os.path.isdir(os.path.join(args.path, './results', args.name)):
os.mkdir(os.path.join(args.path, './results', args.name))
if not os.path.isdir(os.path.join(args.path, './results', args.name,
"log")):
os.mkdir(os.path.join(args.path, './results', args.name, "log"))
# set logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(message)s')
handler = logging.FileHandler(
os.path.join(
args.path, "results/{}/log/{}.log".format(
args.name, time.strftime('%c', time.localtime(time.time())))))
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.addHandler(logging.StreamHandler())
args.logger = logger
# set cuda
if torch.cuda.is_available():
args.logger.info("running on cuda")
args.device = torch.device("cuda")
args.use_cuda = True
else:
args.logger.info("running on cpu")
args.device = torch.device("cpu")
args.use_cuda = False
args.logger.info("[{}] starts".format(args.name))
# 1. load data
adj, features, labels, idx_train, idx_val, idx_test = load_data()
# 2. setup
CORA_NODES = 2708
CORA_FEATURES = 1433
CORA_CLASSES = 7
CITESEER_NODES = 3327
CITESEER_FEATURES = 3703
CITESEER_CLASSES = 6
(num_nodes, feature_dim,
classes) = (CORA_NODES, CORA_FEATURES,
CORA_CLASSES) if args.dataset == 'cora' else (
CITESEER_NODES, CITESEER_FEATURES, CITESEER_CLASSES)
args.logger.info("setting up...")
model = GCN(args, feature_dim, args.hidden, classes,
args.dropout) if args.model == 'gcn' else SpGAT(
args, feature_dim, args.hidden, classes, args.dropout,
args.alpha, args.n_heads)
model.to(args.device)
loss_fn = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.load:
loaded_data = load(args, args.ckpt)
model.load_state_dict(loaded_data['model'])
optimizer.load_state_dict(loaded_data['optimizer'])
# 3. train / test
if not args.test:
# train
args.logger.info("starting training")
train_loss_meter = AverageMeter(args,
name="Loss",
save_all=True,
x_label="epoch")
val_acc_meter = AverageMeter(args,
name="Val Acc",
save_all=True,
x_label="epoch")
earlystop_listener = val_acc_meter.attach_combo_listener(
(lambda prev, new: prev.max >= new.max), threshold=args.patience)
steps = 1
for epoch in range(1, 1 + args.epochs):
spent_time = time.time()
model.train()
train_loss_tmp_meter = AverageMeter(args)
if args.start_from_step is not None:
if steps < args.start_from_step:
steps += 1
continue
optimizer.zero_grad()
batch = len(idx_train)
output = model(features.to(args.device), adj.to(args.device))
loss = loss_fn(output[idx_train],
labels[idx_train].to(args.device))
loss.backward()
optimizer.step()
train_loss_tmp_meter.update(loss, weight=batch)
steps += 1
train_loss_meter.update(train_loss_tmp_meter.avg)
spent_time = time.time() - spent_time
args.logger.info(
"[{}] train loss: {:.3f} took {:.1f} seconds".format(
epoch, train_loss_tmp_meter.avg, spent_time))
model.eval()
spent_time = time.time()
if not args.fastmode:
with torch.no_grad():
output = model(features.to(args.device),
adj.to(args.device))
acc = accuracy(output[idx_val], labels[idx_val]) * 100.0
val_acc_meter.update(acc)
earlystop = earlystop_listener.listen()
spent_time = time.time() - spent_time
args.logger.info(
"[{}] val acc: {:2.1f} % took {:.1f} seconds".format(
epoch, acc, spent_time))
if steps % args.save_period == 0:
save(args, "epoch{}".format(epoch),
{'model': model.state_dict()})
train_loss_meter.plot(scatter=False)
val_acc_meter.plot(scatter=False)
val_acc_meter.save()
if earlystop:
break
else:
# test
args.logger.info("starting test")
model.eval()
with torch.no_grad():
model(features.to(args.device), adj.to(args.device))
acc = accuracy(output[idx_test], labels[idx_test]) * 100
logger.d("test acc: {:2.1f} % took {:.1f} seconds".format(
acc, spent_time))
val_acc_pre = 0
dec_time = 0
for epoch in range(args.epoch):
t = time.time()
gcn.train()
optimizer.zero_grad()
output = gcn(features, norm_adj)
pred = output[train_mask]
ans = torch.argmax(y_train[train_mask],dim=1)
loss = F.cross_entropy(pred, ans)
train_acc = cal_accuracy(output, y_train, train_mask)
loss.backward()
optimizer.step()
#print(torch.min(pred), torch.max(pred))
gcn.eval()
pred = output[val_mask]
ans = torch.argmax(y_train[val_mask],dim=1)
val_loss = F.cross_entropy(pred, ans)
val_acc = cal_accuracy(output, y_val, val_mask)
print("epoch:", epoch, "time:", time.time()-t)
print("train_loss:",float(loss), "train_acc:", float(train_acc))
print("val_loss:", float(val_loss), "val_acc:", float(val_acc))
if val_loss > val_loss_pre and val_acc < val_acc_pre:
dec_time = dec_time + 1
if dec_time>10:
break
else:
dec_time = 0
val_loss_pre = val_loss
