for idx, params in enumerate(HYPER_PARA):
BATCH_SIZE, DROP_RT, LR, EPOCHS = params
best_roc = []
best_prc = []
data_loader = DataLoader(train_data,
batch_size=BATCH_SIZE,
shuffle=True,
collate_fn=collate)
for run in range(1):
model = GCN(NUM_ELEM, EMBEDDING_DIM, HIDDEN_DIM,
EDGE_HIDDEN_DIM, NUM_CLS, NUM_LYS, ADD_ON_FEATS,
F.relu, DROP_RT)
loss_func = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=LR)
device = torch.device('cuda:0')
model.to(device)
last_roc = -1
last_prc = -1
epochs_no_imprv = 0
for epoch in range(EPOCHS):
model.train()
epoch_loss = 0
batch = tqdm(data_loader)
for bg, label in batch:
optimizer.zero_grad()
prediction = model(bg)
# loss = torch.mean(F.cross_entropy(prediction, label, reduction='none')
v = torch.from_numpy(features[1]).to(device)
feature = torch.sparse.FloatTensor(i.t(), v, features[2]).to(device)
i = torch.from_numpy(supports[0]).long().to(device)
v = torch.from_numpy(supports[1]).to(device)
support = torch.sparse.FloatTensor(i.t(), v, supports[2]).float().to(device)
print('x :', feature)
print('sp:', support)
num_features_nonzero = feature._nnz()
feat_dim = feature.shape[1]
net = GCN(feat_dim, num_classes, num_features_nonzero)
net.to(device)
optimizer = optim.Adam(net.parameters(), lr=args.learning_rate)
net.train()
for epoch in range(args.epochs):
out = net((feature, support))
out = out[0]
loss = masked_loss(out, train_label, train_mask)
loss += args.weight_decay * net.l2_loss()
acc = masked_acc(out, train_label, train_mask)
optimizer.zero_grad()
loss.backward()
optimizer.step()
# hyper parameters
epochs = 200
nfeat = features.shape[1]
nhid = 16
nclasses = labels.max().item() + 1
dropout = 0.5
lr = 0.01
weight_decay = 5e-4
# device setting
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Model
model = GCN(nfeat, nhid, nclasses, dropout)
optimzer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
# To device
model = model.to(device)
features = features.to(device)
adj = adj.to(device)
labels = labels.to(device)
idx_train = idx_train.to(device)
idx_val = idx_val.to(device)
idx_test = idx_test.to(device)
def train(epoch):
t = time.time()
model.train()
optimzer.zero_grad()
# print Y_train
## Neural Model
# Loss and Optimizer
nComm = 2
nHid1 = 50
nHid2 = 20
nHid3 = 20
nHid4 = 10
nFeat = X_Tags_Feature.shape[1] #len(Tags_Features)
gcn_model = GCN(nFeat, nHid1, nComm)
#gcn_model = GCN(nFeat, nHid1, nHid2, nHid3, nHid4, nComm)
#gcn_model.load_state_dict(torch.load('gcn_complete2.pkl'))
criterion = nn.CrossEntropyLoss()
gcn_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
gcn_model.parameters()),
lr=args.learning_rate,
weight_decay=args.weight_decay)
epoch_loss = []
epoch_accuracy = []
train_accuracy = []
epochs = []
correct = 0
try:
for epoch in range(1, args.num_epochs + 1):
train_acc = 0
length = 0
loss_train = 0
print("n_train:",n_train)
#norm_adj = normalize(adj)
i = torch.LongTensor([norm_adj.row, norm_adj.col])
v = torch.FloatTensor(norm_adj.data)
norm_adj = torch.sparse.FloatTensor(i,v, adj.shape)
features = torch.tensor(features, dtype=torch.float, requires_grad=False)
y_train = torch.tensor(y_train, dtype=torch.long, requires_grad=False)
y_val = torch.tensor(y_val, dtype=torch.long, requires_grad=False)
y_test = torch.tensor(y_test, dtype=torch.long, requires_grad=False)
load_from = False
gcn = GCN(embsize, hidsize, nclass, weight_decay)
if load_from:
gcn = torch.load(load_from)
optimizer = torch.optim.Adam(gcn.parameters(recurse=True), lr=lr, weight_decay=weight_decay)
train_time = time.time()
val_loss_pre = 1e9
val_acc_pre = 0
dec_time = 0
for epoch in range(args.epoch):
t = time.time()
gcn.train()
optimizer.zero_grad()
output = gcn(features, norm_adj)
pred = output[train_mask]
ans = torch.argmax(y_train[train_mask],dim=1)
loss = F.cross_entropy(pred, ans)
train_acc = cal_accuracy(output, y_train, train_mask)
loss.backward()
class ModelRunner:
def __init__(self, params, logger, data_logger=None, epochs_logger=None):
self._logger = logger
self._epoch_logger = epochs_logger
self._data_logger = EmptyLogger() if data_logger is None else data_logger
self._parameters = params
self._lr = params["lr"]
self._is_nni = params['is_nni']
self._device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
self._mse_loss = self.weighted_mse_loss
self._temporal_loss = self.weighted_mse_loss
self.model = GCN(num_of_features=self._parameters["feature_matrices"][0].shape[1],
hid_size=self._parameters["hid_size"],
num_of_classes=self._parameters["number_of_classes"],
activation=self._parameters["activation"],
dropout=self._parameters["dropout"])
self.model = self.model.to(self._device)
self.opt = self._parameters["optimizer"](self.model.parameters(),
lr=self._parameters['lr'],
weight_decay=self._parameters['weight_decay'])
@property
def logger(self):
return self._logger
@property
def data_logger(self):
return self._data_logger
def weighted_mse_loss(self, pred, target, weights=None):
if weights is None:
return ((pred - target) ** 2).sum(dim=1).sum().to(self._device)
elif self._parameters['loss_weights_type'] == 'sqrt(N/Nj)':
weights = torch.tensor(weights).to(device=self._device, dtype=torch.float)
b = (torch.sqrt((weights).sum() / weights) *(pred - target) ** 2).sum(dim=1).sum().to(self._device)
return b
elif self._parameters['loss_weights_type'] == '1/Njs':
weights = torch.tensor(weights).to(device=self._device, dtype=torch.float)
b = (torch.tensor(1. / weights) * (pred - target) ** 2).sum(dim=1).sum().to(self._device)
return b
def run(self):
train_results_l = []
test_results = []
# train
for epoch in range(self._parameters["epochs"]):
train_results = self.train(epoch)
train_results_l.append(train_results)
if epoch == self._parameters["epochs"]-1: # for grid
test_res = self.test(epoch)
test_results.append(test_res)
if self._is_nni:
nni.report_intermediate_result(test_res["f1_score_macro"][-1])
else:
print(self._parameters["it_num"],
self._parameters["iterations"],
epoch + 1,
self._parameters["epochs"],
self._parameters["lr"],
self._parameters["dropout"],
self._parameters["hid_size"],
self._parameters["weight_decay"],
self._parameters["temporal_pen"],
train_results['loss'].item(),
train_results['tempo_loss'].item(),
train_results['loss'].item() + train_results['tempo_loss'].item(),
train_results['f1_score_macro'][-1],
train_results['f1_score_micro'][-1],
test_res["loss"],
test_res["tempo_loss"],
test_res["loss"] + test_res["tempo_loss"],
test_res["f1_score_macro"][-1],
test_res["f1_score_micro"][-1])
self._logger.debug('Epoch: {:04d} '.format(epoch + 1) +
'lr: {:04f} '.format(self._parameters['lr']) +
'dropout: {:04f} '.format(self._parameters['dropout']) +
'hid_size: {:04f} '.format(self._parameters['hid_size']) +
'weight_decay: {:04f} '.format(self._parameters['weight_decay']) +
'temporal_pen: {:04f} '.format(self._parameters['temporal_pen']) +
'reg_loss_train: {:.4f} '.format(train_results['loss']) +
'temp_loss: {:.4f} '.format(train_results['tempo_loss']))
result = self.test('test', print_to_file=True)
if self._is_nni:
nni.report_final_result(result["f1_score_macro"])
return train_results_l, test_results, result, self._parameters
def train(self, epoch):
z_vals, outputs = [], []
labeled_indices = self._parameters['training_inds']
labels = self._parameters['training_labels']
tempo_loss = 0.
loss_train = 0.
self.model.train()
self.opt.zero_grad()
for idx, adj in enumerate(self._parameters["adj_matrices"]):
input_features = torch.from_numpy(self._parameters["feature_matrices"][idx]).to(dtype=torch.float,
device=self._device)
z, output = self.model(input_features, adj)
output = output[labeled_indices[idx], :]
# Njs are the weights of the loss using the adj mx.
# they should be either used or not.
Nj_s = [sum([labels[u][t][j] for u in range(len(self._parameters["adj_matrices"])) for t in range(len(labels[u]))]) for j in
range(self._parameters['number_of_classes'])]
loss_train += self._mse_loss(output, labels[idx], Nj_s)
# loss_train += self._mse_loss(output, labels[idx].float()) #without weights using the build-in mse
z_vals.append(z) # After 1 GCN layer
outputs.append(output) # Final predictions
# counts the number of cross_year_persons
z_appearances = 0.
for t in range(len(z_vals) - 1):
t_inds = self._parameters['training_inds'][t]
t_plus_one_inds = self._parameters['training_inds'][t + 1]
z_inds = [i for i in t_inds if i in t_plus_one_inds]
z_appearances += len(z_inds)
z_val_t = z_vals[t][z_inds, :]
z_val_t_plus_1 = z_vals[t+1][z_inds, :]
loss = self._temporal_loss(z_val_t_plus_1, z_val_t)
tempo_loss += self._parameters["temporal_pen"] * loss
tempo_loss /= z_appearances
loss_train /= sum([len(labeled_indices[u]) for u in range(len(outputs))])
total_loss = loss_train + tempo_loss
total_loss.backward()
self.opt.step()
f1_score_macro, f1_score_micro = [],[]
if epoch == self._parameters['epochs']-1:
for i in range(len(labels)):
f1_mac, f1_mic, list_real, list_pred = self.accuracy_f1_score(outputs[i], labels[i])
f1_score_macro.append(f1_mac)
f1_score_micro.append(f1_mic)
result = {"loss": loss_train,
"f1_score_macro": f1_score_macro,
"f1_score_micro": f1_score_micro,
"tempo_loss": tempo_loss}
return result
def test(self,epoch, print_to_file=False):
z_vals, outputs = [], []
labeled_indices = self._parameters['test_inds']
labels = self._parameters['test_labels']
tempo_loss = 0.
loss_test = 0.
test_z_appearances = 0.
self.model.eval()
for idx, adj in enumerate(self._parameters["adj_matrices"]):
test_mat = torch.from_numpy(self._parameters["feature_matrices"][idx]).to(self._device)
z, output = self.model(*[test_mat, adj])
output = output[labeled_indices[idx], :]
loss_test += self._mse_loss(output, labels[idx].float())
z_vals.append(z)
outputs.append(output)
if print_to_file:
grid_outputs_folder = str(self._parameters['name'])
self._logger.debug("\nprint to files")
for i in range(len(self._parameters["adj_matrices"])):
np_output = outputs[i].cpu().data.numpy()
products_path = os.path.join(os.getcwd(),'dataset',self._parameters["dataset_name"], "gcn_outputs",grid_outputs_folder)
if not os.path.exists(products_path):
os.makedirs(products_path)
with open(os.path.join("dataset",self._parameters["dataset_name"],"gcn_outputs",grid_outputs_folder, "gcn_" + str(i) + ".pkl"), "wb") as f:
pickle.dump(np_output, f, protocol=pickle.HIGHEST_PROTOCOL)
for t in range(len(z_vals) - 1):
t_inds = self._parameters['test_inds'][t]
t_plus_one_inds = self._parameters['test_inds'][t + 1]
z_inds = [i for i in t_inds if i in t_plus_one_inds]
test_z_appearances += len(z_inds)
z_val_t = z_vals[t][z_inds, :]
z_val_t_plus_1 = z_vals[t + 1][z_inds, :]
tempo_loss += self._parameters["temporal_pen"] * self._temporal_loss(z_val_t_plus_1, z_val_t)
tempo_loss /= test_z_appearances
loss_test /= sum([len(labeled_indices[u]) for u in range(len(outputs))])
f1_score_macro, f1_score_micro = [], []
real,pred = [],[]
if epoch == self._parameters['epochs'] - 1 or epoch == 'test':
for i in range(len(labels)): #running over the years
f1_mac, f1_mic, list_real, list_pred = self.accuracy_f1_score(outputs[i], labels[i])
f1_score_macro.append(f1_mac)
f1_score_micro.append(f1_mic)
real.extend(list_real)
pred.extend(list_pred)
self.confusion_matrix(real, pred) # of all years normalized to 1 for the last epoch test
result = {"loss": loss_test.data.item(),
"f1_score_macro": f1_score_macro,
"f1_score_micro": f1_score_micro,
"tempo_loss": tempo_loss.data.item()}
return result
def accuracy_f1_score(self,output, labels):
pred, real = [],[]
for person in range(labels.size(0)): #range of all persons
for label in range(labels.size(1)):
if labels[person,label]==0:
continue
else:
argmax = output[person].max(0)[1]
real.append(label)
pred.append(argmax.cpu().item())
f1_macro = f1_score(real, pred, average='macro')
f1_micro = f1_score(real, pred, average='micro')
return f1_macro, f1_micro, real,pred
def confusion_matrix(self, list_real, list_pred):
matrix = np.zeros((self._parameters["number_of_classes"], self._parameters["number_of_classes"])) # classes X classes
for i in range(len(list_pred)):
matrix[list_real[i], list_pred[i]] += 1
row_sums = matrix.sum(axis=1, dtype='float')
new_matrix = np.zeros((self._parameters["number_of_classes"], self._parameters["number_of_classes"])) # classes X classes
for i, (row, row_sum) in enumerate(zip(matrix, row_sums)):
if row_sum == 0:
new_matrix[i, :] = 0
else:
new_matrix[i, :] = row / row_sum
new_matrix = np.around(new_matrix, 3)
b = np.asarray(new_matrix)
self._parameters['diag_sum'] = np.trace(b)
self._parameters['diag_elements'] = np.diagonal(b)
print('Diagonal (sum): ', np.trace(b))
print('Diagonal (elements): ', np.diagonal(b))
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(new_matrix, interpolation='nearest')
fig.colorbar(cax)
ax.set_yticks(plt.np.arange(self._parameters["number_of_classes"]))
ax.set_yticklabels(i for i in range(self._parameters["number_of_classes"]))
ax.set_xticks(plt.np.arange(self._parameters["number_of_classes"]))
ax.set_xticklabels(i for i in range(self._parameters["number_of_classes"]))
ax.tick_params(axis='y', labelsize=7)
ax.tick_params(axis='x', labelsize=7, labelbottom=True, labeltop=False)
plt.title('Confusion matrix')
ax.axis('image')
plt.xlabel("Predicted label")
plt.ylabel("Real label")
mypath = "./dataset/"+self._parameters["dataset_name"]+"/figures"
if not os.path.exists(mypath):
os.makedirs(mypath)
plt.savefig("./dataset/"+self._parameters["dataset_name"]+"/figures/cofution_matrix"+str(self._parameters['name'])+time.strftime("%Y%m%d_%H%M%S")+".png")
plt.clf()
plt.close()
return
def train_classification_gcn(self, Adj, features, nfeats, labels, nclasses,
train_mask, val_mask, test_mask, args):
model = GCN(in_channels=nfeats,
hidden_channels=args.hidden,
out_channels=nclasses,
num_layers=args.nlayers,
dropout=args.dropout2,
dropout_adj=args.dropout_adj2,
Adj=Adj,
sparse=args.sparse)
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.w_decay)
bad_counter = 0
best_val = 0
best_model = None
best_loss = 0
best_train_loss = 0
if torch.cuda.is_available():
model = model.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
features = features.cuda()
labels = labels.cuda()
for epoch in range(1, args.epochs + 1):
model.train()
loss, accu = self.get_loss_fixed_adj(model, train_mask, features,
labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch % 10 == 0:
model.eval()
val_loss, accu = self.get_loss_fixed_adj(
model, val_mask, features, labels)
if accu > best_val:
bad_counter = 0
best_val = accu
best_model = copy.deepcopy(model)
best_loss = val_loss
best_train_loss = loss
else:
bad_counter += 1
if bad_counter >= args.patience:
break
print("Val Loss {:.4f}, Val Accuracy {:.4f}".format(
best_loss, best_val))
best_model.eval()
test_loss, test_accu = self.get_loss_fixed_adj(best_model, test_mask,
features, labels)
print("Test Loss {:.4f}, Test Accuracy {:.4f}".format(
test_loss, test_accu))
return best_val, test_accu, best_model
def main():
print('settings: ', args)
setup()
rnd_state = np.random.RandomState(args.seed)
print("Loading data ... ... ...")
dataset = TUDataset('./data/%s/' % args.dataset,
name=args.dataset,
use_node_attr=args.use_cont_node_attr)
train_ids, test_ids = split_ids(rnd_state.permutation(len(dataset)),
folds=args.n_folds)
print("Data loaded !!!")
acc_folds = []
for fold_id in range(args.n_folds):
loaders = []
for split in ['train', 'test']:
gdata = dataset[torch.from_numpy(
(train_ids
if split.find('train') >= 0 else test_ids)[fold_id])]
loader = DataLoader(gdata,
batch_size=args.batch_size,
shuffle=split.find('train') >= 0,
num_workers=args.threads,
collate_fn=collate_batch)
loaders.append(loader)
print('\nFOLD {}, train {}, test {}'.format(fold_id,
len(loaders[0].dataset),
len(loaders[1].dataset)))
model = GCN(in_features=loaders[0].dataset.num_features,
out_features=loaders[0].dataset.num_classes,
n_hidden=args.n_hidden,
filters=args.filters,
dropout=args.dropout).to(args.device)
print('\nInitialize model')
print(model)
train_params = list(
filter(lambda p: p.requires_grad, model.parameters()))
print('N trainable parameters:',
np.sum([p.numel() for p in train_params]))
optimizer = optim.Adam(train_params,
lr=args.lr,
weight_decay=args.wd,
betas=(0.5, 0.999))
scheduler = lr_scheduler.MultiStepLR(optimizer,
args.lr_decay_steps,
gamma=0.1)
for epoch in range(args.epochs):
train(loaders[0],
model=model,
epoch=epoch,
optimizer=optimizer,
scheduler=scheduler)
acc = test(loaders[1], model=model, epoch=epoch)
acc_folds.append(acc)
print(acc_folds)
print('{}-fold cross validation avg acc (+- std): {} ({})'.format(
args.n_folds, np.mean(acc_folds), np.std(acc_folds)))
def main(args):
device = torch.device('cuda' if args.cuda else 'cpu')
adj, features, labels, idx_train, idx_val, idx_test = load_data(
args.dataset)
if args.model == 'gcn':
model = GCN(nfeat=features.size(1),
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout)
print('Model: GCN')
elif args.model == 'gat':
model = GAT(nfeat=features.size(1),
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout,
alpha=args.alpha,
nheads=args.n_heads)
print('Model: GAT')
elif args.model == 'spgcn':
model = SpGCN(nfeat=features.size(1),
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout)
print('Model: SpGCN')
elif args.model == 'spgat':
model = SpGAT(nfeat=features.size(1),
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout,
alpha=args.alpha,
nheads=args.n_heads)
print('Model: SpGAT')
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.cuda:
adj = adj.cuda()
features = features.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
model.cuda()
print(device)
def train(epoch):
model.train()
optimizer.zero_grad()
output = model(features, adj)
loss_train = F.nll_loss(output[idx_train], labels[idx_train])
acc_train = accuracy(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
if not args.fastmode:
model.eval()
output = model(features, adj)
loss_val = F.nll_loss(output[idx_val], labels[idx_val])
acc_val = accuracy(output[idx_val], labels[idx_val])
# print('Epoch: {:04d}'.format(epoch + 1),
# 'loss_train: {:.4f}'.format(loss_train.item()),
# 'acc_train: {:.4f}'.format(acc_train.item()),
# 'loss_val: {:.4f}'.format(loss_val.item()),
# 'acc_val: {:.4f}'.format(acc_val.item()))
pbar.set_description(
'| epoch: {:4d} | loss_train: {:.4f} | acc_train: {:.4f} |'
' loss_val: {:.4f} | acc_val: {:.4f}'.format(
epoch + 1, loss_train.item(), acc_train.item(),
loss_val.item(), acc_val.item()))
return loss_train.item(), loss_val.item()
def test():
model.eval()
output = model(features, adj)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
losses = {}
pbar = tqdm(range(args.epochs))
for epoch in pbar:
loss_train, loss_val = train(epoch)
if epoch % 10 == 0:
if len(losses) == 0:
losses['train'] = [loss_train]
losses['val'] = [loss_val]
else:
losses['train'].append(loss_train)
losses['val'].append(loss_val)
f, ax = plt.subplots()
train_loss = ax.plot(losses['train'], label='Train Loss')
val_loss = ax.plot(losses['val'], label='Validation Loss')
ax.legend()
ax.set_xlabel('Epoch / 10')
ax.set_ylabel('Loss')
plt.savefig('results/loss_{}_{}.png'.format(args.model, args.dataset),
dpi=300)
print('Optimization Finished!')
test()
return loss_test.item(), acc_test.item()
if __name__ == '__main__':
random.seed(seed)
rg = np.random.default_rng(seed)
torch.manual_seed(seed)
# Load data
adj, features, labels, idx_train, idx_val, idx_test = load_data(
dataset=dataset, seed=seed)
# adj, features, labels, idx_train, idx_val, idx_test = load_rand_loadsplit_data(dataset)
# Model and optimizer
model = GCN(nfeat=features.shape[1],
nhid=hidden,
nclass=labels.max().item() + 1,
dropout=dropout)
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
# Train model
t_total = time.time()
for epoch in range(epochs):
train(epoch)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# Testing
res = test()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import matplotlib.pyplot as plt
import torch
import torch.nn.functional as F
from model import GCN
from graph_builder import build_karate_club_graph, draw
import warnings
warnings.filterwarnings("ignore")
# Model
net = GCN(34, 5, 2)
optimizer = torch.optim.Adam(net.parameters(), lr=0.01)
all_logits = []
print("Model structure:", net)
# Data
G = build_karate_club_graph()
inputs = torch.eye(34)
labeled_nodes = torch.tensor([0, 33])
labels = torch.tensor([0, 1])
# Train
net.train()
for epoch in range(30):
optimizer.zero_grad()
def main(args):
# 0. initial setting
# set environmet
cudnn.benchmark = True
if not os.path.isdir(os.path.join(args.path, './ckpt')):
os.mkdir(os.path.join(args.path, './ckpt'))
if not os.path.isdir(os.path.join(args.path, './results')):
os.mkdir(os.path.join(args.path, './results'))
if not os.path.isdir(os.path.join(args.path, './ckpt', args.name)):
os.mkdir(os.path.join(args.path, './ckpt', args.name))
if not os.path.isdir(os.path.join(args.path, './results', args.name)):
os.mkdir(os.path.join(args.path, './results', args.name))
if not os.path.isdir(os.path.join(args.path, './results', args.name,
"log")):
os.mkdir(os.path.join(args.path, './results', args.name, "log"))
# set logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(message)s')
handler = logging.FileHandler(
os.path.join(
args.path, "results/{}/log/{}.log".format(
args.name, time.strftime('%c', time.localtime(time.time())))))
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.addHandler(logging.StreamHandler())
args.logger = logger
# set cuda
if torch.cuda.is_available():
args.logger.info("running on cuda")
args.device = torch.device("cuda")
args.use_cuda = True
else:
args.logger.info("running on cpu")
args.device = torch.device("cpu")
args.use_cuda = False
args.logger.info("[{}] starts".format(args.name))
# 1. load data
adj, features, labels, idx_train, idx_val, idx_test = load_data()
# 2. setup
CORA_NODES = 2708
CORA_FEATURES = 1433
CORA_CLASSES = 7
CITESEER_NODES = 3327
CITESEER_FEATURES = 3703
CITESEER_CLASSES = 6
(num_nodes, feature_dim,
classes) = (CORA_NODES, CORA_FEATURES,
CORA_CLASSES) if args.dataset == 'cora' else (
CITESEER_NODES, CITESEER_FEATURES, CITESEER_CLASSES)
args.logger.info("setting up...")
model = GCN(args, feature_dim, args.hidden, classes,
args.dropout) if args.model == 'gcn' else SpGAT(
args, feature_dim, args.hidden, classes, args.dropout,
args.alpha, args.n_heads)
model.to(args.device)
loss_fn = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.load:
loaded_data = load(args, args.ckpt)
model.load_state_dict(loaded_data['model'])
optimizer.load_state_dict(loaded_data['optimizer'])
# 3. train / test
if not args.test:
# train
args.logger.info("starting training")
train_loss_meter = AverageMeter(args,
name="Loss",
save_all=True,
x_label="epoch")
val_acc_meter = AverageMeter(args,
name="Val Acc",
save_all=True,
x_label="epoch")
earlystop_listener = val_acc_meter.attach_combo_listener(
(lambda prev, new: prev.max >= new.max), threshold=args.patience)
steps = 1
for epoch in range(1, 1 + args.epochs):
spent_time = time.time()
model.train()
train_loss_tmp_meter = AverageMeter(args)
if args.start_from_step is not None:
if steps < args.start_from_step:
steps += 1
continue
optimizer.zero_grad()
batch = len(idx_train)
output = model(features.to(args.device), adj.to(args.device))
loss = loss_fn(output[idx_train],
labels[idx_train].to(args.device))
loss.backward()
optimizer.step()
train_loss_tmp_meter.update(loss, weight=batch)
steps += 1
train_loss_meter.update(train_loss_tmp_meter.avg)
spent_time = time.time() - spent_time
args.logger.info(
"[{}] train loss: {:.3f} took {:.1f} seconds".format(
epoch, train_loss_tmp_meter.avg, spent_time))
model.eval()
spent_time = time.time()
if not args.fastmode:
with torch.no_grad():
output = model(features.to(args.device),
adj.to(args.device))
acc = accuracy(output[idx_val], labels[idx_val]) * 100.0
val_acc_meter.update(acc)
earlystop = earlystop_listener.listen()
spent_time = time.time() - spent_time
args.logger.info(
"[{}] val acc: {:2.1f} % took {:.1f} seconds".format(
epoch, acc, spent_time))
if steps % args.save_period == 0:
save(args, "epoch{}".format(epoch),
{'model': model.state_dict()})
train_loss_meter.plot(scatter=False)
val_acc_meter.plot(scatter=False)
val_acc_meter.save()
if earlystop:
break
else:
# test
args.logger.info("starting test")
model.eval()
with torch.no_grad():
model(features.to(args.device), adj.to(args.device))
acc = accuracy(output[idx_test], labels[idx_test]) * 100
logger.d("test acc: {:2.1f} % took {:.1f} seconds".format(
acc, spent_time))
def main(args, print_fn=print):
print_fn("Experiment arguments: {}".format(args))
if args.random_seed:
torch.manual_seed(args.random_seed)
else:
torch.manual_seed(123)
# Load dataset
if args.dataset.startswith('ogbl'):
graph, split_edge = load_ogb_dataset(args.dataset)
else:
raise NotImplementedError
num_nodes = graph.num_nodes()
# set gpu
if args.gpu_id >= 0 and torch.cuda.is_available():
device = 'cuda:{}'.format(args.gpu_id)
else:
device = 'cpu'
if args.dataset == 'ogbl-collab':
# ogbl-collab dataset is multi-edge graph
use_coalesce = True
else:
use_coalesce = False
# Generate positive and negative edges and corresponding labels
# Sampling subgraphs and generate node labeling features
seal_data = SEALData(g=graph, split_edge=split_edge, hop=args.hop, neg_samples=args.neg_samples,
subsample_ratio=args.subsample_ratio, use_coalesce=use_coalesce, prefix=args.dataset,
save_dir=args.save_dir, num_workers=args.num_workers, print_fn=print_fn)
node_attribute = seal_data.ndata['feat']
edge_weight = seal_data.edata['edge_weight'].float()
train_data = seal_data('train')
val_data = seal_data('valid')
test_data = seal_data('test')
train_graphs = len(train_data.graph_list)
# Set data loader
train_loader = GraphDataLoader(train_data, batch_size=args.batch_size, num_workers=args.num_workers)
val_loader = GraphDataLoader(val_data, batch_size=args.batch_size, num_workers=args.num_workers)
test_loader = GraphDataLoader(test_data, batch_size=args.batch_size, num_workers=args.num_workers)
# set model
if args.model == 'gcn':
model = GCN(num_layers=args.num_layers,
hidden_units=args.hidden_units,
gcn_type=args.gcn_type,
pooling_type=args.pooling,
node_attributes=node_attribute,
edge_weights=edge_weight,
node_embedding=None,
use_embedding=True,
num_nodes=num_nodes,
dropout=args.dropout)
elif args.model == 'dgcnn':
model = DGCNN(num_layers=args.num_layers,
hidden_units=args.hidden_units,
k=args.sort_k,
gcn_type=args.gcn_type,
node_attributes=node_attribute,
edge_weights=edge_weight,
node_embedding=None,
use_embedding=True,
num_nodes=num_nodes,
dropout=args.dropout)
else:
raise ValueError('Model error')
model = model.to(device)
parameters = model.parameters()
optimizer = torch.optim.Adam(parameters, lr=args.lr)
loss_fn = BCEWithLogitsLoss()
print_fn("Total parameters: {}".format(sum([p.numel() for p in model.parameters()])))
# train and evaluate loop
summary_val = []
summary_test = []
for epoch in range(args.epochs):
start_time = time.time()
loss = train(model=model,
dataloader=train_loader,
loss_fn=loss_fn,
optimizer=optimizer,
device=device,
num_graphs=args.batch_size,
total_graphs=train_graphs)
train_time = time.time()
if epoch % args.eval_steps == 0:
val_pos_pred, val_neg_pred = evaluate(model=model,
dataloader=val_loader,
device=device)
test_pos_pred, test_neg_pred = evaluate(model=model,
dataloader=test_loader,
device=device)
val_metric = evaluate_hits(args.dataset, val_pos_pred, val_neg_pred, args.hits_k)
test_metric = evaluate_hits(args.dataset, test_pos_pred, test_neg_pred, args.hits_k)
evaluate_time = time.time()
print_fn("Epoch-{}, train loss: {:.4f}, hits@{}: val-{:.4f}, test-{:.4f}, "
"cost time: train-{:.1f}s, total-{:.1f}s".format(epoch, loss, args.hits_k, val_metric, test_metric,
train_time - start_time,
evaluate_time - start_time))
summary_val.append(val_metric)
summary_test.append(test_metric)
# summary_val = np.array(summary_val)
summary_test = np.array(summary_test)
print_fn("Experiment Results:")
print_fn("Best hits@{}: {:.4f}, epoch: {}".format(args.hits_k, np.max(summary_test), np.argmax(summary_test)))
seed = 9182
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
# load data
gcn_data = GCN_DATA(dataset, preprocess, validation, generalized,
PREFINE_CLASS_path)
# Model and optimizer
model = GCN(input_dim=gcn_data.all_attributes.shape[1],
hiddens_dim=hiddens_dim,
output_dim=gcn_data.true_class_weight.shape[1],
dropout=dropout)
optimizer = optim.Adam(model.parameters(), lr=LR, weight_decay=weight_decay)
loss_fun = torch.nn.MSELoss()
if torch.cuda.is_available():
model = model.cuda()
loss_fun = loss_fun.cuda()
#为了分类，将one-hot表示的标签向量转化为索引表示
def map_label(labels):
index_labels = np.where(labels.astype(int) == 1)[1]
unique_class = np.unique(index_labels)
class_num = len(unique_class)
#print(unique_class,len(unique_class))
mapped_label = np.zeros((labels.shape[0], ))
for i in range(class_num):
def train_gcn(training_features, training_adjs, training_labels, eval_features,
eval_adjs, eval_labels, params, class_weights, activations,
unary, coeffs, graph_params):
device = torch.device(
'cuda:1') if torch.cuda.is_available() else torch.device('cpu')
gcn = GCN(n_features=training_features[0].shape[1],
hidden_layers=params["hidden_layers"],
dropout=params["dropout"],
activations=activations,
p=graph_params["probability"],
normalization=params["edge_normalization"])
gcn.to(device)
opt = params["optimizer"](gcn.parameters(),
lr=params["lr"],
weight_decay=params["regularization"])
n_training_graphs = len(training_labels)
graph_size = graph_params["vertices"]
n_eval_graphs = len(eval_labels)
counter = 0 # For early stopping
min_loss = None
for epoch in range(params["epochs"]):
# -------------------------- TRAINING --------------------------
training_graphs_order = np.arange(n_training_graphs)
np.random.shuffle(training_graphs_order)
for i, idx in enumerate(training_graphs_order):
training_mat = torch.tensor(training_features[idx], device=device)
training_adj, training_lbs = map(
lambda x: torch.tensor(
data=x[idx], dtype=torch.double, device=device),
[training_adjs, training_labels])
gcn.train()
opt.zero_grad()
output_train = gcn(training_mat, training_adj)
output_matrix_flat = (
torch.mm(output_train, output_train.transpose(0, 1)) +
1 / 2).flatten()
training_criterion = gcn_build_weighted_loss(
unary, class_weights, training_lbs)
loss_train = coeffs[0] * training_criterion(output_train.view(output_train.shape[0]), training_lbs) + \
coeffs[1] * gcn_pairwise_loss(output_matrix_flat, training_adj.flatten()) + \
coeffs[2] * gcn_binomial_reg(output_train, graph_params)
loss_train.backward()
opt.step()
# -------------------------- EVALUATION --------------------------
graphs_order = np.arange(n_eval_graphs)
np.random.shuffle(graphs_order)
outputs = torch.zeros(graph_size * n_eval_graphs, dtype=torch.double)
output_xs = torch.zeros(graph_size**2 * n_eval_graphs,
dtype=torch.double)
adj_flattened = torch.tensor(
np.hstack([eval_adjs[idx].flatten() for idx in graphs_order]))
for i, idx in enumerate(graphs_order):
eval_mat = torch.tensor(eval_features[idx], device=device)
eval_adj, eval_lbs = map(
lambda x: torch.tensor(
data=x[idx], dtype=torch.double, device=device),
[eval_adjs, eval_labels])
gcn.eval()
output_eval = gcn(eval_mat, eval_adj)
output_matrix_flat = (
torch.mm(output_eval, output_eval.transpose(0, 1)) +
1 / 2).flatten()
output_xs[i * graph_size**2:(i + 1) *
graph_size**2] = output_matrix_flat.cpu()
outputs[i * graph_size:(i + 1) * graph_size] = output_eval.view(
output_eval.shape[0]).cpu()
all_eval_labels = torch.tensor(np.hstack(
[eval_labels[idx] for idx in graphs_order]),
dtype=torch.double)
eval_criterion = gcn_build_weighted_loss(unary, class_weights,
all_eval_labels)
loss_eval = (
coeffs[0] * eval_criterion(outputs, all_eval_labels) +
coeffs[1] * gcn_pairwise_loss(output_xs, adj_flattened) +
coeffs[2] * gcn_binomial_reg(outputs, graph_params)).item()
if min_loss is None:
current_min_loss = loss_eval
else:
current_min_loss = min(min_loss, loss_eval)
if epoch >= 10 and params[
"early_stop"]: # Check for early stopping during training.
if min_loss is None:
min_loss = current_min_loss
torch.save(gcn.state_dict(),
"tmp_time.pt") # Save the best state.
elif loss_eval < min_loss:
min_loss = current_min_loss
torch.save(gcn.state_dict(),
"tmp_time.pt") # Save the best state.
counter = 0
else:
counter += 1
if counter >= 40: # Patience for learning
break
# After stopping early, our model is the one with the best eval loss.
gcn.load_state_dict(torch.load("tmp_time.pt"))
os.remove("tmp_time.pt")
return gcn
args.cuda = not args.no_cuda and torch.cuda.is_available()
#print(args.cuda)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
adj,features,labels,idx_train,idx_val,idx_test = load_data()
# Model and optimizer
model = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout)
optimizer = optim.Adam(model.parameters(),
lr=args.lr, weight_decay=args.weight_decay)
adj,features,labels,idx_train,idx_val,idx_test = load_data()
model = GCN(nfeat=features.shape[1],nhid = args.hidden,nclass=labels.max().item()+1,dropout=args.dropout)
optimizer = optim.Adam(model.parameters(),lr = args.lr,weight_decay=args.weight_decay)
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
if CD:
net = net.cuda(0)
batch_size = 8
start_epoch = 0
if True:
ckpt = torch.load('/Disk5/junqi/CUB/early_skeleton_93.ckpt')
net.load_state_dict(ckpt['net_state_dict'])
start_epoch = ckpt['epoch'] + 1
datas = CUB()
dataLoader = torch.utils.data.DataLoader(datas,
batch_size=batch_size,
shuffle=True,
num_workers=4)
optimizer = torch.optim.SGD(net.parameters(),
lr=1e-2,
momentum=0.9,
weight_decay=1e-4)
net.train()
for epoch in range(start_epoch, 201):
epoch_loss = 0
correct = 0
for iter, (inputs, mask, labels) in enumerate(dataLoader):
if CD:
inputs = inputs.cuda(0)
mask = mask.cuda(0)
labels = labels.cuda(0)
prediction = net(G, inputs, mask)
_, predict = torch.max(prediction, 1)
def main(opt):
train_dataset = BADataset(opt.dataroot, opt.L, True, False, False)
train_dataloader = BADataloader(train_dataset, batch_size=opt.batchSize, \
shuffle=True, num_workers=opt.workers, drop_last=True)
valid_dataset = BADataset(opt.dataroot, opt.L, False, True, False)
valid_dataloader = BADataloader(valid_dataset, batch_size=opt.batchSize, \
shuffle=True, num_workers=opt.workers, drop_last=True)
test_dataset = BADataset(opt.dataroot, opt.L, False, False, True)
test_dataloader = BADataloader(test_dataset, batch_size=opt.batchSize, \
shuffle=True, num_workers=opt.workers, drop_last=True)
all_dataset = BADataset(opt.dataroot, opt.L, False, False, False)
all_dataloader = BADataloader(all_dataset, batch_size=opt.batchSize, \
shuffle=False, num_workers=opt.workers, drop_last=False)
opt.n_edge_types = train_dataset.n_edge_types
opt.n_node = train_dataset.n_node
net = GCN(opt,
kernel_size=2,
n_blocks=1,
state_dim_bottleneck=opt.state_dim,
annotation_dim_bottleneck=opt.annotation_dim)
net.double()
print(net)
criterion = nn.BCELoss()
if opt.cuda:
net.cuda()
criterion.cuda()
optimizer = optim.Adam(net.parameters(), lr=opt.lr)
early_stopping = EarlyStopping(patience=opt.patience, verbose=True)
os.makedirs(OutputDir, exist_ok=True)
train_loss_ls = []
valid_loss_ls = []
test_loss_ls = []
for epoch in range(0, opt.niter):
train_loss = train(epoch, train_dataloader, net, criterion, optimizer,
opt)
valid_loss = valid(valid_dataloader, net, criterion, opt)
test_loss = test(test_dataloader, net, criterion, opt)
train_loss_ls.append(train_loss)
valid_loss_ls.append(valid_loss)
test_loss_ls.append(test_loss)
early_stopping(valid_loss, net, OutputDir)
if early_stopping.early_stop:
print("Early stopping")
break
df = pd.DataFrame({
'epoch': [i for i in range(1,
len(train_loss_ls) + 1)],
'train_loss': train_loss_ls,
'valid_loss': valid_loss_ls,
'test_loss': test_loss_ls
})
df.to_csv(OutputDir + '/loss.csv', index=False)
net.load_state_dict(torch.load(OutputDir + '/checkpoint.pt'))
inference(all_dataloader, net, criterion, opt, OutputDir)
loss_val = F.nll_loss(output[idx_val], labels[idx_val])
acc_val = accuracy(output[idx_val], labels[idx_val])
print('Epoch:{: 4d}\tloss_train:{:.4f}\tacc_train:{:.4f}\tloss_val:{:.4f}\tacc_val:{:.4f}'.format(epoch+1, loss, acc, loss_val, acc_val))
def test(model, adj, features, labels, idx_test):
model.eval()
output = model.forward(features, adj)
loss = F.nll_loss(output[idx_test], labels[idx_test])
acc = accuracy(output[idx_test], labels[idx_test])
print('Test result\tloss:{:.4f}\tacc:{:.4f}'.format(loss, acc))
if __name__ == '__main__':
seed = 2020
hidden_dim = 16
dropout = 0.5
learning_rate = 0.01
weight_decay = 5e-4
epochs = 200
set_seed(seed)
adj, features, labels, idx_train, idx_val, idx_test = load_data()
model = GCN(input_dim=features.shape[1], hidden_dim=hidden_dim, output_dim=labels.max().item()+1, dropout=dropout)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
for e in range(epochs):
train(e, model, optimizer, adj, features, labels, idx_train, idx_val)
test(model, adj, features, labels, idx_test)
