def main(args):
# load and preprocess dataset
data = load_data(args)
features = torch.FloatTensor(data.features)
labels = torch.LongTensor(data.labels)
train_mask = torch.ByteTensor(data.train_mask)
val_mask = torch.ByteTensor(data.val_mask)
test_mask = torch.ByteTensor(data.test_mask)
num_feats = features.shape[1]
n_classes = data.num_labels
n_edges = data.graph.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes,
train_mask.sum().item(),
val_mask.sum().item(),
test_mask.sum().item()))
if args.gpu < 0:
cuda = False
else:
cuda = True
torch.cuda.set_device(args.gpu)
features = features.cuda()
labels = labels.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
g = data.graph
# add self loop
g.remove_edges_from(g.selfloop_edges())
g = DGLGraph(g)
g.add_edges(g.nodes(), g.nodes())
n_edges = g.number_of_edges()
# create model
heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
model = GAT(g,
args.num_layers,
num_feats,
args.num_hidden,
n_classes,
heads,
F.elu,
args.in_drop,
args.attn_drop,
args.negative_slope,
args.residual)
print(model)
stopper = EarlyStopping(patience=100)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(
model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
# initialize graph
dur = []
for epoch in range(args.epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
train_acc = accuracy(logits[train_mask], labels[train_mask])
if args.fastmode:
val_acc = accuracy(logits[val_mask], labels[val_mask])
else:
val_acc = evaluate(model, features, labels, val_mask)
if stopper.step(val_acc, model):
break
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainAcc {:.4f} |"
" ValAcc {:.4f} | ETputs(KTEPS) {:.2f}".
format(epoch, np.mean(dur), loss.item(), train_acc,
val_acc, n_edges / np.mean(dur) / 1000))
print()
model.load_state_dict(torch.load('es_checkpoint.pt'))
acc = evaluate(model, features, labels, test_mask)
print("Test Accuracy {:.4f}".format(acc))
def main(args):
# load and preprocess dataset
g, graph_labels = load_graphs(
'/yushi/dataset/Amazon2M/Amazon2M_dglgraph.bin')
assert len(g) == 1
g = g[0]
data = g.ndata
features = torch.FloatTensor(data['feat'])
labels = torch.LongTensor(data['label'])
if hasattr(torch, 'BoolTensor'):
train_mask = data['train_mask'].bool()
val_mask = data['val_mask'].bool()
test_mask = data['test_mask'].bool()
num_feats = features.shape[1]
n_classes = 47
n_edges = g.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes, train_mask.int().sum().item(),
val_mask.int().sum().item(), test_mask.int().sum().item()))
if args.gpu < 0:
cuda = False
else:
cuda = True
torch.cuda.set_device(args.gpu)
features = features.cuda()
labels = labels.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
# add self loop
g = add_self_loop(g)
# g.remove_edges_from(nx.selfloop_edges(g))
# g = DGLGraph(g)
# g.add_edges(g.nodes(), g.nodes())
n_edges = g.number_of_edges()
# create model
heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
model = GAT(g, args.num_layers, num_feats, args.num_hidden, n_classes,
heads, F.elu, args.in_drop, args.attn_drop,
args.negative_slope, args.residual)
print(model)
if args.early_stop:
stopper = EarlyStopping(patience=100)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# initialize graph
dur = []
start = time.time()
for epoch in range(args.epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
train_acc = accuracy(logits[train_mask], labels[train_mask])
if args.fastmode:
val_acc = accuracy(logits[val_mask], labels[val_mask])
else:
val_acc = evaluate(model, features, labels, val_mask)
if args.early_stop:
if stopper.step(val_acc, model):
break
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainAcc {:.4f} |"
" ValAcc {:.4f} | ETputs(KTEPS) {:.2f}".format(
epoch, np.mean(dur), loss.item(), train_acc, val_acc,
n_edges / np.mean(dur) / 1000))
print()
if args.early_stop:
model.load_state_dict(torch.load('es_checkpoint.pt'))
acc = evaluate(model, features, labels, test_mask)
print("Test Accuracy {:.4f}".format(acc))
print(f"Time Consuming {np.sum(dur)}, Overall time {time.time() - start}")
def main(args):
# load and preprocess dataset
if args.dataset == 'cora':
data = CoraGraphDataset()
elif args.dataset == 'citeseer':
data = CiteseerGraphDataset()
elif args.dataset == 'pubmed':
data = PubmedGraphDataset()
else:
raise ValueError('Unknown dataset: {}'.format(args.dataset))
g = data[0]
if args.gpu < 0:
cuda = False
else:
cuda = True
g = g.int().to(args.gpu)
features = g.ndata['feat']
labels = g.ndata['label']
train_mask = g.ndata['train_mask']
val_mask = g.ndata['val_mask']
test_mask = g.ndata['test_mask']
num_feats = features.shape[1]
n_classes = data.num_labels
n_edges = data.graph.number_of_edges()
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes, train_mask.int().sum().item(),
val_mask.int().sum().item(), test_mask.int().sum().item()))
# add self loop
g = dgl.remove_self_loop(g)
g = dgl.add_self_loop(g)
n_edges = g.number_of_edges()
# create model
heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
model = GAT(g, args.num_layers, num_feats, args.num_hidden, n_classes,
heads, F.elu, args.in_drop, args.attn_drop,
args.negative_slope, args.residual)
print(model)
if args.early_stop:
stopper = EarlyStopping(patience=100)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# initialize graph
dur = []
for epoch in range(args.epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
train_acc = accuracy(logits[train_mask], labels[train_mask])
if args.fastmode:
val_acc = accuracy(logits[val_mask], labels[val_mask])
else:
val_acc = evaluate(model, features, labels, val_mask)
if args.early_stop:
if stopper.step(val_acc, model):
break
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainAcc {:.4f} |"
" ValAcc {:.4f} | ETputs(KTEPS) {:.2f}".format(
epoch, np.mean(dur), loss.item(), train_acc, val_acc,
n_edges / np.mean(dur) / 1000))
print()
if args.early_stop:
model.load_state_dict(torch.load('es_checkpoint.pt'))
acc = evaluate(model, features, labels, test_mask)
print("Test Accuracy {:.4f}".format(acc))
def main(args):
if args.gpu < 0:
device = torch.device("cpu")
else:
device = torch.device("cuda:" + str(args.gpu))
batch_size = args.batch_size
cur_step = 0
patience = args.patience
best_score = -1
best_loss = 10000
# define loss function
loss_fcn = torch.nn.BCEWithLogitsLoss()
# create the dataset
train_dataset = LegacyPPIDataset(mode='train')
valid_dataset = LegacyPPIDataset(mode='valid')
test_dataset = LegacyPPIDataset(mode='test')
train_dataloader = DataLoader(train_dataset,
batch_size=batch_size,
collate_fn=collate)
valid_dataloader = DataLoader(valid_dataset,
batch_size=batch_size,
collate_fn=collate)
test_dataloader = DataLoader(test_dataset,
batch_size=batch_size,
collate_fn=collate)
n_classes = train_dataset.labels.shape[1]
num_feats = train_dataset.features.shape[1]
g = train_dataset.graph
heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
# define the model
model = GAT(g, args.num_layers, num_feats, args.num_hidden, n_classes,
heads, F.elu, args.in_drop, args.attn_drop, args.alpha,
args.residual)
# define the optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
model = model.to(device)
for epoch in range(args.epochs):
model.train()
loss_list = []
for batch, data in enumerate(train_dataloader):
subgraph, feats, labels = data
feats = feats.to(device)
labels = labels.to(device)
model.g = subgraph
for layer in model.gat_layers:
layer.g = subgraph
logits = model(feats.float())
loss = loss_fcn(logits, labels.float())
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss_list.append(loss.item())
loss_data = np.array(loss_list).mean()
print("Epoch {:05d} | Loss: {:.4f}".format(epoch + 1, loss_data))
if epoch % 5 == 0:
score_list = []
val_loss_list = []
for batch, valid_data in enumerate(valid_dataloader):
subgraph, feats, labels = valid_data
feats = feats.to(device)
labels = labels.to(device)
score, val_loss = evaluate(feats.float(), model, subgraph,
labels.float(), loss_fcn)
score_list.append(score)
val_loss_list.append(val_loss)
mean_score = np.array(score_list).mean()
mean_val_loss = np.array(val_loss_list).mean()
print("F1-Score: {:.4f} ".format(mean_score))
# early stop
if mean_score > best_score or best_loss > mean_val_loss:
if mean_score > best_score and best_loss > mean_val_loss:
val_early_loss = mean_val_loss
val_early_score = mean_score
best_score = np.max((mean_score, best_score))
best_loss = np.min((best_loss, mean_val_loss))
cur_step = 0
else:
cur_step += 1
if cur_step == patience:
break
test_score_list = []
for batch, test_data in enumerate(test_dataloader):
subgraph, feats, labels = test_data
feats = feats.to(device)
labels = labels.to(device)
test_score_list.append(
evaluate(feats, model, subgraph, labels.float(), loss_fcn)[0])
print("F1-Score: {:.4f}".format(np.array(test_score_list).mean()))
def main(args):
# load and preprocess dataset
g, features, labels, n_classes, train_mask, val_mask, test_mask, lp_dict, ind_features, ind_labels = load_reg_data(args)
num_feats = features.shape[1]
n_edges = g.number_of_edges()
print("""----Data statistics------'
#use cuda: %d
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(args.gpu, n_edges, n_classes,
train_mask.int().sum().item(),
val_mask.int().sum().item(),
test_mask.int().sum().item()))
if args.gpu < 0:
cuda = False
else:
cuda = True
torch.cuda.set_device(args.gpu)
features = features.cuda()
ind_features = ind_features.cuda()
labels = labels.cuda()
ind_labels = ind_labels.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
# create model
heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
model = GAT(g,
args.num_layers,
num_feats,
args.num_hidden,
n_classes,
heads,
F.elu,
args.in_drop,
args.attn_drop,
args.negative_slope,
args.residual,
args.bias)
print(model)
if args.early_stop:
stopper = EarlyStopping(patience=100)
if cuda:
model.cuda()
# use optimizer
optimizer = torch.optim.Adam(
model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
# initialize graph
dur = []
for epoch in range(args.epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
pred = model(features)
loss = loss_fcn(pred[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
train_r2 = compute_r2(pred[train_mask], labels[train_mask])
if args.fastmode:
val_r2 = compute_r2(pred[val_mask], labels[val_mask])
else:
val_r2 = evaluate(model, features, labels, val_mask)
if args.early_stop:
if stopper.step(val_r2, model):
break
if epoch > 3:
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainR2 {:.4f} |"
" Val R2 {:.4f} | ETputs(KTEPS) {:.2f}".
format(epoch, np.mean(dur), loss.item(), train_r2,
val_r2, n_edges / np.mean(dur) / 1000))
print()
if args.early_stop:
model.load_state_dict(torch.load('es_checkpoint.pt'))
evaluate_test(model, features, labels, test_mask, lp_dict, meta="2012")
evaluate_test(model, ind_features, ind_labels, test_mask, lp_dict, meta="2016")
def main(args):
# load and preprocess dataset
if args.dataset == 'reddit':
data = RedditDataset()
elif args.dataset in ['photo', "computer"]:
data = MsDataset(args)
else:
data = load_data(args)
features = torch.FloatTensor(data.features)
labels = torch.LongTensor(data.labels)
train_mask = torch.ByteTensor(data.train_mask)
val_mask = torch.ByteTensor(data.val_mask)
test_mask = torch.ByteTensor(data.test_mask)
num_feats = features.shape[1]
n_classes = data.num_labels
n_edges = data.graph.number_of_edges()
current_time = time.strftime('%d_%H:%M:%S', localtime())
writer = SummaryWriter(log_dir='runs/' + current_time + '_' + args.sess, flush_secs=30)
print("""----Data statistics------'
#Edges %d
#Classes %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(n_edges, n_classes,
train_mask.sum().item(),
val_mask.sum().item(),
test_mask.sum().item()))
if args.gpu < 0:
cuda = False
else:
cuda = True
torch.cuda.set_device(args.gpu)
features = features.cuda()
labels = labels.cuda()
train_mask = train_mask.bool().cuda()
val_mask = val_mask.bool().cuda()
test_mask = test_mask.bool().cuda()
g = data.graph
# add self loop
if args.dataset != 'reddit':
g.remove_edges_from(nx.selfloop_edges(g))
g = DGLGraph(g)
g.add_edges(g.nodes(), g.nodes())
n_edges = g.number_of_edges()
print('edge number %d'%(n_edges))
# create model
heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
model = GAT(g,
args.num_layers,
num_feats,
args.num_hidden,
n_classes,
heads,
F.elu,
args.idrop,
args.adrop,
args.alpha,
args.bias,
args.residual, args.l0)
print(model)
if args.early_stop:
stopper = EarlyStopping(patience=150)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
dur = []
time_used = 0
for epoch in range(args.epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
loss_l0 = args.loss_l0*( model.gat_layers[0].loss)
optimizer.zero_grad()
(loss + loss_l0).backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
train_acc = accuracy(logits[train_mask], labels[train_mask])
writer.add_scalar('edge_num/0', model.gat_layers[0].num, epoch)
if args.fastmode:
val_acc, loss = accuracy(logits[val_mask], labels[val_mask], loss_fcn)
else:
val_acc,_ = evaluate(model, features, labels, val_mask, loss_fcn)
if args.early_stop:
if stopper.step(val_acc, model):
break
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainAcc {:.4f} |"
" ValAcc {:.4f} | ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.item(), train_acc,
val_acc, n_edges / np.mean(dur) / 1000))
writer.add_scalar('loss', loss.item(), epoch)
writer.add_scalar('f1/train_f1_mic', train_acc, epoch)
writer.add_scalar('f1/test_f1_mic', val_acc, epoch)
writer.add_scalar('time/time', time_used, epoch)
writer.close()
if args.early_stop:
model.load_state_dict(torch.load('es_checkpoint.pt'))
acc, _ = evaluate(model,features, labels, test_mask, loss_fcn)
print("Test Accuracy {:.4f}".format(acc))
def train(args):
# load and preprocess dataset
#data = load_data(args)
#data = CoraFull()
#data = Coauthor('cs')
#FIRST, CHECK DATASET
path = './dataset/' + str(args.dataset) + '/'
'''
edges = np.loadtxt(path + 'edges.txt')
edges = edges.astype(int)
features = np.loadtxt(path + 'features.txt')
train_mask = np.loadtxt(path + 'train_mask.txt')
train_mask = train_mask.astype(int)
labels = np.loadtxt(path + 'labels.txt')
labels = labels.astype(int)
'''
edges = np.load(path + 'edges.npy')
features = np.load(path + 'features.npy')
train_mask = np.load(path + 'train_mask.npy')
labels = np.load(path + 'labels.npy')
num_edges = edges.shape[0]
num_nodes = features.shape[0]
num_feats = features.shape[1]
n_classes = max(labels) - min(labels) + 1
assert train_mask.shape[0] == num_nodes
print('dataset {}'.format(args.dataset))
print('# of edges : {}'.format(num_edges))
print('# of nodes : {}'.format(num_nodes))
print('# of features : {}'.format(num_feats))
features = torch.FloatTensor(features)
labels = torch.LongTensor(labels)
if hasattr(torch, 'BoolTensor'):
train_mask = torch.BoolTensor(train_mask)
else:
train_mask = torch.ByteTensor(train_mask)
if args.gpu < 0:
cuda = False
else:
cuda = True
torch.cuda.set_device(args.gpu)
features = features.cuda()
labels = labels.cuda()
train_mask = train_mask.cuda()
u = edges[:, 0]
v = edges[:, 1]
#initialize a DGL graph
g = DGLGraph()
g.add_nodes(num_nodes)
g.add_edges(u, v)
# add self loop
if isinstance(g, nx.classes.digraph.DiGraph):
g.remove_edges_from(nx.selfloop_edges(g))
g = DGLGraph(g)
g.add_edges(g.nodes(), g.nodes())
elif isinstance(g, DGLGraph):
g = transform.add_self_loop(g)
n_edges = g.number_of_edges()
# create model
heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
model = GAT(g, args.num_layers, num_feats, args.num_hidden, n_classes,
heads, F.elu, args.in_drop, args.attn_drop,
args.negative_slope, args.residual)
if args.early_stop:
stopper = EarlyStopping(patience=100)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# initialize graph
dur = []
record_time = 0
avg_run_time = 0
Used_memory = 0
for epoch in range(args.num_epochs):
#print('epoch = ', epoch)
#print('mem0 = {}'.format(mem0))
torch.cuda.synchronize()
tf = time.time()
model.train()
if epoch >= 3:
t0 = time.time()
# forward
logits = model(features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
now_mem = torch.cuda.max_memory_allocated(0)
print('now_mem : ', now_mem)
Used_memory = max(now_mem, Used_memory)
tf1 = time.time()
optimizer.zero_grad()
torch.cuda.synchronize()
t1 = time.time()
loss.backward()
torch.cuda.synchronize()
optimizer.step()
t2 = time.time()
run_time_this_epoch = t2 - tf
if epoch >= 3:
dur.append(time.time() - t0)
record_time += 1
avg_run_time += run_time_this_epoch
train_acc = accuracy(logits[train_mask], labels[train_mask])
#log for each step
print(
'Epoch {:05d} | Time(s) {:.4f} | train_acc {:.6f} | Used_Memory {:.6f} mb'
.format(epoch, run_time_this_epoch, train_acc,
(now_mem * 1.0 / (1024**2))))
'''
if args.fastmode:
val_acc = accuracy(logits[val_mask], labels[val_mask])
else:
val_acc = evaluate(model, features, labels, val_mask)
if args.early_stop:
if stopper.step(val_acc, model):
break
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainAcc {:.4f} |"
" ValAcc /{:.4f} | ETputs(KTEPS) {:.2f}".
format(epoch, np.mean(dur), loss.item(), train_acc,
val_acc, n_edges / np.mean(dur) / 1000))
'''
if args.early_stop:
model.load_state_dict(torch.load('es_checkpoint.pt'))
#OUTPUT we need
avg_run_time = avg_run_time * 1. / record_time
Used_memory /= (1024**3)
print('^^^{:6f}^^^{:6f}'.format(Used_memory, avg_run_time))
def main(args):
if args.gpu < 0:
device = torch.device("cpu")
else:
device = torch.device("cuda:" + str(args.gpu))
writer = SummaryWriter()
batch_size = args.batch_size
# cur_step = 0
# patience = args.patience
# best_score = -1
# best_loss = 10000
# define loss function
loss_fcn = torch.nn.BCEWithLogitsLoss()
# create the dataset
train_dataset = LegacyPPIDataset(mode='train')
valid_dataset = LegacyPPIDataset(mode='valid')
test_dataset = LegacyPPIDataset(mode='test')
train_dataloader = DataLoader(train_dataset,
batch_size=batch_size,
collate_fn=collate)
valid_dataloader = DataLoader(valid_dataset,
batch_size=batch_size,
collate_fn=collate)
test_dataloader = DataLoader(test_dataset,
batch_size=batch_size,
collate_fn=collate)
n_classes = train_dataset.labels.shape[1]
num_feats = train_dataset.features.shape[1]
g = train_dataset.graph
heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
# define the model
model = GAT(g, args.num_layers, num_feats, args.num_hidden, n_classes,
heads, F.elu, args.in_drop, args.attn_drop, args.alpha,
args.bias, args.residual, args.l0)
print(model)
# define the optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
model = model.to(device)
best_epoch = 0
dur = []
acc = []
for epoch in range(args.epochs):
num = 0
model.train()
if epoch % 5 == 0:
t0 = time.time()
loss_list = []
for batch, data in enumerate(train_dataloader):
subgraph, feats, labels = data
feats = feats.to(device)
labels = labels.to(device)
model.g = subgraph
for layer in model.gat_layers:
layer.g = subgraph
logits = model(feats.float())
loss = loss_fcn(logits, labels.float())
loss_l0 = args.loss_l0 * (model.gat_layers[0].loss)
optimizer.zero_grad()
(loss + loss_l0).backward()
optimizer.step()
loss_list.append(loss.item())
num += model.gat_layers[0].num
if epoch % 5 == 0:
dur.append(time.time() - t0)
loss_data = np.array(loss_list).mean()
print("Epoch {:05d} | Loss: {:.4f}".format(epoch + 1, loss_data))
writer.add_scalar('edge_num/0', num, epoch)
if epoch % 5 == 0:
score_list = []
val_loss_list = []
for batch, valid_data in enumerate(valid_dataloader):
subgraph, feats, labels = valid_data
feats = feats.to(device)
labels = labels.to(device)
score, val_loss = evaluate(feats.float(), model, subgraph,
labels.float(), loss_fcn)
score_list.append(score)
val_loss_list.append(val_loss)
mean_score = np.array(score_list).mean()
mean_val_loss = np.array(val_loss_list).mean()
print("val F1-Score: {:.4f} ".format(mean_score))
writer.add_scalar('loss', mean_val_loss, epoch)
writer.add_scalar('f1/test_f1_mic', mean_score, epoch)
acc.append(mean_score)
# # early stop
# if mean_score > best_score or best_loss > mean_val_loss:
# if mean_score > best_score and best_loss > mean_val_loss:
# val_early_loss = mean_val_loss
# val_early_score = mean_score
# torch.save(model.state_dict(), '{}.pkl'.format('save_rand'))
# best_epoch = epoch
#
# best_score = np.max((mean_score, best_score))
# best_loss = np.min((best_loss, mean_val_loss))
# cur_step = 0
# else:
# cur_step += 1
# if cur_step == patience:
# break
test_score_list = []
for batch, test_data in enumerate(test_dataloader):
subgraph, feats, labels = test_data
feats = feats.to(device)
labels = labels.to(device)
test_score_list.append(
evaluate(feats, model, subgraph, labels.float(), loss_fcn)[0])
acc = np.array(test_score_list).mean()
print("test F1-Score: {:.4f}".format(acc))
writer.close()
def main(training_file,
dev_file,
test_file,
graph_type=None,
net=None,
epochs=None,
patience=None,
grid_width=None,
image_width=None,
batch_size=None,
num_hidden=None,
heads=None,
gnn_layers=None,
cnn_layers=None,
nonlinearity=None,
residual=None,
lr=None,
weight_decay=None,
in_drop=None,
alpha=None,
attn_drop=None,
cuda=None,
fw='dgl',
index=None,
previous_model=None):
global stop_training
if nonlinearity == 'relu':
nonlinearity = F.relu
elif nonlinearity == 'elu':
nonlinearity = F.elu
loss_fcn = torch.nn.MSELoss() #(reduction='sum')
print('=========================')
print('HEADS', heads)
#print('OUT_HEADS', num_out_heads)
print('GNN LAYERS', gnn_layers)
print('CNN LAYERS', cnn_layers)
print('HIDDEN', num_hidden)
print('RESIDUAL', residual)
print('inDROP', in_drop)
print('atDROP', attn_drop)
print('LR', lr)
print('DECAY', weight_decay)
print('ALPHA', alpha)
print('BATCH', batch_size)
print('GRAPH_ALT', graph_type)
print('ARCHITECTURE', net)
print('=========================')
# create the dataset
time_dataset_a = time.time()
print('Loading training set...')
train_dataset = socnavImg.SocNavDataset(training_file, mode='train')
print('Loading dev set...')
valid_dataset = socnavImg.SocNavDataset(dev_file, mode='valid')
print('Loading test set...')
test_dataset = socnavImg.SocNavDataset(test_file, mode='test')
print('Done loading files')
train_dataloader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate)
valid_dataloader = DataLoader(valid_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate)
test_dataloader = DataLoader(test_dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=collate)
time_dataset_b = time.time()
for _ in range(5):
print(f'TIME {time_dataset_b-time_dataset_a}')
num_rels = len(socnavImg.get_relations())
cur_step = 0
best_loss = -1
n_classes = num_hidden[-1]
print('Number of classes: {}'.format(n_classes))
num_feats = train_dataset.graphs[0].ndata['h'].shape[1]
print('Number of features: {}'.format(num_feats))
g = dgl.batch(train_dataset.graphs)
#heads = ([num_heads] * gnn_layers) + [num_out_heads]
# define the model
if fw == 'dgl':
if net in ['gat']:
model = GAT(
g, # graph
gnn_layers, # gnn_layers
num_feats, # in_dimension
num_hidden, # num_hidden
1,
grid_width, # grid_width
heads, # head
nonlinearity, # activation
in_drop, # feat_drop
attn_drop, # attn_drop
alpha, # negative_slope
residual, # residual
cnn_layers # cnn_layers
)
elif net in ['gatmc']:
model = GATMC(
g, # graph
gnn_layers, # gnn_layers
num_feats, # in_dimension
num_hidden, # num_hidden
grid_width, # grid_width
image_width, # image_width
heads, # head
nonlinearity, # activation
in_drop, # feat_drop
attn_drop, # attn_drop
alpha, # negative_slope
residual, # residual
cnn_layers # cnn_layers
)
elif net in ['rgcn']:
print(
f'CREATING RGCN(GRAPH, gnn_layers:{gnn_layers}, cnn_layers:{cnn_layers}, num_feats:{num_feats}, num_hidden:{num_hidden}, grid_with:{grid_width}, image_width:{image_width}, num_rels:{num_rels}, non-linearity:{nonlinearity}, drop:{in_drop}, num_bases:{num_rels})'
)
model = RGCN(g,
gnn_layers,
cnn_layers,
num_feats,
num_hidden,
grid_width,
image_width,
num_rels,
nonlinearity,
in_drop,
num_bases=num_rels)
else:
print('No valid GNN model specified')
sys.exit(0)
# define the optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
# for name, param in model.named_parameters():
# if param.requires_grad:
# print(name, param.data.shape)
if previous_model is not None:
model.load_state_dict(torch.load(previous_model, map_location=device))
model = model.to(device)
for epoch in range(epochs):
if stop_training:
print("Stopping training. Please wait.")
break
model.train()
loss_list = []
for batch, data in enumerate(train_dataloader):
subgraph, labels = data
subgraph.set_n_initializer(dgl.init.zero_initializer)
subgraph.set_e_initializer(dgl.init.zero_initializer)
feats = subgraph.ndata['h'].to(device)
labels = labels.to(device)
if fw == 'dgl':
model.g = subgraph
for layer in model.layers:
layer.g = subgraph
if net in ['rgcn']:
logits = model(
feats.float(),
subgraph.edata['rel_type'].squeeze().to(device))
else:
logits = model(feats.float())
else:
print('Only DGL is supported at the moment here.')
sys.exit(1)
if net in ['pgat', 'pgcn']:
data = Data(x=feats.float(),
edge_index=torch.stack(
subgraph.edges()).to(device))
else:
data = Data(
x=feats.float(),
edge_index=torch.stack(subgraph.edges()).to(device),
edge_type=subgraph.edata['rel_type'].squeeze().to(
device))
logits = model(data, subgraph)
a = logits ## [getMaskForBatch(subgraph)].flatten()
# print('AA', a.shape)
# print(a)
a = a.flatten()
#print('labels', labels.shape)
b = labels.float()
# print('b')
# print(b)
b = b.flatten()
# print('BB', b.shape)
ad = a.to(device)
bd = b.to(device)
# print(ad.shape, ad.dtype, bd.shape, bd.dtype)
loss = loss_fcn(ad, bd)
optimizer.zero_grad()
a = list(model.parameters())[0].clone()
loss.backward()
optimizer.step()
b = list(model.parameters())[0].clone()
not_learning = torch.equal(a.data, b.data)
if not_learning:
import sys
print('Not learning')
# sys.exit(1)
else:
pass
# print('Diff: ', (a.data-b.data).sum())
# print(loss.item())
loss_list.append(loss.item())
loss_data = np.array(loss_list).mean()
print('Loss: {}'.format(loss_data))
if epoch % 5 == 0:
if epoch % 5 == 0:
print("Epoch {:05d} | Loss: {:.4f} | Patience: {} | ".format(
epoch, loss_data, cur_step),
end='')
score_list = []
val_loss_list = []
for batch, valid_data in enumerate(valid_dataloader):
subgraph, labels = valid_data
subgraph.set_n_initializer(dgl.init.zero_initializer)
subgraph.set_e_initializer(dgl.init.zero_initializer)
feats = subgraph.ndata['h'].to(device)
labels = labels.to(device)
score, val_loss = evaluate(feats.float(), model, subgraph,
labels.float(), loss_fcn, fw, net)
score_list.append(score)
val_loss_list.append(val_loss)
mean_score = np.array(score_list).mean()
mean_val_loss = np.array(val_loss_list).mean()
if epoch % 5 == 0:
print("Score: {:.4f} MEAN: {:.4f} BEST: {:.4f}".format(
mean_score, mean_val_loss, best_loss))
# early stop
if best_loss > mean_val_loss or best_loss < 0:
print('Saving...')
directory = str(index).zfill(5)
os.system('mkdir ' + directory)
best_loss = mean_val_loss
# Save the model
torch.save(model.state_dict(), directory + '/SNGNN2D.tch')
params = {
'loss': best_loss,
'net': net, #str(type(net)),
'fw': fw,
'gnn_layers': gnn_layers,
'cnn_layers': cnn_layers,
'num_feats': num_feats,
'num_hidden': num_hidden,
'graph_type': graph_type,
'n_classes': n_classes,
'heads': heads,
'grid_width': grid_width,
'image_width': image_width,
'F': F.relu,
'in_drop': in_drop,
'attn_drop': attn_drop,
'alpha': alpha,
'residual': residual,
'num_rels': num_rels
}
pickle.dump(params, open(directory + '/SNGNN2D.prms', 'wb'))
cur_step = 0
else:
# print(best_loss, mean_val_loss)
cur_step += 1
if cur_step >= patience:
break
test_score_list = []
for batch, test_data in enumerate(test_dataloader):
subgraph, labels = test_data
subgraph.set_n_initializer(dgl.init.zero_initializer)
subgraph.set_e_initializer(dgl.init.zero_initializer)
feats = subgraph.ndata['h'].to(device)
labels = labels.to(device)
test_score_list.append(
evaluate(feats, model, subgraph, labels.float(), loss_fcn, fw,
net)[1])
print("MSE for the test set {}".format(np.array(test_score_list).mean()))
model.eval()
return best_loss
def main(args):
if args.gpu<0:
device = torch.device("cpu")
else:
device = torch.device("cuda:" + str(args.gpu))
# batch_size = args.batch_size
# cur_step = 0
# patience = args.patience
# best_score = -1
# best_loss = 10000
# # define loss function
# loss_fcn = torch.nn.BCEWithLogitsLoss()
# create the dataset
train_dataset = LegacyPPIDataset(mode='train')
valid_dataset = LegacyPPIDataset(mode='valid')
test_dataset = LegacyPPIDataset(mode='test')
# nxg = valid_dataset.graph.to_networkx().to_undirected()
# comps = [comp for comp in nx.connected_components(nxg) if len(comp)>10]
# print(len(comps))
# exit()
cross_valid_list = []
for i in range(5):
cross_valid_list.append(list(range(4*i, 4*(i + 1))))
cross_train_dataset = copy.copy(train_dataset)
valid_precision = []
valid_recall = []
valid_scores = []
test_precision = []
test_recall = []
test_scores = []
for ind, valid_list in enumerate(cross_valid_list):
batch_size = args.batch_size
cur_step = 0
patience = args.patience
best_score = -1
best_loss = 10000
# define loss function
loss_fcn = torch.nn.BCEWithLogitsLoss()
train_list = [ind for ind in range(20) if ind not in valid_list]
print('Train List: {}'.format(train_list))
print('Valid List: {}'.format(valid_list))
modify(train_dataset, cross_train_dataset, train_list, mode='train', offset=0)
modify(valid_dataset, cross_train_dataset, valid_list, mode='valid', offset=16)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, collate_fn=collate)
valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size, collate_fn=collate)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, collate_fn=collate)
n_classes = train_dataset.labels.shape[1]
num_feats = train_dataset.features.shape[1]
g = train_dataset.graph
heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
# define the model
model = GAT(g,
args.num_layers,
num_feats,
args.num_hidden,
n_classes,
heads,
F.elu,
args.in_drop,
args.attn_drop,
args.alpha,
args.residual)
# define the optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
model = model.to(device)
for epoch in range(args.epochs):
model.train()
loss_list = []
for batch, data in enumerate(train_dataloader):
subgraph, feats, labels = data
feats = feats.to(device)
labels = labels.to(device)
model.g = subgraph
for layer in model.gat_layers:
layer.g = subgraph
logits = model(feats.float())
loss = loss_fcn(logits, labels.float())
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss_list.append(loss.item())
loss_data = np.array(loss_list).mean()
print("Epoch {:05d} | Loss: {:.4f}".format(epoch + 1, loss_data), end=' ')
if epoch % 1 == 0:
score_list = []
val_loss_list = []
for batch, valid_data in enumerate(valid_dataloader):
subgraph, feats, labels = valid_data
feats = feats.to(device)
labels = labels.to(device)
prec, recall, score, val_loss = evaluate(feats.float(), model, subgraph, labels.float(), loss_fcn)
score_list.append([prec, recall, score])
val_loss_list.append(val_loss)
mean_score = np.array(score_list).mean(axis=0)
mean_val_loss = np.array(val_loss_list).mean()
print("| Valid Precision: {:.4f} | Valid Recall: {:.4f} | Valid F1-Score: {:.4f} ".format(mean_score[0], mean_score[1], mean_score[2]), end = ' ')
test_score_list = []
for batch, test_data in enumerate(test_dataloader):
subgraph, feats, labels = test_data
feats = feats.to(device)
labels = labels.to(device)
test_prec, test_rec, test_score, _ = evaluate(feats, model, subgraph, labels.float(), loss_fcn)
test_score_list.append([test_prec, test_rec, test_score])
mean_test_score = np.array(test_score_list).mean(axis=0)
print("| Test Precision: {:.4f} | Test Recall: {:.4f} | Test F1-Score: {:.4f}".format(mean_test_score[0], mean_test_score[1], mean_test_score[2]))
if epoch == args.epochs - 1:
valid_precision.append(round(mean_score[0], 4))
valid_recall.append(round(mean_score[1], 4))
valid_scores.append(round(mean_score[2], 4))
test_precision.append(round(mean_test_score[0], 4))
test_recall.append(round(mean_test_score[1], 4))
test_scores.append(round(mean_test_score[2], 4))
# early stop
if mean_score[2] > best_score or best_loss > mean_val_loss:
if mean_score[2] > best_score and best_loss > mean_val_loss:
val_early_loss = mean_val_loss
val_early_score = mean_score[2]
best_score = np.max((mean_score[2], best_score))
best_loss = np.min((best_loss, mean_val_loss))
cur_step = 0
else:
cur_step += 1
if cur_step == patience:
valid_precision.append(round(mean_score[0], 4))
valid_recall.append(round(mean_score[1], 4))
valid_scores.append(round(mean_score[2], 4))
test_precision.append(round(mean_test_score[0], 4))
test_recall.append(round(mean_test_score[1], 4))
test_scores.append(round(mean_test_score[2], 4))
break
print('Valid Scores: {}'.format(valid_scores))
print('Test Scores: {}'.format(test_scores))
out_matrix = np.stack([valid_precision, valid_recall, valid_scores, test_precision, test_recall, test_scores], axis=1)
np.savetxt('results.csv', out_matrix, delimiter=',')
best_loss = 10000
best_model = None
best_loss_curve = []
val_early_loss = 10000
val_early_score = -1
model = GAT(g, num_feats, 256, n_classes, [4, 4, 6], F.elu, 0.0001, 0.0001,
0.2, True)
loss_fcn = torch.nn.BCEWithLogitsLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=0.005)
model = model.to(device)
save_loss = []
for epoch in range(200):
model.train()
loss_list = []
for train_batch in batch_list:
model.g = g.subgraph(train_batch)
for layer in model.gat_layers:
layer.g = g.subgraph(train_batch)
input_feature = features[train_batch]
logits = model(input_feature)
loss = loss_fcn(logits, labels[train_batch].float())
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss_list.append(loss.item())
loss_data = np.array(loss_list).mean()
save_loss.append(loss_data)
print("Epoch {:05d} | Loss: {:.4f}".format(epoch + 1, loss_data))