def main():
from pre_process import preprocess
feature, a_hat, labels = preprocess()
print("loaded")
selected, unselected = depart(len(labels), 1 - Config.test_ratio)
labels_selected = labels[selected]
labels_unselected = labels[unselected]
feature = torch.from_numpy(feature).float().cuda()
tensor_selected = torch.tensor(labels_selected).long().cuda()
a_hat = torch.tensor(a_hat).float().cuda()
net = GCN(a_hat, feature.shape[1], Config.num_classes, Config.hidden_size,
Config.n_hidden_layer).cuda()
print(net)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=Config.lr)
net.train()
for e in range(Config.num_epochs):
optimizer.zero_grad()
output = net(feature)
loss = criterion(output[selected], tensor_selected)
loss.backward()
optimizer.step()
trained_accuracy = evaluate(output[selected], labels_selected)
untrained_accuracy = evaluate(output[unselected], labels_unselected)
print(
"[Epoch %d]: trained acc: %.7f, untrained acc: %.7f, loss: %.7f" %
(e, trained_accuracy, untrained_accuracy,
loss.detach().cpu().numpy()))
def main():
# Load data
start = time.time()
N, _adj, _feats, _labels, train_adj, train_feats, train_nodes, val_nodes, test_nodes, y_train, y_val, y_test, val_mask, test_mask = utils.load_data(args.dataset)
print('Loaded data in {:.2f} seconds!'.format(time.time() - start))
# Prepare Train Data
start = time.time()
_, parts = utils.partition_graph(train_adj, train_nodes, args.num_clusters_train)
parts = [np.array(pt) for pt in parts]
train_features, train_support, y_train = utils.preprocess_multicluster(train_adj, parts, train_feats, y_train, args.num_clusters_train, args.batch_size)
print('Train Data pre-processed in {:.2f} seconds!'.format(time.time() - start))
# Prepare Test Data
if args.test == 1:
y_test, test_mask = y_val, val_mask
start = time.time()
_, test_features, test_support, y_test, test_mask = utils.preprocess(_adj, _feats, y_test, np.arange(N), args.num_clusters_test, test_mask)
print('Test Data pre-processed in {:.2f} seconds!'.format(time.time() - start))
# Shuffle Batches
batch_idxs = list(range(len(train_features)))
# model
model = GCN(fan_in=_in, fan_out=_out, layers=args.layers, dropout=args.dropout, normalize=True, bias=False).float()
model.cuda()
# Loss Function
criterion = torch.nn.CrossEntropyLoss()
# Optimization Algorithm
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
# Learning Rate Schedule
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=args.lr, steps_per_epoch=int(args.num_clusters_train/args.batch_size), epochs=args.epochs+1, anneal_strategy='linear')
model.train()
# Train
for epoch in range(args.epochs + 1):
np.random.shuffle(batch_idxs)
avg_loss = 0
start = time.time()
for batch in batch_idxs:
loss = train(model.cuda(), criterion, optimizer, train_features[batch], train_support[batch], y_train[batch], dataset=args.dataset)
if args.lr_scheduler == 1:
scheduler.step()
avg_loss += loss.item()
# Write Train stats to tensorboard
writer.add_scalar('time/train', time.time() - start, epoch)
writer.add_scalar('loss/train', avg_loss/len(train_features), epoch)
if args.test == 1:
# Test on cpu
f1 = test(model.cpu(), test_features, test_support, y_test, test_mask, device='cpu')
print('f1: {:.4f}'.format(f1))
def test_memorize_minibatch(self):
for db_name in self.db_names:
db_info = get_db_info(db_name)
train_data, val_data, _ = get_train_val_test_datasets(
dataset_name=db_name,
train_test_split='use_full_train',
encoders=dict(CATEGORICAL='CategoricalOrdinalEnc',
SCALAR='ScalarRobustScalerEnc',
DATETIME='DatetimeScalarEnc',
LATLONG='LatLongScalarEnc',
TEXT='TextSummaryScalarEnc'),
)
train_loader = get_dataloader(
dataset=train_data,
batch_size=256,
sampler_class_name='SequentialSampler',
num_workers=0,
max_nodes_per_graph=False)
writer = DummyWriter()
model = GCN(writer,
db_info=db_info,
hidden_dim=256,
n_init_layers=3,
activation_class_name='SELU',
activation_class_kwargs={},
loss_class_kwargs={},
loss_class_name='CrossEntropyLoss',
p_dropout=0.0,
drop_whole_embeddings=True,
n_layers=3,
readout_class_name='AvgPooling',
readout_kwargs={})
if torch.cuda.is_available():
model.cuda()
model.device = torch.device('cuda:0')
else:
model.device = torch.device('cpu')
model.train()
optimizer = AdamW(model.parameters(), lr=0.001, weight_decay=0.0)
bdgl, features, label = next(iter(train_loader))
recursive_to((bdgl, features, label), model.device)
for _ in tqdm(range(200)):
optimizer.zero_grad()
output = model(bdgl, features)
loss = model.loss_fxn(output, label)
if loss < 1e-4:
break
loss.backward()
optimizer.step()
else:
tqdm.write(f'Loss: {loss}')
self.fail("Didn't memorize minibatch")
def main():
data_generator = DataGenerator(args)
meta_model = GCN(nfeat=args.in_f_d,
nhid=args.hidden,
nclass=args.nclasses,
dropout=args.dropout).to(device)
proto_model = GCN_Proto(args, nfeat=args.hidden, dropout=args.dropout).to(device)
structure_model = GCN_Structure(args, nfeat=args.hidden, nhid=args.structure_dim, dropout=args.dropout).to(
device)
if args.train:
meta_optimiser = torch.optim.Adam(
list(meta_model.parameters()) + list(proto_model.parameters()) + list(structure_model.parameters()),
lr=args.meta_lr, weight_decay=args.weight_decay)
train(args, meta_model, meta_optimiser, proto_model, structure_model,
metatrain_iterations=args.metatrain_iterations,
data_generator=data_generator, fit_function=meta_gradient_step,
fit_function_kwargs={'train': True, 'inner_train_steps': args.inner_train_steps,
'inner_lr': args.inner_lr, 'batch_n': args.batch_n, 'device': device})
else:
if args.test_load_epoch > 0:
meta_model.load_state_dict(
torch.load(args.logdir + '/' + exp_string + '/' + 'model_epoch_{}'.format(args.test_load_epoch)))
proto_model.load_state_dict(
torch.load(
args.logdir + '/' + exp_string + '/' + 'proto_model_epoch_{}'.format(args.test_load_epoch)))
structure_model.load_state_dict(
torch.load(
args.logdir + '/' + exp_string + '/' + 'structure_model_epoch_{}'.format(
args.test_load_epoch)))
meta_optimiser = torch.optim.Adam(list(meta_model.parameters()) + list(proto_model.parameters()),
lr=args.meta_lr, weight_decay=args.weight_decay)
evaluate(args, meta_model, meta_optimiser, proto_model, structure_model, data_generator=data_generator,
fit_function=meta_gradient_step,
fit_function_kwargs={'train': False, 'inner_train_steps': args.inner_train_steps,
'inner_lr': args.inner_lr_test, 'batch_n': args.test_sample_g_n,
'device': device})
def train(self):
"""
Trains the model based on configurations specified in __init__
"""
# Model and optimizer
model = GCN(nfeat=self.features.shape[1],
nhid=self.hidden,
nclass=self.labels.max().item() + 1,
dropout=self.dropout)
optimizer = optim.Adam(model.parameters(),
lr=self.lr,
weight_decay=self.weight_decay)
# Train model
for _ in range(self.epochs):
train(model, self.features, self.adj, self.idx_train, self.labels,
optimizer)
return model
def run(n_trials=100):
start_time = time.time()
# Load data
adj, features, labels, idx_train, idx_test = load_data()
epochs = 200
lr = 0.01
weight_decay = 5e-4
hidden = 16
dropout = 0.5
for i in range(n_trials):
# Model and optimizer
model = GCN(nfeat=features.shape[1],
nhid=hidden,
nclass=labels.max().item() + 1,
dropout=dropout)
optimizer = optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
# Train model
for _ in range(epochs):
train(model, features, adj, idx_train, labels, optimizer)
done_training_time = time.time()
# Testing
metrics = test(model, features, adj, idx_test, labels)
params = model.state_dict()
for k in params:
params[k] = params[k].reshape(-1).data.tolist()
final_results = {**params, **metrics}
done_testing_time = time.time()
# Cache to file
records_filename = "records.csv"
write_results_to_file(records_filename, final_results)
# report_progress(i, start_time, done_training_time,
# done_testing_time, end_time)
end_time = time.time()
speed = n_trials / (end_time - start_time)
print(f"{n_trials} tasks completed in {end_time - start_time}.")
print(f"{round(speed, 3)} tasks/second for non-parallel implementation.")
def main(dataset, times):
adj, features, labels, idx_train, idx_val, idx_test = load_data(dataset)
features = features.to(device)
adj = adj.to(device)
labels = labels.to(device)
idx_train = idx_train.to(device)
idx_val = idx_val.to(device)
idx_test = idx_test.to(device)
nclass = labels.max().item() + 1
acc_lst = list()
for seed in random.sample(range(0, 100000), times):
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
# Model and optimizer
# weight_decay 权值衰减，防止过拟合，调节模型复杂度对损失函数的影响
model = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=nclass,
dropout=args.dropout)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
model.to(device)
# Train model
t_total = time.time()
for epoch in range(args.epochs):
train(epoch, model, optimizer, adj, features, labels, idx_train,
idx_val)
print(f"Total time elapsed: {time.time() - t_total:.4f}s")
# Testing
acc_lst.append(test(model, adj, features, labels, idx_test))
print(acc_lst)
print(np.mean(acc_lst))
aggregation = torch.mm(torch.mm(D_2, A_hat), D_2) #this doesn't
aggregation = torch.mm(torch.inverse(D_hat),
A_hat) #this normalises everything to sum to one
# train model
adj = torch.FloatTensor(aggregation).to_sparse()
model = GCN(dims=[X.shape[1], 4, nclasses],
dropout=dropout,
adj=adj,
nrm_mthd="softmax",
learnable=False,
projection=False,
rand=False)
optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
model(X)
h1 = torch.transpose(model.embeddings_dict['h2'], 0, 1).detach()
#plt.scatter(h1[0], h1[1], marker='o', color=Y_colors)
#plt.show()
for epoch in range(200):
train(epoch)
test()
h1 = torch.transpose(model.embeddings_dict['h2'], 0, 1).detach()
plt.scatter(h1[0], h1[1], marker='o', color=Y_colors)
plt.show()
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:",
"loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
#print ('###in save-load.py###')
#test()
#print ('========')
#exit()
adj, features, labels, idx_train, idx_val, idx_test = load_data()
input = torch.load('input.pt')
adj = torch.sparse.FloatTensor(input['indices']input['values'], input['size'])
features, labels, idx_train, idx_val, idx_test = input['features'], input['labels'], input['idx_train'], input['idx_val'], input['idx_test']
kwargs = {"dropout":0.5, "nfeat":features.shape[1], "nclass":labels.max().item()+1, "nhid":16}
model = GCN(**kwargs)
kwargs = {'params': model.parameters(), 'lr': 0.01, 'weight_decay': 5e-4}
optimizer = optim.Adam(**kwargs)
checkpoint = torch.load('model-optimised.pt')
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
#epoch = checkpoint['epoch']
#testing
model.eval()
test()
# Optionally configure tensorboard
args.run_name = "{}_{}".format(args.run_name, time.strftime("%Y%m%dT%H%M%S"))
tb_logger = None
if not args.no_tensorboard:
tb_logger = TbLogger(os.path.join(args.log_dir, args.run_name))
# model
model = GCN(input_dim=train_dataset[0].x.shape[1],
num_hid_layers=args.num_hid_layers,
hidden_dim=args.hidden_dim,
num_class=2,
dropout=args.dropout)
# optimizer
# optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
optimizer = optim.SGD(model.parameters(), lr=args.lr)
# save pytorch model and track all of the gradients and optionally parameters
wandb.watch(model, log='all') # "gradients", "parameters", "all", or None.
# Train model
t_total = time.time()
# initialize the early_stopping object
early_stopping = EarlyStopping(patience=args.es_patience, verbose=True)
for epoch in range(args.epochs):
running_loss_train = 0
running_acc_train = 0
for i in tqdm(range(len(train_dataset))):
# data with labels based on approximate solutions
nclass=labels.max() + 1,
dropout=0.5)
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
features, adj, labels = Variable(features), Variable(adj), Variable(
labels)
#net = nn.DataParallel(net, device_ids=range(1))
#net.load_state_dict(torch.load(opt.modelIn))
optimizer = optim.Adam(net.parameters(), lr=0.01, weight_decay=5e-4)
net.cuda()
for epoch_n in range(200):
train(epoch_n)
loss_f = nn.CrossEntropyLoss()
net.eval()
output = net(features, adj)
output = F.log_softmax(output, dim=1)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.data[0]),
"accuracy= {:.4f}".format(acc_test.data[0]))
for c in opt.c:
#print (type(adj), type(features))
acc, avg_distort = acc_under_attack(features, adj, labels, net, c,
def train():
acc_test_list = []
for ii in range(args.run_time):
#seed = gen_seeds()
seed = args.seed
print("dataset:{}, epochs:{}, weight_decay:{},lr:{},dropout:{},seed:{}, alpha:{}, features_perturbation: rate1:{},lambda1:{}; adj_pertubation: rate2:{},lambda2:{}".format(
args.dataset, args.epochs, args.weight_decay, args.lr, args.dropout, seed, args.alpha, args.rate1, args.lambda1, args.rate2,args.lambda2))
args.cuda = not args.no_cuda and torch.cuda.is_available()
np.random.seed(seed)
torch.manual_seed(seed)
if args.cuda:
torch.cuda.manual_seed(seed)
#torch.cuda.manual_seed(seed)
if args.dataset == "R8":
adj, features, labels, idx_train, idx_val, idx_test = load_corpus(args.dataset, args.normalization,args.cuda)
else:
adj, features, labels, idx_train, idx_val, idx_test, indices = load_citation(args.dataset, args.normalization, args.cuda)
model = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout)
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
global x,y,z
x,y,z=0,0,0
win = vis.line(X=np.array([x]),
Y=np.array([y]),
opts=dict(title='loss_CE'))
global b_1,b_2
b_0,b_1,b_2=0,0,0
win_b0 = vis.line(X=np.array([x]),
Y=np.array([b_0]),
opts=dict(title='b'))
def train(epoch):
t = time.time()
model.train()
optimizer.zero_grad()
output = model(features, adj)
output = F.log_softmax(output, dim=1)
loss_CE = F.nll_loss(output[idx_train], labels[idx_train])
#l2_reg = sum(torch.sum(param ** 2) for param in model.reg_params)
acc_train = accuracy(output[idx_train], labels[idx_train])
x = epoch
y = loss_CE.detach().cpu().numpy()
vis.line(X=np.array([x]),
Y=np.array([y]),
win=win,
update='append')
#loss_train = loss_CE + args.weight_decay /2 *l2_reg
loss_train = loss_CE
loss_train.backward()
optimizer.step()
if not args.fastmode:
model.eval()
output = model(features, adj)
output = F.log_softmax(output, dim=1)
loss_val = F.nll_loss(output[idx_val], labels[idx_val])
acc_val = accuracy(output[idx_val], labels[idx_val])
if ii == 0:
if epoch%10==0:
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(loss_train.item()),
'loss_CE: {:.4f}'.format(loss_CE.item()),
'acc_train: {:.4f}'.format(acc_train.item()),
#'loss_0: {:.4f}'.format(loss_0.item()),
# 'loss_fx: {:.4f}'.format(loss_fx.item()),
# 'loss_logfx: {:.4f}'.format(loss_logfx.item()),
# 'acc_train: {:.4f}'.format(acc_train.item()),
'loss_val: {:.4f}'.format(loss_val.item()),
'acc_val: {:.4f}'.format(acc_val.item()),
'time: {:.4f}s'.format(time.time() - t))
return loss_val.item()
def test():
model.eval()
output = model(features,adj)
output = F.log_softmax(output, dim=1)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:",
"loss_test= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
return acc_test
t_total = time.time()
loss_values = []
# bad_counter = 0
# best = args.epochs+1
# best_epoch=0
# for epoch in range(args.epochs):
#
# loss_values.append(train(epoch))
# torch.save(model.state_dict(), './checkpoints/{}/{}.pkl'.format(args.dataset, epoch))
# if loss_values[-1] < best:
# best = loss_values[-1]
# best_epoch = epoch
# bad_counter = 0
# else:
# bad_counter += 1
# if bad_counter == args.patience:
# break
# files = glob.glob('./checkpoints/{}/*.pkl'.format(args.dataset)) # =[/checkpoints/{}/{}/0.pkl]
# for file in files:
# epoch_nb = int(file.split('.')[-2].split('/')[-1])
# if epoch_nb < best_epoch:
# os.remove(file)
# files = glob.glob('./checkpoints/{}/*.pkl'.format(args.dataset))
# for file in files:
# epoch_nb = int(file.split('.')[-2].split('/')[-1])
# if epoch_nb > best_epoch:
# os.remove(file)
# print("Optimization Finished!")
# print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
#
# # Testing
# print('Loading {}th epoch'.format(best_epoch))
# model.load_state_dict(torch.load('./checkpoints/{}/{}.pkl'.format(args.dataset, best_epoch)))
for epoch in range(args.epochs):
loss_values.append(train(epoch))
if epoch>args.early_stop and loss_values[-1] > np.mean(loss_values[-(args.early_stop+1):-1]):
print("Early stopping...")
break
#train(epoch)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
acc_test = test()
acc_test_list.append(acc_test)
acc_test = acc_test.view(1, 1)
acc_test = acc_test.cpu().numpy()
with open("./results/{}/{}.txt".format(args.dataset, args.dataset), 'a') as f:
# f.write("不丢弃dropout,加上b= {:.07f}×(output-output_1),epoch : {:04d},weight_decacy={},系数lam ={:.07f}".format
f.write("dataset{} epoch : {:04d},weight_decacy={}, lr{},seed{},dropout{},features_perturbation: rate1:{},lambda1:{}; adj_pertubation: rate2:{},lambda2:{}".format
(args.dataset, args.epochs, args.weight_decay, args.lr, seed, args.dropout,args.rate1,args.lambda1,args.rate2,args.lambda2))
np.savetxt(f, acc_test, fmt="%.6f")
acc_test_list = torch.FloatTensor(acc_test_list)
acc_test_std = torch.std(acc_test_list)
avg_test = torch.mean(acc_test_list)
avg_test = avg_test.view(1, 1)
avg_test = avg_test.cpu().numpy()
acc_test_std = acc_test_std.view(1, 1)
acc_test_std = acc_test_std.cpu().numpy()
print("总共做类{}次实验，平均值为：{:.04f}".format(args.run_time, avg_test.item()))
print("总共做类{}次实验，误差值为：{:.04f}".format(args.run_time,acc_test_std.item()))
with open("./results/{}/{}.txt".format(args.dataset, args.dataset), 'a') as f:
f.write("总共做类{}次实验，平均值为：{:.04f}\n".format(args.run_time, avg_test.item()))
f.write("总共做类{}次实验，误差值为：{:.04f}\n".format(args.run_time, acc_test_std.item()))
def main():
# Make dir
temp = "./tmp"
os.makedirs(temp, exist_ok=True)
# Load data
start = time.time()
(train_adj, full_adj, train_feats, test_feats, y_train, y_val, y_test,
train_mask, val_mask, test_mask, _, val_nodes, test_nodes,
num_data, visible_data) = utils.load_data(args.dataset)
print('Loaded data in {:.2f} seconds!'.format(time.time() - start))
start = time.time()
# Prepare Train Data
if args.batch_size > 1:
start = time.time()
_, parts = utils.partition_graph(train_adj, visible_data, args.num_clusters_train)
print('Partition graph in {:.2f} seconds!'.format(time.time() - start))
parts = [np.array(pt) for pt in parts]
else:
start = time.time()
(parts, features_batches, support_batches, y_train_batches, train_mask_batches) = utils.preprocess(
train_adj, train_feats, y_train, train_mask, visible_data, args.num_clusters_train, diag_lambda=args.diag_lambda)
print('Partition graph in {:.2f} seconds!'.format(time.time() - start))
# Prepare valid Data
start = time.time()
(_, val_features_batches, val_support_batches, y_val_batches, val_mask_batches) = utils.preprocess(
full_adj, test_feats, y_val, val_mask, np.arange(num_data), args.num_clusters_val, diag_lambda=args.diag_lambda)
print('Partition graph in {:.2f} seconds!'.format(time.time() - start))
# Prepare Test Data
start = time.time()
(_, test_features_batches, test_support_batches, y_test_batches, test_mask_batches) = utils.preprocess(
full_adj, test_feats, y_test, test_mask, np.arange(num_data), args.num_clusters_test, diag_lambda=args.diag_lambda)
print('Partition graph in {:.2f} seconds!'.format(time.time() - start))
idx_parts = list(range(len(parts)))
# model
model = GCN(
fan_in=_in, fan_out=_out, layers=args.layers, dropout=args.dropout, normalize=True, bias=False, precalc=True).float()
model.to(torch.device('cuda'))
print(model)
# Loss Function
if args.multilabel:
criterion = torch.nn.BCEWithLogitsLoss()
else:
criterion = torch.nn.CrossEntropyLoss()
# Optimization Algorithm
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
# Learning Rate Schedule
# scheduler = torch.optim.lr_scheduler.OneCycleLR(
# optimizer, max_lr=args.lr, steps_per_epoch=int(args.num_clusters_train/args.batch_size), epochs=args.epochs+1,
# anneal_strategy='linear')
pbar = tqdm.tqdm(total=args.epochs, dynamic_ncols=True)
for epoch in range(args.epochs + 1):
# Train
np.random.shuffle(idx_parts)
start = time.time()
avg_loss = 0
total_correct = 0
n_nodes = 0
if args.batch_size > 1:
(features_batches, support_batches, y_train_batches, train_mask_batches) = utils.preprocess_multicluster(
train_adj, parts, train_feats, y_train, train_mask,
args.num_clusters_train, args.batch_size, args.diag_lambda)
for pid in range(len(features_batches)):
# Use preprocessed batch data
features_b = features_batches[pid]
support_b = support_batches[pid]
y_train_b = y_train_batches[pid]
train_mask_b = train_mask_batches[pid]
loss, pred, labels = train(
model.train(), criterion, optimizer,
features_b, support_b, y_train_b, train_mask_b, torch.device('cuda'))
avg_loss += loss
n_nodes += pred.squeeze().numel()
total_correct += torch.eq(pred.squeeze(), labels.squeeze()).sum().item()
else:
np.random.shuffle(idx_parts)
for pid in idx_parts:
# use preprocessed batch data
features_b = features_batches[pid]
support_b = support_batches[pid]
y_train_b = y_train_batches[pid]
train_mask_b = train_mask_batches[pid]
loss, pred, labels = train(
model.train(), criterion, optimizer,
features_b, support_b, y_train_b, train_mask_b, torch.device('cuda'))
avg_loss = loss.item()
n_nodes += pred.squeeze().numel()
total_correct += torch.eq(pred.squeeze(), labels.squeeze()).sum().item()
train_acc = total_correct / n_nodes
# Write Train stats to tensorboard
writer.add_scalar('time/train', time.time() - start, epoch)
writer.add_scalar('loss/train', avg_loss/len(features_batches), epoch)
writer.add_scalar('acc/train', train_acc, epoch)
# Validation
cost, acc, micro, macro = evaluate(
model.eval(), criterion, val_features_batches, val_support_batches, y_val_batches, val_mask_batches,
val_nodes, torch.device("cuda"))
# Write Valid stats to tensorboard
writer.add_scalar('acc/valid', acc, epoch)
writer.add_scalar('mi_F1/valid', micro, epoch)
writer.add_scalar('ma_F1/valid', macro, epoch)
writer.add_scalar('loss/valid', cost, epoch)
pbar.set_postfix({"t": avg_loss/len(features_batches),"t_acc": train_acc, "v": cost, "v_acc": acc})
pbar.update()
pbar.close()
# Test
if args.test == 1:
# Test on cpu
cost, acc, micro, macro = test(
model.eval(), criterion, test_features_batches, test_support_batches, y_test_batches, test_mask_batches,
torch.device("cpu"))
writer.add_scalar('acc/test', acc, epoch)
writer.add_scalar('mi_F1/test', micro, epoch)
writer.add_scalar('ma_F1/test', macro, epoch)
writer.add_scalar('loss/test', cost, epoch)
print('test: acc: {:.4f}'.format(acc))
print('test: mi_f1: {:.4f}, ma_f1: {:.4f}'.format(micro, macro))
def gcn_train(**kwargs):
"""
GCN training
---
- the folder you need:
- args.path4AffGraph
- args.path4node_feat
- path4partial_label
- these folder would be created:
- data/GCN4DeepLab/Label
- data/GCN4DeepLab/Logit
"""
t_start = time.time()
# update config
args.parse(**kwargs)
device = torch.device("cuda:" + str(kwargs["GPU"]))
print(device)
# tensorboard
if args.use_TB:
time_now = datetime.datetime.today()
time_now = "{}-{}-{}|{}-{}".format(time_now.year, time_now.month,
time_now.day, time_now.hour,
time_now.minute // 30)
keys_ignore = ["start_index", "GPU"]
comment_init = ''
for k, v in kwargs.items():
if k not in keys_ignore:
comment_init += '|{} '.format(v)
writer = SummaryWriter(
logdir='runs/{}/{}'.format(time_now, comment_init))
# initial IoUMetric object for evaluation
IoU = IOUMetric(args.num_class)
# initial dataset
train_dataloader = graph_voc(start_idx=kwargs["start_index"],
end_idx=kwargs["end_index"],
device=device)
# train a seperate GCN for each image
t4epoch = time.time()
for ii, data in enumerate(train_dataloader):
if data is None:
continue
img_label = load_image_label_from_xml(img_name=data["img_name"],
voc12_root=args.path4VOC_root)
img_class = [idx + 1 for idx, f in enumerate(img_label) if int(f) == 1]
num_class = np.max(img_class) + 1
model = GCN(nfeat=data["features_t"].shape[1],
nhid=args.num_hid_unit,
nclass=args.num_class,
dropout=args.drop_rate)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# put data into GPU
if args.cuda:
model.to(device)
data["features_t"] = data["features_t"].to(device)
data["adj_t"] = data["adj_t"].to(device)
data["labels_t"] = data["labels_t"].to(device)
data["label_fg_t"] = data["label_fg_t"].to(device)
data["label_bg_t"] = data["label_bg_t"].to(device)
t_be = time.time()
H, W, C = data["rgbxy_t"].shape
N = H * W
# laplacian
if args.use_lap:
L_mat = compute_lap_test(data, device, radius=2).to(device)
print("Time for laplacian {:3.1f} s".format(time.time() - t_be))
criterion_ent = HLoss()
for epoch in range(args.max_epoch):
model.train()
optimizer.zero_grad()
output = model(data["features_t"], data["adj_t"])
# foreground and background loss
loss_fg = F.nll_loss(output, data["label_fg_t"], ignore_index=255)
loss_bg = F.nll_loss(output, data["label_bg_t"], ignore_index=255)
loss = loss_fg + loss_bg
if args.use_ent:
loss_entmin = criterion_ent(output,
data["labels_t"],
ignore_index=255)
loss += 10. * loss_entmin
if args.use_lap:
loss_lap = torch.trace(
torch.mm(output.transpose(1, 0),
torch.mm(L_mat.type_as(output), output))) / N
gamma = 1e-2
loss += gamma * loss_lap
if loss is None:
print("skip this image: ", data["img_name"])
break
loss_train = loss.cuda()
loss_train.backward()
optimizer.step()
# save predicted mask and IoU at max epoch
if (epoch + 1) % args.max_epoch == 0 and args.save_mask:
t_now = time.time()
evaluate_IoU(model=model,
features=data["features_t"],
adj=data["adj_t"],
img_name=data["img_name"],
img_idx=ii + 1,
writer=writer,
IoU=IoU,
save_prediction_np=True)
print("evaluate time: {:3.1f} s".format(time.time() - t_now))
print("[{}/{}] time: {:.1f}s\n\n".format(
ii + 1, len(train_dataloader), t_now - t4epoch))
t4epoch = t_now
print("======================================")
if writer is not None:
writer.close()
print("training was Finished!")
print("Total time elapsed: {:.0f} h {:.0f} m {:.0f} s\n".format(
(time.time() - t_start) // 3600, (time.time() - t_start) / 60 % 60,
(time.time() - t_start) % 60))
def execute(params, budget=None, max_epoch=243, device='cpu', seed=42):
np.random.seed(seed)
torch.manual_seed(seed)
if device == "cuda":
torch.cuda.manual_seed(seed)
# Load data
if params['dataset'] == "cora":
adj, features, labels, idx_train, idx_val, idx_test = load_data(
dataset=params['dataset'], train_percent=0.052)
if params['dataset'] == "citeseer":
adj, features, labels, idx_train, idx_val, idx_test = load_citeseer(
train_percent=0.036)
# Model and optimizer
model = GCN(nfeat=features.shape[1],
nhid=params['hidden'],
nclass=labels.max().item() + 1,
dropout=params['dropout'])
optimizer = optim.Adam(model.parameters(),
lr=params['lr'],
weight_decay=params['weight_decay'])
if device == "cuda":
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
# train model
if device == "cuda":
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
else:
t1 = time.time_ns()
model.train()
num_epoch = int(budget) if budget != None else max_epoch
for epoch in range(num_epoch):
optimizer.zero_grad()
output = model(features, adj)
loss_train = F.nll_loss(output[idx_train], labels[idx_train])
acc_train = accuracy(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
# evaluation
model.eval()
output = model(features, adj)
loss_val = F.nll_loss(output[idx_val], labels[idx_val])
acc_val = accuracy(output[idx_val], labels[idx_val])
if device == "cuda":
end.record()
torch.cuda.synchronize()
total_time = start.elapsed_time(end) / 1e3
sys.stdout.flush()
acc_val = acc_val.item()
else:
t2 = time.time_ns()
total_time = (t2 - t1) / 1e9
print()
print(
f"dataset={params['dataset']}, num_epoch={num_epoch}, device={next(model.parameters()).device}"
)
print("Validation results:", "loss= {:.4f}".format(loss_val.item()),
"accuracy= {:.4f}".format(acc_val))
print("Total training time: {:.4f} sec".format(total_time))
return 1 - acc_val
def main(args):
start_time_str = time.strftime("_%m_%d_%H_%M_%S", time.localtime())
log_path = os.path.join(args.log_dir, args.model + start_time_str)
if not os.path.exists(log_path):
os.mkdir(log_path)
logging.basicConfig(filename=os.path.join(log_path, 'log_file'),
filemode='w',
format='| %(asctime)s |\n%(message)s',
datefmt='%b %d %H:%M:%S',
level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
logging.info(args)
# load data
if args.model in ['EGNN', 'ECConv', 'GTEA-ST']:
data = Dataset(data_dir=args.data_dir, batch_size=args.batch_size, use_static=True)
else:
data = Dataset(data_dir=args.data_dir, batch_size=args.batch_size)
g = data.g
# features = torch.FloatTensor(data.features)
# labels = torch.LongTensor(data.labels)
train_loader = data.train_loader
val_loader = data.val_loader
test_loader = data.test_loader
num_nodes = data.num_nodes
node_in_dim = args.node_in_dim
num_edges = data.num_edges
edge_in_dim = data.edge_in_dim
edge_timestep_len = data.edge_timestep_len
num_train_samples = data.num_train_samples
num_val_samples = data.num_val_samples
num_test_samples = data.num_test_samples
logging.info("""----Data statistics------'
#Nodes %d
#Edges %d
#Node_feat %d
#Edge_feat %d
#Edge_timestep %d
#Train samples %d
#Val samples %d
#Test samples %d""" %
(num_nodes, num_edges,
node_in_dim, edge_in_dim, edge_timestep_len,
num_train_samples,
num_val_samples,
num_test_samples))
device = torch.device("cuda:"+str(args.gpu) if torch.cuda.is_available() and args.gpu >=0 else "cpu")
infer_device = device if args.infer_gpu else torch.device('cpu')
# g = g.to(device)
# create model
if args.model == 'GCN':
model = GCN(num_nodes=num_nodes,
in_feats=node_in_dim,
n_hidden=args.node_hidden_dim,
n_layers=args.num_layers,
activation=F.relu,
dropout=args.dropout)
elif args.model == 'GraphSAGE':
model = GraphSAGE(num_nodes=num_nodes,
in_feats=node_in_dim,
n_hidden=args.node_hidden_dim,
n_layers=args.num_layers,
activation=F.relu,
dropout=args.dropout)
elif args.model == 'GAT':
model = GAT(num_nodes=num_nodes,
in_dim=node_in_dim,
hidden_dim=args.node_hidden_dim,
num_layers=args.num_layers,
num_heads=args.num_heads)
elif args.model == 'ECConv':
model = ECConv(num_nodes=num_nodes,
node_in_dim=node_in_dim,
edge_in_dim=edge_in_dim,
hidden_dim=args.node_hidden_dim,
num_layers=args.num_layers,
drop_prob=args.dropout,
device=device)
elif args.model == 'EGNN':
model = EGNN(num_nodes=num_nodes,
node_in_dim=node_in_dim,
edge_in_dim=edge_in_dim,
hidden_dim=args.node_hidden_dim,
num_layers=args.num_layers,
drop_prob=args.dropout,
device=device)
elif args.model == 'GTEA-ST':
model = GTEAST(num_nodes=num_nodes,
node_in_dim=node_in_dim,
edge_in_dim=edge_in_dim,
node_hidden_dim=args.node_hidden_dim,
num_layers=args.num_layers,
drop_prob=args.dropout,
device=device)
elif args.model == 'TGAT':
model = TGAT(num_nodes=num_nodes,
node_in_dim=node_in_dim,
node_hidden_dim=args.node_hidden_dim,
edge_in_dim=edge_in_dim-1,
time_hidden_dim=args.time_hidden_dim,
num_class=0,
num_layers=args.num_layers,
num_heads=args.num_heads,
device=device,
drop_prob=args.dropout)
elif args.model == 'GTEA-LSTM':
model = GTEALSTM(num_nodes=num_nodes,
node_in_dim=node_in_dim,
node_hidden_dim=args.node_hidden_dim,
edge_in_dim=edge_in_dim,
num_class=0,
num_layers=args.num_layers,
num_time_layers=args.num_lstm_layers,
bidirectional=args.bidirectional,
device=device,
drop_prob=args.dropout)
elif args.model == 'GTEA-LSTM+T2V':
model = GTEALSTMT2V(num_nodes=num_nodes,
node_in_dim=node_in_dim,
node_hidden_dim=args.node_hidden_dim,
edge_in_dim=edge_in_dim-1,
time_hidden_dim=args.time_hidden_dim,
num_class=0,
num_layers=args.num_layers,
num_time_layers=args.num_lstm_layers,
bidirectional=args.bidirectional,
device=device,
drop_prob=args.dropout)
elif args.model == 'GTEA-Trans':
model = GTEATrans(num_nodes=num_nodes,
node_in_dim=node_in_dim,
node_hidden_dim=args.node_hidden_dim,
edge_in_dim=edge_in_dim,
num_class=0,
num_layers=args.num_layers,
num_heads=args.num_heads,
num_time_layers=args.num_lstm_layers,
device=device,
drop_prob=args.dropout)
elif args.model == 'GTEA-Trans+T2V':
model = GTEATransT2V(num_nodes=num_nodes,
node_in_dim=node_in_dim,
node_hidden_dim=args.node_hidden_dim,
edge_in_dim=edge_in_dim-1,
time_hidden_dim=args.time_hidden_dim,
num_class=0,
num_layers=args.num_layers,
num_heads=args.num_heads,
num_time_layers=args.num_lstm_layers,
device=device,
drop_prob=args.dropout)
else:
logging.info('Model {} not found.'.format(args.model))
exit(0)
# send model to device
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
checkpoint_path = os.path.join(log_path, str(args.model) + '_checkpoint.pt')
trainer = Trainer(g=g,
model=model,
optimizer=optimizer,
epochs=args.epochs,
train_loader=train_loader,
val_loader=val_loader,
test_loader=test_loader,
patience=args.patience,
batch_size=args.batch_size,
num_neighbors=args.num_neighbors,
num_layers=args.num_layers,
num_workers=args.num_workers,
device=device,
infer_device=infer_device,
log_path=log_path,
checkpoint_path=checkpoint_path)
logging.info('Start training')
best_val_result, test_result = trainer.train()
# recording the result
line = [start_time_str[1:]] + [args.model] + ['K=' + str(args.use_K)] + \
[str(x) for x in best_val_result] + [str(x) for x in test_result] + [str(args)]
line = ','.join(line) + '\n'
with open(os.path.join(args.log_dir, str(args.model) + '_result.csv'), 'a') as f:
f.write(line)
# Define the model and optimizer
if (opt.model == 'basic'):
print("| Constructing basic GCN model...")
model = GCN(
nfeat = features.shape[1],
nhid = opt.num_hidden,
nclass = labels.max().item() + 1,
dropout = opt.dropout,
init = opt.init_type
)
else:
raise NotImplementedError
if (opt.optimizer == 'sgd'):
optimizer = optim.SGD(
model.parameters(),
lr = opt.lr,
weight_decay = opt.weight_decay,
momentum = 0.9
)
elif (opt.optimizer == 'adam'):
optimizer = optim.Adam(
model.parameters(),
lr = opt.lr,
weight_decay = opt.weight_decay
)
else:
raise NotImplementedError
if use_gpu:
model.cuda()
def train_gcn(dataset,
test_ratio=0.5,
val_ratio=0.2,
seed=1,
n_hidden=64,
n_epochs=200,
lr=1e-2,
weight_decay=5e-4,
dropout=0.5,
use_embs=False,
verbose=True,
cuda=False):
data = dataset.get_data()
# train text embs
if use_embs:
pad_ix, n_tokens, matrix, pretrained_embs = data['features']
if pretrained_embs is not None:
pretrained_embs = torch.FloatTensor(pretrained_embs)
features = torch.LongTensor(matrix)
else:
pad_ix = None
n_tokens = None
pretrained_embs = None
features = torch.FloatTensor(data['features'])
labels = torch.LongTensor(data['labels'])
n = len(data['ids'])
train_mask, val_mask, test_mask = get_masks(n,
data['main_ids'],
data['main_labels'],
test_ratio=test_ratio,
val_ratio=val_ratio,
seed=seed)
train_mask = torch.BoolTensor(train_mask)
val_mask = torch.BoolTensor(val_mask)
test_mask = torch.BoolTensor(test_mask)
if cuda:
torch.cuda.set_device("cuda:0")
features = features.cuda()
labels = labels.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
g = DGLGraph(data['graph'])
g = dgl.transform.add_self_loop(g)
n_edges = g.number_of_edges()
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
if cuda:
norm = norm.cuda()
g.ndata['norm'] = norm.unsqueeze(1)
if use_embs:
if pretrained_embs is not None:
in_feats = 100
else:
in_feats = 64
else:
in_feats = features.shape[1]
# + 1 for unknown class
n_classes = data['n_classes'] + 1
model = GCN(g,
in_feats=in_feats,
n_hidden=n_hidden,
n_classes=n_classes,
activation=F.relu,
dropout=dropout,
use_embs=use_embs,
pretrained_embs=pretrained_embs,
pad_ix=pad_ix,
n_tokens=n_tokens)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.9,
patience=20,
min_lr=1e-10)
best_f1 = -100
# initialize graph
dur = []
for epoch in range(n_epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
mask_probs = torch.empty(features.shape).uniform_(0, 1)
if cuda:
mask_probs = mask_probs.cuda()
mask_features = torch.where(mask_probs > 0.2, features,
torch.zeros_like(features))
logits = model(mask_features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
f1 = evaluate(model, features, labels, val_mask)
scheduler.step(1 - f1)
if f1 > best_f1:
best_f1 = f1
torch.save(model.state_dict(), 'best_model.pt')
if verbose:
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | F1 {:.4f} | "
"ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur),
loss.item(), f1,
n_edges / np.mean(dur) / 1000))
model.load_state_dict(torch.load('best_model.pt'))
f1 = evaluate(model, features, labels, test_mask)
if verbose:
print()
print("Test F1 {:.2}".format(f1))
return f1
def cross_validation():
##=======================Load Data================================================
# Load data
population = 'PTE'
epidata = np.load(population + '_graphs_gcn.npz')
adj_epi = torch.from_numpy(calc_DAD(epidata)).float().to(
device) # n_subjects*16 *16
features_epi = torch.from_numpy(epidata['features']).float().to(
device) # n_subjectsx16x171
# n_subjects = features_epi.shape[0]
# num_train = int(n_subjects * args.rate)
# train_adj_epi = adj_epi[:num_train, :, :]
# train_features_epi = features_epi[:num_train, :, :]
# test_adj_epi = adj_epi[num_train:, :, :]
# test_features_epi = features_epi[num_train:, :, :]
population = 'NONPTE'
nonepidata = np.load(population + '_graphs_gcn.npz')
adj_non = torch.from_numpy(calc_DAD(nonepidata)).float().to(device)
features_non = torch.from_numpy(nonepidata['features']).float().to(
device) #subjects x 16 x 171
# print("DAD shape:")
# print(adj_non.shape, adj_epi.shape)
## for now we are using the same number of epi , non epi training samples.
# n_subjects_non = features_non.shape[0]
# num_train_non = int(n_subjects_non * args.rate)
# train_adj_non = adj_non[:num_train_non, :, :]
# train_features_non = features_non[:num_train_non, :, :]
# test_adj_non = adj_non[num_train_non:, :, :]
# test_features_non = features_non[num_train_non:, :, :]
features = torch.cat([features_epi, features_non])
adj = torch.cat([adj_epi, adj_non])
labels = torch.from_numpy(
np.hstack((np.ones(adj_epi.shape[0]),
np.zeros(adj_non.shape[0])))).long().to(device)
print(features.shape, adj.shape, labels.shape)
iterations = args.iters
acc_iter = []
auc_iter = []
for i in range(iterations):
kfold = StratifiedKFold(n_splits=36, shuffle=True)
# the folds are made by preserving the percentage of samples for each class.
acc = []
max_epochs = []
test_true = []
probs_fold = []
# epochs_choices = []
features_numpy = features.cpu().numpy()
labels_numpy = labels.cpu().numpy()
adj_numpy = adj.cpu().numpy()
for train_ind, test_ind in kfold.split(features_numpy, labels_numpy):
# Model and optimizer
model = GCN(
nfeat=features_epi.shape[2] // args.ngroup,
nhid=[200, 200, 50],
# nhid=[200, 200, 100, 50],
nclass=2, #labels.max().item() + 1,
dropout=args.dropout)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# optimizer = optim.RMSprop(model.parameters(), lr=args.lr, alpha=0.9)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, T_max=args.epochs, eta_min=1e-7, last_epoch=-1)
model.to(device)
# 72 subjects in total, during CV, training has 70, testing has 2, one epi, one nonepi
train_features = features_numpy[train_ind, :, :]
train_adj = adj_numpy[train_ind, :, :]
train_labels = labels_numpy[train_ind]
state = np.random.get_state()
np.random.shuffle(train_features)
np.random.set_state(state)
np.random.shuffle(train_adj)
np.random.set_state(state)
np.random.shuffle(train_labels)
test_features = features[test_ind, :, :]
test_adj = adj[test_ind, :, :]
test_labels = labels[test_ind]
acc_test = []
start_epoch = 13
gap = 1
mode_on = args.mode
for epoch in range(args.epochs):
train(epoch, model, optimizer, scheduler,
torch.from_numpy(train_features).float().to(device),
torch.from_numpy(train_adj).float().to(device),
torch.from_numpy(train_labels).long().to(device))
if (epoch >= start_epoch) and (epoch % gap == 0) and (mode_on
== True):
acc_test.append(
test(model, test_features, test_adj, test_labels))
##===================
test_prob, test_accur, test_labels_squeezed = test(
model, test_features, test_adj, test_labels)
acc_test.append(test_accur)
# max_epochs.append(np.argmax(acc_test)*gap + start_epoch)
acc.append(np.max(acc_test))
##=============================================
probs_fold.append(test_prob[:, -1])
test_true.append(test_labels_squeezed.cpu().numpy())
# torch.save({'epoch': args.epochs,
# 'model_state_dict': model.state_dict(),
# 'optimizer_state_dict': optimizer.state_dict(),
# }, args.model_path+"model_cv_2002005010050" + str(len(acc)) + ".pth")
del model
del optimizer
# input("any key")
# print(acc, max_epochs)
# with open('../results/accuracy_0.4thr_15e_5layers.txt', 'w') as f:
# f.write(str(acc))
# f.close()
probs = np.array(probs_fold).flatten()
print(probs)
auc = sklearn.metrics.roc_auc_score(
np.array(test_true).flatten(), probs)
print(auc)
# print(np.mean(acc))
acc_iter.append(np.mean(acc))
auc_iter.append(auc)
# print(acc_iter, np.mean(acc_iter), np.std(acc_iter))
print("----------Mean AUC-------------")
print(auc_iter, np.mean(auc_iter), np.std(auc_iter))
print("Accuracy")
print(acc_iter, np.mean(acc_iter), np.std(acc_iter))
def train(**kwargs):
"""
GCN training
---
- the folder you need:
- args.path4AffGraph
- args.path4node_feat
- path4partial_label
- these folder would be created:
- data/GCN_prediction/label
- data/GCN_prediction/logit
"""
# os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(map(str, [0, 1, 2, 3]))
t_start = time.time()
# 根据命令行参数更新配置
args.parse(**kwargs)
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device("cuda:" + str(kwargs["GPU"]))
print(device)
# 把有改動的參數寫到tensorboard名稱上
if kwargs["debug"] is False:
comment_init = ''
for k, v in kwargs.items():
comment_init += '|{} '.format(v)
writer = SummaryWriter(comment=comment_init)
# === set evaluate object for evaluate later
IoU = IOUMetric(args.num_class)
IoU_CRF = IOUMetric(args.num_class)
# === dataset
train_dataloader = graph_voc(start_idx=kwargs["start_index"],
end_idx=kwargs["end_index"],
device=device)
# === for each image, do training and testing in the same graph
# for ii, (adj_t, features_t, labels_t, rgbxy_t, img_name, label_fg_t,
# label_bg_t) in enumerate(train_dataloader):
t4epoch = time.time()
for ii, data in enumerate(train_dataloader):
if data is None:
continue
# === use RGBXY as feature
# if args.use_RGBXY:
# data["rgbxy_t"] = normalize_rgbxy(data["rgbxy_t"])
# features_t = data["rgbxy_t"].clone()
# === only RGB as feature
t_be = time.time()
if args.use_lap:
""" is constructing................ """
H, W, C = data["rgbxy_t"].shape
A = torch.zeros([H * W, H * W], dtype=torch.float64)
def find_neibor(card_x, card_y, H, W, radius=2):
"""
Return idx of neibors of (x,y) in list
---
"""
neibors_idx = []
for idx_x in np.arange(card_x - radius, card_x + radius + 1):
for idx_y in np.arange(card_y - radius,
card_y + radius + 1):
if (-radius < idx_x < H) and (-radius < idx_y < W):
neibors_idx.append(
(idx_x * W + idx_y, idx_x, idx_y))
return neibors_idx
t_start = time.time()
t_start = t4epoch
neibors = dict()
for node_idx in range(H * W):
card_x, card_y = node_idx // W, node_idx % W
neibors = find_neibor(card_x, card_y, H, W, radius=1)
# print("H:{} W:{} | {} -> ({},{})".format(
# H, W, node_idx, card_x, card_y))
for nei in neibors:
# print("nei: ", nei)
diff_rgb = data["rgbxy_t"][
card_x, card_y, :3] - data["rgbxy_t"][nei[1],
nei[2], :3]
diff_xy = data["rgbxy_t"][card_x, card_y,
3:] - data["rgbxy_t"][nei[1],
nei[2], 3:]
A[node_idx, nei[0]] = torch.exp(
-torch.pow(torch.norm(diff_rgb), 2) /
(2. * args.CRF_deeplab["bi_rgb_std"])) + torch.exp(
-torch.pow(torch.norm(diff_xy), 2) /
(2. * args.CRF_deeplab["bi_xy_std"]))
# print("{:3.1f}s".format(time.time() - t_start))
D = torch.diag(A.sum(dim=1))
L_mat = D - A
print("time for Laplacian {:3f} s".format(time.time() - t_be))
# === Model and optimizer
img_label = load_image_label_from_xml(img_name=data["img_name"],
voc12_root=args.path4VOC_root)
img_class = [idx + 1 for idx, f in enumerate(img_label) if int(f) == 1]
num_class = np.max(img_class) + 1
# debug("num_class: {} {}".format(num_class + 1, type(num_class + 1)),
# line=290)
model = GCN(
nfeat=data["features_t"].shape[1],
nhid=args.num_hid_unit,
# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# image label don't have BG
# adaptive num_class should have better performance
nclass=args.num_class, # args.num_class| num_class
# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
dropout=args.drop_rate)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# ==== moving tensor to GPU
if args.cuda:
model.to(device)
data["features_t"] = data["features_t"].to(device)
data["adj_t"] = data["adj_t"].to(device)
data["labels_t"] = data["labels_t"].to(device)
data["label_fg_t"] = data["label_fg_t"].to(device)
data["label_bg_t"] = data["label_bg_t"].to(device)
# L_mat = L_mat.to(device)
# === save the prediction before training
if args.save_mask_before_train:
model.eval()
postprocess_image_save(img_name=data["img_name"],
model_output=model(data["features_t"],
data["adj_t"]).detach(),
epoch=0)
# ==== Train model
# t4epoch = time.time()
criterion_ent = HLoss()
# criterion_sym = symmetricLoss()
for epoch in range(args.max_epoch):
model.train()
optimizer.zero_grad()
output = model(data["features_t"], data["adj_t"])
# === seperate FB/BG label
loss_fg = F.nll_loss(output, data["label_fg_t"], ignore_index=255)
loss_bg = F.nll_loss(output, data["label_bg_t"], ignore_index=255)
# F.log_softmax(label_fg_t, dim=1)
# loss_sym = criterion_sym(output, labels_t, ignore_index=255)
loss = loss_fg + loss_bg
if args.use_ent:
loss_entmin = criterion_ent(output,
data["labels_t"],
ignore_index=255)
loss += 10. * loss_entmin
if args.use_lap:
loss_lap = torch.trace(
torch.mm(output.transpose(1, 0),
torch.mm(L_mat.type_as(output),
output))) / (H * W)
gamma = 1e-2
loss += gamma * loss_lap
# loss = F.nll_loss(output, labels_t, ignore_index=255)
if loss is None:
print("skip this image: ", data["img_name"])
break
# === for normalize cut
# lamda = args.lamda
# n_cut = 0.
# if args.use_regular_NCut:
# W = gaussian_propagator(output)
# d = torch.sum(W, dim=1)
# for k in range(output.shape[1]):
# s = output[idx_test_t, k]
# n_cut = n_cut + torch.mm(
# torch.mm(torch.unsqueeze(s, 0), W),
# torch.unsqueeze(1 - s, 1)) / (torch.dot(d, s))
# === calculus loss & updated parameters
# loss_train = loss.cuda() + lamda * n_cut
loss_train = loss.cuda()
loss_train.backward()
optimizer.step()
# === save predcit mask at max epoch & IoU of img
if (epoch + 1) % args.max_epoch == 0 and args.save_mask:
t_now = time.time()
if not kwargs["debug"]:
evaluate_IoU(model=model,
features=data["features_t"],
adj=data["adj_t"],
img_name=data["img_name"],
epoch=args.max_epoch,
img_idx=ii + 1,
writer=writer,
IoU=IoU,
IoU_CRF=IoU_CRF,
use_CRF=False,
save_prediction_np=True)
print("[{}/{}] time: {:.4f}s\n\n".format(
ii + 1, len(train_dataloader), t_now - t4epoch))
t4epoch = t_now
# end for epoch
# print(
# "loss: {} | loss_fg: {} | loss_bg:{} | loss_entmin: {} | loss_lap: {}"
# .format(loss.data.item(), loss_fg.data.item(), loss_bg.data.item(),
# loss_entmin.data.item(), loss_lap.data.item()))
# end for dataloader
if kwargs["debug"] is False:
writer.close()
print("training was Finished!")
print("Total time elapsed: {:.0f} h {:.0f} m {:.0f} s\n".format(
(time.time() - t_start) // 3600, (time.time() - t_start) / 60 % 60,
(time.time() - t_start) % 60))
import torch
import torch.nn.functional as F
from torch import optim
from models import GCN
from data.utils import build_st_body25_graph
from data.dataloader import *
frames_per_video = 5
model = GCN(3, 16, 100, 0.5, frames_per_video)
# optimizer = torch.optim.Adam(model.parameters(),
# lr=0.01, weight_decay=5e-4)
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)
adj = build_st_body25_graph(frames_per_video)
train_points, train_labels, test_points, test_labels = load_features_for_old(
"../data/features.txt", frames_per_video)
train_points = torch.tensor(train_points, dtype=torch.float32)
train_labels = torch.tensor(train_labels[:, 2], dtype=torch.long).reshape(
(-1, 1)) - 1
test_points = torch.tensor(test_points, dtype=torch.float32)
test_labels = torch.tensor(test_labels[:, 2], dtype=torch.long).reshape(
(-1, 1)) - 1
# train_labels = torch.zeros(train_labels.shape[0], 9).scatter_(1, train_labels, 1)
def train(epoch):
model.train()
for e in range(0, epoch):
outputs = []
neg_node_heat_map = MinMaxScaler(feature_range=(-1, 0)).fit_transform(
node_heat_map * (node_heat_map < 0)).reshape(-1, )
return pos_node_heat_map + neg_node_heat_map
dataset = load_bbbp(hp.N)
random.Random(hp.shuffle_seed).shuffle(dataset)
split_idx = int(np.floor(len(dataset) * hp.train_frac))
test_dataset = dataset[split_idx:]
loader = DataLoader(test_dataset, batch_size=1, shuffle=False)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = GCN(hp.H_0, hp.H_1, hp.H_2, hp.H_3).to(device)
model.load_state_dict(torch.load('gcn_state_dict.pt'))
model.eval()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
print(model)
model.train()
total_loss = 0
for data in tqdm(loader):
# breakpoint()
data = data.to(device)
optimizer.zero_grad()
out = model(data)
loss = F.binary_cross_entropy(out, data.y)
loss.backward()
try:
plot_explanations(model, data)
# except ValueError as e:
except Exception as e:
# adj = sp.csr_matrix(processed_adj)
# adj = sparse_mx_to_torch_sparse_tensor(adj)
# adj = torch.from_numpy(adj).float()
def accuracy(output, labels):
preds = output.max(1)[1].type_as(labels)
correct = preds.eq(labels).double()
correct = correct.sum()
return correct / len(labels)
# model creation and loss seting
model = GCN(nfeat=1, nhid=256, nclass=10).to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
###Training Part
start_time = time.time()
for epoch in range(num_epochs):
for i, (adj, features, masks,
batch_labels) in tqdm(enumerate(train_loader)):
# input = feature[1].view(784,1)
features = features.to(device)
batch_labels = batch_labels.to(device)
adj = adj.to(device)
t = time.time()
output = model(features, adj)
loss_train = criterion(output, batch_labels)
dropout=0.5
)
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
features, adj, labels = Variable(features), Variable(adj), Variable(labels)
#net = nn.DataParallel(net, device_ids=range(1))
#net.load_state_dict(torch.load(opt.modelIn))
optimizer = optim.Adam(net.parameters(),
lr=0.01, weight_decay=5e-4)
net.cuda()
for epoch_n in range(200):
train(epoch_n)
loss_f = nn.CrossEntropyLoss()
net.eval()
output = net(features, adj)
output = F.log_softmax(output, dim =1)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:",
"loss= {:.4f}".format(loss_test.data[0]),
"accuracy= {:.4f}".format(acc_test.data[0]))
for c in opt.c:
edge_dropout=args.edge_dropout,
pre_attn_order=args.pre_attn_order,
post_attn_order=args.post_attn_order,
pre_attn_appnp=args.pre_attn_appnp,
pre_appnp_alpha=args.pre_appnp_alpha,
post_attn_appnp=args.post_attn_appnp,
post_appnp_alpha=args.post_appnp_alpha,
device=device)
else:
model = Linear(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout)
if args.optimizer == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
elif args.optimizer == 'lbfgs':
optimizer = optim.LBFGS(model.parameters())
if args.cuda:
model.cuda()
features = features.cuda()
adj_test = adj_test.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
if inductive:
pass
def experimenter(data_name='cora',
train_ratio=0.03,
cuda=True,
random_seed=42,
hidden=16,
dropout_ratio=0.5,
learning_rate=0.01,
weight_decay=5e-4,
num_epochs=65,
early_stopping=30,
task='classification',
public_splitting=False):
# helper function to run epxeriment
if data_name in ['cora', 'citeseer', 'pubmed']:
print("Loading Classification Datasets")
Tmat, eadj, edge_name, edge_feature_dict, adj, features, edge_features, labels, idx_train, idx_val, idx_test = tqdm(
load_data(data_name=data_name,
train_ratio=train_ratio,
public_splitting=public_splitting))
model = GCN(nfeat_v=features.shape[1],
nfeat_e=edge_features.shape[1],
nhid=hidden,
nclass=labels.max().item() + 1,
dropout=dropout_ratio)
ssl_agent_n = PairwiseDistance(adj,
features,
nhid=args.hidden,
cuda=cuda,
node=True)
ssl_agent_e = PairwiseDistance(eadj,
edge_features,
nhid=args.hidden,
cuda=cuda,
node=False)
else:
ValueError("The input data is not supported! ")
print(">" * 100)
print("Loaded and preprocessed the graph data! ")
print(">" * 100)
optimizer = optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
if cuda:
torch.cuda.manual_seed(random_seed)
torch.set_default_tensor_type('torch.cuda.FloatTensor')
if cuda:
model.cuda()
Tmat = Tmat.cuda()
eadj = eadj.cuda()
adj = adj.cuda()
features = features.cuda()
edge_features = edge_features.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
# pooling = pooling.cuda()
# node_count = node_count.cuda()
if task == "classification":
criteria = F.nll_loss
acc_measure = accuracy
elif task == "regression":
criteria = torch.nn.L1Loss
acc_measure = RMSELoss
# ---------------------------------------
# training function
# ---------------------------------------
# count_time = 0
def train(epoch):
t = time.time()
model.train()
optimizer.zero_grad()
output, n_feature, e_feature = model(features, edge_features, eadj,
adj, Tmat, task)
loss_n = ssl_agent_n.classification_loss(n_feature)
loss_e = ssl_agent_e.classification_loss(e_feature)
loss_train = criteria(output[idx_train], labels[idx_train])
loss_train = loss_train + 0.1 * loss_n + 0.1 * loss_e
acc_train = acc_measure(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
if not args.fastmode:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
model.eval()
output, _, _ = model(features, edge_features, eadj, adj, Tmat,
task)
loss_val = criteria(output[idx_val], labels[idx_val])
acc_val = acc_measure(output[idx_val], labels[idx_val])
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(loss_train.item()),
'acc_train: {:.4f}'.format(acc_train.item()),
'loss_val: {:.4f}'.format(loss_val.item()),
'acc_val: {:.4f}'.format(acc_val.item()),
'time: {:.4f}s'.format(time.time() - t))
return loss_val.item()
# -------------------------------------------
# testing function
# -------------------------------------------
def test():
model.eval()
output, _, _ = model(features, edge_features, eadj, adj, Tmat, task)
loss_test = criteria(output[idx_test], labels[idx_test])
acc_test = acc_measure(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
return acc_test.item()
def pretrain(epoch):
t = time.time()
model.train()
optimizer.zero_grad()
output, node_f, edge_f = model(features, edge_features, eadj, adj,
Tmat, task)
loss_n = ssl_agent_n.classification_loss(node_f)
loss_e = ssl_agent_e.classification_loss(edge_f)
# loss_train = criteria(output[idx_train], labels[idx_train])
loss_train = loss_n + loss_e
acc_train = acc_measure(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
if not args.fastmode:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
model.eval()
output, _, _ = model(features, edge_features, eadj, adj, Tmat,
task)
loss_val = criteria(output[idx_val], labels[idx_val])
acc_val = acc_measure(output[idx_val], labels[idx_val])
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(loss_train.item()),
'acc_train: {:.4f}'.format(acc_train.item()),
'loss_val: {:.4f}'.format(loss_val.item()),
'acc_val: {:.4f}'.format(acc_val.item()),
'time: {:.4f}s'.format(time.time() - t))
return loss_val.item()
# Train model
t_total = time.time()
val_watch = []
pre_val_watch = []
input_idx_train = idx_train
# for epoch in range(50):
# pre_val_watch.append(pretrain(epoch))
# test()
# if epoch > early_stopping and pre_val_watch[-1] > np.mean(pre_val_watch[-(early_stopping + 1):-1]):
# print("Early stopping...")
# break
for epoch in range(num_epochs):
# val_adj, val_fea = ssl_agent.transform_data()
val_watch.append(train(epoch))
test()
if epoch > early_stopping and val_watch[-1] > np.mean(
val_watch[-(early_stopping + 1):-1]):
print("Early stopping...")
break
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
print("Printing the weights : ")
return test()
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
# Load data
adj, features, labels, idx_train, idx_val, idx_test = load_data()
# Model and optimizer
model = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout,
nnz=adj._nnz())
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
gamma = 0
eta = 10**-6 #should be decreasing with epochs
print('!!!!!!!!!!!!CHECK!!!!!!!!!!!!')
print('save (location) of plots')
print('Gamma:', gamma)
print('Eta:', eta)
print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
def get_validation_loss(no_cuda=True,
fastmode=True,
seed=42,
epochs=50,
lr=0.01,
weight_decay=5e-4,
hidden_dim_gcn=32,
output_dim=16,
dropout_prop=0.5,
hidden_dim_fc=32,
verbose=False
):
# Training settings
# help='Weight decay (L2 loss on parameters).')
cuda = not no_cuda and torch.cuda.is_available()
np.random.seed(seed)
torch.manual_seed(seed)
if cuda:
torch.cuda.manual_seed(seed)
# Load data
(training_features,
validation_features,
training_adj,
validation_adj,
training_labels,
validation_labels) = load_all_ligands()
# Model and optimizer
model = GCN(init_dim=11,
hidden_dim_gcn=hidden_dim_gcn,
output_dim=output_dim,
dropout_prop=dropout_prop,
hidden_dim_fc=hidden_dim_fc)
optimizer = optim.Adam(model.parameters(),
lr=lr, weight_decay=weight_decay)
loss = nn.BCELoss()
if cuda:
model.cuda()
training_features = [features.cuda() for features in training_features]
validation_features = [features.cuda() for features in validation_features]
training_adj = [adj.cuda() for adj in training_adj]
validation_adj = [adj.cuda() for adj in validation_adj]
training_adj = [adj.cuda() for adj in training_adj]
validation_adj = [adj.cuda() for adj in validation_adj]
training_labels = [labels.cuda() for labels in training_labels]
validation_labels = [labels.cuda() for labels in validation_labels]
def train(epoch):
model.train()
for i in range(len(training_labels)):
optimizer.zero_grad()
output = model(training_features[i], training_adj[i])
loss_train = loss(output, training_labels[i])
#acc_train = accuracy(output, labels)
if i % 32 == 0:
loss_train.backward()
optimizer.step()
if verbose:
print('Epoch: {:04d}'.format(epoch+1),
'loss_train: {:.4f}'.format(loss_train.item()))
def validate():
model.eval()
for i in range(len(validation_labels)):
output = model(validation_features[i], validation_adj[i])
loss_validation = loss(output, validation_labels[i])
#acc_validation = accuracy(output, labels)
if verbose:
print('Epoch: {:04d}'.format(epoch+1),
'loss_validation: {:.4f}'.format(loss_validation.item()))
return loss_validation.item()
# Train model
t_total = time.time()
for epoch in range(epochs):
train(epoch)
if not fastmode:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
validate()
if verbose:
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# Testing
return validate()
sys.stdout.flush()
# Model and optimizer
# a dummy example to determine the dimension of input data
adj0, features0, labels0, idx_train0, idx_val0, idx_test0, add_children0, or_children0 = load_data(
'0', dataset, and_or=and_or, override_path=ds_path)
model = GCN(
nfeat=features0.shape[1],
nhid=args.hidden,
# nclass=labels.max().item() + 1,
nclass=100,
nhidlayers=args.hidden_layers,
dropout=args.dropout,
indep_weights=indep_weights)
mlp = MLP(dropout=args.dropout)
optimizer = optim.Adam(itertools.chain(model.parameters(), mlp.parameters()),
lr=args.lr,
weight_decay=args.weight_decay)
creterion = torch.nn.TripletMarginLoss(margin=args.margin, p=2)
CE = torch.nn.CrossEntropyLoss()
loss_list = RunningAvg(window_size=200)
loss_list_CE = RunningAvg(window_size=200)
acc_list = RunningAvg(window_size=200)
loss_by_iter = []
if args.cuda:
device = torch.device(args.cuda_device)
model.cuda(device)
mlp.cuda(device)
pseudo_labels = torch.LongTensor([]) #For self-training
acc_val = 0 # For validation
# Model and optimizer
model_S = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout)
model_T = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=0)
student_params = list(model_S.parameters())
teacher_params = list(model_T.parameters())
for param in teacher_params:
param.requires_grad = False
optimizer_S = optim.Adam(model_S.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
optimizer_T = WeightEMA(teacher_params, student_params, alpha=args.alpha)
if args.cuda:
model_S.cuda(args.gpu)
model_T.cuda(args.gpu)
features = features.cuda(args.gpu)
adj_T = adj_T.cuda(args.gpu)
