gcn_optimizer.step()
if epoch > 0:
#print "GCN Weight", gcn_model.gc1.weight.data
print("Total loss", epoch, loss_train)
currcorrect, indexes = evaluate_line(gcn_model, test_loader, Adj, X_Tags_Feature, rowsum)
print "Epoch ", epoch, "'s accuracy is", currcorrect
train_acc = train_acc/float(len(Pairs_train))
print "Train Accuracy", train_acc
epochs.append(epoch)
epoch_loss.append(loss.data[0])
epoch_accuracy.append(currcorrect)
train_accuracy.append(train_acc)
if currcorrect > correct:
correct = currcorrect
torch.save(gcn_model.state_dict(), 'gcnmodel_linegraph.pt')
if epoch == 200 :
with open('Testerrors_asker.txt', 'w') as fout:
for index in indexes[0]:
fout.write(str(index)+"\t"+str(rowsum[index]) + "\n")
#np.savetxt('Testerrors.txt', np.array(degrees).T, fmt = '%3d')
#plt.close("all")
#sns.distplot(degrees)
#plt.xscale('log')
#plt.savefig("figures/TestDegreeDist.png")
#sns.distplot(degrees,hist_kws={'log':True})
#plt.xscale('log')
#plt.savefig("figures/TestDegreeDist_ylog.png")
#plt.show()
def main():
# print('gpu avaliable ' if torch.cuda.is_available() else "gpu not alvliable")
device = torch.device("cuda:0")
print('using device:', device)
config = Config()
train_loader = get_loader(config, config.TRAIN_LIST, 2)
val_loader = get_loader(config, config.VAL_LIST, 2)
model = GCN().to(device)
if not os.path.exists(config.MODEL_PATH):
os.makedirs(config.MODEL_PATH)
criterion0 = LabelSmoothing(0.1)
criterion1 = LabelSmoothing(0.1)
criterion2 = LabelSmoothing(0.1)
criterion3 = LabelSmoothing(0.1)
criterion4 = LabelSmoothing(0.1)
criterion5 = LabelSmoothing(0.1)
criterion6 = LabelSmoothing(0.1)
criterion7 = LabelSmoothing(0.1)
criterion8 = LabelSmoothing(0.1)
criterion9 = LabelSmoothing(0.1)
criterion10 = LabelSmoothing(0.1)
# ckpt_path = os.path.join('./exps/GCN/ckpt/', 'model_epoch_199.ckpt')
# print('loading checkpoint from', ckpt_path)
# checkpoint = torch.load(ckpt_path)
# model.load_state_dict(checkpoint)
params = list(model.parameters())
# optimizer = torch.optim.Adam(params, lr=config.LEARNING_RATE)
A_params = [id(model.A)]
base_params = filter(lambda p: id(p) not in A_params, model.parameters())
optimizer = torch.optim.Adam([{
'params': base_params
}, {
'params': model.A,
'lr': config.LEARNING_RATE * 100
}],
lr=config.LEARNING_RATE)
total_step = len(train_loader)
min_val_loss = float('inf')
# scheduler = lr_scheduler.StepLR(optimizer, step_size=config.EPOCHES/3, gamma=0.1)
scheduler = lr_scheduler.CosineAnnealingLR(optimizer,
T_max=5,
eta_min=4e-08)
train_accs = []
train_loss = []
val_accs = []
val_loss = []
disease_nums = 11
disease_names = [
'主疾病', '结节类型--高/等/低回声', '结节类型--单/多发', '边界', '形态', '纵横比', '皮质类型',
'淋巴门结构', '钙化', '囊性区', '血流'
]
for epoch in range(config.EPOCHES):
running_accs = [0. for _ in range(disease_nums)]
print('TRAINING: Epoch [{}/{}]'.format(epoch, config.EPOCHES))
for ii, (inputs, gt_labels) in enumerate(train_loader):
for i in range(len(inputs)):
# _input = _input.to(torch.float)
inputs[i] = inputs[i].type(torch.FloatTensor)
inputs[i] = inputs[i].to(device)
# for _input in inputs:
# print(_input.device)
for i in range(len(gt_labels)):
gt_labels[i] = gt_labels[i].to(torch.long)
gt_labels[i] = gt_labels[i].to(device)
optimizer.zero_grad()
pred_outputs = model(inputs[0], inputs[1])
loss0 = criterion0(pred_outputs[0], gt_labels[0])
loss1 = criterion1(pred_outputs[1], gt_labels[1])
loss2 = criterion2(pred_outputs[2], gt_labels[2])
loss3 = criterion3(pred_outputs[3], gt_labels[3])
loss4 = criterion4(pred_outputs[4], gt_labels[4])
loss5 = criterion5(pred_outputs[5], gt_labels[5])
loss6 = criterion6(pred_outputs[6], gt_labels[6])
loss7 = criterion7(pred_outputs[7], gt_labels[7])
loss8 = criterion8(pred_outputs[8], gt_labels[8])
loss9 = criterion9(pred_outputs[9], gt_labels[9])
loss10 = criterion10(pred_outputs[10], gt_labels[10])
loss = loss0 + loss1 + 5 * loss2 + loss3 + 5 * loss4 + 5 * loss5 + 5 * loss6 + 5 * loss7 + loss8 + loss9 + 5 * loss10
loss.backward()
optimizer.step()
for i in range(disease_nums):
running_accs[i] += count_correct_nums(pred_outputs[i],
gt_labels[i])
if ii % config.LOG_STEP == 0:
print(' Step [{}/{}], Loss: {:.4f}, '.format(
ii,
total_step,
loss.item(),
))
scheduler.step()
for i in range(len(running_accs)):
running_accs[i] /= len(train_loader) * config.BATCH_SIZE
#保存
train_accs.append(running_accs)
train_loss.append([
l.item() for l in [
loss0, loss1, loss2, loss3, loss4, loss5, loss6, loss7, loss8,
loss9, loss10
]
])
################
###验证
################
running_accs = [0. for _ in range(disease_nums)]
with torch.no_grad():
for ii, (inputs, gt_labels) in enumerate(val_loader):
for i in range(len(inputs)):
# _input = _input.to(torch.float)
inputs[i] = inputs[i].type(torch.FloatTensor)
inputs[i] = inputs[i].to(device)
# for _input in inputs:
# print(_input.device)
for i in range(len(gt_labels)):
gt_labels[i] = gt_labels[i].to(torch.long)
gt_labels[i] = gt_labels[i].to(device)
pred_outputs = model(inputs[0], inputs[1])
loss0 = criterion0(pred_outputs[0], gt_labels[0])
loss1 = criterion1(pred_outputs[1], gt_labels[1])
loss2 = criterion2(pred_outputs[2], gt_labels[2])
loss3 = criterion3(pred_outputs[3], gt_labels[3])
loss4 = criterion4(pred_outputs[4], gt_labels[4])
loss5 = criterion5(pred_outputs[5], gt_labels[5])
loss6 = criterion6(pred_outputs[6], gt_labels[6])
loss7 = criterion7(pred_outputs[7], gt_labels[7])
loss8 = criterion8(pred_outputs[8], gt_labels[8])
loss9 = criterion9(pred_outputs[9], gt_labels[9])
loss10 = criterion10(pred_outputs[10], gt_labels[10])
loss = 10 * loss0 + loss1 + loss2 + loss3 + loss4 + loss5 + loss6 + loss7 + loss8 + loss9 + loss10
for i in range(disease_nums):
running_accs[i] += count_correct_nums(
pred_outputs[i], gt_labels[i])
for i in range(len(running_accs)):
running_accs[i] /= len(train_loader) * config.BATCH_SIZE
print('VALADATION: Epoch [{}/{}],loss:{}'.format(
epoch, config.EPOCHES, loss.item()))
val_accs.append(running_accs)
val_loss.append([
l.item() for l in [
loss0, loss1, loss2, loss3, loss4, loss5, loss6, loss7, loss8,
loss9, loss10
]
])
if loss.item() < min_val_loss:
torch.save(model.state_dict(),
os.path.join(config.MODEL_PATH, 'model_epoch.ckpt'))
print('saving model to ' +
str(os.path.join(config.MODEL_PATH, 'model.ckpt')))
plt.figure(figsize=(24, 8))
temp_train_accs = list(zip(*train_accs))[::-1] #转置
temp_val_accs = list(zip(*val_accs))[::-1] #转置
for i in range(len(disease_names)):
plt.plot(temp_train_accs[i], label='training acc_' + str(i))
plt.plot(temp_val_accs[i], label='validate acc_' + str(i))
plt.legend(frameon=False)
plt.savefig('acc.png')
temp_train_loss = list(zip(*train_loss))[::-1] #转置
temp_val_loss = list(zip(*val_loss))[::-1] #转置
for i in range(len(disease_names)):
plt.plot(temp_train_loss[i], label='training loss_' + str(i))
plt.plot(temp_val_loss[i], label='validate loss_' + str(i))
plt.legend(frameon=False)
plt.savefig('loss.png')
plt.clf()
plt.close()
if epoch > 0:
#print "GCN Weight", gcn_model.gc1.weight.data
print("Total loss", epoch, loss_train)
currcorrect, degrees = evaluate_line(gcn_model, test_loader, Adj,
X_Tags_Feature, rowsum)
print "Epoch ", epoch, "'s accuracy is", currcorrect
train_acc = train_acc.numpy() / float(len(Pairs_train))
print "Train Accuracy", train_acc
epochs.append(epoch)
epoch_loss.append(loss.item())
epoch_accuracy.append(currcorrect)
train_accuracy.append(train_acc)
if currcorrect > correct:
correct = currcorrect
torch.save(gcn_model.state_dict(), 'gcnmodel_linegraph.pt')
if epoch == 3000:
np.savetxt('Testerrors.txt', np.array(degrees).T, fmt='%3d')
plt.close("all")
sns.distplot(degrees)
plt.xscale('log')
plt.savefig("figures/TestDegreeDist.png")
sns.distplot(degrees, hist_kws={'log': True})
plt.xscale('log')
plt.savefig("figures/TestDegreeDist_ylog.png")
#plt.show()
#exit()
except KeyboardInterrupt:
print('-' * 89)
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
# Train model
t_total = time.time()
for epoch in range(args.epochs):
train(epoch)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# Testing
test()
torch.save(model.state_dict(), "models/gcn.pth")
print("Saved model!")
# defining and applying integrated gradients on SeedModel
def custom_forward(features, adj):
# features = torch.tensor(np.random.random(org_shape[0])).double
# adj = torch.tensor(np.random.random(org_shape[0])).double
return model(features, adj)
ig = IntegratedGradients(custom_forward)
attr = ig.attribute(features, additional_forward_args=(adj), target=0)
print(ig)
epoch_loss = 0
correct = 0
for iter, (inputs, mask, labels) in enumerate(dataLoader):
if CD:
inputs = inputs.cuda(0)
mask = mask.cuda(0)
labels = labels.cuda(0)
prediction = net(G, inputs, mask)
_, predict = torch.max(prediction, 1)
loss = F.cross_entropy(prediction, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
iter_loss = loss.detach().item()
correct += torch.sum(predict.data == labels.data)
epoch_loss += iter_loss
print('epoch {}, iter {}/{}, loss {:.4f}'.format(
epoch, iter, len(dataLoader), iter_loss))
epoch_loss /= (iter + 1)
correct = float(correct) / ((iter + 1) * batch_size)
print('Epoch {}, loss {:.4f}, acc {:.4f}'.format(epoch, epoch_loss,
correct))
if epoch % 1 == 0:
torch.save(
{
'epoch': epoch,
'net_state_dict': net.state_dict(),
'optimizer': optimizer
}, '/Disk5/junqi/CUB/early_skeleton_{}_{:.4f}_{:.4f}.ckpt'.format(
epoch, correct, epoch_loss))
def train_gcn(training_features, training_adjs, training_labels, eval_features,
eval_adjs, eval_labels, params, class_weights, activations,
unary, coeffs, graph_params):
device = torch.device(
'cuda:1') if torch.cuda.is_available() else torch.device('cpu')
gcn = GCN(n_features=training_features[0].shape[1],
hidden_layers=params["hidden_layers"],
dropout=params["dropout"],
activations=activations,
p=graph_params["probability"],
normalization=params["edge_normalization"])
gcn.to(device)
opt = params["optimizer"](gcn.parameters(),
lr=params["lr"],
weight_decay=params["regularization"])
n_training_graphs = len(training_labels)
graph_size = graph_params["vertices"]
n_eval_graphs = len(eval_labels)
counter = 0 # For early stopping
min_loss = None
for epoch in range(params["epochs"]):
# -------------------------- TRAINING --------------------------
training_graphs_order = np.arange(n_training_graphs)
np.random.shuffle(training_graphs_order)
for i, idx in enumerate(training_graphs_order):
training_mat = torch.tensor(training_features[idx], device=device)
training_adj, training_lbs = map(
lambda x: torch.tensor(
data=x[idx], dtype=torch.double, device=device),
[training_adjs, training_labels])
gcn.train()
opt.zero_grad()
output_train = gcn(training_mat, training_adj)
output_matrix_flat = (
torch.mm(output_train, output_train.transpose(0, 1)) +
1 / 2).flatten()
training_criterion = gcn_build_weighted_loss(
unary, class_weights, training_lbs)
loss_train = coeffs[0] * training_criterion(output_train.view(output_train.shape[0]), training_lbs) + \
coeffs[1] * gcn_pairwise_loss(output_matrix_flat, training_adj.flatten()) + \
coeffs[2] * gcn_binomial_reg(output_train, graph_params)
loss_train.backward()
opt.step()
# -------------------------- EVALUATION --------------------------
graphs_order = np.arange(n_eval_graphs)
np.random.shuffle(graphs_order)
outputs = torch.zeros(graph_size * n_eval_graphs, dtype=torch.double)
output_xs = torch.zeros(graph_size**2 * n_eval_graphs,
dtype=torch.double)
adj_flattened = torch.tensor(
np.hstack([eval_adjs[idx].flatten() for idx in graphs_order]))
for i, idx in enumerate(graphs_order):
eval_mat = torch.tensor(eval_features[idx], device=device)
eval_adj, eval_lbs = map(
lambda x: torch.tensor(
data=x[idx], dtype=torch.double, device=device),
[eval_adjs, eval_labels])
gcn.eval()
output_eval = gcn(eval_mat, eval_adj)
output_matrix_flat = (
torch.mm(output_eval, output_eval.transpose(0, 1)) +
1 / 2).flatten()
output_xs[i * graph_size**2:(i + 1) *
graph_size**2] = output_matrix_flat.cpu()
outputs[i * graph_size:(i + 1) * graph_size] = output_eval.view(
output_eval.shape[0]).cpu()
all_eval_labels = torch.tensor(np.hstack(
[eval_labels[idx] for idx in graphs_order]),
dtype=torch.double)
eval_criterion = gcn_build_weighted_loss(unary, class_weights,
all_eval_labels)
loss_eval = (
coeffs[0] * eval_criterion(outputs, all_eval_labels) +
coeffs[1] * gcn_pairwise_loss(output_xs, adj_flattened) +
coeffs[2] * gcn_binomial_reg(outputs, graph_params)).item()
if min_loss is None:
current_min_loss = loss_eval
else:
current_min_loss = min(min_loss, loss_eval)
if epoch >= 10 and params[
"early_stop"]: # Check for early stopping during training.
if min_loss is None:
min_loss = current_min_loss
torch.save(gcn.state_dict(),
"tmp_time.pt") # Save the best state.
elif loss_eval < min_loss:
min_loss = current_min_loss
torch.save(gcn.state_dict(),
"tmp_time.pt") # Save the best state.
counter = 0
else:
counter += 1
if counter >= 40: # Patience for learning
break
# After stopping early, our model is the one with the best eval loss.
gcn.load_state_dict(torch.load("tmp_time.pt"))
os.remove("tmp_time.pt")
return gcn
def main(args):
# 0. initial setting
# set environmet
cudnn.benchmark = True
if not os.path.isdir(os.path.join(args.path, './ckpt')):
os.mkdir(os.path.join(args.path, './ckpt'))
if not os.path.isdir(os.path.join(args.path, './results')):
os.mkdir(os.path.join(args.path, './results'))
if not os.path.isdir(os.path.join(args.path, './ckpt', args.name)):
os.mkdir(os.path.join(args.path, './ckpt', args.name))
if not os.path.isdir(os.path.join(args.path, './results', args.name)):
os.mkdir(os.path.join(args.path, './results', args.name))
if not os.path.isdir(os.path.join(args.path, './results', args.name,
"log")):
os.mkdir(os.path.join(args.path, './results', args.name, "log"))
# set logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(message)s')
handler = logging.FileHandler(
os.path.join(
args.path, "results/{}/log/{}.log".format(
args.name, time.strftime('%c', time.localtime(time.time())))))
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.addHandler(logging.StreamHandler())
args.logger = logger
# set cuda
if torch.cuda.is_available():
args.logger.info("running on cuda")
args.device = torch.device("cuda")
args.use_cuda = True
else:
args.logger.info("running on cpu")
args.device = torch.device("cpu")
args.use_cuda = False
args.logger.info("[{}] starts".format(args.name))
# 1. load data
adj, features, labels, idx_train, idx_val, idx_test = load_data()
# 2. setup
CORA_NODES = 2708
CORA_FEATURES = 1433
CORA_CLASSES = 7
CITESEER_NODES = 3327
CITESEER_FEATURES = 3703
CITESEER_CLASSES = 6
(num_nodes, feature_dim,
classes) = (CORA_NODES, CORA_FEATURES,
CORA_CLASSES) if args.dataset == 'cora' else (
CITESEER_NODES, CITESEER_FEATURES, CITESEER_CLASSES)
args.logger.info("setting up...")
model = GCN(args, feature_dim, args.hidden, classes,
args.dropout) if args.model == 'gcn' else SpGAT(
args, feature_dim, args.hidden, classes, args.dropout,
args.alpha, args.n_heads)
model.to(args.device)
loss_fn = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.load:
loaded_data = load(args, args.ckpt)
model.load_state_dict(loaded_data['model'])
optimizer.load_state_dict(loaded_data['optimizer'])
# 3. train / test
if not args.test:
# train
args.logger.info("starting training")
train_loss_meter = AverageMeter(args,
name="Loss",
save_all=True,
x_label="epoch")
val_acc_meter = AverageMeter(args,
name="Val Acc",
save_all=True,
x_label="epoch")
earlystop_listener = val_acc_meter.attach_combo_listener(
(lambda prev, new: prev.max >= new.max), threshold=args.patience)
steps = 1
for epoch in range(1, 1 + args.epochs):
spent_time = time.time()
model.train()
train_loss_tmp_meter = AverageMeter(args)
if args.start_from_step is not None:
if steps < args.start_from_step:
steps += 1
continue
optimizer.zero_grad()
batch = len(idx_train)
output = model(features.to(args.device), adj.to(args.device))
loss = loss_fn(output[idx_train],
labels[idx_train].to(args.device))
loss.backward()
optimizer.step()
train_loss_tmp_meter.update(loss, weight=batch)
steps += 1
train_loss_meter.update(train_loss_tmp_meter.avg)
spent_time = time.time() - spent_time
args.logger.info(
"[{}] train loss: {:.3f} took {:.1f} seconds".format(
epoch, train_loss_tmp_meter.avg, spent_time))
model.eval()
spent_time = time.time()
if not args.fastmode:
with torch.no_grad():
output = model(features.to(args.device),
adj.to(args.device))
acc = accuracy(output[idx_val], labels[idx_val]) * 100.0
val_acc_meter.update(acc)
earlystop = earlystop_listener.listen()
spent_time = time.time() - spent_time
args.logger.info(
"[{}] val acc: {:2.1f} % took {:.1f} seconds".format(
epoch, acc, spent_time))
if steps % args.save_period == 0:
save(args, "epoch{}".format(epoch),
{'model': model.state_dict()})
train_loss_meter.plot(scatter=False)
val_acc_meter.plot(scatter=False)
val_acc_meter.save()
if earlystop:
break
else:
# test
args.logger.info("starting test")
model.eval()
with torch.no_grad():
model(features.to(args.device), adj.to(args.device))
acc = accuracy(output[idx_test], labels[idx_test]) * 100
logger.d("test acc: {:2.1f} % took {:.1f} seconds".format(
acc, spent_time))
