gcn_optimizer.step() if epoch > 0: #print "GCN Weight", gcn_model.gc1.weight.data print("Total loss", epoch, loss_train) currcorrect, indexes = evaluate_line(gcn_model, test_loader, Adj, X_Tags_Feature, rowsum) print "Epoch ", epoch, "'s accuracy is", currcorrect train_acc = train_acc/float(len(Pairs_train)) print "Train Accuracy", train_acc epochs.append(epoch) epoch_loss.append(loss.data[0]) epoch_accuracy.append(currcorrect) train_accuracy.append(train_acc) if currcorrect > correct: correct = currcorrect torch.save(gcn_model.state_dict(), 'gcnmodel_linegraph.pt') if epoch == 200 : with open('Testerrors_asker.txt', 'w') as fout: for index in indexes[0]: fout.write(str(index)+"\t"+str(rowsum[index]) + "\n") #np.savetxt('Testerrors.txt', np.array(degrees).T, fmt = '%3d') #plt.close("all") #sns.distplot(degrees) #plt.xscale('log') #plt.savefig("figures/TestDegreeDist.png") #sns.distplot(degrees,hist_kws={'log':True}) #plt.xscale('log') #plt.savefig("figures/TestDegreeDist_ylog.png") #plt.show()
def main(): # print('gpu avaliable ' if torch.cuda.is_available() else "gpu not alvliable") device = torch.device("cuda:0") print('using device:', device) config = Config() train_loader = get_loader(config, config.TRAIN_LIST, 2) val_loader = get_loader(config, config.VAL_LIST, 2) model = GCN().to(device) if not os.path.exists(config.MODEL_PATH): os.makedirs(config.MODEL_PATH) criterion0 = LabelSmoothing(0.1) criterion1 = LabelSmoothing(0.1) criterion2 = LabelSmoothing(0.1) criterion3 = LabelSmoothing(0.1) criterion4 = LabelSmoothing(0.1) criterion5 = LabelSmoothing(0.1) criterion6 = LabelSmoothing(0.1) criterion7 = LabelSmoothing(0.1) criterion8 = LabelSmoothing(0.1) criterion9 = LabelSmoothing(0.1) criterion10 = LabelSmoothing(0.1) # ckpt_path = os.path.join('./exps/GCN/ckpt/', 'model_epoch_199.ckpt') # print('loading checkpoint from', ckpt_path) # checkpoint = torch.load(ckpt_path) # model.load_state_dict(checkpoint) params = list(model.parameters()) # optimizer = torch.optim.Adam(params, lr=config.LEARNING_RATE) A_params = [id(model.A)] base_params = filter(lambda p: id(p) not in A_params, model.parameters()) optimizer = torch.optim.Adam([{ 'params': base_params }, { 'params': model.A, 'lr': config.LEARNING_RATE * 100 }], lr=config.LEARNING_RATE) total_step = len(train_loader) min_val_loss = float('inf') # scheduler = lr_scheduler.StepLR(optimizer, step_size=config.EPOCHES/3, gamma=0.1) scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=5, eta_min=4e-08) train_accs = [] train_loss = [] val_accs = [] val_loss = [] disease_nums = 11 disease_names = [ '主疾病', '结节类型--高/等/低回声', '结节类型--单/多发', '边界', '形态', '纵横比', '皮质类型', '淋巴门结构', '钙化', '囊性区', '血流' ] for epoch in range(config.EPOCHES): running_accs = [0. for _ in range(disease_nums)] print('TRAINING: Epoch [{}/{}]'.format(epoch, config.EPOCHES)) for ii, (inputs, gt_labels) in enumerate(train_loader): for i in range(len(inputs)): # _input = _input.to(torch.float) inputs[i] = inputs[i].type(torch.FloatTensor) inputs[i] = inputs[i].to(device) # for _input in inputs: # print(_input.device) for i in range(len(gt_labels)): gt_labels[i] = gt_labels[i].to(torch.long) gt_labels[i] = gt_labels[i].to(device) optimizer.zero_grad() pred_outputs = model(inputs[0], inputs[1]) loss0 = criterion0(pred_outputs[0], gt_labels[0]) loss1 = criterion1(pred_outputs[1], gt_labels[1]) loss2 = criterion2(pred_outputs[2], gt_labels[2]) loss3 = criterion3(pred_outputs[3], gt_labels[3]) loss4 = criterion4(pred_outputs[4], gt_labels[4]) loss5 = criterion5(pred_outputs[5], gt_labels[5]) loss6 = criterion6(pred_outputs[6], gt_labels[6]) loss7 = criterion7(pred_outputs[7], gt_labels[7]) loss8 = criterion8(pred_outputs[8], gt_labels[8]) loss9 = criterion9(pred_outputs[9], gt_labels[9]) loss10 = criterion10(pred_outputs[10], gt_labels[10]) loss = loss0 + loss1 + 5 * loss2 + loss3 + 5 * loss4 + 5 * loss5 + 5 * loss6 + 5 * loss7 + loss8 + loss9 + 5 * loss10 loss.backward() optimizer.step() for i in range(disease_nums): running_accs[i] += count_correct_nums(pred_outputs[i], gt_labels[i]) if ii % config.LOG_STEP == 0: print(' Step [{}/{}], Loss: {:.4f}, '.format( ii, total_step, loss.item(), )) scheduler.step() for i in range(len(running_accs)): running_accs[i] /= len(train_loader) * config.BATCH_SIZE #保存 train_accs.append(running_accs) train_loss.append([ l.item() for l in [ loss0, loss1, loss2, loss3, loss4, loss5, loss6, loss7, loss8, loss9, loss10 ] ]) ################ ###验证 ################ running_accs = [0. for _ in range(disease_nums)] with torch.no_grad(): for ii, (inputs, gt_labels) in enumerate(val_loader): for i in range(len(inputs)): # _input = _input.to(torch.float) inputs[i] = inputs[i].type(torch.FloatTensor) inputs[i] = inputs[i].to(device) # for _input in inputs: # print(_input.device) for i in range(len(gt_labels)): gt_labels[i] = gt_labels[i].to(torch.long) gt_labels[i] = gt_labels[i].to(device) pred_outputs = model(inputs[0], inputs[1]) loss0 = criterion0(pred_outputs[0], gt_labels[0]) loss1 = criterion1(pred_outputs[1], gt_labels[1]) loss2 = criterion2(pred_outputs[2], gt_labels[2]) loss3 = criterion3(pred_outputs[3], gt_labels[3]) loss4 = criterion4(pred_outputs[4], gt_labels[4]) loss5 = criterion5(pred_outputs[5], gt_labels[5]) loss6 = criterion6(pred_outputs[6], gt_labels[6]) loss7 = criterion7(pred_outputs[7], gt_labels[7]) loss8 = criterion8(pred_outputs[8], gt_labels[8]) loss9 = criterion9(pred_outputs[9], gt_labels[9]) loss10 = criterion10(pred_outputs[10], gt_labels[10]) loss = 10 * loss0 + loss1 + loss2 + loss3 + loss4 + loss5 + loss6 + loss7 + loss8 + loss9 + loss10 for i in range(disease_nums): running_accs[i] += count_correct_nums( pred_outputs[i], gt_labels[i]) for i in range(len(running_accs)): running_accs[i] /= len(train_loader) * config.BATCH_SIZE print('VALADATION: Epoch [{}/{}],loss:{}'.format( epoch, config.EPOCHES, loss.item())) val_accs.append(running_accs) val_loss.append([ l.item() for l in [ loss0, loss1, loss2, loss3, loss4, loss5, loss6, loss7, loss8, loss9, loss10 ] ]) if loss.item() < min_val_loss: torch.save(model.state_dict(), os.path.join(config.MODEL_PATH, 'model_epoch.ckpt')) print('saving model to ' + str(os.path.join(config.MODEL_PATH, 'model.ckpt'))) plt.figure(figsize=(24, 8)) temp_train_accs = list(zip(*train_accs))[::-1] #转置 temp_val_accs = list(zip(*val_accs))[::-1] #转置 for i in range(len(disease_names)): plt.plot(temp_train_accs[i], label='training acc_' + str(i)) plt.plot(temp_val_accs[i], label='validate acc_' + str(i)) plt.legend(frameon=False) plt.savefig('acc.png') temp_train_loss = list(zip(*train_loss))[::-1] #转置 temp_val_loss = list(zip(*val_loss))[::-1] #转置 for i in range(len(disease_names)): plt.plot(temp_train_loss[i], label='training loss_' + str(i)) plt.plot(temp_val_loss[i], label='validate loss_' + str(i)) plt.legend(frameon=False) plt.savefig('loss.png') plt.clf() plt.close()
if epoch > 0: #print "GCN Weight", gcn_model.gc1.weight.data print("Total loss", epoch, loss_train) currcorrect, degrees = evaluate_line(gcn_model, test_loader, Adj, X_Tags_Feature, rowsum) print "Epoch ", epoch, "'s accuracy is", currcorrect train_acc = train_acc.numpy() / float(len(Pairs_train)) print "Train Accuracy", train_acc epochs.append(epoch) epoch_loss.append(loss.item()) epoch_accuracy.append(currcorrect) train_accuracy.append(train_acc) if currcorrect > correct: correct = currcorrect torch.save(gcn_model.state_dict(), 'gcnmodel_linegraph.pt') if epoch == 3000: np.savetxt('Testerrors.txt', np.array(degrees).T, fmt='%3d') plt.close("all") sns.distplot(degrees) plt.xscale('log') plt.savefig("figures/TestDegreeDist.png") sns.distplot(degrees, hist_kws={'log': True}) plt.xscale('log') plt.savefig("figures/TestDegreeDist_ylog.png") #plt.show() #exit() except KeyboardInterrupt: print('-' * 89)
acc_test = accuracy(output[idx_test], labels[idx_test]) print("Test set results:", "loss= {:.4f}".format(loss_test.item()), "accuracy= {:.4f}".format(acc_test.item())) # Train model t_total = time.time() for epoch in range(args.epochs): train(epoch) print("Optimization Finished!") print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) # Testing test() torch.save(model.state_dict(), "models/gcn.pth") print("Saved model!") # defining and applying integrated gradients on SeedModel def custom_forward(features, adj): # features = torch.tensor(np.random.random(org_shape[0])).double # adj = torch.tensor(np.random.random(org_shape[0])).double return model(features, adj) ig = IntegratedGradients(custom_forward) attr = ig.attribute(features, additional_forward_args=(adj), target=0) print(ig)
epoch_loss = 0 correct = 0 for iter, (inputs, mask, labels) in enumerate(dataLoader): if CD: inputs = inputs.cuda(0) mask = mask.cuda(0) labels = labels.cuda(0) prediction = net(G, inputs, mask) _, predict = torch.max(prediction, 1) loss = F.cross_entropy(prediction, labels) optimizer.zero_grad() loss.backward() optimizer.step() iter_loss = loss.detach().item() correct += torch.sum(predict.data == labels.data) epoch_loss += iter_loss print('epoch {}, iter {}/{}, loss {:.4f}'.format( epoch, iter, len(dataLoader), iter_loss)) epoch_loss /= (iter + 1) correct = float(correct) / ((iter + 1) * batch_size) print('Epoch {}, loss {:.4f}, acc {:.4f}'.format(epoch, epoch_loss, correct)) if epoch % 1 == 0: torch.save( { 'epoch': epoch, 'net_state_dict': net.state_dict(), 'optimizer': optimizer }, '/Disk5/junqi/CUB/early_skeleton_{}_{:.4f}_{:.4f}.ckpt'.format( epoch, correct, epoch_loss))
def train_gcn(training_features, training_adjs, training_labels, eval_features, eval_adjs, eval_labels, params, class_weights, activations, unary, coeffs, graph_params): device = torch.device( 'cuda:1') if torch.cuda.is_available() else torch.device('cpu') gcn = GCN(n_features=training_features[0].shape[1], hidden_layers=params["hidden_layers"], dropout=params["dropout"], activations=activations, p=graph_params["probability"], normalization=params["edge_normalization"]) gcn.to(device) opt = params["optimizer"](gcn.parameters(), lr=params["lr"], weight_decay=params["regularization"]) n_training_graphs = len(training_labels) graph_size = graph_params["vertices"] n_eval_graphs = len(eval_labels) counter = 0 # For early stopping min_loss = None for epoch in range(params["epochs"]): # -------------------------- TRAINING -------------------------- training_graphs_order = np.arange(n_training_graphs) np.random.shuffle(training_graphs_order) for i, idx in enumerate(training_graphs_order): training_mat = torch.tensor(training_features[idx], device=device) training_adj, training_lbs = map( lambda x: torch.tensor( data=x[idx], dtype=torch.double, device=device), [training_adjs, training_labels]) gcn.train() opt.zero_grad() output_train = gcn(training_mat, training_adj) output_matrix_flat = ( torch.mm(output_train, output_train.transpose(0, 1)) + 1 / 2).flatten() training_criterion = gcn_build_weighted_loss( unary, class_weights, training_lbs) loss_train = coeffs[0] * training_criterion(output_train.view(output_train.shape[0]), training_lbs) + \ coeffs[1] * gcn_pairwise_loss(output_matrix_flat, training_adj.flatten()) + \ coeffs[2] * gcn_binomial_reg(output_train, graph_params) loss_train.backward() opt.step() # -------------------------- EVALUATION -------------------------- graphs_order = np.arange(n_eval_graphs) np.random.shuffle(graphs_order) outputs = torch.zeros(graph_size * n_eval_graphs, dtype=torch.double) output_xs = torch.zeros(graph_size**2 * n_eval_graphs, dtype=torch.double) adj_flattened = torch.tensor( np.hstack([eval_adjs[idx].flatten() for idx in graphs_order])) for i, idx in enumerate(graphs_order): eval_mat = torch.tensor(eval_features[idx], device=device) eval_adj, eval_lbs = map( lambda x: torch.tensor( data=x[idx], dtype=torch.double, device=device), [eval_adjs, eval_labels]) gcn.eval() output_eval = gcn(eval_mat, eval_adj) output_matrix_flat = ( torch.mm(output_eval, output_eval.transpose(0, 1)) + 1 / 2).flatten() output_xs[i * graph_size**2:(i + 1) * graph_size**2] = output_matrix_flat.cpu() outputs[i * graph_size:(i + 1) * graph_size] = output_eval.view( output_eval.shape[0]).cpu() all_eval_labels = torch.tensor(np.hstack( [eval_labels[idx] for idx in graphs_order]), dtype=torch.double) eval_criterion = gcn_build_weighted_loss(unary, class_weights, all_eval_labels) loss_eval = ( coeffs[0] * eval_criterion(outputs, all_eval_labels) + coeffs[1] * gcn_pairwise_loss(output_xs, adj_flattened) + coeffs[2] * gcn_binomial_reg(outputs, graph_params)).item() if min_loss is None: current_min_loss = loss_eval else: current_min_loss = min(min_loss, loss_eval) if epoch >= 10 and params[ "early_stop"]: # Check for early stopping during training. if min_loss is None: min_loss = current_min_loss torch.save(gcn.state_dict(), "tmp_time.pt") # Save the best state. elif loss_eval < min_loss: min_loss = current_min_loss torch.save(gcn.state_dict(), "tmp_time.pt") # Save the best state. counter = 0 else: counter += 1 if counter >= 40: # Patience for learning break # After stopping early, our model is the one with the best eval loss. gcn.load_state_dict(torch.load("tmp_time.pt")) os.remove("tmp_time.pt") return gcn
def main(args): # 0. initial setting # set environmet cudnn.benchmark = True if not os.path.isdir(os.path.join(args.path, './ckpt')): os.mkdir(os.path.join(args.path, './ckpt')) if not os.path.isdir(os.path.join(args.path, './results')): os.mkdir(os.path.join(args.path, './results')) if not os.path.isdir(os.path.join(args.path, './ckpt', args.name)): os.mkdir(os.path.join(args.path, './ckpt', args.name)) if not os.path.isdir(os.path.join(args.path, './results', args.name)): os.mkdir(os.path.join(args.path, './results', args.name)) if not os.path.isdir(os.path.join(args.path, './results', args.name, "log")): os.mkdir(os.path.join(args.path, './results', args.name, "log")) # set logger logger = logging.getLogger() logger.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - %(message)s') handler = logging.FileHandler( os.path.join( args.path, "results/{}/log/{}.log".format( args.name, time.strftime('%c', time.localtime(time.time()))))) handler.setFormatter(formatter) logger.addHandler(handler) logger.addHandler(logging.StreamHandler()) args.logger = logger # set cuda if torch.cuda.is_available(): args.logger.info("running on cuda") args.device = torch.device("cuda") args.use_cuda = True else: args.logger.info("running on cpu") args.device = torch.device("cpu") args.use_cuda = False args.logger.info("[{}] starts".format(args.name)) # 1. load data adj, features, labels, idx_train, idx_val, idx_test = load_data() # 2. setup CORA_NODES = 2708 CORA_FEATURES = 1433 CORA_CLASSES = 7 CITESEER_NODES = 3327 CITESEER_FEATURES = 3703 CITESEER_CLASSES = 6 (num_nodes, feature_dim, classes) = (CORA_NODES, CORA_FEATURES, CORA_CLASSES) if args.dataset == 'cora' else ( CITESEER_NODES, CITESEER_FEATURES, CITESEER_CLASSES) args.logger.info("setting up...") model = GCN(args, feature_dim, args.hidden, classes, args.dropout) if args.model == 'gcn' else SpGAT( args, feature_dim, args.hidden, classes, args.dropout, args.alpha, args.n_heads) model.to(args.device) loss_fn = nn.NLLLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.load: loaded_data = load(args, args.ckpt) model.load_state_dict(loaded_data['model']) optimizer.load_state_dict(loaded_data['optimizer']) # 3. train / test if not args.test: # train args.logger.info("starting training") train_loss_meter = AverageMeter(args, name="Loss", save_all=True, x_label="epoch") val_acc_meter = AverageMeter(args, name="Val Acc", save_all=True, x_label="epoch") earlystop_listener = val_acc_meter.attach_combo_listener( (lambda prev, new: prev.max >= new.max), threshold=args.patience) steps = 1 for epoch in range(1, 1 + args.epochs): spent_time = time.time() model.train() train_loss_tmp_meter = AverageMeter(args) if args.start_from_step is not None: if steps < args.start_from_step: steps += 1 continue optimizer.zero_grad() batch = len(idx_train) output = model(features.to(args.device), adj.to(args.device)) loss = loss_fn(output[idx_train], labels[idx_train].to(args.device)) loss.backward() optimizer.step() train_loss_tmp_meter.update(loss, weight=batch) steps += 1 train_loss_meter.update(train_loss_tmp_meter.avg) spent_time = time.time() - spent_time args.logger.info( "[{}] train loss: {:.3f} took {:.1f} seconds".format( epoch, train_loss_tmp_meter.avg, spent_time)) model.eval() spent_time = time.time() if not args.fastmode: with torch.no_grad(): output = model(features.to(args.device), adj.to(args.device)) acc = accuracy(output[idx_val], labels[idx_val]) * 100.0 val_acc_meter.update(acc) earlystop = earlystop_listener.listen() spent_time = time.time() - spent_time args.logger.info( "[{}] val acc: {:2.1f} % took {:.1f} seconds".format( epoch, acc, spent_time)) if steps % args.save_period == 0: save(args, "epoch{}".format(epoch), {'model': model.state_dict()}) train_loss_meter.plot(scatter=False) val_acc_meter.plot(scatter=False) val_acc_meter.save() if earlystop: break else: # test args.logger.info("starting test") model.eval() with torch.no_grad(): model(features.to(args.device), adj.to(args.device)) acc = accuracy(output[idx_test], labels[idx_test]) * 100 logger.d("test acc: {:2.1f} % took {:.1f} seconds".format( acc, spent_time))