"""Testing A Simple Prediction""" #print("Feature vector: %s" % X_test[:1]) print("Label: %s" % str(y_test[0])) print("Predicted: %s" % str(net0.predict(X_test[:1]))) """Metrics""" # layer_info = PrintLayerInfo() # net0.verbose = 3 # net0.initialize() #print layer_info(net0) print "[Classification Report]: " print classification_report(y_test, predicted) print "[Train dataset] Score: ", net0.score(X_train, y_train) print "[Test dataset] Score: ", net0.score(X_test, y_test) plot_matrix(net0, X_test, y_test, filename) valid_accuracies = np.array([i["valid_accuracy"] for i in net0.train_history_]) plot_accuracy(valid_accuracies, filename) train_loss = [row['train_loss'] for row in net0.train_history_] valid_loss = [row['valid_loss'] for row in net0.train_history_] plot_loss(valid_loss, train_loss, filename) y_score = net0.predict_proba(X_test) #[:, 1] y_test_bin = np.array(label_binarize(y_test, classes=np.unique(y))) n_classes = y_test_bin.shape[1] plot_roc_curve(n_classes, y_test_bin, y_score, filename=filename)
#print("reconstruction loss", reconstruction_loss.item(), "sparsity_loss", sparsity_loss.item()) lambda1 = float(config["stain_separation"]["sparsity_weight"]) loss = reconstruction_loss + (lambda1 * sparsity_loss) loss.backward() optimiser_tdss.step() loss_log["reconstruction"].append([epoch, batch_id, reconstruction_loss.item()]) loss_log["sparsity"].append([epoch, batch_id, sparsity_loss.item()]) batch_id += 1 # plot losses plot_loss(loss_log["reconstruction"], "./output/training/reconstruction_loss.png") plot_loss(loss_log["sparsity"], "./output/training/absorption_loss.png") if epoch % 5 == 0: # save images for s, stain_od in enumerate(reconstructed_stains): stain_rgb = OpticalDensity2RGB(stain_od) vutils.save_image( stain_rgb.detach(), "./output/training/reconstructed_stain_{}.png".format(s), normalize=False ) vutils.save_image(
batch_size, shuffle_data, X_train, Y_train, X_val, Y_val, ) train_history_64_layers, val_history_64_layers = trainer_shuffle.train( num_epochs) plt.figure(figsize=(20, 12)) plt.subplot(1, 2, 1) plt.ylim([0.00, 1.00]) utils.plot_loss(train_history_64_layers["loss"], "Training Loss 10 hidden layers", npoints_to_average=10) plt.ylabel("Training Loss") utils.plot_loss(val_history_64_layers["loss"], "Validation Loss 10 hidden layers") plt.ylabel("Validation Loss") utils.plot_loss(train_history["loss"], "Training Loss 2 hidden layers", npoints_to_average=10) utils.plot_loss(val_history["loss"], "Validation Loss 2 hidden layers") plt.legend() plt.subplot(1, 2, 2) plt.ylim([0.92, .97]) utils.plot_loss(val_history_64_layers["accuracy"],
trainer_two = SoftmaxTrainer( momentum_gamma, use_momentum, model_two, learning_rate, batch_size, shuffle_data, X_train, Y_train, X_val, Y_val, ) train_history, val_history = trainer.train(num_epochs) train_history_two, val_history_two = trainer_two.train(num_epochs) plt.figure(figsize=(20, 10)) plt.subplot(1, 2, 1) utils.plot_loss(train_history["accuracy"], "Base model task 3 ") utils.plot_loss(train_history_two["accuracy"], "Ten layer model") plt.ylim([0.85, 1.0]) plt.ylabel("Training Accuracy") plt.legend() plt.subplot(1, 2, 2) plt.ylim([0.85, 1.0]) utils.plot_loss(val_history["accuracy"], "Base model task 3") utils.plot_loss(val_history_two["accuracy"], "Ten layer model") plt.ylabel("Accuracy") plt.legend() plt.savefig("task4e_train_loss.png", dpi=120)
def train(self): # load networks************************************************************************ self.Choose_Model(self.Model_index) utils.print_network(self.model) # optimizer self.momentum = 0.9 self.optimizer = optim.SGD(self.model.parameters(), lr=self.lr, momentum=self.momentum) self.model.float() # loss function if self.loss_func == 'mse': self.loss = nn.MSELoss() elif self.loss_func == 'ssim': self.loss = pytorch_ssim.SSIM(window_size=11) if self.gpu_mode: self.model = nn.DataParallel(self.model) self.model.cuda() self.loss.cuda() # load dataset train_data_loader = self.load_dataset(dataset='train') #val_data_loader = self.load_dataset(dataset='test') # set the logger log_dir = os.path.join(self.save_dir, 'logs') if not os.path.exists(log_dir): os.mkdir(log_dir) logger = Logger(log_dir) ################# Train start################# print('Training is started.') avg_loss = [] step = 0 self.model.train() ### debug ### ### debug end ### for epoch in range(self.num_epochs): epoch_loss = 0 for iter, data in enumerate(train_data_loader): LR = data['img_LR'] HR = data['img_HR'] #only use Y channel input_Y = LR[:, 0:1, :, :] target_Y = HR[:, 0:1, :, :] if self.scale_factor == 4: target_Y = utils.shave( target_Y, border_size=2 * self.scale_factor ) #according to size of the output image passed the network elif self.scale_factor == 6: target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor - 1) elif self.scale_factor == 2: target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor - 1) else: target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor - 2) if self.save_inImg == True: #save the net input image saveinY = (input_Y.numpy()[0, :, :, :].transpose(1, 2, 0) * 255).astype(numpy.uint8) scipy.misc.imsave('lrin.png', saveinY[:, :, 0]) savetarY = ( target_Y.numpy()[0, :, :, :].transpose(1, 2, 0) * 255).astype(numpy.uint8) scipy.misc.imsave('tarin.png', savetarY[:, :, 0]) if self.gpu_mode: target = Variable(target_Y.cuda()) #print("target.size()") #print(target.size()) input = Variable(input_Y.cuda()) #print("input.size():", input.size()) else: target = Variable(target_Y) # target = Variable(utils.shave(target_Y, border_size=2*self.scale_factor)) input = Variable(input_Y) ############## ORIGINAL ############### self.optimizer.zero_grad() recon_image = self.model(input) #print("recon_image.size(): ") #print(recon_image.size()) # if self.scale_factor ==2: # recon_image = recon_image[:,:,1:-1,1:-1] # elif self.scale_factor == 3: # recon_image = recon_image[:, :, 0:-1, 0:-1] #### SSIM loss ############## # loss = 1-self.loss(recon_image, target) loss = self.loss(recon_image, target) # print loss.data loss.backward() self.optimizer.step() # log epoch_loss += loss.data # tensorboard logging logger.scalar_summary('loss', loss.data, step + 1) step += 1 if epoch % self.save_epochs == 0: self.save_model(epoch) # self.validation(epoch, val_data_loader) avg_loss.append(epoch_loss / len(train_data_loader)) print("Epoch: [%2d] [%4d/%4d] loss: %.8f" % ((epoch + 1), (iter + 1), len(train_data_loader), epoch_loss)) # Plot avg. loss utils.plot_loss([avg_loss], self.num_epochs, save_dir=self.save_dir) print("Training is finished.") # Save final trained parameters of model self.save_model(epoch=None)
def train(self): # load networks************************************************************************ self.Choose_Model(self.Model_index) utils.print_network(self.model) # optimizer self.momentum = 0.9 self.optimizer = optim.SGD(self.model.parameters(), lr=self.lr, momentum=self.momentum) self.model.float() # loss function if self.loss_func == 'mse': self.loss = nn.MSELoss() elif self.loss_func == 'ssim': self.loss = pytorch_ssim.SSIM(window_size=11) elif self.loss_func == 'ssim&mse': self.Mseloss = nn.MSELoss() self.SSIMloss = pytorch_ssim.SSIM(window_size=11) # self.loss = 0.97 * nn.MSELoss() + 0.03 * pytorch_ssim.SSIM(window_size = 11) if self.gpu_mode: self.model.cuda() self.Mseloss.cuda() self.SSIMloss.cuda() # load dataset train_data_loader = self.load_dataset(dataset='train') val_data_loader = self.load_dataset(dataset='test') # set the logger log_dir = os.path.join(self.save_dir, 'logs') if not os.path.exists(log_dir): os.mkdir(log_dir) logger = Logger(log_dir) # ckpt_dir = self.ckpt_dir # if not os.path.exists(ckpt_dir): # os.makedirs(ckpt_dir, mode=0o777) ################# Train start################# print('Training is started.') avg_loss = [] step = 0 self.model.train() ### debug ### ### debug end ### for epoch in range(self.num_epochs): epoch_loss = 0 epoch_mseloss = 0 epoch_ssimloss = 0 for iter, data in enumerate(train_data_loader): LR = data['img_LR'] HR = data['img_HR'] #only use Y channel input_Y = LR[:, 0:1, :, :] #print(input_Y.size()) w = input_Y.size()[2] h = input_Y.size()[3] C = input_Y.size()[0] #print(C, w, h) w_new = w + 4 h_new = h + 4 a = numpy.zeros([C, 1, w_new, h_new], dtype='f') numpy_Y = input_Y.numpy() a[:, :, 2:-2, 2:-2] = numpy_Y input_Y_new = torch.from_numpy(a) #print(input_Y_new.size()) target_Y = HR[:, 0:1, :, :] # if self.scale_factor==4: # target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor) #according to size of the output image passed the network # elif self.scale_factor==6: # target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor - 1) # elif self.scale_factor==2: # target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor - 1) # else: # target_Y = utils.shave(target_Y, border_size=2 * self.scale_factor - 2) if self.save_inImg == True: #save the net input image saveinY = (input_Y.numpy()[0, :, :, :].transpose(1, 2, 0) * 255).astype(numpy.uint8) scipy.misc.imsave('lrin.png', saveinY[:, :, 0]) savetarY = ( target_Y.numpy()[0, :, :, :].transpose(1, 2, 0) * 255).astype(numpy.uint8) scipy.misc.imsave('tarin.png', savetarY[:, :, 0]) if self.gpu_mode: target = Variable(target_Y.cuda()) input = Variable(input_Y_new.cuda()) else: target = Variable(target_Y) # target = Variable(utils.shave(target_Y, border_size=2*self.scale_factor)) input = Variable(input_Y_new) ############## ORIGINAL ############### self.optimizer.zero_grad() recon_image = self.model(input) # if self.scale_factor ==2: # recon_image = recon_image[:,:,1:-1,1:-1] # elif self.scale_factor == 3: # recon_image = recon_image[:, :, 0:-1, 0:-1] #### SSIM loss ############## # loss = 1-self.loss(recon_image, target) #print('target.size()', target.size()) #print('recon_image.size()', recon_image.size()) # Caculate total_loss = alpha*mseloss + (1-alpha)*ssimloss alpha = 0.97 mseloss = self.Mseloss(recon_image, target) ssimloss = self.SSIMloss(recon_image, target) loss = alpha * mseloss + (1 - alpha) * ssimloss # loss = self.loss(recon_image, target) # print loss.data loss.backward() self.optimizer.step() # log epoch_mseloss += mseloss.data epoch_ssimloss += ssimloss.data epoch_loss += loss.data # tensorboard logging #print(' loss ', loss.data, ' mseloss ', mseloss.data, ' ssimloss ', ssimloss.data) logger.scalar_summary('loss', loss.data, step + 1) step += 1 print(step, ' loss ', epoch_loss, ' mseloss ', epoch_mseloss, ' ssimloss ', epoch_ssimloss) if epoch % self.save_epochs == 0: #onnx_name = 'x' + str(self.scale_factor) + '_' + self.model_name + '_epoch_' + str(epoch) + '.onnx' #torch.onnx.export(self.model, input, onnx_name, export_params=True, verbose=True) self.save_model(epoch) # save_path = os.path.join(ckpt_dir, "{}_{}.pth".format(self.ckpt_name, epoch)) # torch.save(self.model.state_dict(), save_path) #self.validation(epoch, val_data_loader) avg_loss.append(epoch_loss / len(train_data_loader)) print("Epoch: [%2d] [%4d/%4d] loss: %.8f" % ((epoch + 1), (iter + 1), len(train_data_loader), epoch_loss)) # Plot avg. loss utils.plot_loss([avg_loss], self.num_epochs, save_dir=self.save_dir) print("Training is finished.") # Save final trained parameters of model self.save_model(epoch=None)
def train_model(ques_train_map, ans_train_map, img_train_map, ques_train_ids, ques_to_img_train, ques_val_map, ans_val_map, img_val_map, ques_val_ids, ques_to_img_val, id_to_ans, train_dim, val_dim, ans_types, params): # training parameters num_epochs = params['num_epochs'] batch_size = params['batch_size'] num_batches_train = train_dim // batch_size num_batches_val = val_dim // batch_size eval_every = 5 train_loss, train_acc = [], [] val_loss, val_acc = [], [] eval_acc = [] print "Loading model" model = get_model(dropout_rate=float(params['dropout_rate']), regularization_rate=float(params['regularization_rate']), embedding_size=int(params['embedding_size']), num_classes=int(params['num_answers']), model_name=params['model']) if not ans_types: savedir = "models/%s_%s" % (params['model'], str( params['num_answers'])) else: savedir = "models/%s_%s_%s" % (params['model'], ans_types.replace("/", ""), str(params['num_answers'])) if not os.path.exists(savedir): os.mkdir(savedir) for k in range(num_epochs): loss, acc = train_epoch(k + 1, model, num_batches_train, batch_size, ques_train_map, ans_train_map, img_train_map, ques_train_ids, ques_to_img_train) train_loss.append(loss) train_acc.append(acc) loss, acc = val_epoch(k + 1, model, num_batches_val, batch_size, ques_val_map, ans_val_map, img_val_map, ques_val_ids, ques_to_img_val) val_loss.append(loss) val_acc.append(acc) if (k + 1) % eval_every == 0: model.save_weights("%s/%s_epoch_%d_weights.h5" % (savedir, params['model'], (k + 1)), overwrite=True) eval_accuracy = evaluate(model, vqa_val, batch_size, ques_val_map, img_val_map, id_to_ans, params['ans_types']) print("Eval accuracy: %.2f" % eval_accuracy) eval_acc.append(eval_accuracy) plot_loss(train_loss, val_loss, savedir) plot_accuracy(train_acc, val_acc, savedir) plot_eval_accuracy(eval_acc, savedir) best_epoch = (1 + np.argmax(np.array(eval_acc))) * eval_every print "Best accuracy %.02f on epoch %d" % (max(eval_acc), best_epoch) model.load_weights("%s/%s_epoch_%d_weights.h5" % (savedir, params['model'], best_epoch)) evaluate(model, vqa_val, batch_size, ques_val_map, img_val_map, id_to_ans, params['ans_types'], verbose=True)
# 任务4:理解utils中cross_entropy的实现代码 loss1 = utils.cross_entropy(y_hat, batchy) trainloss.append(loss1) """从后往前迭代更新残差值,并利用残差值来更新各输出层和隐藏层的权重""" error = y_hat - batchy model.backward(error) # 评估模型性能 loss2 = utils.cross_entropy(model.forward(validX), utils.make_onehot(validy, 10)) validloss.append(loss2) #print("iteration:{0}/{1} trainloss:{2:.2f}, validloss:{3:.2f}".format( #j, i, loss1, loss2)) utils.plot_loss(trainloss, validloss) #%% 评估 prediction = np.argmax(model.forward(testX), axis=1) cmatrix = utils.confusion_matrix(prediction, testy.flatten(), 10) print(cmatrix) # accuracy acc = np.diag(cmatrix).sum() / testy.shape[0] print("accuracy: %.4f" % (acc)) # heat map plt.figure() plt.imshow(cmatrix, cmap='gray', interpolation='none') plt.show() # 观察错误
shuffle_data, X_train, Y_train, X_val, Y_val, ) start = time.time() train_history_new, val_history_new = new_trainer.train(num_epochs) end = time.time() print("Elapsed training time (s): ", end - start) plt.subplot(1, 2, 1) utils.plot_loss(train_history["loss"], "Task 2 Model", npoints_to_average=10) utils.plot_loss(train_history_new["loss"], "Task 2 Model - Improved weight, sigmoid and momentum", npoints_to_average=10) plt.ylim([0, .4]) plt.ylabel("Training Cross Entropy Loss") plt.legend() plt.subplot(1, 2, 2) plt.ylim([0.8, .98]) utils.plot_loss(val_history["accuracy"], "Task 2 Model") utils.plot_loss(val_history_new["accuracy"], "Task 2 Model - Improved weight, sigmoid and momentum") plt.ylabel("Validation Accuracy") plt.legend() plt.show()
train_steps_per_epoch, EPOCHS, validation_generator, validation_steps_per_epoch, save_model_filepath=model_file) # Training KerasModel for pretrained models # model_history = k_model.train_learned_model_with_generator(train_generator, # train_steps_per_epoch, # EPOCHS, # validation_generator, # validation_steps_per_epoch, # save_model_filepath='model_transfer_Inceptionv3.h5') # Plotting the model Loss utils.plot_loss(model_history=model_history) # Uncomment these lines to visualize the model's first and second Convolutions for track 1 and # track 2 test images # Track 1 layers visualization # k_model = KerasModel(load=True, model_file='model.h5') # test_image = np.asarray(Image.open( # './assets/Layer_visualization/Track1/center_2018_05_07_18_39_19_350.jpg')) # print(np.array(test_image).shape) # k_model.visualize_layer(test_image, 'Track1 Model First Convolution') # k_model.visualize_layer(test_image, 'Track1 Model Second Convolution', layer_num=4) # Track 2 layers visualization # test_image = np.asarray(Image.open(
def main(args): """Experiment logic""" # Get file separator and construct paths sep = "\t" if args.file_type == "tsv" else "," train_path = os.path.join(args.data_dir, "train.{}".format(args.file_type)) test_path = os.path.join(args.data_dir, "test.{}".format(args.file_type)) # Read column headings headings = pd.read_csv(train_path, sep=sep, nrows=1).columns text, label = "text", "gold_label_{}".format(args.task_type) if args.elmo: from elmo import TabularReader, ElmoLoader # Pretrained urls options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" # Read dataset reader = TabularReader(text, label, sep) loader = ElmoLoader(reader, train_path, test_path, args.batch_dims) # Build model label_map = loader.label_map embedding_size = 1024 model = DAN(to_int(args.layers), len(label_map), embedding_size=embedding_size, elmo_config=(options_file, weight_file)) else: # Build data loader loader = DataLoader( args.data_dir, args.file_type, headings, text, label, to_int(args.batch_dims), (args.glove_type, args.glove_dim), args.temp_dir, ) # Build model vocab, label_map = loader.vocab, loader.label_map model = DAN(to_int(args.layers), len(label_map), vocab_size=len(vocab), embedding_size=args.glove_dim, pretrained_vecs=vocab.vectors) # Define training functions optimiser = optim.SGD(model.parameters(), lr=args.lr) loss_fn = nn.CrossEntropyLoss() # Train logging.info("\n\nStarting training...\n\n") if args.num_processes > 1: model.share_memory() processes = [] for pid in range(args.num_processes): p = mp.Process(target=run.training_process, args=(pid, loader, model, optimiser, loss_fn, (args.num_steps // args.num_processes))) p.start() processes.append(p) for p in processes: p.join() else: report_every = int(args.num_steps * 0.01) losses = run.train(loader, model, optimiser, loss_fn, label_map, args.num_steps, report_every) if args.plot: logging.info("\n\nPlotting training schedule...\n\n") plot_loss(losses, report_every, args.temp_dir) # Save the trained model logging.info("\n\nNow saving...\n\n") torch.save(model, os.path.join(args.temp_dir, "saved_model.pt")) # Test model_acc = run.test(loader, label_map, args.temp_dir) if args.baseline: logging.info( "\n\nComparing with multinomial naive bayes baseline...\n\n") from bayes import multi_nb train, test = pd.read_csv(train_path, sep=sep), pd.read_csv(test_path, sep=sep) train_txt, test_txt = (train[text], train[label]), (test[text], test[label]) base_acc = multi_nb(train_txt, test_txt) logging.info("Model accuracy: {:.6g}".format(model_acc)) logging.info("Baseline accuracy: {:.6g}".format(base_acc)) logging.info("{}".format( "Model wins!" if model_acc > base_acc else "Baseline wins!"))
def run_main(args): ################################################# START SECTION OF LOADING PARAMETERS ################################################# # Read parameters epochs = args.epochs dim_au_out = args.bottleneck #8, 16, 32, 64, 128, 256,512 na = args.missing_value data_path = DATA_MAP[args.target_data] test_size = args.test_size select_drug = args.drug freeze = args.freeze_pretrain valid_size = args.valid_size g_disperson = args.var_genes_disp min_n_genes = args.min_n_genes max_n_genes = args.max_n_genes source_model_path = args.source_model_path target_model_path = args.target_model_path log_path = args.logging_file batch_size = args.batch_size encoder_hdims = args.source_h_dims.split(",") encoder_hdims = list(map(int, encoder_hdims)) source_data_path = args.source_data pretrain = args.pretrain prediction = args.predition data_name = args.target_data label_path = args.label_path reduce_model = args.dimreduce predict_hdims = args.p_h_dims.split(",") predict_hdims = list(map(int, predict_hdims)) leiden_res = args.cluster_res load_model = bool(args.load_target_model) # Misc now = time.strftime("%Y-%m-%d-%H-%M-%S") # Initialize logging and std out out_path = log_path + now + ".err" log_path = log_path + now + ".log" out = open(out_path, "w") sys.stderr = out #Logging infomaion logging.basicConfig( level=logging.INFO, filename=log_path, filemode='a', format= '%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s' ) logging.getLogger('matplotlib.font_manager').disabled = True logging.info(args) # Save arguments args_df = ut.save_arguments(args, now) ################################################# END SECTION OF LOADING PARAMETERS ################################################# ################################################# START SECTION OF SINGLE CELL DATA REPROCESSING ################################################# # Load data and preprocessing adata = pp.read_sc_file(data_path) if data_name == 'GSE117872': adata = ut.specific_process(adata, dataname=data_name, select_origin=args.batch_id) elif data_name == 'GSE122843': adata = ut.specific_process(adata, dataname=data_name) elif data_name == 'GSE110894': adata = ut.specific_process(adata, dataname=data_name) elif data_name == 'GSE112274': adata = ut.specific_process(adata, dataname=data_name) elif data_name == 'GSE116237': adata = ut.specific_process(adata, dataname=data_name) elif data_name == 'GSE108383': adata = ut.specific_process(adata, dataname=data_name) elif data_name == 'GSE140440': adata = ut.specific_process(adata, dataname=data_name) elif data_name == 'GSE129730': adata = ut.specific_process(adata, dataname=data_name) elif data_name == 'GSE149383': adata = ut.specific_process(adata, dataname=data_name) else: adata = adata sc.pp.filter_cells(adata, min_genes=200) sc.pp.filter_genes(adata, min_cells=3) adata = pp.cal_ncount_ngenes(adata) # Show statisctic after QX sc.pl.violin(adata, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt-'], jitter=0.4, multi_panel=True, save=data_name, show=False) sc.pl.scatter(adata, x='total_counts', y='pct_counts_mt-', show=False) sc.pl.scatter(adata, x='total_counts', y='n_genes_by_counts', show=False) if args.remove_genes == 0: r_genes = [] else: r_genes = REMOVE_GENES #Preprocess data by filtering if data_name not in ['GSE112274', 'GSE140440']: adata = pp.receipe_my(adata, l_n_genes=min_n_genes, r_n_genes=max_n_genes, filter_mincells=args.min_c, filter_mingenes=args.min_g, normalize=True, log=True, remove_genes=r_genes) else: adata = pp.receipe_my(adata, l_n_genes=min_n_genes, r_n_genes=max_n_genes, filter_mincells=args.min_c, percent_mito=100, filter_mingenes=args.min_g, normalize=True, log=True, remove_genes=r_genes) # Select highly variable genes sc.pp.highly_variable_genes(adata, min_disp=g_disperson, max_disp=np.inf, max_mean=6) sc.pl.highly_variable_genes(adata, save=data_name, show=False) adata.raw = adata adata = adata[:, adata.var.highly_variable] # Preprocess data if spcific process is required data = adata.X # PCA # Generate neighbor graph sc.tl.pca(adata, svd_solver='arpack') sc.pp.neighbors(adata, n_neighbors=10) # Generate cluster labels sc.tl.leiden(adata, resolution=leiden_res) sc.tl.umap(adata) sc.pl.umap(adata, color=['leiden'], save=data_name + 'umap' + now, show=False) adata.obs['leiden_origin'] = adata.obs['leiden'] adata.obsm['X_umap_origin'] = adata.obsm['X_umap'] data_c = adata.obs['leiden'].astype("long").to_list() ################################################# END SECTION OF SINGLE CELL DATA REPROCESSING ################################################# ################################################# START SECTION OF LOADING SC DATA TO THE TENSORS ################################################# #Prepare to normailize and split target data mmscaler = preprocessing.MinMaxScaler() try: data = mmscaler.fit_transform(data) except: logging.warning("Only one class, no ROC") # Process sparse data data = data.todense() data = mmscaler.fit_transform(data) # Split data to train and valid set # Along with the leiden conditions for CVAE propose Xtarget_train, Xtarget_valid, Ctarget_train, Ctarget_valid = train_test_split( data, data_c, test_size=valid_size, random_state=42) # Select the device of gpu device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # Assuming that we are on a CUDA machine, this should print a CUDA device: logging.info(device) try: torch.cuda.set_device(device) except: logging.warning("No GPU detected, will apply cpu to process") # Construct datasets and data loaders Xtarget_trainTensor = torch.FloatTensor(Xtarget_train).to(device) Xtarget_validTensor = torch.FloatTensor(Xtarget_valid).to(device) # Use leiden label if CVAE is applied Ctarget_trainTensor = torch.LongTensor(Ctarget_train).to(device) Ctarget_validTensor = torch.LongTensor(Ctarget_valid).to(device) X_allTensor = torch.FloatTensor(data).to(device) C_allTensor = torch.LongTensor(data_c).to(device) train_dataset = TensorDataset(Xtarget_trainTensor, Ctarget_trainTensor) valid_dataset = TensorDataset(Xtarget_validTensor, Ctarget_validTensor) Xtarget_trainDataLoader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) Xtarget_validDataLoader = DataLoader(dataset=valid_dataset, batch_size=batch_size, shuffle=True) dataloaders_pretrain = { 'train': Xtarget_trainDataLoader, 'val': Xtarget_validDataLoader } ################################################# START SECTION OF LOADING SC DATA TO THE TENSORS ################################################# ################################################# START SECTION OF LOADING BULK DATA ################################################# # Read source data data_r = pd.read_csv(source_data_path, index_col=0) label_r = pd.read_csv(label_path, index_col=0) label_r = label_r.fillna(na) # Extract labels selected_idx = label_r.loc[:, select_drug] != na label = label_r.loc[selected_idx, select_drug] label = label.values.reshape(-1, 1) if prediction == "regression": lbscaler = preprocessing.MinMaxScaler() label = lbscaler.fit_transform(label) dim_model_out = 1 else: le = preprocessing.LabelEncoder() label = le.fit_transform(label) dim_model_out = 2 # Process source data mmscaler = preprocessing.MinMaxScaler() source_data = mmscaler.fit_transform(data_r) # Split source data Xsource_train_all, Xsource_test, Ysource_train_all, Ysource_test = train_test_split( source_data, label, test_size=test_size, random_state=42) Xsource_train, Xsource_valid, Ysource_train, Ysource_valid = train_test_split( Xsource_train_all, Ysource_train_all, test_size=valid_size, random_state=42) # Transform source data # Construct datasets and data loaders Xsource_trainTensor = torch.FloatTensor(Xsource_train).to(device) Xsource_validTensor = torch.FloatTensor(Xsource_valid).to(device) if prediction == "regression": Ysource_trainTensor = torch.FloatTensor(Ysource_train).to(device) Ysource_validTensor = torch.FloatTensor(Ysource_valid).to(device) else: Ysource_trainTensor = torch.LongTensor(Ysource_train).to(device) Ysource_validTensor = torch.LongTensor(Ysource_valid).to(device) sourcetrain_dataset = TensorDataset(Xsource_trainTensor, Ysource_trainTensor) sourcevalid_dataset = TensorDataset(Xsource_validTensor, Ysource_validTensor) Xsource_trainDataLoader = DataLoader(dataset=sourcetrain_dataset, batch_size=batch_size, shuffle=True) Xsource_validDataLoader = DataLoader(dataset=sourcevalid_dataset, batch_size=batch_size, shuffle=True) dataloaders_source = { 'train': Xsource_trainDataLoader, 'val': Xsource_validDataLoader } ################################################# END SECTION OF LOADING BULK DATA ################################################# ################################################# START SECTION OF MODEL CUNSTRUCTION ################################################# # Construct target encoder if reduce_model == "AE": encoder = AEBase(input_dim=data.shape[1], latent_dim=dim_au_out, h_dims=encoder_hdims) loss_function_e = nn.MSELoss() elif reduce_model == "VAE": encoder = VAEBase(input_dim=data.shape[1], latent_dim=dim_au_out, h_dims=encoder_hdims) elif reduce_model == "CVAE": # Number of condition is equal to the number of clusters encoder = CVAEBase(input_dim=data.shape[1], n_conditions=len(set(data_c)), latent_dim=dim_au_out, h_dims=encoder_hdims) if torch.cuda.is_available(): encoder.cuda() logging.info("Target encoder structure is: ") logging.info(encoder) encoder.to(device) optimizer_e = optim.Adam(encoder.parameters(), lr=1e-2) loss_function_e = nn.MSELoss() exp_lr_scheduler_e = lr_scheduler.ReduceLROnPlateau(optimizer_e) # Load source model before transfer if prediction == "regression": dim_model_out = 1 else: dim_model_out = 2 # Load AE model if reduce_model == "AE": source_model = PretrainedPredictor(input_dim=Xsource_train.shape[1], latent_dim=dim_au_out, h_dims=encoder_hdims, hidden_dims_predictor=predict_hdims, output_dim=dim_model_out, pretrained_weights=None, freezed=freeze) source_model.load_state_dict(torch.load(source_model_path)) source_encoder = source_model # Load VAE model elif reduce_model in ["VAE", "CVAE"]: source_model = PretrainedVAEPredictor( input_dim=Xsource_train.shape[1], latent_dim=dim_au_out, h_dims=encoder_hdims, hidden_dims_predictor=predict_hdims, output_dim=dim_model_out, pretrained_weights=None, freezed=freeze, z_reparam=bool(args.VAErepram)) source_model.load_state_dict(torch.load(source_model_path)) source_encoder = source_model logging.info("Load pretrained source model from: " + source_model_path) source_encoder.to(device) ################################################# END SECTION OF MODEL CUNSTRUCTION ################################################# ################################################# START SECTION OF SC MODEL PRETRAININIG ################################################# # Pretrain target encoder # Pretain using autoencoder is pretrain is not False if (str(pretrain) != '0'): # Pretrained target encoder if there are not stored files in the harddisk train_flag = True pretrain = str(pretrain) if (os.path.exists(pretrain) == True): try: encoder.load_state_dict(torch.load(pretrain)) logging.info("Load pretrained target encoder from " + pretrain) train_flag = False except: logging.warning("Loading failed, procceed to re-train model") if train_flag == True: if reduce_model == "AE": encoder, loss_report_en = t.train_AE_model( net=encoder, data_loaders=dataloaders_pretrain, optimizer=optimizer_e, loss_function=loss_function_e, n_epochs=epochs, scheduler=exp_lr_scheduler_e, save_path=pretrain) elif reduce_model == "VAE": encoder, loss_report_en = t.train_VAE_model( net=encoder, data_loaders=dataloaders_pretrain, optimizer=optimizer_e, n_epochs=epochs, scheduler=exp_lr_scheduler_e, save_path=pretrain) elif reduce_model == "CVAE": encoder, loss_report_en = t.train_CVAE_model( net=encoder, data_loaders=dataloaders_pretrain, optimizer=optimizer_e, n_epochs=epochs, scheduler=exp_lr_scheduler_e, save_path=pretrain) logging.info("Pretrained finished") # Before Transfer learning, we test the performance of using no transfer performance: # Use vae result to predict if (args.dimreduce != "CVAE"): embeddings_pretrain = encoder.encode(X_allTensor) else: embeddings_pretrain = encoder.encode(X_allTensor, C_allTensor) pretrain_prob_prediction = source_model.predict( embeddings_pretrain).detach().cpu().numpy() adata.obs["sens_preds_pret"] = pretrain_prob_prediction[:, 1] adata.obs["sens_label_pret"] = pretrain_prob_prediction.argmax(axis=1) # # Use umap result to predict ## This section is removed because the dim problem and the performance problem # sc.tl.pca(adata, n_comps=max(50,2*dim_au_out),svd_solver='arpack') # sc.tl.umap(adata, n_components=dim_au_out) # embeddings_umap = torch.FloatTensor(adata.obsm["X_umap"]).to(device) # umap_prob_prediction = source_model.predict(embeddings_umap).detach().cpu().numpy() # adata.obs["sens_preds_umap"] = umap_prob_prediction[:,1] # adata.obs["sens_label_umap"] = umap_prob_prediction.argmax(axis=1) # # Use tsne result to predict # #sc.tl.tsne(adata, n_pcs=dim_au_out) # X_pca = adata.obsm["X_pca"] # # Replace tsne by pac beacause TSNE is very slow # X_tsne = adata.obsm["X_umap"] # #X_tsne = TSNE(n_components=dim_au_out,method='exact').fit_transform(X_pca) # embeddings_tsne = torch.FloatTensor(X_tsne).to(device) # tsne_prob_prediction = source_model.predict(embeddings_tsne).detach().cpu().numpy() # adata.obs["sens_preds_tsne"] = tsne_prob_prediction[:,1] # adata.obs["sens_label_tsne"] = tsne_prob_prediction.argmax(axis=1) # adata.obsm["X_tsne_pret"] = X_tsne # Add embeddings to the adata object embeddings_pretrain = embeddings_pretrain.detach().cpu().numpy() adata.obsm["X_pre"] = embeddings_pretrain ################################################# END SECTION OF SC MODEL PRETRAININIG ################################################# ################################################# START SECTION OF TRANSFER LEARNING TRAINING ################################################# # Using ADDA transfer learning if args.transfer == 'ADDA': # Set discriminator model discriminator = Predictor(input_dim=dim_au_out, output_dim=2) discriminator.to(device) loss_d = nn.CrossEntropyLoss() optimizer_d = optim.Adam(encoder.parameters(), lr=1e-2) exp_lr_scheduler_d = lr_scheduler.ReduceLROnPlateau(optimizer_d) # Adversairal trainning discriminator, encoder, report_, report2_ = t.train_ADDA_model( source_encoder, encoder, discriminator, dataloaders_source, dataloaders_pretrain, loss_d, loss_d, # Should here be all optimizer d? optimizer_d, optimizer_d, exp_lr_scheduler_d, exp_lr_scheduler_d, epochs, device, target_model_path) logging.info("Transfer ADDA finished") # DaNN model elif args.transfer == 'DaNN': # Set predictor loss loss_d = nn.CrossEntropyLoss() optimizer_d = optim.Adam(encoder.parameters(), lr=1e-2) exp_lr_scheduler_d = lr_scheduler.ReduceLROnPlateau(optimizer_d) # Set DaNN model DaNN_model = DaNN(source_model=source_encoder, target_model=encoder) DaNN_model.to(device) def loss(x, y, GAMMA=args.GAMMA_mmd): result = mmd.mmd_loss(x, y, GAMMA) return result loss_disrtibution = loss # Tran DaNN model DaNN_model, report_ = t.train_DaNN_model( DaNN_model, dataloaders_source, dataloaders_pretrain, # Should here be all optimizer d? optimizer_d, loss_d, epochs, exp_lr_scheduler_d, dist_loss=loss_disrtibution, load=load_model, weight=args.mmd_weight, save_path=target_model_path + "_DaNN.pkl") encoder = DaNN_model.target_model source_model = DaNN_model.source_model logging.info("Transfer DaNN finished") if (load_model == False): ut.plot_loss(report_[0], path="figures/train_loss_" + now + ".pdf") ut.plot_loss(report_[1], path="figures/mmd_loss_" + now + ".pdf", set_ylim=False) if (args.dimreduce != 'CVAE'): # Attribute test using integrated gradient # Generate a target model including encoder and predictor target_model = TargetModel(source_model, encoder) # Allow require gradients and process label X_allTensor.requires_grad_() # Run integrated gradient check # Return adata and feature integrated gradient ytarget_allPred = target_model(X_allTensor).detach().cpu().numpy() ytarget_allPred = ytarget_allPred.argmax(axis=1) adata, attrp1, senNeu_c0_genes, senNeu_c1_genes = ut.integrated_gradient_differential( net=target_model, input=X_allTensor, clip="positive", target=ytarget_allPred, adata=adata, ig_fc=1, save_name=reduce_model + args.predictor + prediction + select_drug + "sensNeuron" + now) adata, attrn1, resNeu_c0_genes, resNeu_c1_genes = ut.integrated_gradient_differential( net=target_model, input=X_allTensor, clip="negative", target=ytarget_allPred, adata=adata, ig_fc=1, save_name=reduce_model + args.predictor + prediction + select_drug + "restNeuron" + now) sc.pl.heatmap(attrp1, senNeu_c0_genes, groupby='sensitive', cmap='RdBu_r', save=data_name + args.transfer + args.dimreduce + "_seNc0_" + now, show=False) sc.pl.heatmap(attrp1, senNeu_c1_genes, groupby='sensitive', cmap='RdBu_r', save=data_name + args.transfer + args.dimreduce + "_seNc1_" + now, show=False) sc.pl.heatmap(attrn1, resNeu_c0_genes, groupby='sensitive', cmap='RdBu_r', save=data_name + args.transfer + args.dimreduce + "_reNc0_" + now, show=False) sc.pl.heatmap(attrn1, resNeu_c1_genes, groupby='sensitive', cmap='RdBu_r', save=data_name + args.transfer + args.dimreduce + "_reNc1_" + now, show=False) # CHI2 Test on predictive features SFD = SelectFdr(chi2) SFD.fit(adata.raw.X, ytarget_allPred) adata.raw.var['chi2_pval'] = SFD.pvalues_ adata.raw.var['chi2_score'] = SFD.scores_ df_chi2_genes = adata.raw.var[ (SFD.pvalues_ < 0.05) & (adata.raw.var.highly_variable == True) & (adata.raw.var.n_cells > args.min_c)] df_chi2_genes.sort_values(by="chi2_pval", ascending=True, inplace=True) df_chi2_genes.to_csv("saved/results/chi2_pval_genes" + args.predictor + prediction + select_drug + now + '.csv') else: print() ################################################# END SECTION OF TRANSER LEARNING TRAINING ################################################# ################################################# START SECTION OF PREPROCESSING FEATURES ################################################# # Extract feature embeddings # Extract prediction probabilities if (args.dimreduce != "CVAE"): embedding_tensors = encoder.encode(X_allTensor) else: embedding_tensors = encoder.encode(X_allTensor, C_allTensor) prediction_tensors = source_model.predictor(embedding_tensors) embeddings = embedding_tensors.detach().cpu().numpy() predictions = prediction_tensors.detach().cpu().numpy() # Transform predict8ion probabilities to 0-1 labels if (prediction == "regression"): adata.obs["sens_preds"] = predictions else: adata.obs["sens_preds"] = predictions[:, 1] adata.obs["sens_label"] = predictions.argmax(axis=1) adata.obs["sens_label"] = adata.obs["sens_label"].astype('category') adata.obs["rest_preds"] = predictions[:, 0] adata.write("saved/adata/before_ann" + data_name + now + ".h5ad") ################################################# END SECTION OF PREPROCESSING FEATURES ################################################# ################################################# START SECTION OF ANALYSIS AND POST PROCESSING ################################################# # Pipeline of scanpy # Add embeddings to the adata package adata.obsm["X_Trans"] = embeddings #sc.tl.umap(adata) sc.pp.neighbors(adata, n_neighbors=10, use_rep="X_Trans") # Use t-sne on transfer learning features sc.tl.tsne(adata, use_rep="X_Trans") # Leiden on the data # sc.tl.leiden(adata) # Plot tsne sc.pl.tsne(adata, save=data_name + now, color=["leiden"], show=False) # Differenrial expression genes sc.tl.rank_genes_groups(adata, 'leiden', method='wilcoxon') sc.pl.rank_genes_groups(adata, n_genes=args.n_DE_genes, sharey=False, save=data_name + now, show=False) # Differenrial expression genes across 0-1 classes sc.tl.rank_genes_groups(adata, 'sens_label', method='wilcoxon') adata = ut.de_score(adata, clustername='sens_label') # save DE genes between 0-1 class for label in [0, 1]: try: df_degs = get_de_dataframe(adata, label) df_degs.to_csv("saved/results/DEGs_class_" + str(label) + args.predictor + prediction + select_drug + now + '.csv') except: logging.warning("Only one class, no two calsses critical genes") # Generate reports of scores report_df = args_df # Data specific benchmarking sens_pb_pret = adata.obs['sens_preds_pret'] lb_pret = adata.obs['sens_label_pret'] # sens_pb_umap = adata.obs['sens_preds_umap'] # lb_umap = adata.obs['sens_label_umap'] # sens_pb_tsne = adata.obs['sens_preds_tsne'] # lb_tsne = adata.obs['sens_label_tsne'] if ('sensitive' in adata.obs.keys()): report_df = report_df.T Y_test = adata.obs['sensitive'] sens_pb_results = adata.obs['sens_preds'] lb_results = adata.obs['sens_label'] le_sc = LabelEncoder() le_sc.fit(['Resistant', 'Sensitive']) label_descrbie = le_sc.inverse_transform(Y_test) adata.obs['sens_truth'] = label_descrbie color_list = ["sens_truth", "sens_label", 'sens_preds'] color_score_list = [ "Sensitive_score", "Resistant_score", "1_score", "0_score" ] sens_score = pearsonr(adata.obs["sens_preds"], adata.obs["Sensitive_score"])[0] resistant_score = pearsonr(adata.obs["rest_preds"], adata.obs["Resistant_score"])[0] report_df['prob_sens_pearson'] = sens_score report_df['prob_rest_pearson'] = resistant_score try: cluster_score_sens = pearsonr(adata.obs["1_score"], adata.obs["Sensitive_score"])[0] report_df['sens_pearson'] = cluster_score_sens except: logging.warning( "Prediction score 1 not exist, fill adata with 0 values") adata.obs["1_score"] = np.zeros(len(adata)) try: cluster_score_resist = pearsonr(adata.obs["0_score"], adata.obs["Resistant_score"])[0] report_df['rest_pearson'] = cluster_score_resist except: logging.warning( "Prediction score 0 not exist, fill adata with 0 values") adata.obs["0_score"] = np.zeros(len(adata)) #if (data_name in ['GSE110894','GSE117872']): ap_score = average_precision_score(Y_test, sens_pb_results) ap_pret = average_precision_score(Y_test, sens_pb_pret) # ap_umap = average_precision_score(Y_test, sens_pb_umap) # ap_tsne = average_precision_score(Y_test, sens_pb_tsne) report_dict = classification_report(Y_test, lb_results, output_dict=True) f1score = report_dict['weighted avg']['f1-score'] report_df['f1_score'] = f1score classification_report_df = pd.DataFrame(report_dict).T classification_report_df.to_csv("saved/results/clf_report_" + reduce_model + args.predictor + prediction + select_drug + now + '.csv') # report_dict_umap = classification_report(Y_test, lb_umap, output_dict=True) # classification_report_umap_df = pd.DataFrame(report_dict_umap).T # classification_report_umap_df.to_csv("saved/results/clf_umap_report_" + reduce_model + args.predictor+ prediction + select_drug+now + '.csv') report_dict_pret = classification_report(Y_test, lb_pret, output_dict=True) classification_report_pret_df = pd.DataFrame(report_dict_pret).T classification_report_pret_df.to_csv("saved/results/clf_pret_report_" + reduce_model + args.predictor + prediction + select_drug + now + '.csv') # report_dict_tsne = classification_report(Y_test, lb_tsne, output_dict=True) # classification_report_tsne_df = pd.DataFrame(report_dict_tsne).T # classification_report_tsne_df.to_csv("saved/results/clf_tsne_report_" + reduce_model + args.predictor+ prediction + select_drug+now + '.csv') try: auroc_score = roc_auc_score(Y_test, sens_pb_results) auroc_pret = average_precision_score(Y_test, sens_pb_pret) # auroc_umap = average_precision_score(Y_test, sens_pb_umap) # auroc_tsne = average_precision_score(Y_test, sens_pb_tsne) except: logging.warning("Only one class, no ROC") auroc_pret = auroc_umap = auroc_tsne = auroc_score = 0 report_df['auroc_score'] = auroc_score report_df['ap_score'] = ap_score report_df['auroc_pret'] = auroc_pret report_df['ap_pret'] = ap_pret # report_df['auroc_umap'] = auroc_umap # report_df['ap_umap'] = ap_umap # report_df['auroc_tsne'] = auroc_tsne # report_df['ap_tsne'] = ap_tsne ap_title = "ap: " + str(Decimal(ap_score).quantize(Decimal('0.0000'))) auroc_title = "roc: " + str( Decimal(auroc_score).quantize(Decimal('0.0000'))) title_list = ["Ground truth", "Prediction", "Probability"] else: color_list = ["leiden", "sens_label", 'sens_preds'] title_list = ['Cluster', "Prediction", "Probability"] color_score_list = color_list # Simple analysis do neighbors in adata using PCA embeddings #sc.pp.neighbors(adata) # Run UMAP dimension reduction sc.pp.neighbors(adata) sc.tl.umap(adata) # Run leiden clustering # sc.tl.leiden(adata,resolution=leiden_res) # Plot uamp # sc.pl.umap(adata,color=[color_list[0],'sens_label_umap','sens_preds_umap'],save=data_name+args.transfer+args.dimreduce+now,show=False,title=title_list) # Plot transfer learning on umap sc.pl.umap(adata, color=color_list + color_score_list, save=data_name + args.transfer + args.dimreduce + "umap_all" + now, show=False) sc.settings.set_figure_params(dpi=100, frameon=False, figsize=(4, 3), facecolor='white') sc.pl.umap(adata, color=['sensitivity', 'leiden', 'sens_label', 'sens_preds'], title=[ 'Cell sensitivity', 'Cell clusters', 'Transfer learning prediction', 'Prediction probability' ], save=data_name + args.transfer + args.dimreduce + "umap_pred" + now, show=False, ncols=4) sc.pl.umap(adata, color=color_score_list, title=[ 'Sensitive gene score', 'Resistant gene score', 'Sensitive gene score (prediction)', 'Resistant gene score (prediction)' ], save=data_name + args.transfer + args.dimreduce + "umap_scores" + now, show=False, ncols=2) # sc.pl.umap(adata,color=['Sample name'], # save=data_name+args.transfer+args.dimreduce+"umap_sm"+now,show=False,ncols=4) try: sc.pl.umap(adata, color=adata.var.sort_values( "integrated_gradient_sens_class0").head().index, save=data_name + args.transfer + args.dimreduce + "_cgenes0_" + now, show=False) sc.pl.umap(adata, color=adata.var.sort_values( "integrated_gradient_sens_class1").head().index, save=data_name + args.transfer + args.dimreduce + "_cgenes1_" + now, show=False) # c0_genes = df_11_genes.loc[df_11_genes.pval<0.05].head().index # c1_genes = df_00_genes.loc[df_00_genes.pval<0.05].head().index # sc.pl.umap(adata,color=c0_genes,neighbors_key="Trans",save=data_name+args.transfer+args.dimreduce+"_cgenes0_TL"+now,show=False) # sc.pl.umap(adata,color=c1_genes,neighbors_key="Trans",save=data_name+args.transfer+args.dimreduce+"_cgenes1_TL"+now,show=False) except: logging.warning("IG results not avaliable") # Run embeddings using transfered embeddings sc.pp.neighbors(adata, use_rep='X_Trans', key_added="Trans") sc.tl.umap(adata, neighbors_key="Trans") sc.tl.leiden(adata, neighbors_key="Trans", key_added="leiden_trans", resolution=leiden_res) sc.pl.umap(adata, color=color_list, neighbors_key="Trans", save=data_name + args.transfer + args.dimreduce + "_TL" + now, show=False, title=title_list) # Plot cell score on umap sc.pl.umap(adata, color=color_score_list, neighbors_key="Trans", save=data_name + args.transfer + args.dimreduce + "_score_TL" + now, show=False, title=color_score_list) # This tsne is based on transfer learning feature sc.pl.tsne(adata, color=color_list, neighbors_key="Trans", save=data_name + args.transfer + args.dimreduce + "_TL" + now, show=False, title=title_list) # Use tsne origianl version to visualize sc.tl.tsne(adata) # This tsne is based on transfer learning feature # sc.pl.tsne(adata,color=[color_list[0],'sens_label_tsne','sens_preds_tsne'],save=data_name+args.transfer+args.dimreduce+"_original_tsne"+now,show=False,title=title_list) # Plot tsne of the pretrained (autoencoder) embeddings sc.pp.neighbors(adata, use_rep='X_pre', key_added="Pret") sc.tl.umap(adata, neighbors_key="Pret") sc.tl.leiden(adata, neighbors_key="Pret", key_added="leiden_Pret", resolution=leiden_res) sc.pl.umap(adata, color=[color_list[0], 'sens_label_pret', 'sens_preds_pret'], neighbors_key="Pret", save=data_name + args.transfer + args.dimreduce + "_umap_Pretrain_" + now, show=False) # Ari between two transfer learning embedding and sensitivity label ari_score_trans = adjusted_rand_score(adata.obs['leiden_trans'], adata.obs['sens_label']) ari_score = adjusted_rand_score(adata.obs['leiden'], adata.obs['sens_label']) pret_ari_score = adjusted_rand_score(adata.obs['leiden_origin'], adata.obs['leiden_Pret']) transfer_ari_score = adjusted_rand_score(adata.obs['leiden_origin'], adata.obs['leiden_trans']) sc.pl.umap(adata, color=['leiden_origin', 'leiden_trans', 'leiden_Pret'], save=data_name + args.transfer + args.dimreduce + "_comp_Pretrain_" + now, show=False) #report_df = args_df report_df['ari_score'] = ari_score report_df['ari_trans_score'] = ari_score_trans report_df['ari_pre_umap'] = pret_ari_score report_df['ari_trans_umap'] = transfer_ari_score # Trajectory of adata adata, corelations = trajectory(adata, root_key='sensitive', genes_vis=senNeu_c0_genes[:5], root=1, now=now, plot=True) gene_cor = {} # Trajectory for g in np.array(senNeu_c0_genes): gene = g express_vec = adata[:, gene].X corr = pearsonr( np.array(express_vec).ravel(), np.array(adata.obs["dpt_pseudotime"]))[0] gene_cor[gene] = corr try: for k in corelations.keys(): report_df['cor_dpt_' + k] = corelations[k][0] report_df['cor_pvl_' + k] = corelations[k][1] except: logging.warning( "Some of the coorelation cannot be reterived from the dictional") ################################################# END SECTION OF ANALYSIS AND POST PROCESSING ################################################# ################################################# START SECTION OF ANALYSIS FOR BULK DATA ################################################# # bdata = sc.AnnData(data_r) # bdata.obs = label_r # bulk_degs={} # sc.tl.rank_genes_groups(bdata, select_drug, method='wilcoxon') # bdata = ut.de_score(bdata,select_drug) # for label in set(label_r.loc[:,select_drug]): # try: # df_degs = get_de_dataframe(bdata,label) # bulk_degs[label] = df_degs.iloc[:50,:].names # df_degs.to_csv("saved/results/DEGs_bulk_" +str(label)+ args.predictor+ prediction + select_drug+now + '.csv') # except: # logging.warning("Only one class, no two calsses critical genes") # Xsource_allTensor = torch.FloatTensor(data_r.values).to(device) # Ysource_preTensor = source_model(Xsource_allTensor) # Ysource_prediction = Ysource_preTensor.detach().cpu().numpy() # bdata.obs["sens_preds"] = Ysource_prediction[:,1] # bdata.obs["sens_label"] = Ysource_prediction.argmax(axis=1) # bdata.obs["sens_label"] = bdata.obs["sens_label"].astype('category') # bdata.obs["rest_preds"] = Ysource_prediction[:,0] # sc.tl.score_genes(adata, bulk_degs['sensitive'],score_name="bulk_sens_score" ) # sc.tl.score_genes(adata, bulk_degs['resistant'],score_name="bulk_rest_score" ) # sc.pl.umap(adata,color=['bulk_sens_score','bulk_rest_score'],save=data_name+args.transfer+args.dimreduce+"umap_bg_all"+now,show=False) # try: # bulk_score_sens = pearsonr(adata.obs["1_score"],adata.obs["bulk_sens_score"])[0] # report_df['bulk_sens_pearson'] = bulk_score_sens # cluster_score_resist = pearsonr(adata.obs["0_score"],adata.obs["bulk_rest_score"])[0] # report_df['bulk_rest_pearson'] = cluster_score_resist # except: # logging.warning("Bulk level gene score not exist") # Save adata adata.write("saved/adata/" + data_name + now + ".h5ad") # Save report report_df = report_df.T report_df.to_csv("saved/results/report" + reduce_model + args.predictor + prediction + select_drug + now + '.csv')
def train(self): # networks self.model = Net(num_channels=self.num_channels, base_filter=64, num_residuals=16) # weigh initialization self.model.weight_init() # optimizer self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr, betas=(0.9, 0.999), eps=1e-8) # loss function if self.gpu_mode: self.model.cuda() self.L1_loss = nn.L1Loss().cuda() else: self.L1_loss = nn.L1Loss() print('---------- Networks architecture -------------') utils.print_network(self.model) print('----------------------------------------------') # load dataset train_data_loader = self.load_dataset(dataset=self.train_dataset, is_train=True) test_data_loader = self.load_dataset(dataset=self.test_dataset[0], is_train=False) # set the logger log_dir = os.path.join(self.save_dir, 'logs') if not os.path.exists(log_dir): os.makedirs(log_dir) logger = Logger(log_dir) ################# Train ################# print('Training is started.') avg_loss = [] step = 0 # test image test_lr, test_hr, test_bc = test_data_loader.dataset.__getitem__(2) test_lr = test_lr.unsqueeze(0) test_hr = test_hr.unsqueeze(0) test_bc = test_bc.unsqueeze(0) self.model.train() for epoch in range(self.num_epochs): # learning rate is decayed by a factor of 2 every 40 epochs if (epoch + 1) % 40 == 0: for param_group in self.optimizer.param_groups: param_group['lr'] /= 2.0 print('Learning rate decay: lr={}'.format( self.optimizer.param_groups[0]['lr'])) epoch_loss = 0 for iter, (lr, hr, _) in enumerate(train_data_loader): # input data (low resolution image) if self.num_channels == 1: x_ = Variable(hr[:, 0].unsqueeze(1)) y_ = Variable(lr[:, 0].unsqueeze(1)) else: x_ = Variable(hr) y_ = Variable(lr) if self.gpu_mode: x_ = x_.cuda() y_ = y_.cuda() # update network self.optimizer.zero_grad() recon_image = self.model(y_) loss = self.L1_loss(recon_image, x_) loss.backward() self.optimizer.step() # log epoch_loss += loss.data[0] print('Epoch: [%2d] [%4d/%4d] loss: %.8f' % ((epoch + 1), (iter + 1), len(train_data_loader), loss.data[0])) # tensorboard logging logger.scalar_summary('loss', loss.data[0], step + 1) step += 1 # avg. loss per epoch avg_loss.append(epoch_loss / len(train_data_loader)) # prediction if self.num_channels == 1: y_ = Variable(test_lr[:, 0].unsqueeze(1)) else: y_ = Variable(test_lr) if self.gpu_mode: y_ = y_.cuda() recon_img = self.model(y_) sr_img = recon_img[0].cpu().data # save result image save_dir = os.path.join(self.save_dir, 'train_result') utils.save_img(sr_img, epoch + 1, save_dir=save_dir, is_training=True) print('Result image at epoch %d is saved.' % (epoch + 1)) # Save trained parameters of model if (epoch + 1) % self.save_epochs == 0: self.save_model(epoch + 1) # calculate psnrs if self.num_channels == 1: gt_img = test_hr[0][0].unsqueeze(0) lr_img = test_lr[0][0].unsqueeze(0) bc_img = test_bc[0][0].unsqueeze(0) else: gt_img = test_hr[0] lr_img = test_lr[0] bc_img = test_bc[0] bc_psnr = utils.PSNR(bc_img, gt_img) recon_psnr = utils.PSNR(sr_img, gt_img) # plot result images result_imgs = [gt_img, lr_img, bc_img, sr_img] psnrs = [None, None, bc_psnr, recon_psnr] utils.plot_test_result(result_imgs, psnrs, self.num_epochs, save_dir=save_dir, is_training=True) print('Training result image is saved.') # Plot avg. loss utils.plot_loss([avg_loss], self.num_epochs, save_dir=save_dir) print('Training is finished.') # Save final trained parameters of model self.save_model(epoch=None)
0, 2, 3, 1), # convert to BxHxWxC } for key, images in info.items(): img_logger[i].image_summary(key, images, epoch + 1) # Save trained parameters of model torch.save( G_list[i].state_dict(), model_dir + 'generator_' + upsample_list[i] + '_param_epoch_%d.pkl' % (epoch + 1)) torch.save( D_list[i].state_dict(), model_dir + 'discriminator' + upsample_list[i] + '_param_epoch_%d.pkl' % (epoch + 1)) avg_losses = [] for i in range(model_len): # Plot average losses avg_losses.append(D_avg_losses[i]) avg_losses.append(G_avg_losses[i]) utils.plot_loss(avg_losses, num_epochs, upsample_list[i], save=True, save_dir=save_dir) # Save trained parameters of model torch.save(G_list[i].state_dict(), model_dir + 'generator_' + upsample_list[i] + '_param.pkl') torch.save(D_list[i].state_dict(), model_dir + 'discriminator' + upsample_list[i] + '_param.pkl')
def trainNet( datapath='.', nepochs=1, learning_rate=0.001, batch_size=64, cuda=False, savedir='./', lossPlotName='loss.png', num_workers=24, ): ''' Our basic training file ''' if "/" not in savedir[-1]: savedir += "/" if cuda: print(f"Running on GPU") device = torch.device('cuda') net = model.VGG().to(device) else: print(f"Running on CPU") device = torch.device('cpu') net = model.VGG() # Dataset #imagenet = datasets.ImageNet('/research/imgnet/ILSVRC2013_DET_train/', # split='train') t = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), ]) train_images = datasets.CIFAR10('.', train=True, download=True, transform=t) train_data = torch.utils.data.DataLoader(train_images, batch_size=batch_size, shuffle=True, num_workers=num_workers) #criterion = nn.MSELoss() criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=learning_rate) epoch_loss_array = torch.zeros(nepochs) print(f"Train data {len(train_images)}") for epoch in range(nepochs): net.train() epoch_loss = 0 for i, (img, label) in enumerate(train_data): optimizer.zero_grad() out = net(img.to(device)) loss = criterion(out.to(device), label.to(device)) epoch_loss += loss.item() loss.backward() optimizer.step() epoch_loss /= (i + 1) print(f"Epoch {epoch} loss {epoch_loss}") epoch_loss_array[epoch] = epoch_loss utils.plot_loss(epoch_loss_array, savedir + lossPlotName) net.load_state_dict(torch.load('final_model.pt')) # Testing with torch.no_grad(): net.eval() test_loss = 0. test_images = datasets.CIFAR10('.', train=False, download=True, transform=t) test_data = torch.utils.data.DataLoader(test_images, batch_size=batch_size, shuffle=True, num_workers=num_workers) classes = [ 'plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck' ] corrects = np.zeros(len(classes), dtype=np.int) totals = np.zeros(len(classes), dtype=np.int) trues = np.zeros(len(test_images)) preds = np.zeros(len(test_images)) for i, (img, label) in enumerate(test_data): optimizer.zero_grad() out = net(img.to(device)) trues[i * batch_size:(i + 1) * batch_size] = label.to('cpu').numpy() preds[i * batch_size:(i + 1) * batch_size] = [np.argmax(o) for o in out.to('cpu').numpy()] for o, l in zip(out, label): o = o.to('cpu').numpy() l = l.to('cpu').numpy() totals[l] += 1 if np.argmax(o) == l: corrects[l] += 1 loss = criterion(out.to(device), label.to(device)) test_loss += loss.item() if i == 0: utils.plot_examples(img[:9].to('cpu'), out[:9].to('cpu').numpy(), classes, label[:9].to('cpu').numpy(), savedir + "examples.png") test_loss /= (i + 1) print(f"Test loss {test_loss}") utils.confusionMatrix(trues, preds, classes) torch.save(net.state_dict(), "final_model.pt") #print(f"Corrects {corrects}") #print(f"Totals {totals}") print("Accuracy") print(f"Name\tCorrects\tAccuracy") for i in range(len(classes)): print(f"{classes[i]}\t{corrects[i]}\t\t{corrects[i]/totals[i]}") print(30 * "-") print(f"Sum\t{np.sum(corrects)}\t\t{np.sum(corrects)/np.sum(totals)}")
plt.savefig("task4b_softmax_weight_" + str(i) + "_" + str(l) + "_.png") if show_plots: plt.show() else: plt.close() # Save the norm for plotting later norm_list.append(np.linalg.norm(model.w)) # Plotting of accuracy for differente values of lambdas (task 4c) for l, train_history, val_history in zip(l2_lambdas, train_history_list, val_history_list): plt.ylim([0.7, .93]) plt.xlim([-100, 10000]) utils.plot_loss(train_history["accuracy"], "Training Accuracy $\lambda$ = " + str(l)) utils.plot_loss(val_history["accuracy"], "Validation Accuracy $\lambda$ = " + str(l)) plt.xlabel("Number of Training Steps") plt.ylabel("Accuracy") plt.legend() plt.savefig("task4c_l2_reg_accuracy.png") if show_plots: plt.show() else: plt.close() # Task 4d - Plotting of the l2 norm for each weight plt.plot(l2_lambdas, norm_list) plt.title("Plot of length of w vs $\lambda$")
valid_loss, valid_acc, mcc = evaluate(model, valid_iterator, criterion, fields=fields, metric=binary_accuracy) print(f'\t Val. Loss: {valid_loss:.6f} | Val. Accuracy: {valid_acc}') train_accs += [train_acc] train_losses += [train_loss] valid_accs += [valid_acc] valid_losses += [valid_loss] # scheduler.step() early_stopping(valid_loss, model) if valid_acc > 0.55 and mcc > 0.056: torch.save({ 'net': model.state_dict(), 'text': TEXT, }, f'roberta-model-acc-{valid_acc}-{mcc}.pth') if early_stopping.early_stop: print(f"Epochs: {epoch} - Early Stopping...") break plot_acc(train_accs, valid_accs, fname=f"roberta-epochs-{epochs}-acc-{valid_acc}.png") plot_loss(train_losses, valid_losses, fname=f"roberta-epochs-{epochs}-loss-{valid_acc}.png")
model, learning_rate, batch_size, shuffle_dataset, X_train, Y_train, X_val, Y_val, ) train_history, val_history = trainer.train(num_epochs) val_acc.append(val_history["accuracy"]) weights.append(model.w) # Plot accuracy utils.plot_loss(val_history["accuracy"], "Validation Accuracy with $\lambda$ ={}".format(i)) plt.ylim([0.60, .95]) plt.xlabel("Number of Training Steps") plt.ylabel("Accuracy") plt.legend() plt.savefig("task4c_l2_reg_accuracy.png") plt.show() # Plotting of softmax weights (Task 4b) #weight = visualModeltraining() - task4b.py #plt.imsave("task4b_softmax_weight_2.png", weight, cmap="gray") # Plotting of accuracy for difference values of lambdas (task 4c) #plt.savefig("task4c_l2_reg_accuracy.png") # Task 4d - Plotting of the l2 norm for each weight. Could be more elegant, but it's just a bar plot so frankly
D_avg_loss = torch.mean(torch.FloatTensor(D_losses)) G_avg_loss = torch.mean(torch.FloatTensor(G_losses)) # avg loss values for plot D_avg_losses.append(D_avg_loss) G_avg_losses.append(G_avg_loss) # Show result for test image gen_image = G(Variable(test_input.cuda())) gen_image = gen_image.cpu().data utils.plot_test_result(test_input, test_target, gen_image, epoch, save=True, save_dir=save_dir) # Plot average losses utils.plot_loss(D_avg_losses, G_avg_losses, params.num_epochs, save=True, save_dir=save_dir) # Make gif utils.make_gif(params.dataset, params.num_epochs, save_dir=save_dir) # Save trained parameters of model torch.save(G.state_dict(), model_dir + 'generator_param.pkl') torch.save(D.state_dict(), model_dir + 'discriminator_param.pkl')
l2_reg_lambda=l2_reg_lambda) print("Final Train Cross Entropy Loss:", cross_entropy_loss(Y_train, model.forward(X_train))) print("Final Validation Cross Entropy Loss:", cross_entropy_loss(Y_test, model.forward(X_test))) print("Final Test Cross Entropy Loss:", cross_entropy_loss(Y_val, model.forward(X_val))) print("Train accuracy:", calculate_accuracy(X_train, Y_train, model)) print("Validation accuracy:", calculate_accuracy(X_val, Y_val, model)) print("Test accuracy:", calculate_accuracy(X_test, Y_test, model)) # Plot loss utils.plot_loss(train_loss, "Training Loss") utils.plot_loss(val_loss, "Validation Loss") plt.ylim([0, .4]) plt.legend() plt.savefig("binary_train_loss_stopping=%s_epochs=%s.png" % (with_stopping, str(num_epochs))) plt.show() # Plot accuracy plt.ylim([0.93, .99]) utils.plot_loss(train_accuracy, "Training Accuracy") utils.plot_loss(val_accuracy, "Validation Accuracy") plt.legend() plt.savefig("binary_train_accuracy_stopping=%s_epochs=%s.png" % (with_stopping, str(num_epochs)))
os.mkdir(path) except FileExistsError: pass # Model loading and training model_path = os.path.join(path, MODEL_NAME + ".mdl") try: print("Loading model ", MODEL_NAME) model.load_state_dict(torch.load(model_path)) except FileNotFoundError: print("No such model. Training a new one!") train(train_loader1, 1, lf) train(train_loader2, 2, lf) train(train_loader3, 3, lf) train(train_loader4, 4, lf) plot_loss(total_loss, path) torch.save(model.state_dict(), model_path) model.eval() test_loss = 0 results_path = os.path.join(path, "results") try: os.mkdir(results_path) except FileExistsError: pass with torch.no_grad(): for i in range(4): h_views, v_views, i_views, d_views, center, gt, mask, index = test_set.get_scene(i) data_h = torch.tensor(h_views, device=device).float() data_v = torch.tensor(v_views, device=device).float() data_d = torch.tensor(d_views, device=device).float()
momentum_gamma, use_momentum, model_128, learning_rate, batch_size, shuffle_data, X_train, Y_train, X_val, Y_val, ) train_history_128, val_history_128 = trainer_128.train(num_epochs) plt.subplot(1, 2, 1) utils.plot_loss(train_history_32["loss"], "Task 4 Model - 32 Neurons", npoints_to_average=10) utils.plot_loss(train_history_64["loss"], "Task 4 Model - 64 Neurons", npoints_to_average=10) utils.plot_loss(train_history_128["loss"], "Task 4 Model - 128 Neurons", npoints_to_average=10) plt.xlabel("Number of Training Steps") plt.ylabel("Training loss") plt.legend() plt.ylim([0, .4]) plt.subplot(1, 2, 2) plt.ylim([0.9, 1.01]) utils.plot_loss(val_history_32["accuracy"],
def train(self): # networks self.model = Net(num_channels=self.num_channels, scale_factor=self.scale_factor, d=56, s=12, m=4) # weigh initialization self.model.weight_init(mean=0.0, std=0.02) # optimizer self.momentum = 0.9 self.optimizer = optim.SGD(self.model.parameters(), lr=self.lr, momentum=self.momentum) # loss function if self.gpu_mode: self.model.cuda() self.MSE_loss = nn.MSELoss().cuda() else: self.MSE_loss = nn.MSELoss() print('---------- Networks architecture -------------') utils.print_network(self.model) print('----------------------------------------------') # load dataset train_data_loader = self.load_dataset(dataset='train') test_data_loader = self.load_dataset(dataset='test') # set the logger log_dir = os.path.join(self.save_dir, 'logs') if not os.path.exists(log_dir): os.mkdir(log_dir) logger = Logger(log_dir) ################# Train ################# print('Training is started.') avg_loss = [] step = 0 # test image test_input, test_target = test_data_loader.dataset.__getitem__(2) test_input = test_input.unsqueeze(0) test_target = test_target.unsqueeze(0) self.model.train() for epoch in range(self.num_epochs): epoch_loss = 0 for iter, (input, target) in enumerate(train_data_loader): # input data (low resolution image) if self.gpu_mode: x_ = Variable(utils.shave(target, border_size=2*self.scale_factor).cuda()) y_ = Variable(input.cuda()) else: x_ = Variable(utils.shave(target, border_size=2*self.scale_factor)) y_ = Variable(input) # update network self.optimizer.zero_grad() recon_image = self.model(y_) loss = self.MSE_loss(recon_image, x_) loss.backward() self.optimizer.step() # log epoch_loss += loss.data[0] print("Epoch: [%2d] [%4d/%4d] loss: %.8f" % ((epoch + 1), (iter + 1), len(train_data_loader), loss.data[0])) # tensorboard logging logger.scalar_summary('loss', loss.data[0], step + 1) step += 1 # avg. loss per epoch avg_loss.append(epoch_loss / len(train_data_loader)) # prediction recon_imgs = self.model(Variable(test_input.cuda())) recon_img = recon_imgs[0].cpu().data gt_img = utils.shave(test_target[0], border_size=2 * self.scale_factor) lr_img = utils.shave(test_input[0], border_size=2) bc_img = utils.shave(utils.img_interp(test_input[0], self.scale_factor), border_size=2 * self.scale_factor) # calculate psnrs bc_psnr = utils.PSNR(bc_img, gt_img) recon_psnr = utils.PSNR(recon_img, gt_img) # save result images result_imgs = [gt_img, lr_img, bc_img, recon_img] psnrs = [None, None, bc_psnr, recon_psnr] utils.plot_test_result(result_imgs, psnrs, epoch + 1, save_dir=self.save_dir, is_training=True) print("Saving training result images at epoch %d" % (epoch + 1)) # Save trained parameters of model if (epoch + 1) % self.save_epochs == 0: self.save_model(epoch + 1) # Plot avg. loss utils.plot_loss([avg_loss], self.num_epochs, save_dir=self.save_dir) print("Training is finished.") # Save final trained parameters of model self.save_model(epoch=None)
# Initiate the train and test generators with data augumentation train_generator, valid_generator, test_generator = load_data('./data/') # Save the model according to certain conditions # checkpoint = ModelCheckpoint("vgg16_1.h5", monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) early = EarlyStopping(monitor='val_acc', min_delta=0, patience=10, verbose=1, mode='auto') # Train the model history = model_final.fit_generator( train_generator, steps_per_epoch=train_generator.n // batch_size, epochs=epochs, validation_data=valid_generator, validation_steps=valid_generator.n // batch_size, callbacks=[early]) # ,checkpoint]) STEP_SIZE_TEST = test_generator.n // test_generator.batch_size score = model_final.evaluate_generator(generator=test_generator, steps=STEP_SIZE_TEST) print('Test loss:', score[0]) print('Test accuracy:', score[1]) plot_accuracy(history, path.join('./result/', out_file_name)) plot_loss(history, path.join('./result/', out_file_name)) save_model(model_final, path.join('./out/', out_file_name)) save_history(history, path.join('./out/', out_file_name))
criterion, fields=fields, metric=binary_accuracy) print(f'\t Val. Loss: {valid_loss:.6f} | Val. Accuracy: {valid_acc}') train_accs += [train_acc] train_losses += [train_loss] valid_accs += [valid_acc] valid_losses += [valid_loss] # scheduler.step() early_stopping(valid_loss, model) if valid_acc > 0.55 and mcc > 0.056: torch.save( { 'net': model.state_dict(), 'text': TEXT, 'dim': embed_dim, }, f'te-model-acc-{valid_acc}-{mcc}.pth') if early_stopping.early_stop: print(f"Epochs: {epoch} - Early Stopping...") break plot_acc(train_accs, valid_accs, fname=f"ternn-epochs-{epochs}-acc-{valid_acc}.png") plot_loss(train_losses, valid_losses, fname=f"ternn-epochs-{epochs}-loss-{valid_acc}.png")
l1 = DenseLayer((784, num_hidden_neuron), optimizer) bn1 = BatchNorm((num_hidden_neuron,), optimizer) l2 = DenseLayer((num_hidden_neuron, num_hidden_neuron), optimizer) l3 = DenseLayer((num_hidden_neuron, 10), optimizer) def calc_model(x, y=None, train_mode=False): a1 = op.relu(bn1.compute(l1.compute(x), train_mode=train_mode)) a2 = op.relu(l2.compute(a1)) z3 = l3.compute(a2) if not train_mode: return op.softmax(z3) else: return op.softmax_cross_entropy(z3, y) def loss_function(y, m, train_mode=False): if train_mode: return op.sum(m) else: return op.sum(-y * op.log(m)) train_loss_values, test_loss_values, test_score_values = learn(mnist_in_data, mnist_out_data, mnist_test_in_data, mnist_test_out_data, calc_model, loss_function, optimizer, score_func=mnist.score_result, batch_size=128, epoch_number=10) plot_loss(train_loss_values, test_loss_values)
) train_history, val_history = trainer.train(num_epochs) print("Final Train Cross Entropy Loss:", cross_entropy_loss(Y_train, model.forward(X_train))) print("Final Validation Cross Entropy Loss:", cross_entropy_loss(Y_val, model.forward(X_val))) print("Train accuracy:", calculate_accuracy(X_train, Y_train, model)) print("Validation accuracy:", calculate_accuracy(X_val, Y_val, model)) # Plot loss for first model (task 2c) plt.figure(figsize=(16, 10)) plt.subplot(1, 2, 1) plt.ylim([0., .5]) utils.plot_loss(train_history["loss"], "Training Loss", npoints_to_average=10) utils.plot_loss(val_history["loss"], "Validation Loss") plt.legend() plt.xlabel("Number of Training Steps") plt.ylabel("Cross Entropy Loss - Average") # Plot accuracy plt.subplot(1, 2, 2) plt.ylim([0.90, .99]) utils.plot_loss(train_history["accuracy"], "Training Accuracy") utils.plot_loss(val_history["accuracy"], "Validation Accuracy") plt.xlabel("Number of Training Steps") plt.ylabel("Accuracy") plt.legend() plt.savefig("task2c_train_loss.png")
# construct model and ship to GPU hidden_list = list(map(int, args.hiddens.split(','))) model = MADE(xtr.size(1), hidden_list, xtr.size(1) * 2, num_masks=args.num_masks) print("number of model parameters:", sum([np.prod(p.size()) for p in model.parameters()])) model.cuda() # set up the optimizer opt = torch.optim.Adam(model.parameters(), args.learning_rate, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=45, gamma=0.1) # list to store loss loss_tr = [] loss_te = [] loss_od = [] # start the training for epoch in range(args.epoch): scheduler.step(epoch) loss_tr.append(run_epoch('train')) loss_te.append(run_epoch('test')) # run validation, which is pos loss_od.append(run_epoch('ood')) # run test, which is ood print("optimization done") plot_loss(file_name, loss_tr, loss_te, loss_od) # run_epoch('test')
def main(): batch_size = args.batch_size iterations = args.iterations device = args.device # hair_classes, eye_classes = 12, 10 # num_classes = hair_classes + eye_classes hair_class, eye_class, face_class, glass_class = 6, 4, 3, 2 num_classes = hair_class + eye_class + face_class + glass_class latent_dim = 100 smooth = 0.9 config = 'WGANGP_batch{}_steps{}'.format(batch_size, iterations) print('Configuration: {}'.format(config)) root_dir = './{}/images'.format(args.train_dir) tags_file = './{}/cartoon_attr.txt'.format(args.train_dir) # hair_prior = np.load('../{}/hair_prob.npy'.format(args.train_dir)) # eye_prior = np.load('../{}/eye_prob.npy'.format(args.train_dir)) random_sample_dir = '{}/{}/random_generation'.format(args.sample_dir, config) fixed_attribute_dir = '{}/{}/fixed_attributes'.format(args.sample_dir, config) checkpoint_dir = '{}/{}'.format(args.checkpoint_dir, config) if not os.path.exists(random_sample_dir): os.makedirs(random_sample_dir) if not os.path.exists(fixed_attribute_dir): os.makedirs(fixed_attribute_dir) if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) ########## Start Training ########## transform = Transform.Compose([Transform.ToTensor(), Transform.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) dataset = Anime(root_dir = root_dir, tags_file = tags_file, transform = transform) shuffler = Shuffler(dataset = dataset, batch_size = args.batch_size) G = Generator(latent_dim = latent_dim, class_dim = num_classes).to(device) D = Discriminator(hair_classes=hair_class, eye_classes=eye_class, face_classes=face_class, glass_classes=glass_class).to(device) G_optim = optim.Adam(G.parameters(), betas = [args.beta, 0.999], lr = args.lr) D_optim = optim.Adam(D.parameters(), betas = [args.beta, 0.999], lr = args.lr) d_log, g_log, classifier_log = [], [], [] criterion = torch.nn.BCELoss() # criterion = torch.nn.NLLLoss() for step_i in range(1, iterations + 1): real_label = torch.ones(batch_size).to(device) fake_label = torch.zeros(batch_size).to(device) soft_label = torch.Tensor(batch_size).uniform_(smooth, 1).to(device) # we need gradient when training descriminator for p in D.parameters(): p.requires_grad = True for d_iter in range(5): D_optim.zero_grad() # D.zero_grad() # Train discriminator real_img, hair_tags, eye_tags, face_tags, glass_tags = shuffler.get_batch() real_img, hair_tags, eye_tags, face_tags, glass_tags = real_img.to(device), hair_tags.to(device), eye_tags.to(device), face_tags.to(device), glass_tags.to(device) # real_tag = torch.cat((hair_tags, eye_tags), dim = 1) with torch.no_grad(): # totally freeze G , training D z = torch.randn(batch_size, latent_dim).to(device) fake_tag = get_random_label(batch_size = batch_size, hair_classes = hair_class, eye_classes = eye_class, face_classes = face_class, glass_classes = glass_class).to(device) real_img.requires_grad = True real_score, real_hair_predict, real_eye_predict, real_face_predict, real_glass_predict = D(real_img) real_discrim_loss = real_score.mean() # real_discrim_loss = criterion(real_score, soft_label) # fake_discrim_loss = criterion(fake_score, fake_label) real_hair_aux_loss = criterion(real_hair_predict, hair_tags) real_eye_aux_loss = criterion(real_eye_predict, eye_tags) real_face_aux_loss = criterion(real_face_predict, face_tags) real_glass_aux_loss = criterion(real_glass_predict, glass_tags) real_classifier_loss = real_hair_aux_loss + real_eye_aux_loss + real_face_aux_loss + real_glass_aux_loss fake_img = G(z, fake_tag).to(device) fake_score, _ , _ , _ , _ = D(fake_img) fake_discrim_loss = fake_score.mean() gradient_penalty = calculate_gradient_penalty(D, real_img.detach(), fake_img.detach(), batch_size, device) # discrim_loss = real_discrim_loss + fake_discrim_loss discrim_loss = fake_discrim_loss - real_discrim_loss + gradient_penalty classifier_loss = real_classifier_loss * args.classification_weight classifier_log.append(classifier_loss.item()) D_loss = discrim_loss + classifier_loss # D_optim.zero_grad() D_loss.backward() D_optim.step() # Train generator for p in D.parameters(): p.requires_grad = False G_optim.zero_grad() # G.zero_grad() z = torch.randn(batch_size, latent_dim).to(device) z.requires_grad = True fake_tag = get_random_label(batch_size = batch_size, hair_classes = hair_class, eye_classes = eye_class, face_classes = face_class, glass_classes = glass_class).to(device) hair_tag = fake_tag[:, :hair_class] eye_tag = fake_tag[:, 6:10] face_tag = fake_tag[:, 10:13] glass_tag = fake_tag[:, 13:15] fake_img = G(z, fake_tag).to(device) fake_score, hair_predict, eye_predict, face_predict, glass_predict = D(fake_img) discrim_loss = fake_score.mean() G_discrim_loss = -discrim_loss # discrim_loss = criterion(fake_score, real_label) hair_aux_loss = criterion(hair_predict, hair_tag) eye_aux_loss = criterion(eye_predict, eye_tag) face_aux_loss = criterion(face_predict, face_tag) glass_aux_loss = criterion(glass_predict, glass_tag) classifier_loss = hair_aux_loss + eye_aux_loss + face_aux_loss + glass_aux_loss G_loss = classifier_loss * args.classification_weight + G_discrim_loss # G_optim.zero_grad() G_loss.backward() G_optim.step() ########## Updating logs ########## d_log.append(D_loss.item()) g_log.append(G_loss.item()) show_process(total_steps = iterations, step_i = step_i, g_log = g_log, d_log = d_log, classifier_log = classifier_log) ########## Checkpointing ########## if step_i == 1: save_image(denorm(real_img[:16,:,:,:]), os.path.join(random_sample_dir, 'real.png'), nrow=4) if step_i % args.sample == 0: save_image(denorm(fake_img[:16,:,:,:]), os.path.join(random_sample_dir, 'fake_step_{}.png'.format(step_i)), nrow=4) if step_i % args.check == 0: save_model(model = G, optimizer = G_optim, step = step_i, log = tuple(g_log), file_path = os.path.join(checkpoint_dir, 'G_{}.ckpt'.format(step_i))) save_model(model = D, optimizer = D_optim, step = step_i, log = tuple(d_log), file_path = os.path.join(checkpoint_dir, 'D_{}.ckpt'.format(step_i))) plot_loss(g_log = g_log, d_log = d_log, file_path = os.path.join(checkpoint_dir, 'loss.png')) plot_classifier_loss(log = classifier_log, file_path = os.path.join(checkpoint_dir, 'classifier loss.png')) generation_by_attributes(model = G, device = args.device, step = step_i, latent_dim = latent_dim, hair_classes = hair_class, eye_classes = eye_class, face_classes = face_class, glass_classes = glass_class, sample_dir = fixed_attribute_dir)
l2_reg_lambda=0) print("Final Train Cross Entropy Loss:", cross_entropy_loss(Y_train, model.forward(X_train))) print("Final Validation Cross Entropy Loss:", cross_entropy_loss(Y_test, model.forward(X_test))) print("Final Test Cross Entropy Loss:", cross_entropy_loss(Y_val, model.forward(X_val))) print("Final Train accuracy:", calculate_accuracy(X_train, Y_train, model)) print("Final Validation accuracy:", calculate_accuracy(X_val, Y_val, model)) print("Final Test accuracy:", calculate_accuracy(X_test, Y_test, model)) # Plot loss plt.ylim([0.01, .2]) utils.plot_loss(train_loss, "Training Loss") utils.plot_loss(val_loss, "Validation Loss") plt.xlabel("Gradient steps") plt.legend() plt.savefig("softmax_train_loss.png") plt.show() # Plot accuracy plt.ylim([0.8, .95]) utils.plot_loss(train_accuracy, "Training Accuracy") utils.plot_loss(val_accuracy, "Validation Accuracy") plt.xlabel("Gradient steps") plt.legend() plt.show() lambda_values = [0, 0.1]
name = exp_name + '_' + str(date.today()) with open('models/conv_net_'+name+'.pkl', 'wb') as f: cPickle.dump(conv_net, f, -1) conv_net.save_params_to('models/params_'+name) # ----- Train set ---- train_predictions = conv_net.predict_proba(X) make_submission_file(train_predictions[:sample_size], images_id[:sample_size], output_filepath='models/training_'+name+'.csv') # ----- Test set ---- X_test, _, images_id_test = load_numpy_arrays(args['test_file']) print "Test:" print "X_test.shape:", X_test.shape predictions = conv_net.predict_proba(X_test) make_submission_file(predictions, images_id_test, output_filepath='submissions/submission_'+name+'.csv') # ----- Make plots ---- plot_loss(conv_net, "models/loss_"+name+".png", show=False) plot_conv_weights(conv_net.layers_[1], figsize=(4, 4)) plt.savefig('models/weights_'+name+'.png') plot_conv_activity(conv_net.layers_[1], X[0:1]) plt.savefig('models/activity_'+name+'.png') plot_occlusion(conv_net, X[:5], y[:5]) plt.savefig('models/occlusion_'+name+'.png')
train_history, val_history = trainer.train(num_epochs) print("Improved weights") print("Train accuracy:", calculate_accuracy(X_train, Y_train, model)) print("Validation accuracy:", calculate_accuracy(X_val, Y_val, model)) print("Final Validation Cross Entropy Loss:", cross_entropy_loss(Y_val, model.forward(X_val))) print("Final Train Cross Entropy Loss:", cross_entropy_loss(Y_train, model.forward(X_train))) #Plotting plt.figure(figsize=(20, 12)) plt.subplot(1, 2, 1) plt.ylim([0., .5]) utils.plot_loss(train_history["loss"], "Training Loss", npoints_to_average=10) utils.plot_loss(val_history["loss"], "Validation Loss") plt.legend() plt.xlabel("Number of Training Steps") plt.ylabel("Cross Entropy Loss - Average") # Plot accuracy plt.subplot(1, 2, 2) plt.ylim([0.91, 1.01]) utils.plot_loss(train_history["accuracy"], "Training Accuracy") utils.plot_loss(val_history["accuracy"], "Validation Accuracy") plt.xlabel("Number of Training Steps") plt.ylabel("Accuracy") plt.legend() #plt.savefig("task3a_improvedweight.png") plt.show()