def visual(self, env_name): gl._init() plotter = utils.VisdomLinePlotter(env_name=env_name) gl.set_value(env_name, plotter) return gl.get_value(env_name)
def run_train(args_dict): # Set seed for reproducibility torch.manual_seed(args_dict.seed) # Set up device if torch.cuda.is_available(): args_dict.device = torch.device( "cuda:0" ) # you can continue going on here, like cuda:1 cuda:2....etc. print("Running on the GPU") else: args_dict.device = torch.device("cpu") print("Running on the CPU") # Plots global plotter plotter = utils.VisdomLinePlotter(env_name=args_dict.name) # Main process train_model(args_dict)
utils.save_obj( idxq, os.path.join(args.data_dir, 'retrieval_idxq_%s.pckl' % split)) utils.save_obj( labels, os.path.join(args.data_dir, 'retrieval_labels_%s.pckl' % split)) medR1, recall1, medR2, recall2 = rank(scores1, scores2, idxq, labels) logger.info('Accuracy medR {medR:.2f}\t Recall {recall}'.format( medR=medR1, recall=recall1)) if __name__ == "__main__": args = get_params() train_name = 'BertScoring_maxseq%d_%dpairs' % (args.max_seq_length, args.num_pairs) outdir = os.path.join('Training/KnowledgeRetrieval/', train_name) if not os.path.isfile(os.path.join(outdir, 'pytorch_model.bin')): global plotter plotter = utils.VisdomLinePlotter(env_name=train_name) train(args, outdir) if not os.path.exists( os.path.join(args.data_dir, 'retieval_scores_test.pckl')): evaluate(args, outdir, split='test') if not os.path.exists( os.path.join(args.data_dir, 'retieval_scores_val.pckl')): evaluate(args, outdir, split='val') # Do not compute scores for training, we use random samples in the VideoReasoning training part # if not os.path.exists(os.path.join(args.data_dir, 'retieval_scores_train.pckl')): # evaluate(args, outdir, split='train')
model_path = './models' model_name = 'ALV_unet3d_patch64x64x64_1500_3labels_30samples' #remember to include the project title (e.g., ALV) checkpoint_name = 'latest_checkpoint.tar' num_classes = 3 num_channels = 1 num_epochs = 50 num_workers = 6 train_batch_size = 6 val_batch_size = 1 num_batches_to_print = 200 if use_visdom: # set plotter global plotter plotter = utils.VisdomLinePlotter(env_name=model_name) # mkdir 'models' if not os.path.exists(model_path): os.mkdir(model_path) # set dataset training_dataset = CT_Dataset(train_list) val_dataset = CT_Dataset(val_list) train_loader = DataLoader(dataset=training_dataset, batch_size=train_batch_size, shuffle=True, num_workers=num_workers) val_loader = DataLoader(dataset=val_dataset,
from run import run import utils import sys import time import torch import numpy as np torch.backends.cudnn.deterministic = False torch.backends.cudnn.benchmark = False torch.manual_seed(1) np.random.seed(1) start = time.time() print('Take whole Volume!', flush=True) plotter = utils.VisdomLinePlotter(env_name='Plots') path_to_net = './Network/' label_dir = './label/' nii_dir = './numpy/' run(path_to_net, label_dir, nii_dir, plotter) print('Whole run took ', time.time() - start, flush=True) print('Done!', flush=True)
train=False, download=True, transform=transform) testloader = torch.utils.data.DataLoader(testset, batch_size=512, shuffle=False, num_workers=2) # Now, let's start the training process! print('Training...') for epoch in range(100): # Compute a training epoch trainEpoch(trainloader, model, criterion, optimizer, epoch) # Compute a validation epoch lossval = valEpoch(testloader, model, criterion, epoch) # Print validation accuracy and best validation accuracy best_val = max(lossval, best_val) print '** Validation: %f (best) - %f (current)' % (best_val, lossval) if __name__ == "__main__": # Plots global plotter plotter = utils.VisdomLinePlotter(env_name='Tutorial Plots') # Training process trainProcess()
num_workers=1, pin_memory=True) print('train_loader', len(train_loader)) print('test_loader', len(test_loader)) #%% model = VAE().to(device) # optimizer = optim.Adam(model.parameters(), lr=1e-3) optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999), weight_decay=0.0005) #%% epochs = 100 viz = Visdom() global plotter, recon plotter = utils.VisdomLinePlotter(env_name='main') sample_image = utils.VisdomImage(env_name='main') recon = utils.VisdomImage(env_name='main') for epoch in range(1, epochs + 1): with torch.no_grad(): sample = torch.randn(32, 32).to(device) sample = model.decode(sample).cpu() print("save image: " + 'results/sample_' + str(epoch) + '.png') save_image(sample, 'results/sample_' + str(epoch) + '.png') sample_image.display_image(sample, 0, 'SAMPLE RECON') train(batch_size, epoch, model, train_loader, device, optimizer, plotter) test(batch_size, epoch, model, test_loader, device, optimizer, plotter, recon)
def main(): args = get_arguments() # expriment name if not args.exp_name: args.exp_name = '_'.join([args.dataset, args.model]) print("# Experiment: ", args.exp_name) # output folder output_folder = os.path.join(args.output_root, args.dataset, args.exp_name) os.makedirs(output_folder, exist_ok=True) print("# Output path: ", output_folder) # visdom global plotter if args.use_visdom: logging_folder = os.path.join(args.logging_root, args.dataset, args.exp_name) os.makedirs(logging_folder, exist_ok=True) plotter = utils.VisdomLinePlotter(env_name=args.exp_name, logging_path=os.path.join(logging_folder, 'vis.log')) print("# Visdom path: ", logging_folder) # dataset print("# Load datasets") train_datasets, val_datasets, test_datasets = get_datasets(args.dataset, args.dataset_folder, args.batch_size) num_classes = train_datasets[0].num_classes vocab = set(train_datasets[0].vocab) vocab = vocab.union(set(val_datasets[0].vocab)) vocab = vocab.union(set(test_datasets[0].vocab)) # pre-trained word2vec print("# Load pre-trained word2vec") pretrained_word2vec_cache = os.path.join(os.path.dirname(args.w2v_file), args.dataset + '_w2v.pkl') if os.path.isfile(pretrained_word2vec_cache): with open(pretrained_word2vec_cache, 'rb') as f: pretrained_word2vec = pickle.load(f) else: pretrained_word2vec = PretrainedWord2Vec(vocab, args.w2v_file) with open(pretrained_word2vec_cache, 'wb') as f: pickle.dump(pretrained_word2vec, f) # train print("# Start training") for cv, (train_dataset, val_dataset, test_dataset) in enumerate(zip(train_datasets, val_datasets, test_datasets)): # fix random seed utils.fix_random_seed(seed=const.RANDOM_SEED) # model cnn = get_model(args.model, num_classes, pretrained_word2vec) if torch.cuda.is_available(): cnn.cuda() # dataloader train_loader = DataLoader(train_dataset, args.batch_size, shuffle=True, collate_fn=sentence_collate_fn) val_loader = DataLoader(val_dataset, batch_size=1, shuffle=False, collate_fn=sentence_collate_fn) test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, collate_fn=sentence_collate_fn) # optimizer optim = Adadelta(cnn.parameters(), rho=0.95, eps=1e-6) # criterion criterion = CrossEntropyLoss() # training if plotter: plotter.set_cv(cv) output_path = os.path.join(output_folder, 'cv_%d_best.pkl' % cv) train(args.num_epochs, cnn, train_loader, optim, criterion, val_loader, output_path) # evaluation utils.load_model(output_path, cnn) find_most_similar_words(cnn) accuracy = eval(cnn, test_loader) print('cross_val:', cv, '\taccuracy:', accuracy)
parser.add_argument('--lr_steps', default=[80], nargs='+', type=int) parser.add_argument('--dtype', choices=['pickle', 'image', 'hdf5'], default='pickle') parser.add_argument('--load_epoch', type=int, default=-1) parser.add_argument('--save_epoch', type=int, default=1) args = parser.parse_args() save_path = 'results/%s' % (args.dataset) save_path = save_path + '/' + args.model use_pretrain = 'pretrain' in args.model if not os.path.exists(save_path): os.makedirs(save_path) plotter = utils.VisdomLinePlotter(env_name='Experiment1') torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) random.seed(args.seed) np.random.seed(args.seed) dataset = utils.__dict__['ImageDataset_' + args.dtype] def save_checkpoint(): checkpoint = [model.state_dict(), opt.state_dict()] torch.save(checkpoint, '%s/checkpoint_%d_%d.pth' % (save_path, args.seed, epoch))
def training(): gettrace = getattr(sys, 'gettrace', None) if gettrace(): # If debugging trajectory_timesteps = 100 total_episodes = 20 epochs = 10000 batch_size = 10 adam_lr = 1e-3 print_freq = 1 else: trajectory_timesteps = 100 total_episodes = 1000 epochs = 15000 batch_size = 20 adam_lr = 5e-4 print_freq = 25 model = DELIP_model() optimizer = optim.Adam(model.parameters(), lr=adam_lr) loss_func = nn.MSELoss() kld_loss = nn.KLDivLoss() print("initializing dataset") train_dataset = RobotDoorsDataset(total_episodes, trajectory_timesteps) train_dataloader = DataLoader(train_dataset, batch_size=batch_size, num_workers=0, collate_fn=cat_collate, shuffle=True) plotter = utils.VisdomLinePlotter() try: print("beginning training") print("dataset size: {}".format(len(train_dataset))) model.train() model.cuda() # for batch_idx, (data, _) in enumerate(train_loader): for epoch in range(1, epochs + 1): obs_loss_total = 0 rew_loss_total = 0 next_state_loss_total = 0 KLD_loss_total = 0 data_iter = iter(train_dataloader) for data in data_iter: optimizer.zero_grad() last_data = data next_state, obs, rew, mu, logvar, sample = model( (data[:, :, :4], data[:, :, 4].unsqueeze(-1))) obs_loss = loss_func(obs, data[:, :, :3]) rew_loss = loss_func(rew, data[:, :, 3, ].unsqueeze(-1)) next_state_loss = loss_func(next_state[:, :-1, :], sample[:, 1:, :]) KLD_loss = -0.0 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) obs_loss_total += obs_loss.detach().item() rew_loss_total += rew_loss.detach().item() next_state_loss_total += next_state_loss.detach().item() KLD_loss_total += KLD_loss.detach().item() total_loss = obs_loss + rew_loss + next_state_loss # + KLD_loss total_loss.backward() optimizer.step() if epoch % print_freq == 0: print( "Epoch: {} Obs_loss: {} Rew_loss: {} Next_state_loss: {} KLD_loss: {}" .format(epoch, obs_loss_total, rew_loss_total, next_state_loss_total, KLD_loss_total)) print("Sample next_state") print(next_state[0][10][:].detach().cpu().numpy()) print(sample[0][6][:].detach().cpu().numpy()) print("Sample obs") print(obs[0][10][:].detach().cpu().numpy()) print(last_data[0][10][:3].detach().cpu().numpy()) print("Sample rew") print(rew[0][10][:].detach().cpu().numpy()) print(last_data[0][10][3].detach().cpu().numpy()) print("Logvar") print(logvar[0][10][:].detach().cpu().numpy()) plotter.plot("obs_loss", epoch, obs_loss_total) plotter.plot("rew_loss", epoch, rew_loss_total) plotter.plot("next_state_loss", epoch, next_state_loss_total) plotter.plot("KLD_loss", epoch, KLD_loss_total, title="KLD_losses") for param_group in optimizer.param_groups: param_group['lr'] = adam_lr * (1 - (epoch / epochs)) except KeyboardInterrupt: print("Saving model") torch.save(model.state_dict(), "./delip_model") torch.save(model.state_dict(), "./delip_model") return model
def train(self): if self.visdom: random.seed(time.time()) today = date.today() vis = utils.VisdomLinePlotter(env_name='Cobo_depth_Train-Plots_' + str(today) + '_' + self.seed) visValidation = utils.VisdomLinePlotter( env_name='Cobo_depth_Train-Plots_' + str(today) + '_' + self.seed) visEpoch = utils.VisdomLineTwoPlotter( env_name='Cobo_depth_Train-Plots_' + str(today) + '_' + self.seed) visImages = utils.VisdomImagePlotter( env_name='Cobo_depth_Images_' + str(today) + '_' + self.seed) visImagesTest = utils.VisdomImagePlotter( env_name='Cobo_depth_ImagesTest_' + str(today) + '_' + self.seed) visLossGTest = utils.VisdomLinePlotter( env_name='Cobo_depth_Train-Plots_' + str(today) + '_' + self.seed) visLossGValidation = utils.VisdomLinePlotter( env_name='Cobo_depth_Train-Plots_' + str(today) + '_' + self.seed) visLossDTest = utils.VisdomLinePlotter( env_name='Cobo_depth_Train-Plots_' + str(today) + '_' + self.seed) visLossDValidation = utils.VisdomLinePlotter( env_name='Cobo_depth_Train-Plots_' + str(today) + '_' + self.seed) self.train_hist = {} self.epoch_hist = {} self.details_hist = {} self.train_hist['D_loss_train'] = [] self.train_hist['G_loss_train'] = [] self.train_hist['D_loss_Validation'] = [] self.train_hist['G_loss_Validation'] = [] self.train_hist['per_epoch_time'] = [] self.train_hist['total_time'] = [] self.details_hist['G_T_Comp_im'] = [] self.details_hist['G_T_BCE_fake_real'] = [] self.details_hist['G_T_Cycle'] = [] self.details_hist['G_zCR'] = [] self.details_hist['G_V_Comp_im'] = [] self.details_hist['G_V_BCE_fake_real'] = [] self.details_hist['G_V_Cycle'] = [] self.details_hist['D_T_BCE_fake_real_R'] = [] self.details_hist['D_T_BCE_fake_real_F'] = [] self.details_hist['D_zCR'] = [] self.details_hist['D_bCR'] = [] self.details_hist['D_V_BCE_fake_real_R'] = [] self.details_hist['D_V_BCE_fake_real_F'] = [] self.epoch_hist['D_loss_train'] = [] self.epoch_hist['G_loss_train'] = [] self.epoch_hist['D_loss_Validation'] = [] self.epoch_hist['G_loss_Validation'] = [] ##Para poder tomar el promedio por epoch iterIniTrain = 0 iterFinTrain = 0 iterIniValidation = 0 iterFinValidation = 0 maxIter = self.data_loader.dataset.__len__() // self.batch_size maxIterVal = self.data_Validation.dataset.__len__() // self.batch_size if (self.WGAN): one = torch.tensor(1, dtype=torch.float).cuda() mone = one * -1 else: self.y_real_ = torch.ones(self.batch_size, 1) self.y_fake_ = torch.zeros(self.batch_size, 1) if self.gpu_mode: self.y_real_, self.y_fake_ = self.y_real_.cuda( ), self.y_fake_.cuda() print('training start!!') start_time = time.time() for epoch in range(self.epoch): if (epoch < self.epochVentaja): ventaja = True else: ventaja = False self.G.train() if not self.onlyGen: self.D.train() epoch_start_time = time.time() # TRAIN!!! for iter, data in enumerate(self.data_loader): x_im = data.get('x_im') x_dep = data.get('x_dep') y_im = data.get('y_im') y_dep = data.get('y_dep') y_ = data.get('y_') # x_im = imagenes normales # x_dep = profundidad de images # y_im = imagen con el angulo cambiado # y_ = angulo de la imagen = tengo que tratar negativos # Aumento mi data if (self.CR): x_im_aug, y_im_aug = augmentData(x_im, y_im) x_im_vanilla = x_im if self.gpu_mode: x_im_aug, y_im_aug = x_im_aug.cuda(), y_im_aug.cuda() if iter >= maxIter: break if self.gpu_mode: x_im, y_, y_im, x_dep, y_dep = x_im.cuda(), y_.cuda( ), y_im.cuda(), x_dep.cuda(), y_dep.cuda() # update D network if not ventaja and not self.onlyGen: for p in self.D.parameters(): # reset requires_grad p.requires_grad = True # they are set to False below in netG update self.D_optimizer.zero_grad() # Real Images D_real, D_features_real = self.D( y_im, x_im, y_dep, y_) ## Es la funcion forward `` g(z) x # Fake Images G_, G_dep = self.G(y_, x_im, y_dep) D_fake, D_features_fake = self.D(G_, x_im, G_dep, y_) # Losses # GAN Loss if (self.WGAN): # de WGAN D_loss_real_fake_R = -torch.mean(D_real) D_loss_real_fake_F = torch.mean(D_fake) #D_loss_real_fake_R = - D_loss_real_fake_R_positive else: # de Gan normal D_loss_real_fake_R = self.BCEWithLogitsLoss( D_real, self.y_real_) D_loss_real_fake_F = self.BCEWithLogitsLoss( D_fake, self.y_fake_) D_loss = D_loss_real_fake_F + D_loss_real_fake_R if self.CR: # Fake Augmented Images bCR x_im_aug_bCR, G_aug_bCR = augmentData( x_im_vanilla, G_.data.cpu()) if self.gpu_mode: G_aug_bCR, x_im_aug_bCR = G_aug_bCR.cuda( ), x_im_aug_bCR.cuda() D_fake_bCR, D_features_fake_bCR = self.D( G_aug_bCR, x_im_aug_bCR, G_dep, y_) D_real_bCR, D_features_real_bCR = self.D( y_im_aug, x_im_aug, y_dep, y_) # Fake Augmented Images zCR G_aug_zCR, G_dep_aug_zCR = self.G(y_, x_im_aug, x_dep) D_fake_aug_zCR, D_features_fake_aug_zCR = self.D( G_aug_zCR, x_im_aug, G_dep_aug_zCR, y_) # bCR Loss (*) D_loss_real = self.MSE(D_features_real, D_features_real_bCR) D_loss_fake = self.MSE(D_features_fake, D_features_fake_bCR) D_bCR = (D_loss_real + D_loss_fake) * self.bFactor # zCR Loss D_zCR = self.MSE( D_features_fake, D_features_fake_aug_zCR) * self.zDisFactor D_CR_losses = D_bCR + D_zCR #D_CR_losses.backward(retain_graph=True) D_loss += D_CR_losses self.details_hist['D_bCR'].append( D_bCR.detach().item()) self.details_hist['D_zCR'].append( D_zCR.detach().item()) else: self.details_hist['D_bCR'].append(0) self.details_hist['D_zCR'].append(0) self.train_hist['D_loss_train'].append( D_loss.detach().item()) self.details_hist['D_T_BCE_fake_real_R'].append( D_loss_real_fake_R.detach().item()) self.details_hist['D_T_BCE_fake_real_F'].append( D_loss_real_fake_F.detach().item()) if self.visdom: visLossDTest.plot('Discriminator_losses', [ 'D_T_BCE_fake_real_R', 'D_T_BCE_fake_real_F', 'D_bCR', 'D_zCR' ], 'train', self.details_hist) #if self.WGAN: # D_loss_real_fake_F.backward(retain_graph=True) # D_loss_real_fake_R_positive.backward(mone) #else: # D_loss_real_fake.backward() D_loss.backward() self.D_optimizer.step() #WGAN if (self.WGAN): for p in self.D.parameters(): p.data.clamp_( -self.clipping, self.clipping ) #Segun paper si el valor es muy chico lleva al banishing gradient # Si se aplicaria la mejora en las WGANs tendiramos que sacar los batch normalizations de la red # update G network self.G_optimizer.zero_grad() G_, G_dep = self.G(y_, x_im, x_dep) if not ventaja and not self.onlyGen: for p in self.D.parameters(): p.requires_grad = False # to avoid computation # Fake images D_fake, _ = self.D(G_, x_im, G_dep, y_) if (self.WGAN): G_loss_fake = -torch.mean(D_fake) #de WGAN else: G_loss_fake = self.BCEWithLogitsLoss( D_fake, self.y_real_) # loss between images (*) #G_join = torch.cat((G_, G_dep), 1) #y_join = torch.cat((y_im, y_dep), 1) G_loss_Comp = self.L1(G_, y_im) if self.depth: G_loss_Comp += self.L1(G_dep, y_dep) G_loss_Dif_Comp = G_loss_Comp * self.lambdaL1 reverse_y = -y_ + 1 reverse_G, reverse_G_dep = self.G(reverse_y, G_, G_dep) G_loss_Cycle = self.L1(reverse_G, x_im) if self.depth: G_loss_Cycle += self.L1(reverse_G_dep, x_dep) G_loss_Cycle = G_loss_Cycle * self.lambdaL1 / 2 if (self.CR): # Fake images augmented G_aug, G_dep_aug = self.G(y_, x_im_aug, x_dep) D_fake_aug, _ = self.D(G_aug, x_im, G_dep_aug, y_) if (self.WGAN): G_loss_fake = -(torch.mean(D_fake) + torch.mean(D_fake_aug)) / 2 else: G_loss_fake = (self.BCEWithLogitsLoss( D_fake, self.y_real_) + self.BCEWithLogitsLoss( D_fake_aug, self.y_real_)) / 2 # loss between images (*) #y_aug_join = torch.cat((y_im_aug, y_dep), 1) #G_aug_join = torch.cat((G_aug, G_dep_aug), 1) G_loss_Comp_Aug = self.L1(G_aug, y_im_aug) if self.depth: G_loss_Comp_Aug += self.L1(G_dep_aug, y_dep) G_loss_Dif_Comp = (G_loss_Comp + G_loss_Comp_Aug) / 2 * self.lambdaL1 G_loss = G_loss_fake + G_loss_Dif_Comp + G_loss_Cycle self.details_hist['G_T_BCE_fake_real'].append( G_loss_fake.detach().item()) self.details_hist['G_T_Comp_im'].append( G_loss_Dif_Comp.detach().item()) self.details_hist['G_T_Cycle'].append( G_loss_Cycle.detach().item()) self.details_hist['G_zCR'].append(0) else: G_loss = self.L1(G_, y_im) if self.depth: G_loss += self.L1(G_dep, y_dep) G_loss = G_loss * self.lambdaL1 self.details_hist['G_T_Comp_im'].append( G_loss.detach().item()) self.details_hist['G_T_BCE_fake_real'].append(0) self.details_hist['G_T_Cycle'].append(0) self.details_hist['G_zCR'].append(0) G_loss.backward() self.G_optimizer.step() self.train_hist['G_loss_train'].append(G_loss.detach().item()) if self.onlyGen: self.train_hist['D_loss_train'].append(0) iterFinTrain += 1 if self.visdom: visLossGTest.plot('Generator_losses', [ 'G_T_Comp_im', 'G_T_BCE_fake_real', 'G_zCR', 'G_T_Cycle' ], 'train', self.details_hist) vis.plot('loss', ['D_loss_train', 'G_loss_train'], 'train', self.train_hist) ##################Validation#################################### with torch.no_grad(): self.G.eval() if not self.onlyGen: self.D.eval() for iter, data in enumerate(self.data_Validation): # Aumento mi data x_im = data.get('x_im') x_dep = data.get('x_dep') y_im = data.get('y_im') y_dep = data.get('y_dep') y_ = data.get('y_') # x_im = imagenes normales # x_dep = profundidad de images # y_im = imagen con el angulo cambiado # y_ = angulo de la imagen = tengo que tratar negativos # x_im = torch.Tensor(list(x_im)) # x_dep = torch.Tensor(x_dep) # y_im = torch.Tensor(y_im) # print(y_.shape[0]) if iter == maxIterVal: # print ("Break") break # print (y_.type(torch.LongTensor).unsqueeze(1)) # print("y_vec_", y_vec_) # print ("z_", z_) if self.gpu_mode: x_im, y_, y_im, x_dep, y_dep = x_im.cuda(), y_.cuda( ), y_im.cuda(), x_dep.cuda(), y_dep.cuda() # D network if not ventaja and not self.onlyGen: # Real Images D_real, _ = self.D( y_im, x_im, y_dep, y_) ## Es la funcion forward `` g(z) x # Fake Images G_, G_dep = self.G(y_, x_im, x_dep) D_fake, _ = self.D(G_, x_im, G_dep, y_) # Losses # GAN Loss if (self.WGAN): # de WGAN D_loss_real_fake_R = -torch.mean(D_real) D_loss_real_fake_F = torch.mean(D_fake) else: # de Gan normal D_loss_real_fake_R = self.BCEWithLogitsLoss( D_real, self.y_real_) D_loss_real_fake_F = self.BCEWithLogitsLoss( D_fake, self.y_fake_) D_loss_real_fake = D_loss_real_fake_F + D_loss_real_fake_R D_loss = D_loss_real_fake self.train_hist['D_loss_Validation'].append( D_loss.item()) self.details_hist['D_V_BCE_fake_real_R'].append( D_loss_real_fake_R.item()) self.details_hist['D_V_BCE_fake_real_F'].append( D_loss_real_fake_F.item()) if self.visdom: visLossDValidation.plot( 'Discriminator_losses', ['D_V_BCE_fake_real_R', 'D_V_BCE_fake_real_F'], 'Validation', self.details_hist) # G network G_, G_dep = self.G(y_, x_im, x_dep) if not ventaja and not self.onlyGen: # Fake images D_fake, _ = self.D(G_, x_im, G_dep, y_) #Loss GAN if (self.WGAN): G_loss = -torch.mean(D_fake) # porWGAN else: G_loss = self.BCEWithLogitsLoss( D_fake, self.y_real_) #de GAN NORMAL self.details_hist['G_V_BCE_fake_real'].append( G_loss.item()) #Loss comparation #G_join = torch.cat((G_, G_dep), 1) #y_join = torch.cat((y_im, y_dep), 1) G_loss_Comp = self.L1(G_, y_im) if self.depth: G_loss_Comp += self.L1(G_dep, y_dep) G_loss_Comp = G_loss_Comp * self.lambdaL1 reverse_y = -y_ + 1 reverse_G, reverse_G_dep = self.G(reverse_y, G_, G_dep) G_loss_Cycle = self.L1(reverse_G, x_im) if self.depth: G_loss_Cycle += self.L1(reverse_G_dep, x_dep) G_loss_Cycle = G_loss_Cycle * self.lambdaL1 / 2 G_loss += G_loss_Comp + G_loss_Cycle self.details_hist['G_V_Comp_im'].append( G_loss_Comp.item()) self.details_hist['G_V_Cycle'].append( G_loss_Cycle.detach().item()) else: G_loss = self.L1(G_, y_im) if self.depth: G_loss += self.L1(G_dep, y_dep) G_loss = G_loss * self.lambdaL1 self.details_hist['G_V_Comp_im'].append(G_loss.item()) self.details_hist['G_V_BCE_fake_real'].append(0) self.details_hist['G_V_Cycle'].append(0) self.train_hist['G_loss_Validation'].append(G_loss.item()) if self.onlyGen: self.train_hist['D_loss_Validation'].append(0) iterFinValidation += 1 if self.visdom: visLossGValidation.plot( 'Generator_losses', ['G_V_Comp_im', 'G_V_BCE_fake_real', 'G_V_Cycle'], 'Validation', self.details_hist) visValidation.plot( 'loss', ['D_loss_Validation', 'G_loss_Validation'], 'Validation', self.train_hist) ##Vis por epoch if ventaja or self.onlyGen: self.epoch_hist['D_loss_train'].append(0) self.epoch_hist['D_loss_Validation'].append(0) else: #inicioTr = (epoch - self.epochVentaja) * (iterFinTrain - iterIniTrain) #inicioTe = (epoch - self.epochVentaja) * (iterFinValidation - iterIniValidation) self.epoch_hist['D_loss_train'].append( mean(self.train_hist['D_loss_train'][iterIniTrain:-1])) self.epoch_hist['D_loss_Validation'].append( mean(self.train_hist['D_loss_Validation'] [iterIniValidation:-1])) self.epoch_hist['G_loss_train'].append( mean(self.train_hist['G_loss_train'] [iterIniTrain:iterFinTrain])) self.epoch_hist['G_loss_Validation'].append( mean(self.train_hist['G_loss_Validation'] [iterIniValidation:iterFinValidation])) if self.visdom: visEpoch.plot('epoch', epoch, [ 'D_loss_train', 'G_loss_train', 'D_loss_Validation', 'G_loss_Validation' ], self.epoch_hist) self.train_hist['D_loss_train'] = self.train_hist['D_loss_train'][ -1:] self.train_hist['G_loss_train'] = self.train_hist['G_loss_train'][ -1:] self.train_hist['D_loss_Validation'] = self.train_hist[ 'D_loss_Validation'][-1:] self.train_hist['G_loss_Validation'] = self.train_hist[ 'G_loss_Validation'][-1:] self.train_hist['per_epoch_time'] = self.train_hist[ 'per_epoch_time'][-1:] self.train_hist['total_time'] = self.train_hist['total_time'][-1:] self.details_hist['G_T_Comp_im'] = self.details_hist[ 'G_T_Comp_im'][-1:] self.details_hist['G_T_BCE_fake_real'] = self.details_hist[ 'G_T_BCE_fake_real'][-1:] self.details_hist['G_T_Cycle'] = self.details_hist['G_T_Cycle'][ -1:] self.details_hist['G_zCR'] = self.details_hist['G_zCR'][-1:] self.details_hist['G_V_Comp_im'] = self.details_hist[ 'G_V_Comp_im'][-1:] self.details_hist['G_V_BCE_fake_real'] = self.details_hist[ 'G_V_BCE_fake_real'][-1:] self.details_hist['G_V_Cycle'] = self.details_hist['G_V_Cycle'][ -1:] self.details_hist['D_T_BCE_fake_real_R'] = self.details_hist[ 'D_T_BCE_fake_real_R'][-1:] self.details_hist['D_T_BCE_fake_real_F'] = self.details_hist[ 'D_T_BCE_fake_real_F'][-1:] self.details_hist['D_zCR'] = self.details_hist['D_zCR'][-1:] self.details_hist['D_bCR'] = self.details_hist['D_bCR'][-1:] self.details_hist['D_V_BCE_fake_real_R'] = self.details_hist[ 'D_V_BCE_fake_real_R'][-1:] self.details_hist['D_V_BCE_fake_real_F'] = self.details_hist[ 'D_V_BCE_fake_real_F'][-1:] ##Para poder tomar el promedio por epoch iterIniTrain = 1 iterFinTrain = 1 iterIniValidation = 1 iterFinValidation = 1 self.train_hist['per_epoch_time'].append(time.time() - epoch_start_time) if epoch % 10 == 0: self.save(str(epoch)) with torch.no_grad(): if self.visdom: self.visualize_results(epoch, dataprint=self.dataprint, visual=visImages) self.visualize_results(epoch, dataprint=self.dataprint_test, visual=visImagesTest) else: imageName = self.model_name + '_' + 'Train' + '_' + str( self.seed) + '_' + str(epoch) self.visualize_results(epoch, dataprint=self.dataprint, name=imageName) self.visualize_results(epoch, dataprint=self.dataprint_test, name=imageName) self.train_hist['total_time'].append(time.time() - start_time) print("Avg one epoch time: %.2f, total %d epochs time: %.2f" % (np.mean(self.train_hist['per_epoch_time']), self.epoch, self.train_hist['total_time'][0])) print("Training finish!... save training results") self.save()
def main(): main_arg_parser = argparse.ArgumentParser(description="options") main_arg_parser.add_argument("-e,", "--epochs", type=int, default=400) main_arg_parser.add_argument("-lr", "--learning-rate", type=float, default=0.001) main_arg_parser.add_argument("--weight-decay", help="L2 regularization coefficient", type=float, default=0) main_arg_parser.add_argument("--cuda", action="store_true") main_arg_parser.add_argument( "--test-set-size", help="proportion of dataset to allocate as test set [0..1]", type=float, default=0.1) main_arg_parser.add_argument( "--aflw-path", help="path to aflw dir (should contain aflw_{12,14}.t7)", default="EX2_data/aflw") main_arg_parser.add_argument("--negatives-path", help="path to VOC2007 mined negatives", default="EX2_data/negative_mines_subset.pth") main_arg_parser.add_argument("--batch-size", type=int, default=64) # submitted convergence plot obtained from visdom using this flag (everything else default) main_arg_parser.add_argument("--visdom-plot", action="store_true") main_arg_parser.add_argument("--seed", help="random seed for torch", type=int, default=42) main_arg_parser.add_argument("--continue-from", help="checkpoint to continue from") args = main_arg_parser.parse_args() np.random.seed(args.seed) torch.manual_seed(args.seed) # cuda only if asked and exists cuda = args.cuda and torch.cuda.is_available() if cuda: torch.cuda.manual_seed(args.seed) print("Using CUDA!") else: import time print( "Not using CUDA. Add --cuda or this may take a while. Have a moment to hit ctrl-c" ) #time.sleep(3) if args.visdom_plot: plotter = utils.VisdomLinePlotter('24Net Loss') else: plotter = None # load data dataset = load_24net_data(args.aflw_path, args.negatives_path) # data is ordered by class, so shuffle and split to test/train indices_shuffled = list(torch.randperm(len(dataset))) first_test_index = int((1 - args.test_set_size) * len(indices_shuffled)) # we keep lists of indices as the test/train division. This respects torch's seed # and we can sample out of these separate lists at test and train train_subset = indices_shuffled[:first_test_index] test_subset = indices_shuffled[first_test_index:] # train and test loss_criterion = nn.BCELoss() net = Net24() optimizer = Adam(net.parameters(), args.learning_rate, weight_decay=args.weight_decay) if args.continue_from: print("continuing from {}".format(args.continue_from)) loaded = torch.load(args.continue_from) net.load_state_dict(loaded['state_dict']) optimizer.load_state_dict(loaded['optimizer']) if cuda: net.cuda() if args.epochs > 0: train(net, loss_criterion, dataset, optimizer, plotter=plotter, epochs=args.epochs, train_subset=train_subset, test_subset=test_subset, batch_size=args.batch_size, cuda=cuda) precisions, recalls = calc_precision_recall(net, loss_criterion, dataset, test_subset, batch_size=args.batch_size, cuda=cuda) if args.visdom_plot: import visdom viz = visdom.Visdom() viz.line(X=np.array(recalls), Y=np.array(precisions), opts=dict(title="Precision-Recall Curve", xlabel="Recall", ylabel="Precision"), env="main") # find first threshold below 99% recall for idx in range(len(recalls)): if recalls[idx] < 0.99: break best_index = idx - 1 # one before we dropped below 99% print( "threshold {} to get recall >99% ({}). Resulting precision {}".format( best_index / len(recalls), recalls[best_index], precisions[best_index])) torch.save( { 'state_dict': net.state_dict(), 'optimizer': optimizer.state_dict(), }, "q3_checkpoint.pth.tar")
'state_dict': model.state_dict(), 'best_val': best_val, 'optimizer': optimizer.state_dict(), 'valtrack': valtrack, 'freeVision': args_dict.freeVision, 'curr_val': lossval, }) print '** Validation: %f (best) - %d (valtrack)' % (best_val, valtrack) if __name__ == "__main__": # Set the correct system encoding to read the csv files reload(sys) sys.setdefaultencoding('Cp1252') # Load parameters parser = get_parser() args_dict, unknown = parser.parse_known_args() # Set seed for reproducibility torch.manual_seed(args_dict.seed) if args_dict.use_gpu: torch.cuda.manual_seed(args_dict.seed) # Plots global plotter plotter = utils.VisdomLinePlotter(env_name=args_dict.train_name) # Training process trainProcess(args_dict)
x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in ['train', 'val', 'test'] } dataset_sizes = { x: len(image_datasets[x]) for x in ['train', 'val', 'test'] } class_names = image_datasets['train'].classes if args.mode == "train": # Visdom for plots global plotter plotter = utils.VisdomLinePlotter(env_name='NameThatDog Plots') # Extract args num_epochs = args.num_epochs learning_rate = args.learning_rate momentum = args.momentum decay = args.decay gamma = args.gamma step_size = args.step_size using_pretrained = args.using_pretrained print( "Training new final layer of Resnet18 model for {} Epochs with hyperparams: LR: {}, Momentum: {}, Decay: {}, Gamma: {}, Step Size: {}, Batch Size: {}" .format(num_epochs, learning_rate, momentum, decay, gamma, step_size, batch_size))