iter_start_time = time.time() epoch_iter += args.batch_size model.set_input( data['image'], data['segmentation_outlined'], data['image_name'] ) model.optimize_parameters() if epoch_iter % args.print_freq == 0: errors = model.get_current_errors() t = (time.time() - iter_start_time) / args.batch_size visualizer.print_current_errors(epoch, epoch_iter, errors, t) if epoch_iter % args.display_freq == 0: visuals = model.get_current_visuals() visualizer.display_current_results(visuals, epoch) if epoch_iter % args.save_freq == 0: model.save("latest") print( "Saving current model to {}".format(checkpoints_dir) ) if epoch - 1 != 0 and epoch % args.lr_decay_freq == 0: model.update_learning_rate(args.lr_decay) print('End of epoch %d / %d \t Time Taken: %d sec' % (epoch, args.num_epochs, time.time() - epoch_start_time)) model.save(epoch) print( "Saving current model to {}".format(checkpoints_dir)
def train(self): # if self.config.visualize: visualizer = Visualizer() """Train StarGAN within a single dataset.""" # Set dataloader self.data_loader = self.train_loader # The number of iterations per epoch iters_per_epoch = len(self.data_loader) fixed_x = [] real_c = [] for i, (imgs, labels, _) in enumerate(self.data_loader): fixed_x.append(imgs[0]) real_c.append(labels) if i == 0: break # Fixed inputs and target domain labels for debugging fixed_x = torch.cat(fixed_x, dim=0) fixed_x = self.to_var(fixed_x, volatile=True) real_c = torch.cat(real_c, dim=0) fixed_c_list = self.make_celeb_labels(self.config.batch_size) # lr cache for decaying g_lr = self.config.g_lr d_lr = self.config.d_lr # Start with trained model if exists if self.config.pretrained_model: start = int(self.config.pretrained_model.split('_')[0]) - 1 else: start = 0 # Start training self.loss = {} start_time = time.time() for e in range(start, self.config.num_epochs): self.test(e) for i, (images, real_label, identity) in enumerate(self.data_loader): real_x = images[0] if self.config.use_si: real_ox = self.to_var(images[1]) real_oo = self.to_var(images[2]) if self.config.id_cls_loss == 'cross': identity = identity.squeeze() # Generate fake labels randomly (target domain labels) rand_idx = torch.randperm(real_label.size(0)) fake_label = real_label[rand_idx] real_c = real_label.clone() fake_c = fake_label.clone() # Convert tensor to variable real_x = self.to_var(real_x) real_c = self.to_var(real_c) # input for the generator fake_c = self.to_var(fake_c) real_label = self.to_var( real_label ) # this is same as real_c if dataset == 'CelebA' fake_label = self.to_var(fake_label) identity = self.to_var(identity) # ================== Train D ================== # # Compute loss with real images if self.config.loss_id_cls: out_src, out_cls, out_id_real = self.D(real_x) else: out_src, out_cls = self.D(real_x) d_loss_real = -torch.mean(out_src) d_loss_cls = F.binary_cross_entropy_with_logits( out_cls, real_label, size_average=False) / real_x.size(0) if self.config.loss_id_cls: d_loss_id_cls = self.id_cls_criterion( out_id_real, identity) self.loss[ 'D/loss_id_cls'] = self.config.lambda_id_cls * d_loss_id_cls.data[ 0] else: d_loss_id_cls = 0.0 # Compute classification accuracy of the discriminator if (i + 1) % self.config.log_step == 0: accuracies = self.compute_accuracy(out_cls.detach(), real_label, self.config.dataset) log = [ "{:.2f}".format(acc) for acc in accuracies.data.cpu().numpy() ] print('Classification Acc (20 classes): ') print(log) print('\n') # Compute loss with fake images if self.config.use_gpb: fake_x, _ = self.G(real_x, fake_c) else: fake_x = self.G(real_x, fake_c) fake_x = Variable(fake_x.data) if self.config.loss_id_cls: out_src, out_cls, _ = self.D(fake_x.detach()) else: out_src, out_cls = self.D(fake_x.detach()) d_loss_fake = torch.mean(out_src) # Backward + Optimize d_loss = d_loss_real + d_loss_fake + self.config.lambda_cls * d_loss_cls + d_loss_id_cls * self.config.lambda_id_cls self.reset_grad() d_loss.backward() self.d_optimizer.step() # Compute gradient penalty alpha = torch.rand(real_x.size(0), 1, 1, 1).cuda().expand_as(real_x) interpolated = Variable(alpha * real_x.data + (1 - alpha) * fake_x.data, requires_grad=True) if self.config.loss_id_cls: out, out_cls, _ = self.D(interpolated) else: out, out_cls = self.D(interpolated) grad = torch.autograd.grad(outputs=out, inputs=interpolated, grad_outputs=torch.ones( out.size()).cuda(), retain_graph=True, create_graph=True, only_inputs=True)[0] grad = grad.view(grad.size(0), -1) grad_l2norm = torch.sqrt(torch.sum(grad**2, dim=1)) d_loss_gp = torch.mean((grad_l2norm - 1)**2) # Backward + Optimize d_loss = self.config.lambda_gp * d_loss_gp self.reset_grad() d_loss.backward() self.d_optimizer.step() # Logging self.loss['D/loss_real'] = d_loss_real.data[0] self.loss['D/loss_fake'] = d_loss_fake.data[0] self.loss[ 'D/loss_cls'] = self.config.lambda_cls * d_loss_cls.data[0] self.loss[ 'D/loss_gp'] = self.config.lambda_gp * d_loss_gp.data[0] # ================== Train G ================== # if (i + 1) % self.config.d_train_repeat == 0: self.img = {} # Original-to-target and target-to-original domain if self.config.use_gpb: fake_x, id_vector_real_in_x = self.G(real_x, fake_c) rec_x, id_vector_fake_in_x = self.G( fake_x.detach(), real_c) else: fake_x = self.G(real_x, fake_c) rec_x = self.G(fake_x.detach(), real_c) # Compute losses if self.config.loss_id_cls: out_src, out_cls, out_id_fake = self.D(fake_x) else: out_src, out_cls = self.D(fake_x) g_loss_fake = -torch.mean(out_src) g_loss_rec = torch.mean(torch.abs(real_x - rec_x)) ### siamese loss if self.config.use_si: if self.config.use_gpb: # feedforward fake_ox, id_vector_ox = self.G(real_ox, fake_c) fake_oo, id_vector_oo = self.G(real_oo, fake_c) id_vector_ox = id_vector_ox.detach() id_vector_oo = id_vector_oo.detach() mdist = 1.0 - torch.mean( torch.abs(id_vector_real_in_x - id_vector_oo)) mdist = torch.clamp(mdist, min=0.0) g_loss_si = 0.5 * (torch.pow( torch.mean( torch.abs(id_vector_real_in_x - id_vector_ox)), 2) + torch.pow(mdist, 2)) # backward _, id_vector_ox = self.G(fake_ox.detach(), real_c) _, id_vector_oo = self.G(fake_oo.detach(), real_c) id_vector_ox = id_vector_ox.detach() id_vector_oo = id_vector_oo.detach() mdist = 1.0 - torch.mean( torch.abs(id_vector_fake_in_x - id_vector_oo)) mdist = torch.clamp(mdist, min=0.0) g_loss_si += 0.5 * (torch.pow( torch.mean( torch.abs(id_vector_fake_in_x - id_vector_ox)), 2) + torch.pow(mdist, 2)) self.loss['G/g_loss_si'] = g_loss_si.data[0] else: fake_ox = self.G(real_ox, fake_c).detach() fake_ooc = fake_c.data.cpu().numpy().copy() fake_ooc = np.roll(fake_ooc, np.random.randint( self.config.c_dim), axis=1) fake_ooc = self.to_var(torch.FloatTensor(fake_ooc)) fake_oo = self.G(real_oo, fake_ooc).detach() mdist = 1.0 - torch.mean( torch.abs(fake_x - fake_oo)) mdist = torch.clamp(mdist, min=0.0) g_loss_si = 0.5 * (torch.pow( torch.mean(torch.abs(fake_x - fake_ox)), 2) + torch.pow(mdist, 2)) self.loss['G/g_loss_si'] = g_loss_si.data[0] else: g_loss_si = 0.0 ### id cls loss if self.config.loss_id_cls: g_loss_id_cls = self.id_cls_criterion( out_id_fake, identity) self.loss[ 'G/g_loss_id_cls'] = self.config.lambda_id_cls * g_loss_id_cls.data[ 0] else: g_loss_id_cls = 0.0 ### sym loss if self.config.loss_symmetry: g_loss_sym_fake = self.find_sym_img_and_cal_loss( fake_x, fake_c, True) # cal. over samples w/ specific labels g_loss_sym_rec = self.find_sym_img_and_cal_loss( rec_x, real_c, True) lap_fake_x = self.take_laplacian(fake_x) lap_rec_x = self.take_laplacian(rec_x) g_loss_sym_lap_fake = self.find_sym_img_and_cal_loss( lap_fake_x, None, False) # cal. over all samples g_loss_sym_lap_rec = self.find_sym_img_and_cal_loss( lap_rec_x, None, False) sym_loss = (g_loss_sym_fake + g_loss_sym_rec + g_loss_sym_lap_fake + g_loss_sym_lap_rec) self.loss[ 'G/g_loss_sym'] = self.config.lambda_symmetry * sym_loss.data[ 0] else: sym_loss = 0 ###id loss if self.config.loss_id: if self.config.use_gpb: idx, _ = self.G(real_x, real_c) else: idx = self.G(real_x, real_c) self.img['idx'] = idx g_loss_id = torch.mean(torch.abs(real_x - idx)) self.loss[ 'G/g_loss_id'] = self.config.lambda_idx * g_loss_id.data[ 0] else: g_loss_id = 0 ###identity loss if self.config.loss_identity: real_x_f, real_x_p = self.get_feature(real_x) fake_x_f, fake_x_p = self.get_feature(fake_x) g_loss_identity = torch.mean( torch.abs(real_x_f - fake_x_f)) g_loss_identity += torch.mean( torch.abs(real_x_p - fake_x_p)) self.loss[ 'G/g_loss_identity'] = self.config.lambda_identity * g_loss_identity.data[ 0] else: g_loss_identity = 0 ###total var loss if self.config.loss_tv: g_tv_loss = (self.total_variation_loss(fake_x) + self.total_variation_loss(rec_x)) / 2 self.loss[ 'G/tv_loss'] = self.config.lambda_tv * g_tv_loss.data[ 0] else: g_tv_loss = 0 ### D's cls loss g_loss_cls = F.binary_cross_entropy_with_logits( out_cls, fake_label, size_average=False) / fake_x.size(0) # Backward + Optimize g_loss = g_loss_fake +\ self.config.lambda_rec * g_loss_rec +\ self.config.lambda_cls * g_loss_cls+\ self.config.lambda_idx * g_loss_id+\ self.config.lambda_identity*g_loss_identity+\ self.config.lambda_tv*g_tv_loss+\ self.config.lambda_symmetry*sym_loss+\ self.config.lambda_id_cls * g_loss_id_cls+\ self.config.lambda_si * g_loss_si self.reset_grad() g_loss.backward() self.g_optimizer.step() # Logging self.img['real_x'] = real_x self.img['fake_x'] = fake_x self.img['rec_x'] = rec_x self.loss['G/loss_fake'] = g_loss_fake.data[0] self.loss[ 'G/loss_rec'] = self.config.lambda_rec * g_loss_rec.data[ 0] self.loss[ 'G/loss_cls'] = self.config.lambda_cls * g_loss_cls.data[ 0] # # Print out log info if (i + 1) % self.config.log_step == 0: elapsed = time.time() - start_time elapsed = str(datetime.timedelta(seconds=elapsed)) log = "Elapsed [{}], Epoch [{}/{}], Iter [{}/{}]".format( elapsed, e + 1, self.config.num_epochs, i + 1, iters_per_epoch) for tag, value in self.loss.items(): log += ", {}: {}".format(tag, value) print(log) if self.config.use_tensorboard: for tag, value in self.loss.items(): self.logger.scalar_summary( tag, value, e * iters_per_epoch + i + 1) # Translate fixed images for debugging if (i) % self.config.sample_step == 0: fake_image_list = [fixed_x] for fixed_c in fixed_c_list: if self.config.use_gpb: fake_image_list.append(self.G(fixed_x, fixed_c)[0]) else: fake_image_list.append(self.G(fixed_x, fixed_c)) fake_images = torch.cat(fake_image_list, dim=3) if not self.config.log_space: save_image(self.denorm(fake_images.data), os.path.join( self.config.sample_path, '{}_{}_fake.png'.format(e + 1, i + 1)), nrow=1, padding=0) else: fake_images = self.denorm(fake_images.data) * 255.0 fake_images = torch.pow( 2.71828182846, fake_images / 255.0 * np.log(256.0)) - 1.0 fake_images = fake_images / 255.0 fake_images = fake_images.clamp(0.0, 1.0) save_image(fake_images, os.path.join( self.config.sample_path, '{}_{}_fake.png'.format(e + 1, i + 1)), nrow=1, padding=0) print('Translated images and saved into {}..!'.format( self.config.sample_path)) # Save model checkpoints if (i + 1) % self.config.model_save_step == 0: torch.save( self.G.state_dict(), os.path.join(self.config.model_save_path, '{}_{}_G.pth'.format(e + 1, i + 1))) torch.save( self.D.state_dict(), os.path.join(self.config.model_save_path, '{}_{}_D.pth'.format(e + 1, i + 1))) if self.config.visualize and (i + 1) % self.config.display_f == 0: visualizer.display_current_results(self.img) visualizer.plot_current_errors( e, float(i + 1) / iters_per_epoch, self.loss) # Decay learning rate if (e + 1) > (self.config.num_epochs - self.config.num_epochs_decay): g_lr -= (self.config.g_lr / float(self.config.num_epochs_decay)) d_lr -= (self.config.d_lr / float(self.config.num_epochs_decay)) self.update_lr(g_lr, d_lr) print('Decay learning rate to g_lr: {}, d_lr: {}.'.format( g_lr, d_lr))
('content_loss0_edge', content_loss0_edge.detach().cpu().float().numpy()), ('content_loss1_edge', content_loss1_edge.detach().cpu().float().numpy()), ('L_img_edge', L_edge.detach().cpu().float().numpy()), ('my_psnr', my_psnr / (ccnt)) ]) t = (time.time() - iter_start_time) / opt.batchSize trainLogger.write( visualizer.print_current_losses(epoch, epoch_iter, losses, t, t_data) + '\n') visualizer.print_current_losses(epoch, epoch_iter, losses, t, t_data) r = float(epoch_iter) / (dataset_size * opt.batchSize) if opt.display_port != -1: visualizer.display_current_results(current_visuals, epoch, False) visualizer.plot_current_losses(epoch, r, opt, losses) netG.train() netEdge.train() # save model and test performance in validation set if epoch % 1 == 0: print('hit') my_file = open("./" + opt.name + "_" + "evaluation.txt", 'a+') torch.save(netG.state_dict(), '%s/netG_epoch_%d.pth' % (opt.exp, epoch)) torch.save(netEdge.state_dict(), '%s/netEdge_epoch_%d.pth' % (opt.exp, epoch)) vcnt = 0
visualizer.reset() # reset the visualizer: make sure it saves the results to HTML at least once every epoch model.update_learning_rate() # update learning rates in the beginning of every epoch. for i, data in enumerate(dataset): # inner loop within one epoch iter_start_time = time.time() # timer for computation per iteration if total_iters % opt.print_freq == 0: t_data = iter_start_time - iter_data_time total_iters += opt.batch_size epoch_iter += opt.batch_size model.set_input(data) # unpack data from dataset and apply preprocessing model.optimize_parameters() # calculate loss functions, get gradients, update network weights if total_iters % opt.display_freq == 0: # display images on visdom and save images to a HTML file save_result = total_iters % opt.update_html_freq == 0 model.compute_visuals() visualizer.display_current_results(model.get_current_visuals(), epoch, save_result) if total_iters % opt.print_freq == 0: # print training losses and save logging information to the disk losses = model.get_current_losses() t_comp = (time.time() - iter_start_time) / opt.batch_size visualizer.print_current_losses(epoch, epoch_iter, losses, t_comp, t_data) if opt.display_id > 0: visualizer.plot_current_losses(epoch, float(epoch_iter) / dataset_size, losses) if total_iters % opt.save_latest_freq == 0: # cache our latest model every <save_latest_freq> iterations print('saving the latest model (epoch %d, total_iters %d)' % (epoch, total_iters)) save_suffix = 'iter_%d' % total_iters if opt.save_by_iter else 'latest' model.save_networks(save_suffix) iter_data_time = time.time() if epoch % opt.save_epoch_freq == 0: # cache our model every <save_epoch_freq> epochs
def train(self): self.genA2B.train(), self.genB2A.train(), self.disGA.train( ), self.disGB.train(), self.disLA.train(), self.disLB.train() start_iter = 1 if self.resume: model_list = glob( os.path.join(self.result_dir, self.dataset, 'model', '*.pt')) if not len(model_list) == 0: model_list.sort() start_iter = int(model_list[-1].split('_')[-1].split('.')[0]) self.load(os.path.join(self.result_dir, self.dataset, 'model'), start_iter) print(" [*] Load SUCCESS") if self.decay_flag and start_iter > (self.iteration // 2): self.G_optim.param_groups[0]['lr'] -= ( self.lr / (self.iteration // 2)) * (start_iter - self.iteration // 2) self.D_optim.param_groups[0]['lr'] -= ( self.lr / (self.iteration // 2)) * (start_iter - self.iteration // 2) # training loop print('training start !') # visualize training process... visualizer = Visualizer( self.opt) # create a visualizer that display/save images and plots start_time = time.time() for step in range(start_iter, self.iteration + 1): if self.decay_flag and step > (self.iteration // 2): self.G_optim.param_groups[0]['lr'] -= (self.lr / (self.iteration // 2)) self.D_optim.param_groups[0]['lr'] -= (self.lr / (self.iteration // 2)) try: self.real_A, _ = trainA_iter.next() except: trainA_iter = iter(self.trainA_loader) self.real_A, _ = trainA_iter.next() try: self.real_B, _ = trainB_iter.next() except: trainB_iter = iter(self.trainB_loader) self.real_B, _ = trainB_iter.next() self.real_A, self.real_B = self.real_A.to( self.device), self.real_B.to(self.device) # Update D self.D_optim.zero_grad() self.fake_A2B, _, _ = self.genA2B(self.real_A) self.fake_B2A, _, _ = self.genB2A(self.real_B) real_GA_logit, real_GA_cam_logit, _ = self.disGA(self.real_A) real_LA_logit, real_LA_cam_logit, _ = self.disLA(self.real_A) real_GB_logit, real_GB_cam_logit, _ = self.disGB(self.real_B) real_LB_logit, real_LB_cam_logit, _ = self.disLB(self.real_B) fake_GA_logit, fake_GA_cam_logit, _ = self.disGA(self.fake_B2A) fake_LA_logit, fake_LA_cam_logit, _ = self.disLA(self.fake_B2A) fake_GB_logit, fake_GB_cam_logit, _ = self.disGB(self.fake_A2B) fake_LB_logit, fake_LB_cam_logit, _ = self.disLB(self.fake_A2B) D_ad_loss_GA = self.MSE_loss( real_GA_logit, torch.ones_like(real_GA_logit).to( self.device)) + self.MSE_loss( fake_GA_logit, torch.zeros_like(fake_GA_logit).to(self.device)) D_ad_cam_loss_GA = self.MSE_loss( real_GA_cam_logit, torch.ones_like(real_GA_cam_logit).to( self.device)) + self.MSE_loss( fake_GA_cam_logit, torch.zeros_like(fake_GA_cam_logit).to(self.device)) D_ad_loss_LA = self.MSE_loss( real_LA_logit, torch.ones_like(real_LA_logit).to( self.device)) + self.MSE_loss( fake_LA_logit, torch.zeros_like(fake_LA_logit).to(self.device)) D_ad_cam_loss_LA = self.MSE_loss( real_LA_cam_logit, torch.ones_like(real_LA_cam_logit).to( self.device)) + self.MSE_loss( fake_LA_cam_logit, torch.zeros_like(fake_LA_cam_logit).to(self.device)) D_ad_loss_GB = self.MSE_loss( real_GB_logit, torch.ones_like(real_GB_logit).to( self.device)) + self.MSE_loss( fake_GB_logit, torch.zeros_like(fake_GB_logit).to(self.device)) D_ad_cam_loss_GB = self.MSE_loss( real_GB_cam_logit, torch.ones_like(real_GB_cam_logit).to( self.device)) + self.MSE_loss( fake_GB_cam_logit, torch.zeros_like(fake_GB_cam_logit).to(self.device)) D_ad_loss_LB = self.MSE_loss( real_LB_logit, torch.ones_like(real_LB_logit).to( self.device)) + self.MSE_loss( fake_LB_logit, torch.zeros_like(fake_LB_logit).to(self.device)) D_ad_cam_loss_LB = self.MSE_loss( real_LB_cam_logit, torch.ones_like(real_LB_cam_logit).to( self.device)) + self.MSE_loss( fake_LB_cam_logit, torch.zeros_like(fake_LB_cam_logit).to(self.device)) D_loss_A = self.adv_weight * (D_ad_loss_GA + D_ad_cam_loss_GA + D_ad_loss_LA + D_ad_cam_loss_LA) D_loss_B = self.adv_weight * (D_ad_loss_GB + D_ad_cam_loss_GB + D_ad_loss_LB + D_ad_cam_loss_LB) Discriminator_loss = D_loss_A + D_loss_B Discriminator_loss.backward() self.D_optim.step() # Update G self.G_optim.zero_grad() self.fake_A2B, fake_A2B_cam_logit, _ = self.genA2B(self.real_A) self.fake_B2A, fake_B2A_cam_logit, _ = self.genB2A(self.real_B) self.fake_A2B2A, _, _ = self.genB2A(self.fake_A2B) self.fake_B2A2B, _, _ = self.genA2B(self.fake_B2A) self.fake_A2A, fake_A2A_cam_logit, _ = self.genB2A(self.real_A) self.fake_B2B, fake_B2B_cam_logit, _ = self.genA2B(self.real_B) fake_GA_logit, fake_GA_cam_logit, _ = self.disGA(self.fake_B2A) fake_LA_logit, fake_LA_cam_logit, _ = self.disLA(self.fake_B2A) fake_GB_logit, fake_GB_cam_logit, _ = self.disGB(self.fake_A2B) fake_LB_logit, fake_LB_cam_logit, _ = self.disLB(self.fake_A2B) G_ad_loss_GA = self.MSE_loss( fake_GA_logit, torch.ones_like(fake_GA_logit).to(self.device)) G_ad_cam_loss_GA = self.MSE_loss( fake_GA_cam_logit, torch.ones_like(fake_GA_cam_logit).to(self.device)) G_ad_loss_LA = self.MSE_loss( fake_LA_logit, torch.ones_like(fake_LA_logit).to(self.device)) G_ad_cam_loss_LA = self.MSE_loss( fake_LA_cam_logit, torch.ones_like(fake_LA_cam_logit).to(self.device)) G_ad_loss_GB = self.MSE_loss( fake_GB_logit, torch.ones_like(fake_GB_logit).to(self.device)) G_ad_cam_loss_GB = self.MSE_loss( fake_GB_cam_logit, torch.ones_like(fake_GB_cam_logit).to(self.device)) G_ad_loss_LB = self.MSE_loss( fake_LB_logit, torch.ones_like(fake_LB_logit).to(self.device)) G_ad_cam_loss_LB = self.MSE_loss( fake_LB_cam_logit, torch.ones_like(fake_LB_cam_logit).to(self.device)) G_recon_loss_A = self.L1_loss(self.fake_A2B2A, self.real_A) G_recon_loss_B = self.L1_loss(self.fake_B2A2B, self.real_B) G_identity_loss_A = self.L1_loss(self.fake_A2A, self.real_A) G_identity_loss_B = self.L1_loss(self.fake_B2B, self.real_B) G_cam_loss_A = self.BCE_loss( fake_B2A_cam_logit, torch.ones_like(fake_B2A_cam_logit).to( self.device)) + self.BCE_loss( fake_A2A_cam_logit, torch.zeros_like(fake_A2A_cam_logit).to(self.device)) G_cam_loss_B = self.BCE_loss( fake_A2B_cam_logit, torch.ones_like(fake_A2B_cam_logit).to( self.device)) + self.BCE_loss( fake_B2B_cam_logit, torch.zeros_like(fake_B2B_cam_logit).to(self.device)) G_loss_A = self.adv_weight * ( G_ad_loss_GA + G_ad_cam_loss_GA + G_ad_loss_LA + G_ad_cam_loss_LA ) + self.cycle_weight * G_recon_loss_A + self.identity_weight * G_identity_loss_A + self.cam_weight * G_cam_loss_A G_loss_B = self.adv_weight * ( G_ad_loss_GB + G_ad_cam_loss_GB + G_ad_loss_LB + G_ad_cam_loss_LB ) + self.cycle_weight * G_recon_loss_B + self.identity_weight * G_identity_loss_B + self.cam_weight * G_cam_loss_B Generator_loss = G_loss_A + G_loss_B Generator_loss.backward() self.G_optim.step() # clip parameter of AdaILN and ILN, applied after optimizer step self.genA2B.apply(self.Rho_clipper) self.genB2A.apply(self.Rho_clipper) print("[%5d/%5d] time: %4.4f d_loss: %.8f, g_loss: %.8f" % (step, self.iteration, time.time() - start_time, Discriminator_loss, Generator_loss)) if step % self.print_freq == 0: train_sample_num = 5 test_sample_num = 5 A2B = np.zeros((self.img_size * 7, 0, 3)) B2A = np.zeros((self.img_size * 7, 0, 3)) self.genA2B.eval(), self.genB2A.eval(), self.disGA.eval( ), self.disGB.eval(), self.disLA.eval(), self.disLB.eval() for _ in range(train_sample_num): try: self.real_A, _ = trainA_iter.next() except: trainA_iter = iter(self.trainA_loader) self.real_A, _ = trainA_iter.next() try: self.real_B, _ = trainB_iter.next() except: trainB_iter = iter(self.trainB_loader) self.real_B, _ = trainB_iter.next() self.real_A, self.real_B = self.real_A.to( self.device), self.real_B.to(self.device) self.fake_A2B, _, fake_A2B_heatmap = self.genA2B( self.real_A) self.fake_B2A, _, fake_B2A_heatmap = self.genB2A( self.real_B) self.fake_A2B2A, _, fake_A2B2A_heatmap = self.genB2A( self.fake_A2B) self.fake_B2A2B, _, fake_B2A2B_heatmap = self.genA2B( self.fake_B2A) self.fake_A2A, _, fake_A2A_heatmap = self.genB2A( self.real_A) self.fake_B2B, _, fake_B2B_heatmap = self.genA2B( self.real_B) A2B = np.concatenate( (A2B, np.concatenate( (RGB2BGR(tensor2numpy(denorm(self.real_A[0]))), cam(tensor2numpy(fake_A2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(self.fake_A2A[0]))), cam(tensor2numpy(fake_A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(self.fake_A2B[0]))), cam(tensor2numpy(fake_A2B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm( self.fake_A2B2A[0])))), 0)), 1) B2A = np.concatenate( (B2A, np.concatenate( (RGB2BGR(tensor2numpy(denorm(self.real_B[0]))), cam(tensor2numpy(fake_B2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(self.fake_B2B[0]))), cam(tensor2numpy(fake_B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(self.fake_B2A[0]))), cam(tensor2numpy(fake_B2A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm( self.fake_B2A2B[0])))), 0)), 1) for _ in range(test_sample_num): try: self.real_A, _ = testA_iter.next() except: testA_iter = iter(self.testA_loader) self.real_A, _ = testA_iter.next() try: self.real_B, _ = testB_iter.next() except: testB_iter = iter(self.testB_loader) self.real_B, _ = testB_iter.next() self.real_A, self.real_B = self.real_A.to( self.device), self.real_B.to(self.device) self.fake_A2B, _, fake_A2B_heatmap = self.genA2B( self.real_A) self.fake_B2A, _, fake_B2A_heatmap = self.genB2A( self.real_B) self.fake_A2B2A, _, fake_A2B2A_heatmap = self.genB2A( self.fake_A2B) self.fake_B2A2B, _, fake_B2A2B_heatmap = self.genA2B( self.fake_B2A) self.fake_A2A, _, fake_A2A_heatmap = self.genB2A( self.real_A) self.fake_B2B, _, fake_B2B_heatmap = self.genA2B( self.real_B) A2B = np.concatenate( (A2B, np.concatenate( (RGB2BGR(tensor2numpy(denorm(self.real_A[0]))), cam(tensor2numpy(fake_A2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(self.fake_A2A[0]))), cam(tensor2numpy(fake_A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(self.fake_A2B[0]))), cam(tensor2numpy(fake_A2B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm( self.fake_A2B2A[0])))), 0)), 1) B2A = np.concatenate( (B2A, np.concatenate( (RGB2BGR(tensor2numpy(denorm(self.real_B[0]))), cam(tensor2numpy(fake_B2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(self.fake_B2B[0]))), cam(tensor2numpy(fake_B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(self.fake_B2A[0]))), cam(tensor2numpy(fake_B2A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm( self.fake_B2A2B[0])))), 0)), 1) cv2.imwrite( os.path.join(self.result_dir, self.dataset, 'img', 'A2B_%07d.png' % step), A2B * 255.0) cv2.imwrite( os.path.join(self.result_dir, self.dataset, 'img', 'B2A_%07d.png' % step), B2A * 255.0) self.genA2B.train(), self.genB2A.train(), self.disGA.train( ), self.disGB.train(), self.disLA.train(), self.disLB.train() if step % self.opt.display_freq == 0: # self.compute_visuals() # in pytorch_cyclegan,only used in colorization_model visualizer.display_current_results(self.get_current_visuals(), step, True) if step % self.save_freq == 0: self.save(os.path.join(self.result_dir, self.dataset, 'model'), step) if step % 1000 == 0: params = {} params['genA2B'] = self.genA2B.state_dict() params['genB2A'] = self.genB2A.state_dict() params['disGA'] = self.disGA.state_dict() params['disGB'] = self.disGB.state_dict() params['disLA'] = self.disLA.state_dict() params['disLB'] = self.disLB.state_dict() torch.save( params, os.path.join(self.result_dir, self.dataset + '_params_latest.pt')) visualizer.reset( ) # reset the visualizer: make sure it saves the results to HTML at least once every epoch
# (2) Update G network: maximize log(D(G(z))) ########################### netG.zero_grad() label.fill_(real_label) # fake labels are real for generator cost output = netD(fake) errG = criterion(output, label) errG.backward() D_G_z2 = output.mean().item() optimizerG.step() if i % 100 == 0: visuals = OrderedDict() visuals["real"] = real_cpu.detach() visuals["fake"] = fake.detach() losses = OrderedDict() losses["D_loss"] = errD.detach() losses["G_loss"] = errG.detach() losses["D"] = D_x losses["D_G_z1"] = D_G_z1 losses["D_G_z2"] = D_G_z2 vis.display_current_results(visuals, epoch, True) vis.plot_current_losses(epoch, i / len(dataloader), losses) print( '[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f' % (epoch, opt.niter, i, len(dataloader), errD.item(), errG.item(), D_x, D_G_z1, D_G_z2)) # do checkpointing torch.save(netG.state_dict(), '%s/netG_epoch_%d.pth' % (opt.outf, epoch)) torch.save(netD.state_dict(), '%s/netD_epoch_%d.pth' % (opt.outf, epoch))
def test_sample(folder_name, frame_no, list_of_boxes): global ROOT_DIR global vis if not vis: from visualizer import Visualizer settings = Object() settings.display_winsize = 8 settings.display_id = 1 settings.print_freq = 200 settings.port_id = 8097 settings.name = 'pcn' vis = Visualizer(settings) args = get_parameters() args.device = torch.device("cuda:%d" % (args.gpu_id) if torch.cuda.is_available() else "cpu") # for selecting device for chamfer loss args.checkpoints_dir = args.save_path torch.cuda.set_device(args.gpu_id) #os.makedirs(os.path.join('plots'), exist_ok=True) #os.makedirs(os.path.join('completions'), exist_ok=True) if (Kitti): data_dir = os.path.join( ROOT_DIR, "..", "test_dataset/{}/bin_data/{:010d}.bin".format( folder_name, int(frame_no))) elif (Argo): data_dir = os.path.join( ROOT_DIR, "..", "test_dataset/{}/sample/argoverse/lidar/{:03d}.bin".format( folder_name, int(frame_no))) #pcd_dir = os.path.join(ROOT_DIR, "..", "test_dataset", folder_name, "cars") #bbox_dir = os.path.join(ROOT_DIR, "..", "test_dataset", folder_name, "bboxes") results_dir = os.path.join(ROOT_DIR, "..", "test_dataset", "results") print("data_dir ", data_dir) print(args) total_time = 0 total_points = 0 plot_id = 0 c = np.cos s = np.sin rot_mat = lambda x: [[c(x), -s(x), 0], [s(x), c(x), 0], [0, 0, 1.0]] # avg_ = np.array([-5.5450159e-03, 3.1248237e-05, -1.7545074e-02]) # var_ = np.array([0.06499612, 0.01349955, 0.00575584]) # min_ = np.array([-0.4482678174972534, -0.17956072092056274, -0.12795628607273102]) # max_ = np.array([0.4482678174972534, 0.17956072092056274, 0.12795628607273102]) points = np.fromfile(data_dir, dtype=np.float32).reshape((-1, 4))[:, :3] print("Points look like: ") print(points[:10, :]) #for i, car_id in enumerate(car_ids): concat_complete = list() for j in range(len(list_of_boxes)): car_id = folder_name + '_frame_%d_car_%d' % (int(frame_no), j) cur_box = list_of_boxes[j] bbox = list( getCorners(0, cur_box['width'] + 0.3 * cur_box['width'], cur_box['length'] + 0.3 * cur_box['length'], cur_box['center']['x'], cur_box['center']['y'], cur_box['height'] / 2, -cur_box['angle'], rotation=True)) bbox = np.array(bbox) # original pointcloud partial_xyz = np.array( [point for point in points if within_bbox(point, bbox)]) total_points += partial_xyz.shape[0] plot_path = os.path.join(results_dir, 'plots', '%s.png' % car_id) # plot_pcd_three_views(plot_path, [partial_xyz], ['input'],'%d input points' % partial_xyz.shape[0], [5]) # Calculate center, rotation and scale center = (bbox.min(0) + bbox.max(0)) / 2 bbox -= center yaw = np.arctan2(bbox[3, 1] - bbox[0, 1], bbox[3, 0] - bbox[0, 0]) rotation = np.array([[np.cos(yaw), -np.sin(yaw), 0], [np.sin(yaw), np.cos(yaw), 0], [0, 0, 1]]) bbox = np.dot(bbox, rotation) # scale = bbox[3, 0] - bbox[0, 0] # bbox /= scale # Scope 0 # Scale and rotate to make cannonical, Rotate from y axis to x axis partial_xyz_center = np.dot(partial_xyz - center, rotation) #/ scale partial_xyz_center = np.dot(partial_xyz_center, np.array(rot_mat(90 * 3.14 / 180))) # Scope 1 # Ground removal partial_xyz_center_gr = partial_xyz_center[ partial_xyz_center[:, 2] > min(partial_xyz_center[:, 2]) + 0.2] # Scope 2 # Height based point removal base_pt = np.min(partial_xyz_center_gr[:, 2]) ht_pt = base_pt + 1.6 partial_xyz_center_ht = partial_xyz_center_gr[ partial_xyz_center_gr[:, 2] < ht_pt] max_x, max_y, max_z = np.max(partial_xyz_center_ht, axis=0) min_x, min_y, min_z = np.min(partial_xyz_center_ht, axis=0) mheight = min_z - max_z mheight = mheight if mheight > 1.3 else 1.6 mcenter_height = max_z + mheight / 2 # Scope 3 # Scale input min_ = np.array([ -0.4482678174972534, -0.17956072092056274, -0.12795628607273102 ]) # shapenet specific change according to the traning dataset max_ = np.array( [0.4482678174972534, 0.17956072092056274, 0.12795628607273102]) maxx = np.max(partial_xyz_center_ht, axis=0) minn = np.min(partial_xyz_center_ht, axis=0) avg_ = np.array([-5.5450159e-03, 3.1248237e-05, -1.7545074e-02]) var_ = np.array([0.06499612, 0.01349955, 0.00575584]) partial_xyz_center_ht[:, 2] += 1.6 + (1) / 2 # print(mcenter_height) # partial_xyz_center_ht[:, 2] -=mcenter_height act_mean = np.mean(partial_xyz_center_ht) act_var = np.var(partial_xyz_center_ht, axis=0) # partial_xyz_center_gr_ht_scaled = avg_ + (partial_xyz_center_ht - act_mean)* var_/act_var partial_xyz_center_gr_ht_scaled = ((partial_xyz_center_ht) / (maxx[2] - minn[2])) * (0.38) # Get output start = time.time() completion_xyz_center_out = main( args, input_data=partial_xyz_center_gr_ht_scaled) if completion_xyz_center_out.shape[0] == 3: completion_xyz_center_out = completion_xyz_center_out.transpose() total_time += time.time() - start # completion_xyz = np.dot(completion_xyz_center * scale, rotation.T) + center # Scale output back completion_xyz_center = ( (completion_xyz_center_out) / 0.295) * (maxx[2] - minn[2]) # completion_xyz_center = act_mean + (completion_xyz_center_out - avg_)* act_var/var_ completion_xyz_center[:, 2] -= 1.3 + (1) / 2 # completion_xyz_center[:, 2] +=mcenter_height # Rotate output back completion_xyz_center = np.dot(completion_xyz_center, np.array(rot_mat(-90 * 3.14 / 180))) # Rotate translate output back completion_xyz = np.dot(completion_xyz_center, rotation.T) + center pcd_path = os.path.join(results_dir, 'completions_rl', '%s.pcd' % car_id) if vis: print("plotting") # Scope 0 # Check axis x and y. Check if at (0,0) visuals = OrderedDict([ ('Partial_pc partial_xyz_center ', partial_xyz_center) ]) # ('Complete Predicted_pc ', completion) vis.display_current_results(visuals, 0, plot_id) plot_id += 1 # Scope 1 # Check ground removed points visuals = OrderedDict([ ('Partial_pc partial_xyz_center_gr ', partial_xyz_center_gr) ]) # ('Complete Predicted_pc ', completion) vis.display_current_results(visuals, 0, plot_id) plot_id += 1 # Scope 2 # Check height removed points visuals = OrderedDict([ ('Partial_pc partial_xyz_center_ht ', partial_xyz_center_ht) ]) # ('Complete Predicted_pc ', completion) vis.display_current_results(visuals, 0, plot_id) plot_id += 1 # Scope 3 # Check height scaled points visuals = OrderedDict([ ('Partial_pc partial_xyz_center_gr_ht_scaled ', partial_xyz_center_gr_ht_scaled) ]) # ('Complete Predicted_pc ', completion) vis.display_current_results(visuals, 0, plot_id) plot_id += 1 # Check direct output visuals = OrderedDict([('Complete Predicted_pc ', completion_xyz_center_out)]) vis.display_current_results(visuals, 0, plot_id) plot_id += 1 # Check output scaled visuals = OrderedDict([('Complete Predicted_pc ', completion_xyz_center)]) vis.display_current_results(visuals, 0, plot_id) plot_id += 1 # Check translated and rotaed to original poistion visuals = OrderedDict([('Complete Predicted_pc ', completion_xyz)]) vis.display_current_results(visuals, 0, plot_id) plot_id += 1 both_scaled = np.concatenate( (partial_xyz_center_gr_ht_scaled, completion_xyz_center_out)) both_final = np.concatenate((partial_xyz_center, completion_xyz)) visuals = OrderedDict([('Both pcs scaled', both_scaled)]) vis.display_current_results(visuals, 0, plot_id) plot_id += 1 visuals = OrderedDict([('Both pcs original', both_final)]) vis.display_current_results(visuals, 0, plot_id) plot_id += 1 concat_complete.append(completion_xyz) print('Average # input points:', total_points / len(list_of_boxes)) print('Average time:', total_time / len(list_of_boxes)) return concat_complete
def main(): opt = parser.parse_args() torch.manual_seed(opt.seed) # set gpu_ids str_ids = opt.gpu_ids.split(',') opt.gpu_ids = [] for str_id in str_ids: id = int(str_id) if id >= 0: opt.gpu_ids.append(id) if len(opt.gpu_ids) > 0: torch.cuda.set_device(opt.gpu_ids[0]) cudnn.benchmark = True print(opt) if opt.SR_name == 'IDN': model = IDNModel() train_dataset = DatasetFromNpyTrain(opt, rgb=False, input_up=True) test_dataset = DatasetFromNpyTest(opt, rgb=False, input_up=True) elif opt.SR_name == 'RCAN': model = RCANModel() train_dataset = DatasetFromNpyTrain(opt, rgb=True) test_dataset = DatasetFromNpyTest(opt, rgb=True) elif opt.SR_name == 'EDSR': model = EDSRModel() train_dataset = DatasetFromNpyTrain(opt, rgb=True) test_dataset = DatasetFromNpyTest(opt, rgb=True) elif opt.SR_name == 'RDN': model = RDNModel() train_dataset = DatasetFromNpyTrain(opt, rgb=True) test_dataset = DatasetFromNpyTest(opt, rgb=True) else: raise NotImplementedError('%s is not supported!' % opt.SR_name) visualizer = Visualizer() print(len(train_dataset)) train_loader = DataLoader(train_dataset, batch_size=opt.batch_size) test_loader = DataLoader(test_dataset, batch_size=1) model.initialize(opt) if opt.epoch > 0: model.load_model(opt.epoch, opt.SR_name) for epoch in range(opt.epoch + 1, opt.nEpochs + 1): model.set_mode(train=True) for i, data in enumerate(train_loader, 1): model.set_input(data) loss = model.train() if i % 50 == 0: images = {'input': model.input, 'output': model.output, 'target': model.target} visualizer.display_current_results(images, k=0) if i % 10 == 0: print('epoch: {}, iteration: {}/{}, loss: {}'.format(epoch, i, len(train_loader), loss.item())) visualizer.plot_current_loss(loss.item()) model.scheduler.step(epoch) # update learning rate print('Learning rate: %f' % model.scheduler.get_lr()[0]) if epoch % opt.save_epoch == 0: print('a') model.set_mode(train=False) average_psnr = [] average_ssim = [] for i, data in enumerate(test_loader): model.set_eval_input(data) outputs = model.eval() psnr_, ssim_ = model.comput_PSNR_SSIM(outputs['output'], outputs['target'], shave_border=opt.sr_factor) average_psnr.append(psnr_) average_ssim.append(ssim_) img_type = ['input', 'output', 'target'] for name, img in zip(img_type, outputs): save_name = os.path.join(model.result_dir, '%d_%s.png' % (i, name)) model.save_image(outputs[img], save_name) average_psnr = np.average(average_psnr) average_ssim = np.average(average_ssim) log = 'Epoch %d: Average psnr: %f , ssim: %f \n' % (epoch, average_psnr, average_ssim) print(log) model.log_file.write(log) model.save_model(epoch, opt.SR_name) model.log_file.close()