class UGATIT(object): def __init__(self, args): self.light = args.light if self.light: self.model_name = 'UGATIT_light' else: self.model_name = 'UGATIT' self.result_dir = args.result_dir self.dataset = args.dataset self.iteration = args.iteration self.decay_flag = args.decay_flag self.batch_size = args.batch_size self.print_freq = args.print_freq self.save_freq = args.save_freq self.lr = args.lr self.weight_decay = args.weight_decay self.ch = args.ch """ Weight """ self.adv_weight = args.adv_weight self.cycle_weight = args.cycle_weight self.identity_weight = args.identity_weight self.cam_weight = args.cam_weight """ Generator """ self.n_res = args.n_res """ Discriminator """ self.n_dis = args.n_dis self.img_size = args.img_size self.img_ch = args.img_ch self.device = args.device self.benchmark_flag = args.benchmark_flag self.resume = args.resume if torch.backends.cudnn.enabled and self.benchmark_flag: print('set benchmark !') torch.backends.cudnn.benchmark = True print() print("##### Information #####") print("# light : ", self.light) print("# dataset : ", self.dataset) print("# batch_size : ", self.batch_size) print("# iteration per epoch : ", self.iteration) print() print("##### Generator #####") print("# residual blocks : ", self.n_res) print() print("##### Discriminator #####") print("# discriminator layer : ", self.n_dis) print() print("##### Weight #####") print("# adv_weight : ", self.adv_weight) print("# cycle_weight : ", self.cycle_weight) print("# identity_weight : ", self.identity_weight) print("# cam_weight : ", self.cam_weight) ################################################################################## # Model ################################################################################## def build_model(self): """ DataLoader """ train_transform = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.Resize((self.img_size + 30, self.img_size+30)), transforms.RandomCrop(self.img_size), transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)) ]) test_transform = transforms.Compose([ transforms.Resize((self.img_size, self.img_size)), transforms.ToTensor(), transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)) ]) self.trainA = ImageFolder(os.path.join('dataset', self.dataset, 'trainA'), train_transform) self.trainB = ImageFolder(os.path.join('dataset', self.dataset, 'trainB'), train_transform) self.testA = ImageFolder(os.path.join('dataset', self.dataset, 'testA'), test_transform) self.testB = ImageFolder(os.path.join('dataset', self.dataset, 'testB'), test_transform) self.trainA_loader = DataLoader(self.trainA, batch_size=self.batch_size, shuffle=True) self.trainB_loader = DataLoader(self.trainB, batch_size=self.batch_size, shuffle=True) self.testA_loader = DataLoader(self.testA, batch_size=1, shuffle=False) self.testB_loader = DataLoader(self.testB, batch_size=1, shuffle=False) """ Define Generator, Discriminator """ self.genA2B = ResnetGenerator(input_nc=3, output_nc=3, ngf=self.ch, n_blocks=self.n_res, img_size=self.img_size, light=self.light).to(self.device) self.genB2A = ResnetGenerator(input_nc=3, output_nc=3, ngf=self.ch, n_blocks=self.n_res, img_size=self.img_size, light=self.light).to(self.device) self.disGA = Discriminator(input_nc=3, ndf=self.ch, n_layers=7).to(self.device) self.disGB = Discriminator(input_nc=3, ndf=self.ch, n_layers=7).to(self.device) self.disLA = Discriminator(input_nc=3, ndf=self.ch, n_layers=5).to(self.device) self.disLB = Discriminator(input_nc=3, ndf=self.ch, n_layers=5).to(self.device) """ Define Loss """ self.L1_loss = nn.L1Loss().to(self.device) self.MSE_loss = nn.MSELoss().to(self.device) self.BCE_loss = nn.BCEWithLogitsLoss().to(self.device) """ Trainer """ self.G_optim = torch.optim.Adam(itertools.chain(self.genA2B.parameters(), self.genB2A.parameters()), lr=self.lr, betas=(0.5, 0.999), weight_decay=self.weight_decay) self.D_optim = torch.optim.Adam(itertools.chain(self.disGA.parameters(), self.disGB.parameters(), self.disLA.parameters(), self.disLB.parameters()), lr=self.lr, betas=(0.5, 0.999), weight_decay=self.weight_decay) """ Define Rho clipper to constraint the value of rho in AdaILN and ILN""" self.Rho_clipper = RhoClipper(0, 1) def train(self): self.genA2B.train(), self.genB2A.train() self.disGA.train(), self.disGB.train() self.disLA.train(), self.disLB.train() start_iter = 1 if self.resume: model_list = glob(os.path.join(self.result_dir, self.dataset, 'model', '*.pt')) if not len(model_list) == 0: model_list.sort() start_iter = int(model_list[-1].split('_')[-1].split('.')[0]) self.load(os.path.join(self.result_dir, self.dataset, 'model'), start_iter) print(" [*] Load SUCCESS") if self.decay_flag and start_iter > (self.iteration // 2): self.G_optim.param_groups[0]['lr'] -= (self.lr / (self.iteration // 2)) \ * (start_iter - self.iteration // 2) self.D_optim.param_groups[0]['lr'] -= (self.lr / (self.iteration // 2)) \ * (start_iter - self.iteration // 2) # training loop print('training start !') start_time = time.time() for step in range(start_iter, self.iteration + 1): if self.decay_flag and step > (self.iteration // 2): self.G_optim.param_groups[0]['lr'] -= ( self.lr / (self.iteration // 2)) self.D_optim.param_groups[0]['lr'] -= ( self.lr / (self.iteration // 2)) try: real_A, _ = trainA_iter.next() # noqa: F821 except Exception: trainA_iter = iter(self.trainA_loader) real_A, _ = trainA_iter.next() try: real_B, _ = trainB_iter.next() # noqa: F821 except Exception: trainB_iter = iter(self.trainB_loader) real_B, _ = trainB_iter.next() real_A, real_B = real_A.to(self.device), real_B.to(self.device) # Update D self.D_optim.zero_grad() fake_A2B, _, _ = self.genA2B(real_A) fake_B2A, _, _ = self.genB2A(real_B) real_GA_logit, real_GA_cam_logit, _ = self.disGA(real_A) real_LA_logit, real_LA_cam_logit, _ = self.disLA(real_A) real_GB_logit, real_GB_cam_logit, _ = self.disGB(real_B) real_LB_logit, real_LB_cam_logit, _ = self.disLB(real_B) fake_GA_logit, fake_GA_cam_logit, _ = self.disGA(fake_B2A) fake_LA_logit, fake_LA_cam_logit, _ = self.disLA(fake_B2A) fake_GB_logit, fake_GB_cam_logit, _ = self.disGB(fake_A2B) fake_LB_logit, fake_LB_cam_logit, _ = self.disLB(fake_A2B) D_ad_loss_GA = self.MSE_loss(real_GA_logit, torch.ones_like(real_GA_logit).to( self.device)) + self.MSE_loss(fake_GA_logit, torch.zeros_like(fake_GA_logit).to(self.device)) D_ad_cam_loss_GA = self.MSE_loss(real_GA_cam_logit, torch.ones_like(real_GA_cam_logit).to( self.device)) + self.MSE_loss(fake_GA_cam_logit, torch.zeros_like(fake_GA_cam_logit).to(self.device)) D_ad_loss_LA = self.MSE_loss(real_LA_logit, torch.ones_like(real_LA_logit).to( self.device)) + self.MSE_loss(fake_LA_logit, torch.zeros_like(fake_LA_logit).to(self.device)) D_ad_cam_loss_LA = self.MSE_loss(real_LA_cam_logit, torch.ones_like(real_LA_cam_logit).to( self.device)) + self.MSE_loss(fake_LA_cam_logit, torch.zeros_like(fake_LA_cam_logit).to(self.device)) D_ad_loss_GB = self.MSE_loss(real_GB_logit, torch.ones_like(real_GB_logit).to( self.device)) + self.MSE_loss(fake_GB_logit, torch.zeros_like(fake_GB_logit).to(self.device)) D_ad_cam_loss_GB = self.MSE_loss(real_GB_cam_logit, torch.ones_like(real_GB_cam_logit).to( self.device)) + self.MSE_loss(fake_GB_cam_logit, torch.zeros_like(fake_GB_cam_logit).to(self.device)) D_ad_loss_LB = self.MSE_loss(real_LB_logit, torch.ones_like(real_LB_logit).to( self.device)) + self.MSE_loss(fake_LB_logit, torch.zeros_like(fake_LB_logit).to(self.device)) D_ad_cam_loss_LB = self.MSE_loss(real_LB_cam_logit, torch.ones_like(real_LB_cam_logit).to( self.device)) + self.MSE_loss(fake_LB_cam_logit, torch.zeros_like(fake_LB_cam_logit).to(self.device)) D_loss_A = self.adv_weight * \ (D_ad_loss_GA + D_ad_cam_loss_GA + D_ad_loss_LA + D_ad_cam_loss_LA) D_loss_B = self.adv_weight * \ (D_ad_loss_GB + D_ad_cam_loss_GB + D_ad_loss_LB + D_ad_cam_loss_LB) Discriminator_loss = D_loss_A + D_loss_B Discriminator_loss.backward() self.D_optim.step() # Update G self.G_optim.zero_grad() fake_A2B, fake_A2B_cam_logit, _ = self.genA2B(real_A) fake_B2A, fake_B2A_cam_logit, _ = self.genB2A(real_B) fake_A2B2A, _, _ = self.genB2A(fake_A2B) fake_B2A2B, _, _ = self.genA2B(fake_B2A) fake_A2A, fake_A2A_cam_logit, _ = self.genB2A(real_A) fake_B2B, fake_B2B_cam_logit, _ = self.genA2B(real_B) fake_GA_logit, fake_GA_cam_logit, _ = self.disGA(fake_B2A) fake_LA_logit, fake_LA_cam_logit, _ = self.disLA(fake_B2A) fake_GB_logit, fake_GB_cam_logit, _ = self.disGB(fake_A2B) fake_LB_logit, fake_LB_cam_logit, _ = self.disLB(fake_A2B) G_ad_loss_GA = self.MSE_loss( fake_GA_logit, torch.ones_like(fake_GA_logit).to(self.device)) G_ad_cam_loss_GA = self.MSE_loss( fake_GA_cam_logit, torch.ones_like(fake_GA_cam_logit).to(self.device)) G_ad_loss_LA = self.MSE_loss( fake_LA_logit, torch.ones_like(fake_LA_logit).to(self.device)) G_ad_cam_loss_LA = self.MSE_loss( fake_LA_cam_logit, torch.ones_like(fake_LA_cam_logit).to(self.device)) G_ad_loss_GB = self.MSE_loss( fake_GB_logit, torch.ones_like(fake_GB_logit).to(self.device)) G_ad_cam_loss_GB = self.MSE_loss( fake_GB_cam_logit, torch.ones_like(fake_GB_cam_logit).to(self.device)) G_ad_loss_LB = self.MSE_loss( fake_LB_logit, torch.ones_like(fake_LB_logit).to(self.device)) G_ad_cam_loss_LB = self.MSE_loss( fake_LB_cam_logit, torch.ones_like(fake_LB_cam_logit).to(self.device)) G_recon_loss_A = self.L1_loss(fake_A2B2A, real_A) G_recon_loss_B = self.L1_loss(fake_B2A2B, real_B) G_identity_loss_A = self.L1_loss(fake_A2A, real_A) G_identity_loss_B = self.L1_loss(fake_B2B, real_B) G_cam_loss_A = self.BCE_loss(fake_B2A_cam_logit, torch.ones_like(fake_B2A_cam_logit).to( self.device)) + self.BCE_loss(fake_A2A_cam_logit, torch.zeros_like(fake_A2A_cam_logit).to(self.device)) G_cam_loss_B = self.BCE_loss(fake_A2B_cam_logit, torch.ones_like(fake_A2B_cam_logit).to( self.device)) + self.BCE_loss(fake_B2B_cam_logit, torch.zeros_like(fake_B2B_cam_logit).to(self.device)) G_loss_A = self.adv_weight * (G_ad_loss_GA + G_ad_cam_loss_GA + G_ad_loss_LA + G_ad_cam_loss_LA) + \ self.cycle_weight * G_recon_loss_A + self.identity_weight * \ G_identity_loss_A + self.cam_weight * G_cam_loss_A G_loss_B = self.adv_weight * (G_ad_loss_GB + G_ad_cam_loss_GB + G_ad_loss_LB + G_ad_cam_loss_LB) + \ self.cycle_weight * G_recon_loss_B + self.identity_weight * \ G_identity_loss_B + self.cam_weight * G_cam_loss_B Generator_loss = G_loss_A + G_loss_B Generator_loss.backward() self.G_optim.step() # clip parameter of AdaILN and ILN, applied after optimizer step self.genA2B.apply(self.Rho_clipper) self.genB2A.apply(self.Rho_clipper) msg = "[%5d/%5d] time: %4.4f d_loss: %.8f, g_loss: %.8f" % (step, self.iteration, time.time() - start_time, Discriminator_loss, Generator_loss) print(msg) if step % self.print_freq == 0: train_sample_num = 5 test_sample_num = 5 A2B = np.zeros((self.img_size * 7, 0, 3)) B2A = np.zeros((self.img_size * 7, 0, 3)) self.genA2B.eval(), self.genB2A.eval() self.disGA.eval(), self.disGB.eval() self.disLA.eval(), self.disLB.eval() for _ in range(train_sample_num): try: real_A, _ = trainA_iter.next() except Exception: trainA_iter = iter(self.trainA_loader) real_A, _ = trainA_iter.next() try: real_B, _ = trainB_iter.next() except Exception: trainB_iter = iter(self.trainB_loader) real_B, _ = trainB_iter.next() real_A, real_B = real_A.to(self.device), real_B.to(self.device) fake_A2B, _, fake_A2B_heatmap = self.genA2B(real_A) fake_B2A, _, fake_B2A_heatmap = self.genB2A(real_B) fake_A2B2A, _, fake_A2B2A_heatmap = self.genB2A(fake_A2B) fake_B2A2B, _, fake_B2A2B_heatmap = self.genA2B(fake_B2A) fake_A2A, _, fake_A2A_heatmap = self.genB2A(real_A) fake_B2B, _, fake_B2B_heatmap = self.genA2B(real_B) A2B = np.concatenate((A2B, np.concatenate((RGB2BGR(tensor2numpy(denorm(real_A[0]))), cam(tensor2numpy(fake_A2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2A[0]))), cam(tensor2numpy(fake_A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B[0]))), cam(tensor2numpy(fake_A2B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B2A[0])))), 0)), 1) B2A = np.concatenate((B2A, np.concatenate((RGB2BGR(tensor2numpy(denorm(real_B[0]))), cam(tensor2numpy(fake_B2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2B[0]))), cam(tensor2numpy(fake_B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A[0]))), cam(tensor2numpy(fake_B2A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A2B[0])))), 0)), 1) for _ in range(test_sample_num): try: real_A, _ = testA_iter.next() # noqa: F821 except Exception: testA_iter = iter(self.testA_loader) real_A, _ = testA_iter.next() try: real_B, _ = testB_iter.next() # noqa: F821 except Exception: testB_iter = iter(self.testB_loader) real_B, _ = testB_iter.next() real_A, real_B = real_A.to(self.device), real_B.to(self.device) fake_A2B, _, fake_A2B_heatmap = self.genA2B(real_A) fake_B2A, _, fake_B2A_heatmap = self.genB2A(real_B) fake_A2B2A, _, fake_A2B2A_heatmap = self.genB2A(fake_A2B) fake_B2A2B, _, fake_B2A2B_heatmap = self.genA2B(fake_B2A) fake_A2A, _, fake_A2A_heatmap = self.genB2A(real_A) fake_B2B, _, fake_B2B_heatmap = self.genA2B(real_B) A2B = np.concatenate((A2B, np.concatenate((RGB2BGR(tensor2numpy(denorm(real_A[0]))), cam(tensor2numpy(fake_A2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2A[0]))), cam(tensor2numpy(fake_A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B[0]))), cam(tensor2numpy(fake_A2B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B2A[0])))), 0)), 1) B2A = np.concatenate((B2A, np.concatenate((RGB2BGR(tensor2numpy(denorm(real_B[0]))), cam(tensor2numpy(fake_B2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2B[0]))), cam(tensor2numpy(fake_B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A[0]))), cam(tensor2numpy(fake_B2A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A2B[0])))), 0)), 1) cv2.imwrite(os.path.join(self.result_dir, self.dataset, 'img', 'A2B_%07d.png' % step), A2B * 255.0) cv2.imwrite(os.path.join(self.result_dir, self.dataset, 'img', 'B2A_%07d.png' % step), B2A * 255.0) self.genA2B.train(), self.genB2A.train() self.disGA.train(), self.disGB.train() self.disLA.train(), self.disLB.train() if step % self.save_freq == 0: self.save(os.path.join(self.result_dir, self.dataset, 'model'), step) if step % 1000 == 0: params = {} params['genA2B'] = self.genA2B.state_dict() params['genB2A'] = self.genB2A.state_dict() params['disGA'] = self.disGA.state_dict() params['disGB'] = self.disGB.state_dict() params['disLA'] = self.disLA.state_dict() params['disLB'] = self.disLB.state_dict() torch.save(params, os.path.join(self.result_dir, self.dataset + '_params_latest.pt')) def save(self, dir, step): params = {} params['genA2B'] = self.genA2B.state_dict() params['genB2A'] = self.genB2A.state_dict() params['disGA'] = self.disGA.state_dict() params['disGB'] = self.disGB.state_dict() params['disLA'] = self.disLA.state_dict() params['disLB'] = self.disLB.state_dict() torch.save(params, os.path.join(dir, self.dataset + '_params_%07d.pt' % step)) def load(self, dir, step): params = torch.load(os.path.join(dir, self.dataset + '_params_%07d.pt' % step)) self.genA2B.load_state_dict(params['genA2B']) self.genB2A.load_state_dict(params['genB2A']) self.disGA.load_state_dict(params['disGA']) self.disGB.load_state_dict(params['disGB']) self.disLA.load_state_dict(params['disLA']) self.disLB.load_state_dict(params['disLB']) def test(self): model_list = glob(os.path.join(self.result_dir, self.dataset, 'model', '*.pt')) if not len(model_list) == 0: model_list.sort() iter = int(model_list[-1].split('_')[-1].split('.')[0]) self.load(os.path.join(self.result_dir, self.dataset, 'model'), iter) print(" [*] Load SUCCESS") else: print(" [*] Load FAILURE") return self.genA2B.eval(), self.genB2A.eval() for n, (real_A, _) in enumerate(self.testA_loader): real_A = real_A.to(self.device) fake_A2B, _, fake_A2B_heatmap = self.genA2B(real_A) fake_A2B2A, _, fake_A2B2A_heatmap = self.genB2A(fake_A2B) fake_A2A, _, fake_A2A_heatmap = self.genB2A(real_A) A2B = np.concatenate((RGB2BGR(tensor2numpy(denorm(real_A[0]))), cam(tensor2numpy(fake_A2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2A[0]))), cam(tensor2numpy(fake_A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B[0]))), cam(tensor2numpy(fake_A2B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B2A[0])))), 0) cv2.imwrite(os.path.join(self.result_dir, self.dataset, 'test', 'A2B_%d.png' % (n + 1)), A2B * 255.0) for n, (real_B, _) in enumerate(self.testB_loader): real_B = real_B.to(self.device) fake_B2A, _, fake_B2A_heatmap = self.genB2A(real_B) fake_B2A2B, _, fake_B2A2B_heatmap = self.genA2B(fake_B2A) fake_B2B, _, fake_B2B_heatmap = self.genA2B(real_B) B2A = np.concatenate((RGB2BGR(tensor2numpy(denorm(real_B[0]))), cam(tensor2numpy(fake_B2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2B[0]))), cam(tensor2numpy(fake_B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A[0]))), cam(tensor2numpy(fake_B2A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A2B[0])))), 0) cv2.imwrite(os.path.join(self.result_dir, self.dataset, 'test', 'B2A_%d.png' % (n + 1)), B2A * 255.0)
class OurModel(nn.Module): def __init__(self, hp, class_emb_vis, class_emb_all): super(OurModel, self).__init__() self.hp = hp self.Em_vis = nn.Embedding.from_pretrained(class_emb_vis).cuda() self.Em_vis.weight.requires_grad = False self.Em_all = nn.Embedding.from_pretrained(class_emb_all).cuda() self.Em_all.weight.requires_grad = False self.prior = np.ones((hp['dis']['out_dim_cls'] - 1)) for k in range(hp['dis']['out_dim_cls'] - hp['num_unseen'] - 1, hp['dis']['out_dim_cls'] - 1): self.prior[k] = self.prior[k] + hp['gen_unseen_rate'] self.prior_ = self.prior / np.linalg.norm(self.prior, ord=1) self.gen = Generator(hp['gen']) self.dis = Discriminator(hp['dis']) self.back = DeepLabV2_ResNet101_local_MSC(hp['back']) self.discLoss, self.contentLoss, self.clsLoss = init_loss(hp) def forward(self, data, gt, mode): assert (mode == 'step1' or mode == 'step2') self.init_all(mode) flag = 1 try: ignore_mask = (gt != self.hp['ignore_index']).cuda() if not (ignore_mask.sum() > 0): # meaningless batch raise MeaninglessError() if mode == 'step1': # step1 self.set_mode('step1') self.loss_KLD, self.target_all, self.target, self.contextual = self.back( data, ignore_mask) self.target_shape_all = [x.shape for x in self.target_all] self.gt_all = [ resize_target(gt, x[2]).cuda() for x in self.target_shape_all ] self.ignore_mask_all = [(x != self.hp['ignore_index']).cuda() for x in self.gt_all] if not all([x.sum() > 0 for x in self.ignore_mask_all ]): # meaningless batch raise MeaninglessError() # self.target_shape = self.target.shape # self.contextual_shape = self.contextual.shape self.gt = resize_target(gt, self.target.shape[2]).cuda() self.ignore_mask = (self.gt != self.hp['ignore_index']).cuda() if not (self.ignore_mask.sum() > 0): # meaningless batch raise MeaninglessError() condition = self.Em_vis(self.gt).permute(0, 3, 1, 2).contiguous() self.sample = torch.cat((condition, self.contextual), dim=1) self.predict = self.gen(self.sample.detach()) else: # step2 self.set_mode('step2') with torch.no_grad(): _, _, self.target, self.contextual = self.back( data, ignore_mask) self.target_shape = self.target.shape self.contextual_shape = self.contextual.shape self.gt = torch.LongTensor( np.random.choice( #a=range(self.Em_all.shape[0]), a=range(self.hp['dis']['out_dim_cls'] - 1), size=(self.target_shape[0], self.target_shape[2], self.target_shape[3]), replace=True, p=self.prior_)).cuda() self.ignore_mask = (self.gt != self.hp['ignore_index']).cuda() if not (self.ignore_mask.sum() > 0): # meaningless batch raise MeaninglessError() condition = self.Em_all(self.gt).permute(0, 3, 1, 2).contiguous() random_noise = torch.randn(self.contextual_shape).cuda() self.sample = torch.cat((condition, random_noise), dim=1) self.predict = self.gen(self.sample.detach()) except MeaninglessError: flag = -1 assert (flag == 1 or flag == -1) if flag == 1: self.get_loss_D(mode) if self.hp['update_back'] == 't': self.get_loss_B() self.get_loss_G(mode) return self.get_losses(flag, mode) def test(self, data, gt): with torch.no_grad(): self.set_mode('test') flag = 1 try: ignore_mask = (gt != self.hp['ignore_index']).cuda() _, _, self.target, _ = self.back(data, ignore_mask) self.gt = resize_target(gt, self.target.shape[2]).cuda() self.ignore_mask = (self.gt != self.hp['ignore_index']).cuda() if not (self.ignore_mask.sum() > 0): # meaningless batch raise MeaninglessError() except MeaninglessError: flag = -1 assert (flag == 1 or flag == -1) if flag == 1: self.get_loss_D('test') return self.get_losses(flag, 'test') def get_loss_D(self, mode): assert (mode == 'step1' or mode == 'step2' or mode == 'test') if mode == 'step1': self.loss_D_GAN, self.loss_D_real, self.loss_D_fake, self.loss_D_gp = \ self.discLoss(self.dis, self.predict.detach(), self.target.detach(), self.ignore_mask) self.loss_cls_fake, self.acc_cls_fake, _, _ = self.clsLoss( self.dis, self.predict, self.gt, self.ignore_mask) for (target, gt, ignore_mask) in zip(self.target_all, self.gt_all, self.ignore_mask_all): loss_cls_real, acc_cls_real, _, _ = self.clsLoss( self.dis, target, gt, ignore_mask) # backward to backbone, no detach self.loss_cls_real += loss_cls_real self.acc_cls_real += acc_cls_real total = len(self.target_all) self.loss_cls_real /= total self.acc_cls_real /= total self.loss_D_cls_fake = self.loss_cls_fake * self.hp[ 'lambda_D_cls_fake'] self.loss_D_cls_real = self.loss_cls_real * self.hp[ 'lambda_D_cls_real'] self.loss_D_cls = self.loss_D_cls_fake + self.loss_D_cls_real self.loss_D = self.loss_D_GAN + self.loss_D_cls elif mode == 'step2': self.loss_cls_fake, self.acc_cls_fake, _, _ = self.clsLoss( self.dis, self.predict, self.gt, self.ignore_mask) # backward to generator, no detach self.loss_D_cls_fake = self.loss_cls_fake * self.hp[ 'lambda_D_cls_fake_transfer'] self.loss_D_cls = self.loss_D_cls_fake self.loss_D = self.loss_D_cls else: with torch.no_grad(): _, _, self.pred_cls_real, self.sorted_indices = self.clsLoss( self.dis, self.target, self.gt, self.ignore_mask) def get_loss_G(self, mode): assert (mode == 'step1' or mode == 'step2') if mode == 'step1': self.loss_G_GAN = self.discLoss.get_g_loss(self.dis, self.predict, self.ignore_mask) loss_G_Content = self.contentLoss(self.predict, self.target.detach(), self.gt, self.ignore_mask) self.loss_G_Content = loss_G_Content * self.hp['lambda_G_Content'] self.loss_G_cls = self.loss_cls_fake * self.hp['lambda_G_cls'] self.loss_G = self.loss_G_GAN * self.hp[ 'lambda_G_GAN'] + self.loss_G_Content + self.loss_G_cls else: self.loss_G_cls = self.loss_cls_fake * self.hp[ 'lambda_G_cls_transfer'] self.loss_G = self.loss_G_cls def get_loss_B(self): self.loss_B_KLD = self.loss_KLD * self.hp['lambda_B_KLD'] self.loss_B_cls = self.loss_cls_real * self.hp['lambda_B_cls'] self.loss_B = self.loss_B_KLD + self.loss_B_cls def set_mode(self, mode): assert (mode == 'step1' or mode == 'step2' or mode == 'test') if mode == 'step1': self.train() self.back.freeze_bn() elif mode == 'step2': self.train() self.back.eval() else: self.eval() self.dis.eval() self.back.eval() self.gen.eval() self.Em_vis.eval() self.Em_all.eval() def init_all(self, mode): assert (mode == 'step1' or mode == 'step2') if mode == 'step1': self.loss_G_GAN = 0 self.loss_G_Content = 0 self.loss_G_cls = 0 self.loss_G = 0 self.loss_B_KLD = 0 self.loss_B_cls = 0 self.loss_B = 0 self.loss_D_real = 0 self.loss_D_fake = 0 self.loss_D_gp = 0 self.loss_D_GAN = 0 self.loss_D_cls_real = 0 self.loss_D_cls_fake = 0 self.loss_D_cls = 0 self.loss_D = 0 self.loss_cls_real = 0 self.loss_cls_fake = 0 self.acc_cls_real = 0 self.acc_cls_fake = 0 else: self.loss_G_cls = 0 self.loss_G = 0 self.loss_D_cls_fake = 0 self.loss_D_cls = 0 self.loss_D = 0 self.loss_cls_fake = 0 self.acc_cls_fake = 0 def get_losses(self, flag, mode): assert (mode == 'step1' or mode == 'step2' or mode == 'test') zero_tensor = torch.from_numpy(np.array(0)).cuda() if mode == 'step1': if flag == 1: return torch.from_numpy(np.array(flag)).long().cuda(),\ self.loss_G_GAN,\ self.loss_G_Content,\ self.loss_G_cls,\ self.loss_G,\ self.loss_B_KLD if self.hp['update_back'] == 't' else zero_tensor,\ self.loss_B_cls if self.hp['update_back'] == 't' else zero_tensor,\ self.loss_B if self.hp['update_back'] == 't' else zero_tensor,\ self.loss_D_real,\ self.loss_D_fake,\ self.loss_D_gp if self.loss_D_gp != None else zero_tensor,\ self.loss_D_GAN,\ self.loss_D_cls_real,\ self.loss_D_cls_fake,\ self.loss_D_cls,\ self.loss_D,\ self.loss_cls_real,\ self.loss_cls_fake,\ self.acc_cls_real,\ self.acc_cls_fake else: return torch.from_numpy(np.array(flag)).long().cuda(),\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor elif mode == 'step2': if flag == 1: return torch.from_numpy(np.array(flag)).long().cuda(),\ self.loss_G_cls,\ self.loss_G,\ self.loss_D_cls_fake,\ self.loss_D_cls,\ self.loss_D,\ self.loss_cls_fake,\ self.acc_cls_fake else: return torch.from_numpy(np.array(flag)).long().cuda(),\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor else: with torch.no_grad(): if flag == 1: return torch.from_numpy(np.array(flag)).long().cuda(),\ self.pred_cls_real,\ self.sorted_indices,\ self.gt,\ self.ignore_mask # original label and corresponding ignore mask else: return torch.from_numpy(np.array(flag)).long().cuda(),\ zero_tensor,\ zero_tensor,\ zero_tensor,\ zero_tensor
class fgan(object): """ This class ensembles data generating process of Huber's contamination model and training process for estimating center parameter via F-GAN. Usage: >> f = fgan(p=100, eps=0.2, device=device, tol=1e-5) >> f.dist_init(true_type='Gaussian', cont_type='Gaussian', cont_mean=5.0, cont_var=1.) >> f.data_init(train_size=50000, batch_size=500) >> f.net_init(d_hidden_units=[20], elliptical=False, activation_D1='LeakyReLU') >> f.optimizer_init(lr_d=0.2, lr_g=0.02, d_steps=5, g_steps=1) >> f.fit(floss='js', epochs=150, avg_epochs=25, verbose=50, show=True) Please refer to the Demo.ipynb for more examples. """ def __init__(self, p, eps, device=None, tol=1e-5): """Set parameters for Huber's model epsilon X i.i.d ~ (1-eps) P(mu, Sigma) + eps Q, where P is the real distribution, mu is the center parameter we want to estimate, Q is the contamination distribution and eps is the contamination ratio. Args: p: dimension. eps: contamination ratio. tol: make sure the denominator is not zero. device: If no device is provided, it will automatically choose cpu or cuda. """ self.p = p self.eps = eps self.tol = tol self.device = device if device is not None \ else torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') def dist_init(self, true_type='Gaussian', cont_type='Gaussian', true_mean=0.0, cont_mean=0.0, cont_var=1, cont_covmat=None): """ Set parameters for distribution under Huber contaminaton models. We assume the center parameter of the true distribution mu is 0 and the covariance is indentity martix. Args: true_type : Type of real distribution P. 'Gaussian', 'Cauchy'. cont_type : Type of contamination distribution Q, 'Gaussian', 'Cauchy'. cont_mean: center parameter for Q cont_var: If scatter (covariance) matrix of Q is diagonal, cont_var gives the diagonal element. cont_covmat: Other scatter matrix can be provided (as torch.tensor format). If cont_covmat is not None, cont_var will be ignored. """ self.true_type = true_type self.cont_type = cont_type ## settings for true distribution sampler self.true_mean = torch.ones(self.p) * true_mean if true_type == 'Gaussian': self.t_d = MultivariateNormal(self.true_mean, covariance_matrix=torch.eye(self.p)) elif true_type == 'Cauchy': self.t_normal_d = MultivariateNormal(torch.zeros(self.p), covariance_matrix=torch.eye( self.p)) self.t_chi2_d = Chi2(df=1) else: raise NameError('True type must be Gaussian or Cauchy!') ## settings for contamination distribution sampler if cont_covmat is not None: self.cont_covmat = cont_covmat else: self.cont_covmat = torch.eye(self.p) * cont_var self.cont_mean = torch.ones(self.p) * cont_mean if cont_type == 'Gaussian': self.c_d = MultivariateNormal(torch.zeros(self.p), covariance_matrix=self.cont_covmat) elif cont_type == 'Cauchy': self.c_normal_d = MultivariateNormal( torch.zeros(self.p), covariance_matrix=self.cont_covmat) self.c_chi2_d = Chi2(df=1) else: raise NameError('Cont type must be Gaussian or Cauchy!') def _sampler(self, n): """ Sampler and it will return a [n, p] torch tensor. """ if self.true_type == 'Gaussian': t_x = self.t_d.sample((n, )) elif self.true_type == 'Cauchy': t_normal_x = self.t_normal_d.sample((n, )) t_chi2_x = self.t_chi2_d.sample((n, )) t_x = t_normal_x / (torch.sqrt(t_chi2_x.view(-1, 1)) + self.tol) if self.cont_type == 'Gaussian': c_x = self.c_d.sample((n, )) + self.cont_mean.view(1, -1) elif self.cont_type == 'Cauchy': c_normal_x = self.c_normal_d.sample((n, )) c_chi2_x = self.c_chi2_d.sample((n, )) c_x = c_normal_x / (torch.sqrt(c_chi2_x.view(-1, 1)) + self.tol) +\ self.cont_mean.view(1, -1) s = (torch.rand(n) < self.eps).float() x = (t_x.transpose(1, 0) * (1 - s) + c_x.transpose(1, 0) * s).transpose(1, 0) return x def data_init(self, train_size=50000, batch_size=100): self.Xtr = self._sampler(train_size) self.batch_size = batch_size self.poolset = PoolSet(self.Xtr) self.dataloader = DataLoader(self.poolset, batch_size=self.batch_size, shuffle=True) def net_init(self, d_hidden_units, use_logistic_regression=False, init_weights=None, init_eta=0.0, use_median_init_G=True, elliptical=False, g_input_dim=10, g_hidden_units=[10, 10], activation_D1='Sigmoid', verbose=True): """ Settings for Discriminator and Generator. Args: d_hidden_units: a list of hidden units for Discriminator, e.g. d_hidden_units=[10, 5], then the discrimintor has structure p (input) - 10 - 5 - 1 (output). elliptical: Boolean. If elliptical == False, G_1(x|b) = x + b, where b will be learned and x ~ Gaussian/Cauchy(0, I_p) according to the true distribution. If elliptical = True, G_2(t, u|b) = g_2(t)u + b, where G_2(t, x|b) generates the family of elliptical distribution, t ~ Normal(0, I) and u ~ Uniform(\\|u\\|_2 = 1) g_input_dim: (Even) number. When elliptical == True, the dimension of input for g_2(t) need to be provided. g_hidden_units: A list of hidden units for g_2(t). When elliptical == True, structure of g_2(t) need to be provided. e.g. g_hidden_units = [24, 12, 8], then g_2(t) has structure g_input_dim - 24 - 12 - 8 - p. activation_D1: 'Sigmoid', 'ReLU' or 'LeakyReLU'. The first activation function after the input layer. Especially when true_type == 'Cauchy', Sigmoid activation is preferred. verbose: Boolean. If verbose == True, initial error \\|\\hat{\\mu}_0 - \\mu\\|_2 will be printed. """ self.elliptical = elliptical self.g_input_dim = g_input_dim if self.elliptical: assert (g_input_dim % 2 == 0), 'g_input_dim should be an even number' self.netGXi = GeneratorXi(input_dim=g_input_dim, hidden_units=g_hidden_units).to( self.device) self.netG = Generator(p=self.p, elliptical=self.elliptical).to(self.device) # Initialize center parameter with sample median. if use_median_init_G: self.netG.bias.data = torch.median(self.Xtr, dim=0)[0].to(self.device) else: self.netG.bias.data = (torch.ones(self.p) * init_eta).to( self.device) self.mean_err_init = np.linalg.norm(self.netG.bias.data.cpu().numpy() -\ self.true_mean.numpy()) if verbose: print('Initialize Mean Error: %.4f' % self.mean_err_init) ## Initialize discrminator and g_2(t) when ellpitical == True if use_logistic_regression: self.netD = LogisticRegression(p=self.p).to(self.device) else: self.netD = Discriminator(p=self.p, hidden_units=d_hidden_units, activation_1=activation_D1).to( self.device) weights_init_netD = partial(weights_init, value=init_weights) self.netD.apply(weights_init_netD) if (self.elliptical): self.netGXi.apply(weights_init_xavier) def optimizer_init(self, lr_d, lr_g, d_steps, g_steps, type_opt='SGD'): """ Settings for optimizer. Args: lr_d: learning rate for discrimintaor. lr_g: learning rate for generator. d_steps: number of steps of discriminator per discriminator iteration. g_steps: number of steps of generator per generator iteration. """ if type_opt == 'SGD': self.optG = optim.SGD(self.netG.parameters(), lr=lr_g) if self.elliptical: self.optGXi = optim.SGD(self.netGXi.parameters(), lr=lr_g) self.optD = optim.SGD(self.netD.parameters(), lr=lr_d) else: self.optG = optim.Adam(self.netG.parameters(), lr=lr_g) if self.elliptical: self.optGXi = optim.Adam(self.netGXi.parameters(), lr=lr_g) self.optD = optim.Adam(self.netD.parameters(), lr=lr_d) self.g_steps = g_steps self.d_steps = d_steps def fit(self, floss='js', epochs=20, avg_epochs=10, use_inverse_gaussian=True, verbose=25): """ Training process. Args: floss: 'js' or 'tv'. For JS-GAN, we consider the original GAN with Jensen-Shannon divergence and for TV-GAN, total variation will be used. epochs: Number. Number of epochs for training. avg_epochs: Number. An average estimation using the last certain epochs. use_use_inverse_gaussian: Boolean. If elliptical == True, \\xi generator, g_2(t) takes random vector t as input and outputs \\xi samples. If use_use_inverse_gaussian == True, we take t = (t1, t2), where t1 ~ Normal(0, I_(d/2)) and t2 ~ 1/Normal(0, I_(d/2)), otherwise, t ~ Normal(0, I_d). verbose: Number. Print intermediate result every certain epochs. show: Boolean. If show == True, final result will be printed after training. """ assert floss in ['js', 'tv'], 'floss must be \'js\' or \'tv\'' if floss == 'js': criterion = nn.BCEWithLogitsLoss() self.floss = floss self.loss_D = [] self.loss_G = [] self.mean_err_record = [] self.mean_est_record = [] current_d_step = 1 for ep in range(epochs): loss_D_ep = [] loss_G_ep = [] for _, data in enumerate(self.dataloader): ## update D self.netD.train() self.netD.zero_grad() ## discriminator loss x_real = data.to(self.device) feat_real, d_real_score = self.netD(x_real) if (floss == 'js'): one_b = torch.ones_like(d_real_score).to(self.device) d_real_loss = criterion(d_real_score, one_b) elif floss == 'tv': d_real_loss = -torch.sigmoid(d_real_score).mean() #d_real_loss = criterion(d_real_score, one_b) ## generator loss z_b = torch.zeros(data.shape[0], self.p).to(self.device) if self.elliptical: if use_inverse_gaussian: xi_b1 = torch.zeros(data.shape[0], self.g_input_dim // 2).to(self.device) xi_b2 = torch.zeros(data.shape[0], self.g_input_dim // 2).to(self.device) else: xi_b = torch.zeros(data.shape[0], self.g_input_dim).to(self.device) if self.elliptical: z_b.normal_() z_b.div_(z_b.norm(2, dim=1).view(-1, 1) + self.tol) if use_inverse_gaussian: xi_b1.normal_() xi_b2.normal_() xi_b2.data = 1 / (torch.abs(xi_b2.data) + self.tol) xi = self.netGXi(torch.cat([xi_b1, xi_b2], dim=1)).view( self.batch_size, -1) else: xi_b.normal_() xi = self.netGXi(xi_b).view(self.batch_size, -1) x_fake = self.netG(z_b, xi).detach() elif (self.true_type == 'Cauchy'): z_b.normal_() z_b.data.div_( torch.sqrt(self.t_chi2_d.sample((self.batch_size, 1))).to(self.device) + self.tol) x_fake = self.netG(z_b).detach() elif self.true_type == 'Gaussian': x_fake = self.netG(z_b.normal_()).detach() feat_fake, d_fake_score = self.netD(x_fake) if floss == 'js': one_b = torch.ones_like(d_fake_score).to(self.device) d_fake_loss = criterion(d_fake_score, 1 - one_b) elif floss == 'tv': d_fake_loss = torch.sigmoid(d_fake_score).mean() d_loss = d_real_loss + d_fake_loss d_loss.backward() loss_D_ep.append(d_loss.cpu().item()) self.optD.step() if current_d_step < self.d_steps: current_d_step += 1 continue else: current_d_step = 1 ## update G self.netD.eval() for _ in range(self.g_steps): self.netG.zero_grad() if self.elliptical: self.netGXi.zero_grad() z_b.normal_() z_b.div_(z_b.norm(2, dim=1).view(-1, 1) + self.tol) if use_inverse_gaussian: xi_b1.normal_() xi_b2.normal_() xi_b2.data = 1 / (torch.abs(xi_b2.data) + self.tol) xi = self.netGXi(torch.cat([xi_b1, xi_b2], dim=1)).view( self.batch_size, -1) else: xi_b.normal_() xi = self.netGXi(xi_b).view(self.batch_size, -1) x_fake = self.netG(z_b, xi) elif self.true_type == 'Gaussian': x_fake = self.netG(z_b.normal_()) elif (self.true_type == 'Cauchy'): z_b.normal_() z_b.data.div_( torch.sqrt( self.t_chi2_d.sample((self.batch_size, 1))).to(self.device) + self.tol) x_fake = self.netG(z_b) feat_fake, g_fake_score = self.netD(x_fake) if (floss == 'js'): one_b = torch.ones_like(g_fake_score).to(self.device) g_fake_loss = -criterion(g_fake_score, 1 - one_b) g_fake_loss.backward() loss_G_ep.append(-g_fake_loss.cpu().item()) elif floss == 'tv': g_fake_loss = -torch.sigmoid(g_fake_score).mean() g_fake_loss.backward() loss_G_ep.append(g_fake_loss.cpu().item()) self.optG.step() if self.elliptical: self.optGXi.step() ## Record intermediate error during training for monitoring. self.mean_err_record.append( (self.netG.bias.data - self.true_mean.to(self.device)).norm(2).item()) ## Record intermediate estimation during training for averaging. if (ep >= (epochs - avg_epochs)): self.mean_est_record.append(self.netG.bias.data.clone().cpu()) self.loss_D.append(np.mean(loss_D_ep)) self.loss_G.append(np.mean(loss_G_ep)) ## Print intermediate result every verbose epoch. if ((ep + 1) % verbose == 0): print('Epoch:%d, LossD/G:%.4f/%.4f, Error(Mean):%.4f' % (ep + 1, self.loss_D[-1], self.loss_G[-1], self.mean_err_record[-1])) ## Final results self.mean_avg = sum(self.mean_est_record[-avg_epochs:])/\ len(self.mean_est_record[-avg_epochs:]) self.mean_err_avg = (self.mean_avg - self.true_mean.cpu()).norm(2).item() self.mean_err_last = (self.netG.bias.data - self.true_mean.to(self.device)).norm(2).item() def report_results(self, figsize=(6, 4), show_plots=True, save_g_loss=None, save_d_loss=None, save_error=None, save_distribution=None): ## Print the final results. self.netD.eval() ## Scores of true distribution from 10,000 samples. if self.true_type == 'Gaussian': t_x = self.t_d.sample((10000, )) elif self.true_type == 'Cauchy': t_normal_x = self.t_normal_d.sample((10000, )) t_chi2_x = self.t_chi2_d.sample((10000, )) t_x = t_normal_x / (torch.sqrt(t_chi2_x.view(-1, 1)) + self.tol) self.true_D = self.netD(t_x.to(self.device))[1].detach().cpu().numpy() ## Scores of contamination distribution from 10,000 samples. if self.cont_type == 'Gaussian': c_x = self.c_d.sample((10000, )) + self.cont_mean.view(1, -1) elif self.cont_type == 'Cauchy': c_normal_x = self.c_normal_d.sample((10000, )) c_chi2_x = self.c_chi2_d.sample((10000, )) c_x = c_normal_x / (torch.sqrt(c_chi2_x.view(-1, 1)) + self.tol) +\ self.cont_mean.view(1, -1) self.cont_D = self.netD(c_x.to(self.device))[1].detach().cpu().numpy() ## Scores of 10,000 generating samples. if self.elliptical: t_z = torch.randn(10000, self.p).to(self.device) t_z.div_(t_z.norm(2, dim=1).view(-1, 1) + self.tol) if use_inverse_gaussian: t_xi1 = torch.randn(10000, self.g_input_dim // 2).to(self.device) t_xi2 = torch.randn(10000, self.g_input_dim // 2).to(self.device) t_xi2 = 1 / (torch.abs(t_xi2.data) + self.tol) xi = self.netGXi(torch.cat([t_xi1, t_xi2], dim=1)).view(10000, -1) else: t_xi = torch.randn(10000, self.g_input_dim).to(self.device) xi = self.netGXi(t_xi).view(10000, -1) g_x = self.netG(t_z, xi).detach() elif self.true_type == 'Gaussian': g_x = self.netG(torch.randn(10000, self.p).to(self.device)) elif (self.true_type == 'Cauchy'): g_z = torch.randn(10000, self.p).to(self.device) g_z.data.div_( torch.sqrt(self.t_chi2_d.sample((10000, 1))).to(self.device) + self.tol) g_x = self.netG(g_z) self.gene_D = self.netD(g_x)[1].detach().cpu().numpy() ## Some useful prints and plots print('Avg error: %.4f, Last error: %.4f' % (self.mean_err_avg, self.mean_err_last)) grand_mean = (1 - self.eps) * self.true_mean + self.eps * self.cont_mean grand_mean_err = (grand_mean.to(self.device) - self.true_mean.to(self.device)).norm(2).item() grand_mean_err_record = [ grand_mean_err for i in range(len(self.mean_err_record)) ] if self.p == 1: print("True mean = %.4f" % (self.true_mean.item())) print("Contamination mean = %.4f" % (self.cont_mean.item())) print("Result mean = %.4f" % (self.netG.bias.data.item())) print("Grand mean = %.4f" % (grand_mean.item())) loss_type = 'Total Variation' if self.floss == 'tv' else 'Jensen-Shannon' fig, ax = plt.subplots(figsize=figsize) ax.plot(self.loss_D) ax.grid(True) ax.set_title(f'Discriminator loss, type = {loss_type}') ax.set_xlabel("epoch num") ax.set_ylabel("Loss") if save_d_loss is not None: plt.savefig(save_d_loss) if show_plots: plt.show() else: plt.close(fig) fig, ax = plt.subplots(figsize=figsize) ax.plot(self.loss_G) ax.grid(True) ax.set_title(f'Generator loss, type = {loss_type}') ax.set_xlabel("epoch num") ax.set_ylabel("Loss") if save_g_loss is not None: plt.savefig(save_g_loss) if show_plots: plt.show() else: plt.close(fig) fig, ax = plt.subplots(figsize=figsize) ax.plot(self.mean_err_record, label='mean error process') ax.plot(grand_mean_err_record, label='grand mean error') ax.legend() ax.grid(True) ax.set_title( r'$\ell_{2}$ error in prediction of mean for true distribution') ax.set_xlabel("epoch num") ax.set_ylabel(r"$\|\eta_{est} - \eta_{true}\|_{2}$") if save_error is not None: plt.savefig(save_error) if show_plots: plt.show() else: plt.close(fig) fig, ax = plt.subplots(figsize=figsize) d_distributions = {} d_distributions['true distribution'] = self.true_D[(self.true_D < 25) & (self.true_D > -25)] d_distributions['generated distribution'] = self.gene_D[ (self.gene_D < 25) & (self.gene_D > -25)] d_distributions['contamination distribution'] = self.cont_D[ (self.cont_D < 25) & (self.cont_D > -25)] g = sns.kdeplot(ax=ax, data=d_distributions) ax.set_xlabel(r"$D(x)$") ax.set_ylabel("Density") ax.grid(True) ax.set_title(r'Discriminator distribution, $D(x)$') if save_distribution is not None: plt.savefig(save_distribution) if show_plots: plt.show() else: plt.close(fig)
class UGATIT(object): def __init__(self, args): self.light = args.light if self.light: self.model_name = 'UGATIT_light' else: self.model_name = 'UGATIT' self.result_dir = args.result_dir self.dataset = args.dataset self.iteration = args.iteration self.decay_flag = args.decay_flag self.batch_size = args.batch_size self.print_freq = args.print_freq self.save_freq = args.save_freq self.lr = args.lr self.weight_decay = args.weight_decay self.ch = args.ch """ Weight """ self.adv_weight = args.adv_weight self.cycle_weight = args.cycle_weight self.identity_weight = args.identity_weight self.cam_weight = args.cam_weight """ Generator """ self.n_res = args.n_res """ Discriminator """ self.n_dis = args.n_dis self.img_size = args.img_size self.img_ch = args.img_ch self.device = args.device self.benchmark_flag = args.benchmark_flag self.resume = args.resume ################################################################################## # Model ################################################################################## def optimizer_setting(self, parameters): lr = 0.0001 optimizer = fluid.optimizer.Adam( learning_rate=lr, parameter_list=parameters, beta1=0.5, beta2=0.999, regularization=fluid.regularizer.L2Decay(self.weight_decay)) return optimizer def build_model(self): """ DataLoader """ train_transform = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.Resize((self.img_size + 30, self.img_size + 30)), transforms.RandomCrop(self.img_size), transforms.ToTensor(), transforms.Normalize(mean=0.5, std=0.5) ]) test_transform = transforms.Compose([ transforms.Resize((self.img_size, self.img_size)), transforms.ToTensor(), transforms.Normalize(mean=0.5, std=0.5) ]) self.trainA_loader = paddle.batch( a_reader(shuffle=True, transforms=train_transform), self.batch_size)() self.trainB_loader = paddle.batch( b_reader(shuffle=True, transforms=train_transform), self.batch_size)() self.testA_loader = a_test_reader(transforms=test_transform) self.testB_loader = b_test_reader(transforms=test_transform) """ Define Generator, Discriminator """ self.genA2B = ResnetGenerator(input_nc=3, output_nc=3, ngf=self.ch, n_blocks=self.n_res, img_size=self.img_size, light=self.light) self.genB2A = ResnetGenerator(input_nc=3, output_nc=3, ngf=self.ch, n_blocks=self.n_res, img_size=self.img_size, light=self.light) self.disGA = Discriminator(input_nc=3, ndf=self.ch, n_layers=7) self.disGB = Discriminator(input_nc=3, ndf=self.ch, n_layers=7) self.disLA = Discriminator(input_nc=3, ndf=self.ch, n_layers=5) self.disLB = Discriminator(input_nc=3, ndf=self.ch, n_layers=5) """ Define Loss """ self.L1_loss = L1Loss() self.MSE_loss = MSELoss() self.BCE_loss = BCEWithLogitsLoss() """ Trainer """ self.G_optim = self.optimizer_setting(self.genA2B.parameters() + self.genB2A.parameters()) self.D_optim = self.optimizer_setting(self.disGA.parameters() + self.disGB.parameters() + self.disLA.parameters() + self.disLB.parameters()) """ Define Rho clipper to constraint the value of rho in AdaILN and ILN""" self.Rho_clipper = RhoClipper(0, 1) def train(self): self.genA2B.train(), self.genB2A.train(), self.disGA.train( ), self.disGB.train(), self.disLA.train(), self.disLB.train() start_iter = 1 if self.resume: model_list = os.listdir( os.path.join(self.result_dir, self.dataset, 'model')) if not len(model_list) == 0: model_list.sort() iter = int(model_list[-1]) print("[*]load %d" % (iter)) self.load(os.path.join(self.result_dir, self.dataset, 'model'), iter) print("[*] Load SUCCESS") # training loop print('training start !') start_time = time.time() for step in range(start_iter, self.iteration + 1): real_A = next(self.trainA_loader) real_B = next(self.trainB_loader) real_A = np.array([real_A[0].reshape(3, 256, 256)]).astype("float32") real_B = np.array([real_B[0].reshape(3, 256, 256)]).astype("float32") real_A = to_variable(real_A) real_B = to_variable(real_B) # Update D fake_A2B, _, _ = self.genA2B(real_A) fake_B2A, _, _ = self.genB2A(real_B) real_GA_logit, real_GA_cam_logit, _ = self.disGA(real_A) real_LA_logit, real_LA_cam_logit, _ = self.disLA(real_A) real_GB_logit, real_GB_cam_logit, _ = self.disGB(real_B) real_LB_logit, real_LB_cam_logit, _ = self.disLB(real_B) fake_GA_logit, fake_GA_cam_logit, _ = self.disGA(fake_B2A) fake_LA_logit, fake_LA_cam_logit, _ = self.disLA(fake_B2A) fake_GB_logit, fake_GB_cam_logit, _ = self.disGB(fake_A2B) fake_LB_logit, fake_LB_cam_logit, _ = self.disLB(fake_A2B) D_ad_loss_GA = self.MSE_loss( real_GA_logit, ones_like(real_GA_logit)) + self.MSE_loss( fake_GA_logit, zeros_like(fake_GA_logit)) D_ad_cam_loss_GA = self.MSE_loss( real_GA_cam_logit, ones_like(real_GA_cam_logit)) + self.MSE_loss( fake_GA_cam_logit, zeros_like(fake_GA_cam_logit)) D_ad_loss_LA = self.MSE_loss( real_LA_logit, ones_like(real_LA_logit)) + self.MSE_loss( fake_LA_logit, zeros_like(fake_LA_logit)) D_ad_cam_loss_LA = self.MSE_loss( real_LA_cam_logit, ones_like(real_LA_cam_logit)) + self.MSE_loss( fake_LA_cam_logit, zeros_like(fake_LA_cam_logit)) D_ad_loss_GB = self.MSE_loss( real_GB_logit, ones_like(real_GB_logit)) + self.MSE_loss( fake_GB_logit, zeros_like(fake_GB_logit)) D_ad_cam_loss_GB = self.MSE_loss( real_GB_cam_logit, ones_like(real_GB_cam_logit)) + self.MSE_loss( fake_GB_cam_logit, zeros_like(fake_GB_cam_logit)) D_ad_loss_LB = self.MSE_loss( real_LB_logit, ones_like(real_LB_logit)) + self.MSE_loss( fake_LB_logit, zeros_like(fake_LB_logit)) D_ad_cam_loss_LB = self.MSE_loss( real_LB_cam_logit, ones_like(real_LB_cam_logit)) + self.MSE_loss( fake_LB_cam_logit, zeros_like(fake_LB_cam_logit)) D_loss_A = self.adv_weight * (D_ad_loss_GA + D_ad_cam_loss_GA + D_ad_loss_LA + D_ad_cam_loss_LA) D_loss_B = self.adv_weight * (D_ad_loss_GB + D_ad_cam_loss_GB + D_ad_loss_LB + D_ad_cam_loss_LB) Discriminator_loss = D_loss_A + D_loss_B Discriminator_loss.backward() self.D_optim.minimize(Discriminator_loss) self.genB2A.clear_gradients() self.genA2B.clear_gradients() self.disGA.clear_gradients() self.disLA.clear_gradients() self.disGB.clear_gradients() self.disLB.clear_gradients() self.D_optim.clear_gradients() # Update G fake_A2B, fake_A2B_cam_logit, _ = self.genA2B(real_A) fake_B2A, fake_B2A_cam_logit, _ = self.genB2A(real_B) fake_A2B2A, _, _ = self.genB2A(fake_A2B) fake_B2A2B, _, _ = self.genA2B(fake_B2A) fake_A2A, fake_A2A_cam_logit, _ = self.genB2A(real_A) fake_B2B, fake_B2B_cam_logit, _ = self.genA2B(real_B) fake_GA_logit, fake_GA_cam_logit, _ = self.disGA(fake_B2A) fake_LA_logit, fake_LA_cam_logit, _ = self.disLA(fake_B2A) fake_GB_logit, fake_GB_cam_logit, _ = self.disGB(fake_A2B) fake_LB_logit, fake_LB_cam_logit, _ = self.disLB(fake_A2B) G_ad_loss_GA = self.MSE_loss(fake_GA_logit, ones_like(fake_GA_logit)) G_ad_cam_loss_GA = self.MSE_loss(fake_GA_cam_logit, ones_like(fake_GA_cam_logit)) G_ad_loss_LA = self.MSE_loss(fake_LA_logit, ones_like(fake_LA_logit)) G_ad_cam_loss_LA = self.MSE_loss(fake_LA_cam_logit, ones_like(fake_LA_cam_logit)) G_ad_loss_GB = self.MSE_loss(fake_GB_logit, ones_like(fake_GB_logit)) G_ad_cam_loss_GB = self.MSE_loss(fake_GB_cam_logit, ones_like(fake_GB_cam_logit)) G_ad_loss_LB = self.MSE_loss(fake_LB_logit, ones_like(fake_LB_logit)) G_ad_cam_loss_LB = self.MSE_loss(fake_LB_cam_logit, ones_like(fake_LB_cam_logit)) G_recon_loss_A = self.L1_loss(fake_A2B2A, real_A) G_recon_loss_B = self.L1_loss(fake_B2A2B, real_B) G_identity_loss_A = self.L1_loss(fake_A2A, real_A) G_identity_loss_B = self.L1_loss(fake_B2B, real_B) G_cam_loss_A = self.BCE_loss( fake_B2A_cam_logit, ones_like(fake_B2A_cam_logit)) + self.BCE_loss( fake_A2A_cam_logit, zeros_like(fake_A2A_cam_logit)) G_cam_loss_B = self.BCE_loss( fake_A2B_cam_logit, ones_like(fake_A2B_cam_logit)) + self.BCE_loss( fake_B2B_cam_logit, zeros_like(fake_B2B_cam_logit)) G_loss_A = self.adv_weight * ( G_ad_loss_GA + G_ad_cam_loss_GA + G_ad_loss_LA + G_ad_cam_loss_LA ) + self.cycle_weight * G_recon_loss_A + self.identity_weight * G_identity_loss_A + self.cam_weight * G_cam_loss_A G_loss_B = self.adv_weight * ( G_ad_loss_GB + G_ad_cam_loss_GB + G_ad_loss_LB + G_ad_cam_loss_LB ) + self.cycle_weight * G_recon_loss_B + self.identity_weight * G_identity_loss_B + self.cam_weight * G_cam_loss_B Generator_loss = G_loss_A + G_loss_B Generator_loss.backward() self.G_optim.minimize(Generator_loss) self.genB2A.clear_gradients() self.genA2B.clear_gradients() self.disGA.clear_gradients() self.disLA.clear_gradients() self.disGB.clear_gradients() self.disLB.clear_gradients() self.G_optim.clear_gradients() self.Rho_clipper(self.genA2B) self.Rho_clipper(self.genB2A) print("[%5d/%5d] time: %4.4f d_loss: %.8f, g_loss: %.8f" % (step, self.iteration, time.time() - start_time, Discriminator_loss, Generator_loss)) if step % self.print_freq == 0: train_sample_num = 5 test_sample_num = 5 A2B = np.zeros((self.img_size * 7, 0, 3)) B2A = np.zeros((self.img_size * 7, 0, 3)) self.genA2B.eval(), self.genB2A.eval(), self.disGA.eval( ), self.disGB.eval(), self.disLA.eval(), self.disLB.eval() for _ in range(train_sample_num): real_A = next(self.trainA_loader) real_B = next(self.trainB_loader) real_A = np.array([real_A[0].reshape(3, 256, 256) ]).astype("float32") real_B = np.array([real_B[0].reshape(3, 256, 256) ]).astype("float32") real_A = to_variable(real_A) real_B = to_variable(real_B) fake_A2B, _, fake_A2B_heatmap = self.genA2B(real_A) fake_B2A, _, fake_B2A_heatmap = self.genB2A(real_B) fake_A2B2A, _, fake_A2B2A_heatmap = self.genB2A(fake_A2B) fake_B2A2B, _, fake_B2A2B_heatmap = self.genA2B(fake_B2A) fake_A2A, _, fake_A2A_heatmap = self.genB2A(real_A) fake_B2B, _, fake_B2B_heatmap = self.genA2B(real_B) A2B = np.concatenate( (A2B, np.concatenate( (RGB2BGR(tensor2numpy(denorm(real_A[0]))), cam(tensor2numpy(fake_A2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2A[0]))), cam(tensor2numpy(fake_A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B[0]))), cam(tensor2numpy(fake_A2B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B2A[0])))), 0)), 1) B2A = np.concatenate( (B2A, np.concatenate( (RGB2BGR(tensor2numpy(denorm(real_B[0]))), cam(tensor2numpy(fake_B2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2B[0]))), cam(tensor2numpy(fake_B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A[0]))), cam(tensor2numpy(fake_B2A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A2B[0])))), 0)), 1) for _ in range(test_sample_num): real_A = next(self.testA_loader()) real_B = next(self.testB_loader()) real_A = np.array([real_A[0].reshape(3, 256, 256) ]).astype("float32") real_B = np.array([real_B[0].reshape(3, 256, 256) ]).astype("float32") real_A = to_variable(real_A) real_B = to_variable(real_B) fake_A2B, _, fake_A2B_heatmap = self.genA2B(real_A) fake_B2A, _, fake_B2A_heatmap = self.genB2A(real_B) fake_A2B2A, _, fake_A2B2A_heatmap = self.genB2A(fake_A2B) fake_B2A2B, _, fake_B2A2B_heatmap = self.genA2B(fake_B2A) fake_A2A, _, fake_A2A_heatmap = self.genB2A(real_A) fake_B2B, _, fake_B2B_heatmap = self.genA2B(real_B) A2B = np.concatenate( (A2B, np.concatenate( (RGB2BGR(tensor2numpy(denorm(real_A[0]))), cam(tensor2numpy(fake_A2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2A[0]))), cam(tensor2numpy(fake_A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B[0]))), cam(tensor2numpy(fake_A2B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B2A[0])))), 0)), 1) B2A = np.concatenate( (B2A, np.concatenate( (RGB2BGR(tensor2numpy(denorm(real_B[0]))), cam(tensor2numpy(fake_B2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2B[0]))), cam(tensor2numpy(fake_B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A[0]))), cam(tensor2numpy(fake_B2A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A2B[0])))), 0)), 1) cv2.imwrite( os.path.join(self.result_dir, self.dataset, 'img', 'A2B_%07d.png' % step), A2B * 255.0) cv2.imwrite( os.path.join(self.result_dir, self.dataset, 'img', 'B2A_%07d.png' % step), B2A * 255.0) self.genA2B.train(), self.genB2A.train(), self.disGA.train( ), self.disGB.train(), self.disLA.train(), self.disLB.train() if step % self.save_freq == 0: self.save(os.path.join(self.result_dir, self.dataset, 'model'), step) if step % 1000 == 0: fluid.save_dygraph( self.genA2B.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/genA2B")) fluid.save_dygraph( self.genB2A.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/genB2A")) fluid.save_dygraph( self.disGA.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/disGA")) fluid.save_dygraph( self.disGB.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/disGB")) fluid.save_dygraph( self.disLA.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/disLA")) fluid.save_dygraph( self.disLB.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/disLB")) fluid.save_dygraph( self.D_optim.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/D_optim")) fluid.save_dygraph( self.G_optim.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/G_optim")) fluid.save_dygraph( self.genA2B.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/D_optim")) fluid.save_dygraph( self.genB2A.state_dict(), os.path.join(self.result_dir, self.dataset + "/latest/new/G_optim")) def save(self, result_dir, step): fluid.save_dygraph(self.genA2B.state_dict(), os.path.join(result_dir, "{}/genA2B".format(step))) fluid.save_dygraph(self.genB2A.state_dict(), os.path.join(result_dir, "{}/genB2A".format(step))) fluid.save_dygraph(self.disGA.state_dict(), os.path.join(result_dir, "{}/disGA".format(step))) fluid.save_dygraph(self.disGB.state_dict(), os.path.join(result_dir, "{}/disGB".format(step))) fluid.save_dygraph(self.disLA.state_dict(), os.path.join(result_dir, "{}/disLA".format(step))) fluid.save_dygraph(self.disLB.state_dict(), os.path.join(result_dir, "{}/disLB".format(step))) fluid.save_dygraph(self.genA2B.state_dict(), os.path.join(result_dir, "{}/D_optim".format(step))) fluid.save_dygraph(self.genB2A.state_dict(), os.path.join(result_dir, "{}/G_optim".format(step))) fluid.save_dygraph(self.D_optim.state_dict(), os.path.join(result_dir, "{}/D_optim".format(step))) fluid.save_dygraph(self.G_optim.state_dict(), os.path.join(result_dir, "{}/G_optim".format(step))) def load(self, dir, step): genA2B, _ = fluid.load_dygraph( os.path.join(dir, "{}/genA2B".format(step))) genB2A, _ = fluid.load_dygraph( os.path.join(dir, "{}/genB2A".format(step))) disGA, _ = fluid.load_dygraph( os.path.join(dir, "{}/disGA".format(step))) disGB, _ = fluid.load_dygraph( os.path.join(dir, "{}/disGB".format(step))) disLA, _ = fluid.load_dygraph( os.path.join(dir, "{}/disLA".format(step))) disLB, _ = fluid.load_dygraph( os.path.join(dir, "{}/disLB".format(step))) _, D_optim = fluid.load_dygraph( os.path.join(dir, "{}/D_optim".format(step))) _, G_optim = fluid.load_dygraph( os.path.join(dir, "{}/G_optim".format(step))) self.genA2B.load_dict(genA2B) self.genB2A.load_dict(genB2A) self.disGA.load_dict(disGA) self.disGB.load_dict(disGB) self.disLA.load_dict(disLA) self.disLB.load_dict(disLB) self.G_optim.set_dict(G_optim) self.D_optim.set_dict(D_optim) def test(self): model_list = os.listdir( os.path.join(self.result_dir, self.dataset, 'model')) if not len(model_list) == 0: model_list.sort() iter = int(model_list[-1]) self.load(os.path.join(self.result_dir, self.dataset, 'model'), iter) print("[*] Load SUCCESS") else: print("[*] Load FAILURE") return self.genA2B.eval(), self.genB2A.eval() for n, (real_A, _) in enumerate(self.testA_loader()): real_A = np.array([real_A.reshape(3, 256, 256)]).astype("float32") real_A = to_variable(real_A) fake_A2B, _, fake_A2B_heatmap = self.genA2B(real_A) fake_A2B2A, _, fake_A2B2A_heatmap = self.genB2A(fake_A2B) fake_A2A, _, fake_A2A_heatmap = self.genB2A(real_A) A2B = np.concatenate( (RGB2BGR(tensor2numpy(denorm(real_A[0]))), cam(tensor2numpy(fake_A2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2A[0]))), cam(tensor2numpy(fake_A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B[0]))), cam(tensor2numpy(fake_A2B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_A2B2A[0])))), 0) cv2.imwrite( os.path.join(self.result_dir, self.dataset, 'test', 'A2B_%d.png' % (n + 1)), A2B * 255.0) for n, (real_B, _) in enumerate(self.testB_loader()): real_B = np.array([real_B.reshape(3, 256, 256)]).astype("float32") real_B = to_variable(real_B) fake_B2A, _, fake_B2A_heatmap = self.genB2A(real_B) fake_B2A2B, _, fake_B2A2B_heatmap = self.genA2B(fake_B2A) fake_B2B, _, fake_B2B_heatmap = self.genA2B(real_B) B2A = np.concatenate( (RGB2BGR(tensor2numpy(denorm(real_B[0]))), cam(tensor2numpy(fake_B2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2B[0]))), cam(tensor2numpy(fake_B2A_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A[0]))), cam(tensor2numpy(fake_B2A2B_heatmap[0]), self.img_size), RGB2BGR(tensor2numpy(denorm(fake_B2A2B[0])))), 0) cv2.imwrite( os.path.join(self.result_dir, self.dataset, 'test', 'B2A_%d.png' % (n + 1)), B2A * 255.0) def test_change(self): model_list = os.listdir( os.path.join(self.result_dir, self.dataset, 'model')) if not len(model_list) == 0: model_list.sort() iter = int(model_list[-1].split('/')[-1]) self.load(os.path.join(self.result_dir, self.dataset, 'model'), iter) print("[*] Load SUCCESS") else: print("[*] Load FAILURE") return self.genA2B.eval(), self.genB2A.eval() for n, (real_A, fname) in enumerate(self.testA_loader()): real_A = np.array([real_A[0].reshape(3, 256, 256)]).astype("float32") real_A = to_variable(real_A) fake_A2B, _, _ = self.genA2B(real_A) A2B = RGB2BGR(tensor2numpy(denorm(fake_A2B[0]))) cv2.imwrite( os.path.join( self.result_dir, self.dataset, 'test', 'testA2B', '%s_fake.%s' % (fname.split('.')[0], fname.split('.')[-1])), A2B * 255.0) for n, (real_B, fname) in enumerate(self.testB_loader()): real_B = np.array([real_B[0].reshape(3, 256, 256)]).astype("float32") real_B = to_variable(real_B) fake_B2A, _, _ = self.genB2A(real_B) B2A = RGB2BGR(tensor2numpy(denorm(fake_B2A[0]))) cv2.imwrite( os.path.join( self.result_dir, self.dataset, 'test', 'testB2A', '%s_fake.%s' % (fname.split('.')[0], fname.split('.')[-1])), B2A * 255.0)