class SRGAN(): def __init__(self): logger.info('Set Data Loader') self.dataset = AnimeFaceDataset( avatar_tag_dat_path=avatar_tag_dat_path, transform=transforms.Compose([ToTensor()])) self.data_loader = torch.utils.data.DataLoader(self.dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True) checkpoint, checkpoint_name = self.load_checkpoint(model_dump_path) if checkpoint == None: logger.info( 'Don\'t have pre-trained model. Ignore loading model process.') logger.info('Set Generator and Discriminator') self.G = Generator().to(device) self.D = Discriminator().to(device) logger.info('Initialize Weights') self.G.apply(initital_network_weights).to(device) self.D.apply(initital_network_weights).to(device) logger.info('Set Optimizers') self.optimizer_G = torch.optim.Adam(self.G.parameters(), lr=learning_rate, betas=(beta_1, 0.999)) self.optimizer_D = torch.optim.Adam(self.D.parameters(), lr=learning_rate, betas=(beta_1, 0.999)) self.epoch = 0 else: logger.info('Load Generator and Discriminator') self.G = Generator().to(device) self.D = Discriminator().to(device) logger.info('Load Pre-Trained Weights From Checkpoint'.format( checkpoint_name)) self.G.load_state_dict(checkpoint['G']) self.D.load_state_dict(checkpoint['D']) logger.info('Load Optimizers') self.optimizer_G = torch.optim.Adam(self.G.parameters(), lr=learning_rate, betas=(beta_1, 0.999)) self.optimizer_D = torch.optim.Adam(self.D.parameters(), lr=learning_rate, betas=(beta_1, 0.999)) self.optimizer_G.load_state_dict(checkpoint['optimizer_G']) self.optimizer_D.load_state_dict(checkpoint['optimizer_D']) self.epoch = checkpoint['epoch'] logger.info('Set Criterion') self.label_criterion = nn.BCEWithLogitsLoss().to(device) self.tag_criterion = nn.MultiLabelSoftMarginLoss().to(device) def load_checkpoint(self, model_dir): models_path = read_newest_model(model_dir) if len(models_path) == 0: return None, None models_path.sort() new_model_path = os.path.join(model_dump_path, models_path[-1]) checkpoint = torch.load(new_model_path) return checkpoint, new_model_path def train(self): iteration = -1 label = Variable(torch.FloatTensor(batch_size, 1.0)).to(device) logging.info('Current epoch: {}. Max epoch: {}.'.format( self.epoch, max_epoch)) while self.epoch <= max_epoch: # dump checkpoint torch.save( { 'epoch': self.epoch, 'D': self.D.state_dict(), 'G': self.G.state_dict(), 'optimizer_D': self.optimizer_D.state_dict(), 'optimizer_G': self.optimizer_G.state_dict(), }, '{}/checkpoint_{}.tar'.format(model_dump_path, str(self.epoch).zfill(4))) logger.info('Checkpoint saved in: {}'.format( '{}/checkpoint_{}.tar'.format(model_dump_path, str(self.epoch).zfill(4)))) msg = {} adjust_learning_rate(self.optimizer_G, iteration) adjust_learning_rate(self.optimizer_D, iteration) for i, (avatar_tag, avatar_img) in enumerate(self.data_loader): iteration += 1 if verbose: if iteration % verbose_T == 0: msg['epoch'] = int(self.epoch) msg['step'] = int(i) msg['iteration'] = iteration avatar_img = Variable(avatar_img).to(device) avatar_tag = Variable(torch.FloatTensor(avatar_tag)).to(device) # D : G = 2 : 1 # 1. Training D # 1.1. use really image for discriminating self.D.zero_grad() label_p, tag_p = self.D(avatar_img) label.data.fill_(1.0) # 1.2. real image's loss real_label_loss = self.label_criterion(label_p, label) real_tag_loss = self.tag_criterion(tag_p, avatar_tag) real_loss_sum = real_label_loss * lambda_adv / 2.0 + real_tag_loss * lambda_adv / 2.0 real_loss_sum.backward() if verbose: if iteration % verbose_T == 0: msg['discriminator real loss'] = float(real_loss_sum) # 1.3. use fake image for discriminating g_noise, fake_tag = fake_generator() fake_feat = torch.cat([g_noise, fake_tag], dim=1) fake_img = self.G(fake_feat).detach() fake_label_p, fake_tag_p = self.D(fake_img) label.data.fill_(.0) # 1.4. fake image's loss fake_label_loss = self.label_criterion(fake_label_p, label) fake_tag_loss = self.tag_criterion(fake_tag_p, fake_tag) fake_loss_sum = fake_label_loss * lambda_adv / 2.0 + fake_tag_loss * lambda_adv / 2.0 fake_loss_sum.backward() if verbose: if iteration % verbose_T == 0: msg['discriminator fake loss'] = float(fake_loss_sum) # 1.5. gradient penalty # https://github.com/jfsantos/dragan-pytorch/blob/master/dragan.py alpha_size = [1] * avatar_img.dim() alpha_size[0] = avatar_img.size(0) alpha = torch.rand(alpha_size).to(device) x_hat = Variable(alpha * avatar_img.data + (1 - alpha) * \ (avatar_img.data + 0.5 * avatar_img.data.std() * Variable(torch.rand(avatar_img.size())).to(device)), requires_grad=True).to(device) pred_hat, pred_tag = self.D(x_hat) gradients = grad(outputs=pred_hat, inputs=x_hat, grad_outputs=torch.ones( pred_hat.size()).to(device), create_graph=True, retain_graph=True, only_inputs=True)[0].view(x_hat.size(0), -1) gradient_penalty = lambda_gp * ( (gradients.norm(2, dim=1) - 1)**2).mean() gradient_penalty.backward() if verbose: if iteration % verbose_T == 0: msg['discriminator gradient penalty'] = float( gradient_penalty) # 1.6. update optimizer self.optimizer_D.step() # 2. Training G # 2.1. generate fake image self.G.zero_grad() g_noise, fake_tag = fake_generator() fake_feat = torch.cat([g_noise, fake_tag], dim=1) fake_img = self.G(fake_feat) fake_label_p, fake_tag_p = self.D(fake_img) label.data.fill_(1.0) # 2.2. calc loss label_loss_g = self.label_criterion(fake_label_p, label) tag_loss_g = self.tag_criterion(fake_tag_p, fake_tag) loss_g = label_loss_g * lambda_adv / 2.0 + tag_loss_g * lambda_adv / 2.0 loss_g.backward() if verbose: if iteration % verbose_T == 0: msg['generator loss'] = float(loss_g) # 2.2. update optimizer self.optimizer_G.step() if verbose: if iteration % verbose_T == 0: logger.info( '------------------------------------------') for key in msg.keys(): logger.info('{} : {}'.format(key, msg[key])) # save intermediate file if iteration % verbose_T == 0: vutils.save_image( avatar_img.data.view(batch_size, 3, avatar_img.size(2), avatar_img.size(3)), os.path.join( tmp_path, 'real_image_{}.png'.format( str(iteration).zfill(8)))) g_noise, fake_tag = fake_generator() fake_feat = torch.cat([g_noise, fake_tag], dim=1) fake_img = self.G(fake_feat) vutils.save_image( fake_img.data.view(batch_size, 3, avatar_img.size(2), avatar_img.size(3)), os.path.join( tmp_path, 'fake_image_{}.png'.format( str(iteration).zfill(8)))) logger.info('Saved intermediate file in {}'.format( os.path.join( tmp_path, 'fake_image_{}.png'.format( str(iteration).zfill(8))))) self.epoch += 1
hr_test, lr_test = next(iter(test_dataloader)) vutils.save_image(hr_test, f"{results_dir}/hr.png", normalize=True) vutils.save_image(lr_test, f"{results_dir}/lr.png", normalize=True) with open(f"{models_dir}/args.yml", "w") as f: yaml.dump(args, f) for epoch in range(args.epochs): training_bar = tqdm(train_dataloader) stats: defaultdict = defaultdict(float) for i, (hr_sample, lr_sample) in enumerate(training_bar, 1): gen_net = gen_net.train() if epoch >= args.start_discriminator: hr_sample = hr_sample.to(device) lr_sample = lr_sample.to(device) sr_sample = gen_net(lr_sample) dis_net.zero_grad() sr_dis = dis_net(sr_sample).mean() hr_dis = dis_net(hr_sample).mean() gradient_penalty = ( calc_grad_pen(dis_net, hr_sample, sr_sample, device) * args.wgan_gp_lambda) d_loss = sr_dis + 1 - hr_dis d_loss_total = d_loss + gradient_penalty d_loss_total.backward() opt_dis.step() else: d_loss = 0 gradient_penalty = 0 gen_net.zero_grad()
# realLabelGT, fakeLabelGT = fakeLabelGT, realLabelGT # Prepare a (batch_size * 1 * w * h) tensor indicates class for discriminator of CGAN repeatRealClass = None if args.GAN_TYPE == "CGAN": repeatRealClass = F.one_hot(realClass, num_classes=args.NUM_CLASSES) repeatRealClass = repeatRealClass.unsqueeze(-1).unsqueeze(-1) repeatRealClass = repeatRealClass.repeat(1, 1, *realImage.size()[-2:]) repeatRealClass = repeatRealClass.type(torch.FloatTensor) repeatRealClass = repeatRealClass.to(device) ### Update Discriminator ### # Train with real discriminator.zero_grad() realImage = realImage.to(device) pred = discriminator(realImage, repeatRealClass) if args.GAN_TYPE == "ACGAN": predLabel, predClass = pred else: predLabel = pred realClass = realClass.to(device) realLabel = realLabel.to(device) # Compute loss of D with real inputs lossRealLabelD = criterionLabel(predLabel, realLabel) lossRealClassD = criterionClass(predClass, realClass) if args.GAN_TYPE == "ACGAN" else 0 lossRealD = lossRealLabelD + lossRealClassD
class SRGAN(): def __init__(self): logger.info('Set Data Loader') self.dataset = FoodDataset(transform=transforms.Compose([ToTensor()])) self.data_loader = torch.utils.data.DataLoader(self.dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True) checkpoint, checkpoint_name = self.load_checkpoint(model_dump_path) if checkpoint == None: logger.info( 'Don\'t have pre-trained model. Ignore loading model process.') logger.info('Set Generator and Discriminator') self.G = Generator(tag=tag_size).to(device) self.D = Discriminator(tag=tag_size).to(device) logger.info('Initialize Weights') self.G.apply(initital_network_weights).to(device) self.D.apply(initital_network_weights).to(device) logger.info('Set Optimizers') self.optimizer_G = torch.optim.Adam(self.G.parameters(), lr=learning_rate, betas=(beta_1, 0.999)) self.optimizer_D = torch.optim.Adam(self.D.parameters(), lr=learning_rate, betas=(beta_1, 0.999)) self.epoch = 0 else: logger.info('Load Generator and Discriminator') self.G = Generator(tag=tag_size).to(device) self.D = Discriminator(tag=tag_size).to(device) logger.info('Load Pre-Trained Weights From Checkpoint'.format( checkpoint_name)) self.G.load_state_dict(checkpoint['G']) self.D.load_state_dict(checkpoint['D']) logger.info('Load Optimizers') self.optimizer_G = torch.optim.Adam(self.G.parameters(), lr=learning_rate, betas=(beta_1, 0.999)) self.optimizer_D = torch.optim.Adam(self.D.parameters(), lr=learning_rate, betas=(beta_1, 0.999)) self.optimizer_G.load_state_dict(checkpoint['optimizer_G']) self.optimizer_D.load_state_dict(checkpoint['optimizer_D']) self.epoch = checkpoint['epoch'] logger.info('Set Criterion') self.a_D = alexnet.alexnet(num_classes=tag_size).to(device) self.optimizer_a_D = torch.optim.Adam(self.a_D.parameters(), lr=learning_rate, betas=(beta_1, .999)) # self.label_criterion = nn.BCEWithLogitsLoss().to(device) # self.tag_criterion = nn.BCEWithLogitsLoss().to(device) def load_checkpoint(self, model_dir): models_path = utils.read_newest_model(model_dir) if len(models_path) == 0: return None, None models_path.sort() new_model_path = os.path.join(model_dump_path, models_path[-1]) if torch.cuda.is_available(): checkpoint = torch.load(new_model_path) else: checkpoint = torch.load( new_model_path, map_location='cuda' if torch.cuda.is_available() else 'cpu') return checkpoint, new_model_path def train(self): iteration = -1 label = Variable(torch.FloatTensor(batch_size, 1)).to(device) logging.info('Current epoch: {}. Max epoch: {}.'.format( self.epoch, max_epoch)) while self.epoch <= max_epoch: msg = {} adjust_learning_rate(self.optimizer_G, iteration) adjust_learning_rate(self.optimizer_D, iteration) for i, (food_tag, food_img) in enumerate(self.data_loader): iteration += 1 if food_img.shape[0] != batch_size: logging.warn('Batch size not satisfied. Ignoring.') continue if verbose: if iteration % verbose_T == 0: msg['epoch'] = int(self.epoch) msg['step'] = int(i) msg['iteration'] = iteration food_img = Variable(food_img).to(device) # 0. training assistant D self.a_D.zero_grad() a_D_feat = self.a_D(food_img) # 1. Training D # 1.1. use really image for discriminating self.D.zero_grad() label_p = self.D(food_img) label.data.fill_(1.0) # 1.2. real image's loss # real_label_loss = self.label_criterion(label_p, label) real_label_loss = F.binary_cross_entropy(label_p, label) real_loss_sum = real_label_loss real_loss_sum.backward() if verbose: if iteration % verbose_T == 0: msg['discriminator real loss'] = float(real_loss_sum) # 1.3. use fake image for discriminating g_noise, fake_tag = utils.fake_generator( batch_size, noise_size, device) fake_feat = torch.cat([g_noise, fake_tag], dim=1) fake_img = self.G(fake_feat).detach() fake_label_p = self.D(fake_img) label.data.fill_(.0) # 1.4. fake image's loss # fake_label_loss = self.label_criterion(fake_label_p, label) fake_label_loss = F.binary_cross_entropy(fake_label_p, label) # TODO: fake_loss_sum = fake_label_loss fake_loss_sum.backward() if verbose: if iteration % verbose_T == 0: print('predicted fake label: {}'.format(fake_label_p)) msg['discriminator fake loss'] = float(fake_loss_sum) # 1.6. update optimizer self.optimizer_D.step() # 2. Training G # 2.1. generate fake image self.G.zero_grad() g_noise, fake_tag = utils.fake_generator( batch_size, noise_size, device) fake_feat = torch.cat([g_noise, fake_tag], dim=1) fake_img = self.G(fake_feat) fake_label_p = self.D(fake_img) label.data.fill_(1.0) a_D_feat = self.a_D(fake_img) feat_loss = F.binary_cross_entropy(a_D_feat, fake_tag) # 2.2. calc loss # label_loss_g = self.label_criterion(fake_label_p, label) label_loss_g = F.binary_cross_entropy(fake_label_p, label) loss_g = label_loss_g loss_g.backward() if verbose: if iteration % verbose_T == 0: msg['generator loss'] = float(loss_g) # 2.2. update optimizer self.optimizer_G.step() if verbose: if iteration % verbose_T == 0: logger.info( '------------------------------------------') for key in msg.keys(): logger.info('{} : {}'.format(key, msg[key])) # save intermediate file if iteration % 10000 == 0: torch.save( { 'epoch': self.epoch, 'D': self.D.state_dict(), 'G': self.G.state_dict(), 'optimizer_D': self.optimizer_D.state_dict(), 'optimizer_G': self.optimizer_G.state_dict(), }, '{}/checkpoint_{}.tar'.format(model_dump_path, str(iteration).zfill(8))) logger.info('Checkpoint saved in: {}'.format( '{}/checkpoint_{}.tar'.format( model_dump_path, str(iteration).zfill(8)))) if iteration % verbose_T == 0: vutils.save_image( food_img.data.view(batch_size, 3, food_img.size(2), food_img.size(3)), os.path.join( tmp_path, 'real_image_{}.png'.format( str(iteration).zfill(8)))) g_noise, fake_tag = utils.fake_generator( batch_size, noise_size, device) fake_feat = torch.cat([g_noise, fake_tag], dim=1) fake_img = self.G(fake_feat) vutils.save_image( fake_img.data.view(batch_size, 3, food_img.size(2), food_img.size(3)), os.path.join( tmp_path, 'fake_image_{}.png'.format( str(iteration).zfill(8)))) logger.info('Saved intermediate file in {}'.format( os.path.join( tmp_path, 'fake_image_{}.png'.format( str(iteration).zfill(8))))) # dump checkpoint torch.save( { 'epoch': self.epoch, 'D': self.D.state_dict(), 'G': self.G.state_dict(), 'optimizer_D': self.optimizer_D.state_dict(), 'optimizer_G': self.optimizer_G.state_dict(), }, '{}/checkpoint_{}.tar'.format(model_dump_path, str(self.epoch).zfill(4))) logger.info('Checkpoint saved in: {}'.format( '{}/checkpoint_{}.tar'.format(model_dump_path, str(self.epoch).zfill(4)))) self.epoch += 1