class Model(): def __init__(self, args): self.args = args self.pretrained = False self.epoch = 0 self.G = Generator() self.D = Discriminator() self.g_optimizer = optim.Adam(self.G.parameters(), lr=1E-4) self.d_optimizer = optim.Adam(self.D.parameters(), lr=1E-4) self.g_scheduler = optim.lr_scheduler.StepLR(self.g_optimizer, step_size=40) self.d_scheduler = optim.lr_scheduler.StepLR(self.d_optimizer, step_size=40) self.train_losses = [] self.val_losses = [] if args.load_model: self._load_state(args.load_model) # extract all layers prior to the last softmax of VGG-19 vgg19_layers = list(models.vgg19(pretrained=True).features)[:36] self.vgg19 = nn.Sequential(*vgg19_layers).eval() for param in self.vgg19.parameters(): param.requires_grad = False self.mse_loss = torch.nn.MSELoss() self.bce_loss = torch.nn.BCELoss() def train(self, train_dataloader, val_dataloader=None): self.D.to(device) self.G.to(device) self.vgg19.to(device) """ Pretrain Generator """ if not self.pretrained: log_message("Starting pretraining") self._pretrain(train_dataloader) self._save_state() if val_dataloader: val_g_loss, _ = self.evaluate(val_dataloader) log_message("Pretrain G loss: {:.4f}".format(val_g_loss)) """ Real Training """ log_message("Starting training") while self.epoch < self.args.epochs: # Train one epoch self.D.train() self.G.train() g_loss, d_loss = self._run_epoch(train_dataloader, train=True) self.train_losses.append([g_loss, d_loss]) self.g_scheduler.step() self.d_scheduler.step() self.epoch += 1 log_message("Epoch: {}/{}".format(self.epoch, self.args.epochs)) # Print evaluation train_string = "Train G loss: {:.4f} | Train D loss: {:.4f}".format( g_loss, d_loss) if self.epoch % self.args.eval_epochs == 0: if val_dataloader: val_g_loss, val_d_loss = self.evaluate(val_dataloader) self.val_losses.append([val_g_loss, val_d_loss]) train_string += " | Val G loss: {:.4f} | Val D loss: {:.4f}".format( val_g_loss, val_d_loss) log_message(train_string) # Save the model if self.epoch % self.args.save_epochs == 0: self._save_state() log_message("Finished training") self._save_state() def evaluate(self, dataloader): self.D.eval() self.G.eval() with torch.no_grad(): return self._run_epoch(dataloader, train=False) def generate(self, dataloader): def to_image(tensor): array = tensor.data.cpu().numpy() array = array.transpose((1, 2, 0)) array = np.clip(255.0 * (array + 1) / 2, 0, 255) array = np.uint8(array) return Image.fromarray(array) self.D.eval() self.G.eval() if not os.path.exists(self.args.generate_dir): os.mkdir(self.args.generate_dir) with torch.no_grad(): for batch in dataloader: low_res = batch['low_res'].to(device) hi_res = batch['high_res'] generated = self.G(low_res) for i in range(len(generated)): naive = np.clip( 255.0 * low_res[i].data.cpu().numpy().transpose( (1, 2, 0)), 0, 255) naive = Image.fromarray(np.uint8(naive)) naive = naive.resize((96, 96), Image.BICUBIC) fake_im = to_image(generated[i]) real_im = to_image(hi_res[i]) naive.save( os.path.join(self.args.generate_dir, "{}_naive.png".format(i))) fake_im.save( os.path.join(self.args.generate_dir, "{}_fake.png".format(i))) real_im.save( os.path.join(self.args.generate_dir, "{}_real.png".format(i))) if i > 10: return def _load_state(self, fname): if torch.cuda.is_available(): map_location = lambda storage, loc: storage.cuda() else: map_location = 'cpu' state = torch.load(fname, map_location=map_location) self.pretrained = state["pretrained"] self.epoch = state["epoch"] self.train_losses = state["train_losses"] self.val_losses = state["val_losses"] self.G.load_state_dict(state["G"]) self.D.load_state_dict(state["D"]) self.g_optimizer.load_state_dict(state["g_optimizer"]) self.d_optimizer.load_state_dict(state["d_optimizer"]) self.g_scheduler.load_state_dict(state["g_scheduler"]) self.d_scheduler.load_state_dict(state["d_scheduler"]) for state in self.d_optimizer.state.values(): for k, v in state.items(): if torch.is_tensor(v): state[k] = v.to(device) for state in self.g_optimizer.state.values(): for k, v in state.items(): if torch.is_tensor(v): state[k] = v.to(device) def _save_state(self): if not os.path.exists(self.args.save_dir): os.mkdir(self.args.save_dir) fname = "%s/save_%d.pkl" % (self.args.save_dir, self.epoch) state = { "pretrained": self.pretrained, "epoch": self.epoch, "G": self.G.state_dict(), "D": self.D.state_dict(), "g_optimizer": self.g_optimizer.state_dict(), "d_optimizer": self.d_optimizer.state_dict(), "g_scheduler": self.g_scheduler.state_dict(), "d_scheduler": self.d_scheduler.state_dict(), "train_losses": self.train_losses, "val_losses": self.val_losses } torch.save(state, fname) def _pretrain(self, dataloader): self.G.train() for i in range(self.args.pretrain_epochs): log_message("Pretrain Epoch: {}/{}".format( i, self.args.pretrain_epochs)) for batch in dataloader: low_res = batch['low_res'].to(device) high_res = batch['high_res'].to(device) self.g_optimizer.zero_grad() generated = self.G(low_res) # Optimize pixel loss g_loss = self.mse_loss(generated, high_res) g_loss.backward() self.g_optimizer.step() self.pretrained = True def _run_epoch(self, dataloader, train): g_losses, d_losses = [], [] for batch in dataloader: low_res = batch['low_res'].to(device) high_res = batch['high_res'].to(device) batch_size = high_res.size(0) real = torch.ones((batch_size, 1), requires_grad=False).to(device) fake = torch.zeros((batch_size, 1), requires_grad=False).to(device) """ Discriminator """ generated = self.G(low_res) self.d_optimizer.zero_grad() real_loss = self.bce_loss(self.D(high_res), real) fake_loss = self.bce_loss(self.D(generated), fake) d_loss = real_loss + fake_loss d_losses.append(d_loss.item()) if train: d_loss.backward() self.d_optimizer.step() """ Generator """ generated = self.G(low_res) self.g_optimizer.zero_grad() # take a [B, C, W, H] batch of [-1, 1] images, normalize, then run through vgg19 def vgg_features(image): mean = torch.tensor( [0.485, 0.456, 0.406]).unsqueeze(0).unsqueeze(2).unsqueeze(3).to(device) std = torch.tensor( [0.229, 0.224, 0.225]).unsqueeze(0).unsqueeze(2).unsqueeze(3).to(device) image = (image + 1) / 2 image = (image - mean) / std return self.vgg19(image) pixel_loss = self.mse_loss(high_res, generated) content_loss = self.mse_loss(vgg_features(high_res), vgg_features(generated)) adversarial_loss = self.bce_loss(self.D(generated), real) g_loss = pixel_loss + 0.006 * content_loss + 1E-3 * adversarial_loss g_losses.append(g_loss.item()) if train: g_loss.backward() self.g_optimizer.step() return np.mean(g_losses), np.mean(d_losses)
class Solver(object): def __init__(self, config, data_loader): self.generator = None self.discriminator = None self.g_optimizer = None self.d_optimizer = None self.g_conv_dim = config.g_conv_dim self.d_conv_dim = config.d_conv_dim self.z_dim = config.z_dim self.beta1 = config.beta1 self.beta2 = config.beta2 self.image_size = config.image_size self.data_loader = data_loader self.num_epochs = config.num_epochs self.batch_size = config.batch_size self.sample_size = config.sample_size self.lr = config.lr self.log_step = config.log_step self.sample_step = config.sample_step self.sample_path = config.sample_path self.model_path = config.model_path self.epoch = config.epoch self.build_model() self.plotter = Plotter() def build_model(self): """Build generator and discriminator.""" self.generator = Generator(z_dim=self.z_dim) print(count_parameters(self.generator)) self.discriminator = Discriminator() print(count_parameters(self.discriminator)) self.g_optimizer = optim.Adam(self.generator.parameters(), self.lr, (self.beta1, self.beta2)) self.d_optimizer = optim.Adam(self.discriminator.parameters(), self.lr*1, (self.beta1, self.beta2)) if self.epoch: g_path = os.path.join(self.model_path, 'generator-%d.pkl' % self.epoch) d_path = os.path.join(self.model_path, 'discriminator-%d.pkl' % self.epoch) g_optim_path = os.path.join(self.model_path, 'gen-optim-%d.pkl' % self.epoch) d_optim_path = os.path.join(self.model_path, 'dis-optim-%d.pkl' % self.epoch) self.generator.load_state_dict(torch.load(g_path)) self.discriminator.load_state_dict(torch.load(d_path)) self.g_optimizer.load_state_dict(torch.load(g_optim_path)) self.d_optimizer.load_state_dict(torch.load(d_optim_path)) if torch.cuda.is_available(): self.generator.cuda() self.discriminator.cuda() def to_variable(self, x): """Convert tensor to variable.""" if torch.cuda.is_available(): x = x.cuda() return Variable(x) def to_data(self, x): """Convert variable to tensor.""" if torch.cuda.is_available(): x = x.cpu() return x.data def reset_grad(self): """Zero the gradient buffers.""" self.discriminator.zero_grad() self.generator.zero_grad() def denorm(self, x): """Convert range (-1, 1) to (0, 1)""" out = (x + 1) / 2 return out.clamp(0, 1) def train(self): """Train generator and discriminator.""" fixed_noise = self.to_variable(torch.randn(self.batch_size, self.z_dim)) total_step = len(self.data_loader) for epoch in range(self.epoch, self.epoch + self.num_epochs) if self.epoch else range(self.num_epochs): for i, images in enumerate(self.data_loader): if len(images) != self.batch_size: continue # self.plotter.draw_kernels(self.discriminator) for p in self.discriminator.parameters(): p.requires_grad = True #===================== Train D =====================# images = self.to_variable(images) images.retain_grad() batch_size = images.size(0) noise = self.to_variable(torch.randn(batch_size, self.z_dim)) # Train D to recognize real images as real. outputs = self.discriminator(images) real_loss = torch.mean((outputs - 1) ** 2) # L2 loss instead of Binary cross entropy loss (this is optional for stable training) # real_loss = torch.mean(outputs - 1) # Train D to recognize fake images as fake. fake_images = self.generator(noise) fake_images.retain_grad() outputs = self.discriminator(fake_images) fake_loss = torch.mean(outputs ** 2) # fake_loss = torch.mean(outputs) # gradient penalty gp_loss = calc_gradient_penalty(self.discriminator, images, fake_images) # Backprop + optimize d_loss = fake_loss + real_loss + gp_loss self.reset_grad() d_loss.backward() self.d_optimizer.step() if i % 10 == 0: self.plotter.draw_activations(fake_images.grad[0], original=fake_images[0]) g_losses = [] for p in self.discriminator.parameters(): p.requires_grad = False #===================== Train G =====================# for g_batch in range(5): noise = self.to_variable(torch.randn(batch_size, self.z_dim)) # Train G so that D recognizes G(z) as real. fake_images = self.generator(noise) outputs = self.discriminator(fake_images) g_loss = torch.mean((outputs - 1) ** 2) # g_loss = -torch.mean(outputs) # Backprop + optimize self.reset_grad() g_loss.backward() # if g_loss.item() < 0.5 * d_loss.item(): # break self.g_optimizer.step() g_losses.append("%.3f"%g_loss.clone().item()) # print the log info if (i+1) % self.log_step == 0: print('Epoch [%d/%d], Step[%d/%d], d_real_loss: %.4f, ' 'd_fake_loss: %.4f, gp_loss: %s, g_loss: %s' %(epoch+1, self.num_epochs, i+1, total_step, real_loss.item(), fake_loss.item(), gp_loss.item(), ", ".join(g_losses))) # save the sampled images # print((i+1)%self.sample_step) if (i) % self.sample_step == 0: print("saving samples") fake_images = self.generator(fixed_noise) if not os.path.exists(self.sample_path): os.makedirs(self.sample_path) torchvision.utils.save_image(self.denorm(fake_images.data), os.path.join(self.sample_path, 'fake_samples-%d-%d.png' %(epoch+1, i+1))) # save the model parameters for each epoch if epoch % 5 == 0: if not os.path.exists(self.model_path): os.mkdir(self.model_path) g_path = os.path.join(self.model_path, 'generator-%d.pkl' %(epoch+1)) d_path = os.path.join(self.model_path, 'discriminator-%d.pkl' %(epoch+1)) g_optim_path = os.path.join(self.model_path, 'gen-optim-%d.pkl' % (epoch + 1)) d_optim_path = os.path.join(self.model_path, 'dis-optim-%d.pkl' % (epoch + 1)) torch.save(self.generator.state_dict(), g_path) torch.save(self.discriminator.state_dict(), d_path) torch.save(self.g_optimizer.state_dict(), g_optim_path) torch.save(self.d_optimizer.state_dict(), d_optim_path) def sample(self): # Load trained parameters g_path = os.path.join(self.model_path, 'generator-%d.pkl' % self.num_epochs) d_path = os.path.join(self.model_path, 'discriminator-%d.pkl' % self.num_epochs) self.generator.load_state_dict(torch.load(g_path)) self.discriminator.load_state_dict(torch.load(d_path)) self.generator.eval() self.discriminator.eval() # Sample the images noise = self.to_variable(torch.randn(self.sample_size, self.z_dim)) fake_images = self.generator(noise) sample_path = os.path.join(self.sample_path, 'fake_samples-final.png') torchvision.utils.save_image(self.denorm(fake_images.data), sample_path, nrow=12) print("Saved sampled images to '%s'" %sample_path)