def get_initsamples(self, dataloader, sigma_begin=None, inpainting=False, data_iter=None, bs=None): if inpainting: data_iter = iter(dataloader) refer_images, _ = next(data_iter) refer_images = refer_images.to(self.args.device) width = int(np.sqrt(self.config.sampling.batch_size)) init_samples = torch.rand(width, width, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.args.device) init_samples = data_transform(self.config.data, init_samples) return init_samples, refer_images elif self.config.sampling.data_init: _return_iter = data_iter is not None if data_iter is None: data_iter = iter(dataloader) try: samples, _ = next(data_iter) except StopIteration: data_iter = iter(dataloader) samples, _ = next(data_iter) if bs is not None: samples = samples[:bs] samples = samples.to(self.args.device) samples = data_transform(self.config.data, samples) init_samples = samples if not self.config.sampling.noise_first: init_samples += sigma_begin * torch.randn_like(samples) if _return_iter: return init_samples, data_iter return init_samples else: bs = self.config.sampling.batch_size if bs is None else bs # fid else config.fast_fid.batch_size init_samples = torch.rand(bs, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.args.device) init_samples = data_transform(self.config.data, init_samples) return init_samples
def sample_interpolation(self, S): dataset, test_dataset = get_dataset(self.args, self.config) train_loader = data.DataLoader( dataset, batch_size=2, shuffle=True, num_workers=self.config.data.num_workers, ) for i, (x, y) in enumerate(train_loader): images = tvu.make_grid(x, nrow=2, padding=1, pad_value=1, normalize=False) tvu.save_image(images, os.path.join(self.args.image_folder, "reals.png")) break noise = torch.randn( 1, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.device, ).repeat(2, 1, 1, 1) * np.sqrt(0.1) x = noise + data_transform(self.config, x.to(self.device)) coef = torch.linspace(0, 1, 10).view(-1, 1, 1, 1) coef = coef.to(self.device) x = x[[1]] * coef + x[[0]] * (1 - coef) x = sbp_stage2_interpolation(x, S, self.config, sigma_sq=0.1, record=True) images = tvu.make_grid(inverse_data_transform(self.config, x), nrow=10, padding=1, pad_value=1, normalize=False) tvu.save_image(images, os.path.join(self.args.image_folder, "2-final.png"))
def test(self): score = get_model(self.config) score = torch.nn.DataParallel(score) sigmas = get_sigmas(self.config) dataset, test_dataset = get_dataset(self.args, self.config) test_dataloader = DataLoader( test_dataset, batch_size=self.config.test.batch_size, shuffle=True, num_workers=self.config.data.num_workers, drop_last=True, ) verbose = False for ckpt in tqdm.tqdm( range(self.config.test.begin_ckpt, self.config.test.end_ckpt + 1, 5000), desc="processing ckpt:", ): states = torch.load( os.path.join(self.args.log_path, f"checkpoint_{ckpt}.pth"), map_location=self.config.device, ) if self.config.model.ema: ema_helper = EMAHelper(mu=self.config.model.ema_rate) ema_helper.register(score) ema_helper.load_state_dict(states[-1]) ema_helper.ema(score) else: score.load_state_dict(states[0]) score.eval() step = 0 mean_loss = 0.0 mean_grad_norm = 0.0 average_grad_scale = 0.0 for x, y in test_dataloader: step += 1 x = x.to(self.config.device) x = data_transform(self.config, x) test_loss = anneal_sliced_score_estimation_vr( score, x, sigmas, None, self.config.training.anneal_power ) if verbose: logging.info( "step: {}, test_loss: {}".format(step, test_loss.item()) ) mean_loss += test_loss.item() mean_loss /= step mean_grad_norm /= step average_grad_scale /= step logging.info("ckpt: {}, average test loss: {}".format(ckpt, mean_loss))
def sample_inpainting(self, S): dataset, test_dataset = get_dataset(self.args, self.config) train_loader = data.DataLoader( dataset, batch_size=4, shuffle=True, num_workers=self.config.data.num_workers, ) for i, (x, y) in enumerate(train_loader): images = tvu.make_grid(x, nrow=1, padding=1, pad_value=1, normalize=False) tvu.save_image(images, os.path.join(self.args.image_folder, "reals.png")) break mask = torch.zeros( 4, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.device, ) mask[:, :, :, :16] += 1 # 0 for missing pixels x_occluded = x.to(self.device) * mask images = tvu.make_grid(x_occluded, nrow=1, padding=1, pad_value=1, normalize=False) tvu.save_image(images, os.path.join(self.args.image_folder, "2-occluded.png")) torch.manual_seed(1) x = data_transform(self.config, x.to(self.device)) x = sbp_stage2_inpainting(x, mask, S, self.config, sigma_sq=self.args.sigma_sq, record=True) images = tvu.make_grid(inverse_data_transform(self.config, x), nrow=1, padding=1, pad_value=1, normalize=False) tvu.save_image(images, os.path.join(self.args.image_folder, "2-final.png"))
def fast_ensemble_fid(self): from evaluation.fid_score import get_fid, get_fid_stats_path import pickle num_ensembles = 5 scores = [NCSN(self.config).to(self.config.device) for _ in range(num_ensembles)] scores = [torch.nn.DataParallel(score) for score in scores] sigmas_th = get_sigmas(self.config) sigmas = sigmas_th.cpu().numpy() fids = {} for ckpt in tqdm.tqdm(range(self.config.fast_fid.begin_ckpt, self.config.fast_fid.end_ckpt + 1, 5000), desc="processing ckpt"): begin_ckpt = max(self.config.fast_fid.begin_ckpt, ckpt - (num_ensembles - 1) * 5000) index = 0 for i in range(begin_ckpt, ckpt + 5000, 5000): states = torch.load(os.path.join(self.args.log_path, f'checkpoint_{i}.pth'), map_location=self.config.device) scores[index].load_state_dict(states[0]) scores[index].eval() index += 1 def scorenet(x, labels): num_ckpts = (ckpt - begin_ckpt) // 5000 + 1 return sum([scores[i](x, labels) for i in range(num_ckpts)]) / num_ckpts num_iters = self.config.fast_fid.num_samples // self.config.fast_fid.batch_size output_path = os.path.join(self.args.image_folder, 'ckpt_{}'.format(ckpt)) os.makedirs(output_path, exist_ok=True) for i in range(num_iters): init_samples = torch.rand(self.config.fast_fid.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device) init_samples = data_transform(self.config, init_samples) all_samples = anneal_Langevin_dynamics(init_samples, scorenet, sigmas, self.config.fast_fid.n_steps_each, self.config.fast_fid.step_lr, verbose=self.config.fast_fid.verbose, denoise=self.config.sampling.denoise) final_samples = all_samples[-1] for id, sample in enumerate(final_samples): sample = sample.view(self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) save_image(sample, os.path.join(output_path, 'sample_{}.png'.format(id))) stat_path = get_fid_stats_path(self.args, self.config, download=True) fid = get_fid(stat_path, output_path) fids[ckpt] = fid print("ckpt: {}, fid: {}".format(ckpt, fid)) with open(os.path.join(self.args.image_folder, 'fids.pickle'), 'wb') as handle: pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def train_d(self): D = DensityRatioEstNet( self.config.model.ngf_d, self.config.data.image_size, self.config.data.channels ).to(self.device) optimizerD = optim.Adam( D.parameters(), lr=self.config.optim_d.lr, weight_decay=self.config.optim_d.weight_decay, betas=(self.config.optim_d.beta1, self.config.optim_d.beta2) ) D.train() dataset, test_dataset = get_dataset(self.args, self.config) train_loader = data.DataLoader( dataset, batch_size=self.config.training.batch_size, shuffle=True, num_workers=self.config.data.num_workers, ) start_epoch, step = 0, 0 for epoch in range(start_epoch, self.config.training.n_epochs_d): for i, (x, y) in enumerate(train_loader): step += 1 x = x.to(self.device) x = data_transform(self.config, x) e = torch.randn_like(x) x_real = x + e * np.sqrt(self.args.sigma_sq) z = torch.randn_like(x).to(self.device) * np.sqrt(self.args.tau) real_score = D(x_real) fake_score = D(z) optimizerD.zero_grad() loss_d_real = torch.log(1 + torch.exp(-real_score)).mean() loss_d_fake = torch.log(1 + torch.exp(fake_score)).mean() loss_d = loss_d_real + loss_d_fake loss_d.backward() optimizerD.step() if step >= self.config.training.n_iters_d: break if not step % 100: logging.info(f"step: {step}, loss: {loss_d}") if step >= self.config.training.n_iters_d: break state = {'D': D.state_dict()} torch.save(state, os.path.join(self.args.log_path, f"ckpt_DRE_{self.args.sigma_sq}_{self.args.tau}.pth"))
def forward(self, x): if not self.source_logit_transform and not self.source_rescaled: h = 2 * x - 1. else: h = x output = self.begin_conv(h) layer1 = self._compute_cond_module(self.res1, output) layer2 = self._compute_cond_module(self.res2, layer1) layer3 = self._compute_cond_module(self.res3, layer2) layer4 = self._compute_cond_module(self.res4, layer3) ref1 = self.refine1([layer4], layer4.shape[2:]) ref2 = self.refine2([layer3, ref1], layer3.shape[2:]) ref3 = self.refine3([layer2, ref2], layer2.shape[2:]) output = self.refine4([layer1, ref3], layer1.shape[2:]) output = self.normalizer(output) output = self.act(output) output = self.end_conv(output) output = torch.sigmoid(output) return data_transform(self.config.target, output)
def fast_fid(self): ### Test the fids of ensembled checkpoints. ### Shouldn't be used for pretrained with ema if self.config.fast_fid.ensemble: if self.config.model.ema: raise RuntimeError( "Cannot apply ensembling to pretrained with EMA.") self.fast_ensemble_fid() return from ncsn.evaluation.fid_score import get_fid, get_fid_stats_path import pickle score = get_model(self.config) score = torch.nn.DataParallel(score) sigmas_th = get_sigmas(self.config) sigmas = sigmas_th.cpu().numpy() fids = {} for ckpt in tqdm.tqdm(range(self.config.fast_fid.begin_ckpt, self.config.fast_fid.end_ckpt + 1, 5000), desc="processing ckpt"): states = torch.load(os.path.join(self.args.log_path, f'checkpoint_{ckpt}.pth'), map_location=self.config.device) if self.config.model.ema: ema_helper = EMAHelper(mu=self.config.model.ema_rate) ema_helper.register(score) ema_helper.load_state_dict(states[-1]) ema_helper.ema(score) else: score.load_state_dict(states[0]) score.eval() num_iters = self.config.fast_fid.num_samples // self.config.fast_fid.batch_size output_path = os.path.join(self.args.image_folder, 'ckpt_{}'.format(ckpt)) os.makedirs(output_path, exist_ok=True) for i in range(num_iters): init_samples = torch.rand(self.config.fast_fid.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device) init_samples = data_transform(self.config, init_samples) all_samples = anneal_Langevin_dynamics( init_samples, score, sigmas, self.config.fast_fid.n_steps_each, self.config.fast_fid.step_lr, verbose=self.config.fast_fid.verbose, denoise=self.config.sampling.denoise) final_samples = all_samples[-1] for id, sample in enumerate(final_samples): sample = sample.view(self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) save_image( sample, os.path.join(output_path, 'sample_{}.png'.format(id))) stat_path = get_fid_stats_path(self.args, self.config, download=True) fid = get_fid(stat_path, output_path) fids[ckpt] = fid print("ckpt: {}, fid: {}".format(ckpt, fid)) with open(os.path.join(self.args.image_folder, 'fids.pickle'), 'wb') as handle: pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def train(self): dataset, test_dataset = get_dataset(self.args, self.config) dataloader = DataLoader(dataset, batch_size=self.config.training.batch_size, shuffle=True, num_workers=self.config.data.num_workers) test_loader = DataLoader(test_dataset, batch_size=self.config.training.batch_size, shuffle=True, num_workers=self.config.data.num_workers, drop_last=True) test_iter = iter(test_loader) self.config.input_dim = self.config.data.image_size**2 * self.config.data.channels tb_logger = self.config.tb_logger score = get_model(self.config) score = torch.nn.DataParallel(score) optimizer = get_optimizer(self.config, score.parameters()) start_epoch = 0 step = 0 if self.config.model.ema: ema_helper = EMAHelper(mu=self.config.model.ema_rate) ema_helper.register(score) if self.args.resume_training: states = torch.load( os.path.join(self.args.log_path, 'checkpoint.pth')) score.load_state_dict(states[0]) ### Make sure we can resume with different eps states[1]['param_groups'][0]['eps'] = self.config.optim.eps optimizer.load_state_dict(states[1]) start_epoch = states[2] step = states[3] if self.config.model.ema: ema_helper.load_state_dict(states[4]) sigmas = get_sigmas(self.config) if self.config.training.log_all_sigmas: ### Commented out training time logging to save time. test_loss_per_sigma = [None for _ in range(len(sigmas))] def hook(loss, labels): # for i in range(len(sigmas)): # if torch.any(labels == i): # test_loss_per_sigma[i] = torch.mean(loss[labels == i]) pass def tb_hook(): # for i in range(len(sigmas)): # if test_loss_per_sigma[i] is not None: # tb_logger.add_scalar('test_loss_sigma_{}'.format(i), test_loss_per_sigma[i], # global_step=step) pass def test_hook(loss, labels): for i in range(len(sigmas)): if torch.any(labels == i): test_loss_per_sigma[i] = torch.mean(loss[labels == i]) def test_tb_hook(): for i in range(len(sigmas)): if test_loss_per_sigma[i] is not None: tb_logger.add_scalar('test_loss_sigma_{}'.format(i), test_loss_per_sigma[i], global_step=step) else: hook = test_hook = None def tb_hook(): pass def test_tb_hook(): pass for epoch in range(start_epoch, self.config.training.n_epochs): for i, (X, y) in enumerate(dataloader): score.train() step += 1 X = X.to(self.config.device) X = data_transform(self.config, X) loss = anneal_dsm_score_estimation( score, X, sigmas, None, self.config.training.anneal_power, hook) tb_logger.add_scalar('loss', loss, global_step=step) tb_hook() logging.info("step: {}, loss: {}".format(step, loss.item())) optimizer.zero_grad() loss.backward() optimizer.step() if self.config.model.ema: ema_helper.update(score) if step >= self.config.training.n_iters: return 0 if step % 100 == 0: if self.config.model.ema: test_score = ema_helper.ema_copy(score) else: test_score = score test_score.eval() try: test_X, test_y = next(test_iter) except StopIteration: test_iter = iter(test_loader) test_X, test_y = next(test_iter) test_X = test_X.to(self.config.device) test_X = data_transform(self.config, test_X) with torch.no_grad(): test_dsm_loss = anneal_dsm_score_estimation( test_score, test_X, sigmas, None, self.config.training.anneal_power, hook=test_hook) tb_logger.add_scalar('test_loss', test_dsm_loss, global_step=step) test_tb_hook() logging.info("step: {}, test_loss: {}".format( step, test_dsm_loss.item())) del test_score if step % self.config.training.snapshot_freq == 0: states = [ score.state_dict(), optimizer.state_dict(), epoch, step, ] if self.config.model.ema: states.append(ema_helper.state_dict()) torch.save( states, os.path.join(self.args.log_path, 'checkpoint_{}.pth'.format(step))) torch.save( states, os.path.join(self.args.log_path, 'checkpoint.pth')) if self.config.training.snapshot_sampling: if self.config.model.ema: test_score = ema_helper.ema_copy(score) else: test_score = score test_score.eval() ## Different part from NeurIPS 2019. ## Random state will be affected because of sampling during training time. init_samples = torch.rand(36, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device) init_samples = data_transform(self.config, init_samples) all_samples = anneal_Langevin_dynamics( init_samples, test_score, sigmas.cpu().numpy(), self.config.sampling.n_steps_each, self.config.sampling.step_lr, final_only=True, verbose=True, denoise=self.config.sampling.denoise) sample = all_samples[-1].view( all_samples[-1].shape[0], self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid(sample, 6) save_image( image_grid, os.path.join(self.args.log_sample_path, 'image_grid_{}.png'.format(step))) torch.save( sample, os.path.join(self.args.log_sample_path, 'samples_{}.pth'.format(step))) del test_score del all_samples
def sample(self, return_NCSN=False): if self.config.sampling.ckpt_id is None: states = torch.load(os.path.join(self.args.log_path, 'checkpoint.pth'), map_location=self.config.device) else: states = torch.load(os.path.join( self.args.log_path, f'checkpoint_{self.config.sampling.ckpt_id}.pth'), map_location=self.config.device) score = get_model(self.config) score = torch.nn.DataParallel(score) score.load_state_dict(states[0], strict=True) if self.config.model.ema: ema_helper = EMAHelper(mu=self.config.model.ema_rate) ema_helper.register(score) ema_helper.load_state_dict(states[-1]) ema_helper.ema(score) sigmas_th = get_sigmas(self.config) sigmas = sigmas_th.cpu().numpy() dataset, _ = get_dataset(self.args, self.config) dataloader = DataLoader(dataset, batch_size=self.config.sampling.batch_size, shuffle=True, num_workers=4) score.eval() if (return_NCSN): return score if not self.config.sampling.fid: if self.config.sampling.inpainting: data_iter = iter(dataloader) refer_images, _ = next(data_iter) refer_images = refer_images.to(self.config.device) width = int(np.sqrt(self.config.sampling.batch_size)) init_samples = torch.rand(width, width, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device) init_samples = data_transform(self.config, init_samples) all_samples = anneal_Langevin_dynamics_inpainting( init_samples, refer_images[:width, ...], score, sigmas, self.config.data.image_size, self.config.sampling.n_steps_each, self.config.sampling.step_lr) torch.save( refer_images[:width, ...], os.path.join(self.args.image_folder, 'refer_image.pth')) refer_images = refer_images[:width, None, ...].expand( -1, width, -1, -1, -1).reshape(-1, *refer_images.shape[1:]) save_image(refer_images, os.path.join(self.args.image_folder, 'refer_image.png'), nrow=width) if not self.config.sampling.final_only: for i, sample in enumerate(tqdm.tqdm(all_samples)): sample = sample.view(self.config.sampling.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid( sample, int(np.sqrt(self.config.sampling.batch_size))) save_image( image_grid, os.path.join(self.args.image_folder, 'image_grid_{}.png'.format(i))) torch.save( sample, os.path.join(self.args.image_folder, 'completion_{}.pth'.format(i))) else: sample = all_samples[-1].view( self.config.sampling.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid( sample, int(np.sqrt(self.config.sampling.batch_size))) save_image( image_grid, os.path.join( self.args.image_folder, 'image_grid_{}.png'.format( self.config.sampling.ckpt_id))) torch.save( sample, os.path.join( self.args.image_folder, 'completion_{}.pth'.format( self.config.sampling.ckpt_id))) elif self.config.sampling.interpolation: if self.config.sampling.data_init: data_iter = iter(dataloader) samples, _ = next(data_iter) samples = samples.to(self.config.device) samples = data_transform(self.config, samples) init_samples = samples + sigmas_th[0] * torch.randn_like( samples) else: init_samples = torch.rand(self.config.sampling.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device) init_samples = data_transform(self.config, init_samples) all_samples = anneal_Langevin_dynamics_interpolation( init_samples, score, sigmas, self.config.sampling.n_interpolations, self.config.sampling.n_steps_each, self.config.sampling.step_lr, verbose=True, final_only=self.config.sampling.final_only) if not self.config.sampling.final_only: for i, sample in tqdm.tqdm(enumerate(all_samples), total=len(all_samples), desc="saving image samples"): sample = sample.view(sample.shape[0], self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid( sample, nrow=self.config.sampling.n_interpolations) save_image( image_grid, os.path.join(self.args.image_folder, 'image_grid_{}.png'.format(i))) torch.save( sample, os.path.join(self.args.image_folder, 'samples_{}.pth'.format(i))) else: sample = all_samples[-1].view(all_samples[-1].shape[0], self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid( sample, self.config.sampling.n_interpolations) save_image( image_grid, os.path.join( self.args.image_folder, 'image_grid_{}.png'.format( self.config.sampling.ckpt_id))) torch.save( sample, os.path.join( self.args.image_folder, 'samples_{}.pth'.format( self.config.sampling.ckpt_id))) else: if self.config.sampling.data_init: data_iter = iter(dataloader) samples, _ = next(data_iter) samples = samples.to(self.config.device) samples = data_transform(self.config, samples) init_samples = samples + sigmas_th[0] * torch.randn_like( samples) else: init_samples = torch.rand(self.config.sampling.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device) init_samples = data_transform(self.config, init_samples) all_samples = anneal_Langevin_dynamics( init_samples, score, sigmas, self.config.sampling.n_steps_each, self.config.sampling.step_lr, verbose=True, final_only=self.config.sampling.final_only, denoise=self.config.sampling.denoise) if not self.config.sampling.final_only: for i, sample in tqdm.tqdm(enumerate(all_samples), total=len(all_samples), desc="saving image samples"): sample = sample.view(sample.shape[0], self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid( sample, int(np.sqrt(self.config.sampling.batch_size))) save_image( image_grid, os.path.join(self.args.image_folder, 'image_grid_{}.png'.format(i))) torch.save( sample, os.path.join(self.args.image_folder, 'samples_{}.pth'.format(i))) else: sample = all_samples[-1].view(all_samples[-1].shape[0], self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid( sample, int(np.sqrt(self.config.sampling.batch_size))) save_image( image_grid, os.path.join( self.args.image_folder, 'image_grid_{}.png'.format( self.config.sampling.ckpt_id))) torch.save( sample, os.path.join( self.args.image_folder, 'samples_{}.pth'.format( self.config.sampling.ckpt_id))) else: total_n_samples = self.config.sampling.num_samples4fid n_rounds = total_n_samples // self.config.sampling.batch_size if self.config.sampling.data_init: dataloader = DataLoader( dataset, batch_size=self.config.sampling.batch_size, shuffle=True, num_workers=4) data_iter = iter(dataloader) img_id = 0 for _ in tqdm.tqdm( range(n_rounds), desc= 'Generating image samples for FID/inception score evaluation' ): if self.config.sampling.data_init: try: samples, _ = next(data_iter) except StopIteration: data_iter = iter(dataloader) samples, _ = next(data_iter) samples = samples.to(self.config.device) samples = data_transform(self.config, samples) samples = samples + sigmas_th[0] * torch.randn_like( samples) else: samples = torch.rand(self.config.sampling.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device) samples = data_transform(self.config, samples) all_samples = anneal_Langevin_dynamics( samples, score, sigmas, self.config.sampling.n_steps_each, self.config.sampling.step_lr, verbose=False, denoise=self.config.sampling.denoise) samples = all_samples[-1] for img in samples: img = inverse_data_transform(self.config, img) save_image( img, os.path.join(self.args.image_folder, 'image_{}.png'.format(img_id))) img_id += 1
def train(self): source_dataset, source_test_dataset = get_dataset(self.args, self.config.source) source_loader = DataLoader(source_dataset, batch_size=self.config.training.batch_size, shuffle=True, num_workers=self.config.source.data.num_workers, drop_last=True) source_batches = iter(source_loader) target_dataset, target_test_dataset = get_dataset(self.args, self.config.target) target_loader = DataLoader(target_dataset, batch_size=self.config.training.batch_size, shuffle=True, num_workers=self.config.target.data.num_workers, drop_last=True) target_batches = iter(target_loader) if self.config.compatibility.ckpt_id is None: states = torch.load(os.path.join('baryproj', self.config.compatibility.log_path, 'checkpoint.pth'), map_location=self.config.device) else: states = torch.load(os.path.join('baryproj', self.config.compatibility.log_path, f'checkpoint_{self.config.compatibility.ckpt_id}.pth'), map_location=self.config.device) cpat = get_compatibility(self.config) cpat.load_state_dict(states[0]) cpat.eval() baryproj = get_bary(self.config) bp_opt = get_optimizer(self.config, baryproj.parameters()) if(self.args.resume_training): states = torch.load(os.path.join(self.args.log_path, 'checkpoint.pt')) baryproj.load_state_dict(states[0]) bp_opt.load_state_dict(states[1]) logging.info(f"Resuming training after {states[2]} steps.") logging.info("Optimizing the barycentric projection of the OT map.") with tqdm(total=self.config.training.n_iters) as progress: for d_step in range(self.config.training.n_iters): try: (Xs, ys) = next(source_batches) (Xt, yt) = next(target_batches) except StopIteration: # Refresh after one epoch source_batches = iter(source_loader) target_batches = iter(target_loader) (Xs, ys) = next(source_batches) (Xt, yt) = next(target_batches) Xs = data_transform(self.config.source, Xs) Xs = Xs.to(self.config.device) Xt = data_transform(self.config.target, Xt) Xt = Xt.to(self.config.device) obj = bp_opt.step(lambda: self._bp_closure(Xs, Xt, cpat, baryproj, bp_opt)) obj_val = round(obj.item(), 5) progress.update(1) progress.set_description_str(f"L2 Error: {obj_val}") self.config.tb_logger.add_scalars('Optimization', { 'Objective': obj_val }, d_step) if(d_step % self.config.training.sample_freq == 0): with torch.no_grad(): samples = baryproj(Xs) img_grid1 = torchvision.utils.make_grid(torch.clamp(samples, 0, 1)) img_grid2 = torchvision.utils.make_grid(torch.clamp(Xs, 0, 1)) self.config.tb_logger.add_image('Samples', img_grid1, d_step) self.config.tb_logger.add_image('Sources', img_grid2, d_step) if(d_step % self.config.training.snapshot_freq == 0): states = [ baryproj.state_dict(), d_step ] torch.save(states, os.path.join(self.args.log_path, f'checkpoint_{d_step}.pth')) torch.save(states, os.path.join(self.args.log_path, f'checkpoint.pth'))
def train(self): args, config = self.args, self.config vdl_logger = self.config.vdl_logger dataset, test_dataset = get_dataset(args, config) train_loader = data.DataLoader( dataset, batch_size=config.training.batch_size, shuffle=True, num_workers=config.data.num_workers, use_shared_memory=False, ) model = Model(config) model = model model = paddle.DataParallel(model) optimizer = get_optimizer(self.config, model.parameters()) if self.config.model.ema: ema_helper = EMAHelper(mu=self.config.model.ema_rate) ema_helper.register(model) else: ema_helper = None start_epoch, step = 0, 0 if self.args.resume_training: states = paddle.load(os.path.join(self.args.log_path, "ckpt.pdl")) model.set_state_dict({ k.split("$model_")[-1]: v for k, v in states.items() if "$model_" in k }) optimizer.set_state_dict({ k.split("$optimizer_")[-1]: v for k, v in states.items() if "$optimizer_" in k }) optimizer._epsilon = self.config.optim.eps start_epoch = states["$epoch"] step = states["$step"] if self.config.model.ema: ema_helper.set_state_dict({ k.split("$ema_")[-1]: v for k, v in states.items() if "$ema_" in k }) for epoch in range(start_epoch, self.config.training.n_epochs): data_start = time.time() data_time = 0 for i, (x, y) in enumerate(train_loader): n = x.shape[0] data_time += time.time() - data_start model.train() step += 1 x = data_transform(self.config, x) e = paddle.randn(x.shape) b = self.betas # antithetic sampling t = paddle.randint(low=0, high=self.num_timesteps, shape=(n // 2 + 1, )) t = paddle.concat([t, self.num_timesteps - t - 1], 0)[:n] loss = loss_registry[config.model.type](model, x, t, e, b) vdl_logger.add_scalar("loss", loss, step=step) logging.info( f"step: {step}, loss: {loss.numpy()[0]}, data time: {data_time / (i+1)}" ) optimizer.clear_grad() loss.backward() optimizer.step() if self.config.model.ema: ema_helper.update(model) if step % self.config.training.snapshot_freq == 0 or step == 1: states = dict( **{ "$model_" + k: v for k, v in model.state_dict().items() }, **{ "$optimizer_" + k: v for k, v in optimizer.state_dict().items() }, **{"$epoch": epoch}, **{"$step": step}, ) if self.config.model.ema: states.update({ "$ema_" + k: v for k, v in ema_helper.state_dict().items() }) paddle.save( states, os.path.join(self.args.log_path, "ckpt_{}.pdl".format(step)), ) paddle.save(states, os.path.join(self.args.log_path, "ckpt.pdl")) data_start = time.time()
def calculate_fid(self): import fid, pickle import tensorflow as tf stats_path = "fid_stats_cifar10_train.npz" # training set statistics inception_path = fid.check_or_download_inception( "./tmp/" ) # download inception network score = get_model(self.config) score = torch.nn.DataParallel(score) sigmas_th = get_sigmas(self.config) sigmas = sigmas_th.cpu().numpy() fids = {} for ckpt in tqdm.tqdm( range( self.config.fast_fid.begin_ckpt, self.config.fast_fid.end_ckpt + 1, 5000 ), desc="processing ckpt", ): states = torch.load( os.path.join(self.args.log_path, f"checkpoint_{ckpt}.pth"), map_location=self.config.device, ) if self.config.model.ema: ema_helper = EMAHelper(mu=self.config.model.ema_rate) ema_helper.register(score) ema_helper.load_state_dict(states[-1]) ema_helper.ema(score) else: score.load_state_dict(states[0]) score.eval() num_iters = ( self.config.fast_fid.num_samples // self.config.fast_fid.batch_size ) output_path = os.path.join(self.args.image_folder, "ckpt_{}".format(ckpt)) os.makedirs(output_path, exist_ok=True) for i in range(num_iters): init_samples = torch.rand( self.config.fast_fid.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device, ) init_samples = data_transform(self.config, init_samples) all_samples = anneal_Langevin_dynamics( init_samples, score, sigmas, self.config.fast_fid.n_steps_each, self.config.fast_fid.step_lr, verbose=self.config.fast_fid.verbose, ) final_samples = all_samples[-1] for id, sample in enumerate(final_samples): sample = sample.view( self.config.data.channels, self.config.data.image_size, self.config.data.image_size, ) sample = inverse_data_transform(self.config, sample) save_image( sample, os.path.join(output_path, "sample_{}.png".format(id)) ) # load precalculated training set statistics f = np.load(stats_path) mu_real, sigma_real = f["mu"][:], f["sigma"][:] f.close() fid.create_inception_graph( inception_path ) # load the graph into the current TF graph final_samples = ( (final_samples - final_samples.min()) / (final_samples.max() - final_samples.min()).data.cpu().numpy() * 255 ) final_samples = np.transpose(final_samples, [0, 2, 3, 1]) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) mu_gen, sigma_gen = fid.calculate_activation_statistics( final_samples, sess, batch_size=100 ) fid_value = fid.calculate_frechet_distance( mu_gen, sigma_gen, mu_real, sigma_real ) print("FID: %s" % fid_value) with open(os.path.join(self.args.image_folder, "fids.pickle"), "wb") as handle: pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def sample(self): if self.config.ncsn.sampling.ckpt_id is None: ncsn_states = torch.load(os.path.join( 'scones', self.config.ncsn.sampling.log_path, 'checkpoint.pth'), map_location=self.config.device) else: ncsn_states = torch.load(os.path.join( 'scones', self.config.ncsn.sampling.log_path, f'checkpoint_{self.config.ncsn.sampling.ckpt_id}.pth'), map_location=self.config.device) score = get_scorenet(self.config) score = torch.nn.DataParallel(score) sigmas_th = get_sigmas(self.config.ncsn) sigmas = sigmas_th.cpu().numpy() if ("module.sigmas" in ncsn_states[0].keys()): ncsn_states[0]["module.sigmas"] = sigmas_th score.load_state_dict(ncsn_states[0], strict=True) score.eval() baryproj_data_init = (hasattr(self.config, "baryproj") and self.config.ncsn.sampling.data_init) if (baryproj_data_init): if (self.config.baryproj.ckpt_id is None): bproj_states = torch.load(os.path.join( 'scones', self.config.baryproj.log_path, 'checkpoint.pth'), map_location=self.config.device) else: bproj_states = torch.load(os.path.join( 'scones', self.config.baryproj.log_path, f'checkpoint_{self.config.baryproj.ckpt_id}.pth'), map_location=self.config.device) bproj = get_bary(self.config) bproj.load_state_dict(bproj_states[0]) bproj = torch.nn.DataParallel(bproj) bproj.eval() if self.config.compatibility.ckpt_id is None: cpat_states = torch.load(os.path.join( 'scones', self.config.compatibility.log_path, 'checkpoint.pth'), map_location=self.config.device) else: cpat_states = torch.load(os.path.join( 'scones', self.config.compatibility.log_path, f'checkpoint_{self.config.compatibility.ckpt_id}.pth'), map_location=self.config.device) cpat = get_compatibility(self.config) cpat.load_state_dict(cpat_states[0]) if self.config.ncsn.model.ema: ema_helper = EMAHelper(mu=self.config.ncsn.model.ema_rate) ema_helper.register(score) ema_helper.load_state_dict(ncsn_states[-1]) ema_helper.ema(score) source_dataset, _ = get_dataset(self.args, self.config.source) dataloader = DataLoader( source_dataset, batch_size=self.config.ncsn.sampling.sources_per_batch, shuffle=True, num_workers=self.config.source.data.num_workers) data_iter = iter(dataloader) (Xs, labels) = next(data_iter) Xs_global = torch.cat([Xs] * self.config.ncsn.sampling.samples_per_source, dim=0).to(self.config.device) Xs_global = data_transform(self.config.source, Xs_global) if (hasattr(self.config.ncsn.sampling, "n_sigmas_skip")): n_sigmas_skip = self.config.ncsn.sampling.n_sigmas_skip else: n_sigmas_skip = 0 if not self.config.ncsn.sampling.fid: if self.config.ncsn.sampling.inpainting: ''' NCSN INPAINTING CODE. EITHER PATCH THIS FOR SCONES OR REMOVE IT. data_iter = iter(dataloader) refer_images, _ = next(data_iter) refer_images = refer_images.to(self.config.device) width = int(np.sqrt(self.config.sampling.batch_size)) init_samples = torch.rand(width, width, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device) init_samples = data_transform(self.config, init_samples) all_samples = anneal_Langevin_dynamics_inpainting(init_samples, refer_images[:width, ...], score, sigmas, self.config.data.image_size, self.config.sampling.n_steps_each, self.config.sampling.step_lr) torch.save(refer_images[:width, ...], os.path.join(self.args.image_folder, 'refer_image.pth')) refer_images = refer_images[:width, None, ...].expand(-1, width, -1, -1, -1).reshape(-1, *refer_images.shape[ 1:]) save_image(refer_images, os.path.join(self.args.image_folder, 'refer_image.png'), nrow=width) if not self.config.sampling.final_only: for i, sample in enumerate(tqdm.tqdm(all_samples)): sample = sample.view(self.config.sampling.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid(sample, int(np.sqrt(self.config.sampling.batch_size))) save_image(image_grid, os.path.join(self.args.image_folder, 'image_grid_{}.png'.format(i))) torch.save(sample, os.path.join(self.args.image_folder, 'completion_{}.pth'.format(i))) else: sample = all_samples[-1].view(self.config.sampling.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid(sample, int(np.sqrt(self.config.sampling.batch_size))) save_image(image_grid, os.path.join(self.args.image_folder, 'image_grid_{}.png'.format(self.config.ncsn.sampling.ckpt_id))) torch.save(sample, os.path.join(self.args.image_folder, 'completion_{}.pth'.format(self.config.sampling.ckpt_id))) ''' raise NotImplementedError( "Inpainting with SCONES is not currently implemented.") elif self.config.ncsn.sampling.interpolation: ''' NCSN INTERPOLATION CODE. EITHER PATCH THIS FOR SCONES OR REMOVE IT. if self.config.sampling.data_init: data_iter = iter(dataloader) samples, _ = next(data_iter) samples = samples.to(self.config.device) samples = data_transform(self.config, samples) init_samples = samples + sigmas_th[0] * torch.randn_like(samples) else: init_samples = torch.rand(self.config.sampling.batch_size, self.config.data.channels, self.config.data.image_size, self.config.data.image_size, device=self.config.device) init_samples = data_transform(self.config, init_samples) all_samples = anneal_Langevin_dynamics_interpolation(init_samples, score, sigmas, self.config.sampling.n_interpolations, self.config.sampling.n_steps_each, self.config.sampling.step_lr, verbose=True, final_only=self.config.sampling.final_only) if not self.config.sampling.final_only: for i, sample in tqdm.tqdm(enumerate(all_samples), total=len(all_samples), desc="saving image samples"): sample = sample.view(sample.shape[0], self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid(sample, nrow=self.config.sampling.n_interpolations) save_image(image_grid, os.path.join(self.args.image_folder, 'image_grid_{}.png'.format(i))) torch.save(sample, os.path.join(self.args.image_folder, 'samples_{}.pth'.format(i))) else: sample = all_samples[-1].view(all_samples[-1].shape[0], self.config.data.channels, self.config.data.image_size, self.config.data.image_size) sample = inverse_data_transform(self.config, sample) image_grid = make_grid(sample, self.config.sampling.n_interpolations) save_image(image_grid, os.path.join(self.args.image_folder, 'image_grid_{}.png'.format(self.config.sampling.ckpt_id))) torch.save(sample, os.path.join(self.args.image_folder, 'samples_{}.pth'.format(self.config.sampling.ckpt_id))) ''' raise NotImplementedError( "Interpolation with SCONES is not currently implemented.") else: if self.config.ncsn.sampling.data_init: if (baryproj_data_init): with torch.no_grad(): init_Xt = (bproj(Xs_global) + sigmas_th[n_sigmas_skip] * torch.randn_like(Xs_global)).detach() else: init_Xt = Xs_global + sigmas_th[ n_sigmas_skip] * torch.randn_like(Xs_global) init_Xt.requires_grad = True init_Xt = init_Xt.to(self.config.device) else: init_Xt = torch.rand( self.config.ncsn.sampling.sources_per_batch * self.config.ncsn.sampling.samples_per_source, self.config.target.data.channels, self.config.target.data.image_size, self.config.target.data.image_size, device=self.config.device) init_Xt = data_transform(self.config.target, init_Xt) init_Xt.requires_grad = True init_Xt = init_Xt.to(self.config.device) all_samples = anneal_Langevin_dynamics( init_Xt, Xs_global, score, cpat, sigmas, self.config.ncsn.sampling.n_steps_each, self.config.ncsn.sampling.step_lr, verbose=True, final_only=self.config.ncsn.sampling.final_only, denoise=self.config.ncsn.sampling.denoise, n_sigmas_skip=n_sigmas_skip) all_samples = torch.stack(all_samples, dim=0) if not self.config.ncsn.sampling.final_only: all_samples = all_samples.view( (-1, self.config.ncsn.sampling.sources_per_batch, self.config.ncsn.sampling.samples_per_source, self.config.target.data.channels, self.config.target.data.image_size, self.config.target.data.image_size)) np.save( os.path.join(self.args.image_folder, 'all_samples.npy'), all_samples.detach().cpu().numpy()) sample = all_samples[-1].view( self.config.ncsn.sampling.sources_per_batch * self.config.ncsn.sampling.samples_per_source, self.config.target.data.channels, self.config.target.data.image_size, self.config.target.data.image_size) sample = inverse_data_transform(self.config.target, sample) image_grid = make_grid( sample, nrow=self.config.ncsn.sampling.sources_per_batch) save_image( image_grid, os.path.join(self.args.image_folder, 'sample_grid.png')) source_grid = make_grid( Xs, nrow=self.config.ncsn.sampling.sources_per_batch) save_image( source_grid, os.path.join(self.args.image_folder, 'source_grid.png')) bproj_of_source = make_grid( bproj(Xs), nrow=self.config.ncsn.sampling.sources_per_batch) save_image( bproj_of_source, os.path.join(self.args.image_folder, 'bproj_sources.png')) np.save(os.path.join(self.args.image_folder, 'sources.npy'), Xs.detach().cpu().numpy()) np.save( os.path.join(self.args.image_folder, 'source_labels.npy'), labels.detach().cpu().numpy()) np.save(os.path.join(self.args.image_folder, 'bproj.npy'), bproj(Xs).detach().cpu().numpy()) np.save(os.path.join(self.args.image_folder, 'samples.npy'), sample.detach().cpu().numpy()) else: batch_size = self.config.ncsn.sampling.sources_per_batch * self.config.ncsn.sampling.samples_per_source total_n_samples = self.config.ncsn.sampling.num_samples4fid n_rounds = total_n_samples // batch_size if self.config.ncsn.sampling.data_init: dataloader = DataLoader( source_dataset, batch_size=self.config.ncsn.sampling.sources_per_batch, shuffle=True, num_workers=self.config.source.data.num_workers) data_iter = iter(dataloader) img_id = 0 for r in tqdm.tqdm( range(n_rounds), desc= 'Generating image samples for FID/inception score evaluation' ): if self.config.ncsn.sampling.data_init: try: init_samples, labels = next(data_iter) init_samples = torch.cat( [init_samples] * self.config.ncsn.sampling.samples_per_source, dim=0) labels = torch.cat( [labels] * self.config.ncsn.sampling.samples_per_source, dim=0) except StopIteration: data_iter = iter(dataloader) init_samples, labels = next(data_iter) init_samples = torch.cat( [init_samples] * self.config.ncsn.sampling.samples_per_source, dim=0) labels = torch.cat( [labels] * self.config.ncsn.sampling.samples_per_source, dim=0) init_samples = init_samples.to(self.config.device) init_samples = data_transform(self.config.target, init_samples) if (baryproj_data_init): with torch.no_grad(): bproj_samples = bproj(init_samples).detach() else: bproj_samples = torch.clone(init_samples).detach() samples = bproj_samples + sigmas_th[ n_sigmas_skip] * torch.randn_like(bproj_samples) samples.requires_grad = True samples = samples.to(self.config.device) else: samples = torch.rand(batch_size, self.config.target.data.channels, self.config.target.data.image_size, self.config.target.data.image_size, device=self.config.device) init_samples = torch.clone(samples) samples = data_transform(self.config.target, samples) samples.requires_grad = True samples = samples.to(self.config.device) all_samples = anneal_Langevin_dynamics( samples, Xs_global, score, cpat, sigmas, self.config.ncsn.sampling.n_steps_each, self.config.ncsn.sampling.step_lr, verbose=True, final_only=self.config.ncsn.sampling.final_only, denoise=self.config.ncsn.sampling.denoise, n_sigmas_skip=n_sigmas_skip) samples = all_samples[-1] for img in samples: img = inverse_data_transform(self.config.target, img) save_image( img, os.path.join(self.args.image_folder, 'image_{}.png'.format(img_id))) img_id += 1 if (self.args.save_labels): save_path = os.path.join(self.args.image_folder, 'labels') np.save(os.path.join(save_path, f'sources_{r}.npy'), init_samples.detach().cpu().numpy()) np.save(os.path.join(save_path, f'source_labels_{r}.npy'), labels.detach().cpu().numpy()) np.save(os.path.join(save_path, f"bproj_{r}.npy"), bproj_samples.detach().cpu().numpy()) np.save(os.path.join(save_path, f"samples_{r}.npy"), samples.detach().cpu().numpy())
def fast_ensemble_fid(self): from ncsn.evaluation.fid_score import get_fid, get_fid_stats_path import pickle num_ensembles = 5 scores = [ NCSN(self.config.ncsn).to(self.config.device) for _ in range(num_ensembles) ] scores = [torch.nn.DataParallel(score) for score in scores] sigmas_th = get_sigmas(self.config.ncsn) sigmas = sigmas_th.cpu().numpy() if self.config.compatibility.ckpt_id is None: cpat_states = torch.load(os.path.join( 'scones', self.config.compatibility.log_path, 'checkpoint.pth'), map_location=self.config.device) else: cpat_states = torch.load(os.path.join( 'scones', self.config.compatibility.log_path, f'checkpoint_{self.config.compatibility.ckpt_id}.pth'), map_location=self.config.device) cpat = get_compatibility(self.config) cpat.load_state_dict(cpat_states[0]) source_dataset, _ = get_dataset(self.args, self.config.source) source_dataloader = DataLoader( source_dataset, batch_size=self.config.ncsn.sampling.sources_per_batch, shuffle=True, num_workers=self.config.source.data.num_workers) source_iter = iter(source_dataloader) fids = {} for ckpt in tqdm.tqdm(range(self.config.ncsn.fast_fid.begin_ckpt, self.config.ncsn.fast_fid.end_ckpt + 1, 5000), desc="processing ckpt"): begin_ckpt = max(self.config.ncsn.fast_fid.begin_ckpt, ckpt - (num_ensembles - 1) * 5000) index = 0 for i in range(begin_ckpt, ckpt + 5000, 5000): states = torch.load(os.path.join(self.args.log_path, f'checkpoint_{i}.pth'), map_location=self.config.device) scores[index].load_state_dict(states[0]) scores[index].eval() index += 1 def scorenet(x, labels): num_ckpts = (ckpt - begin_ckpt) // 5000 + 1 return sum([scores[i](x, labels) for i in range(num_ckpts)]) / num_ckpts num_iters = self.config.ncsn.fast_fid.num_samples // self.config.ncsn.fast_fid.batch_size output_path = os.path.join(self.args.image_folder, 'ckpt_{}'.format(ckpt)) os.makedirs(output_path, exist_ok=True) for i in range(num_iters): try: (Xs, _) = next(source_iter) Xs_global = torch.cat( [Xs] * self.config.ncsn.sampling.samples_per_source, dim=0).to(self.config.device) except StopIteration: source_iter = iter(source_dataloader) (Xs, _) = next(source_iter) Xs_global = torch.cat( [Xs] * self.config.ncsn.sampling.samples_per_source, dim=0).to(self.config.device) init_samples = torch.rand(self.config.ncsn.fast_fid.batch_size, self.config.target.data.channels, self.config.target.data.image_size, self.config.target.data.image_size, device=self.config.device) init_samples = data_transform(self.config.target, init_samples) init_samples.requires_grad = True init_samples = init_samples.to(self.config.device) all_samples = anneal_Langevin_dynamics( init_samples, Xs_global, scorenet, cpat, sigmas, self.config.ncsn.fast_fid.n_steps_each, self.config.ncsn.fast_fid.step_lr, verbose=self.config.ncsn.fast_fid.verbose, final_only=self.config.ncsn.sampling.final_only, denoise=self.config.ncsn.sampling.denoise) final_samples = all_samples[-1] for id, sample in enumerate(final_samples): sample = sample.view(self.config.target.data.channels, self.config.target.data.image_size, self.config.target.data.image_size) sample = inverse_data_transform(self.config.target, sample) save_image( sample, os.path.join(output_path, 'sample_{}.png'.format(id))) stat_path = get_fid_stats_path(self.args, self.config.ncsn, download=True) fid = get_fid(stat_path, output_path) fids[ckpt] = fid print("ckpt: {}, fid: {}".format(ckpt, fid)) with open(os.path.join(self.args.image_folder, 'fids.pickle'), 'wb') as handle: pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def fast_fid(self): ### Test the fids of ensembled checkpoints. ### Shouldn't be used for pretrained with ema if self.config.ncsn.fast_fid.ensemble: if self.config.ncsn.model.ema: raise RuntimeError( "Cannot apply ensembling to pretrained with EMA.") self.fast_ensemble_fid() return from ncsn.evaluation.fid_score import get_fid, get_fid_stats_path import pickle source_dataset, _ = get_dataset(self.args, self.config.source) source_dataloader = DataLoader( source_dataset, batch_size=self.config.ncsn.sampling.sources_per_batch, shuffle=True, num_workers=self.config.source.data.num_workers) source_iter = iter(source_dataloader) score = get_scorenet(self.config.ncsn) score = torch.nn.DataParallel(score) if self.config.compatibility.ckpt_id is None: cpat_states = torch.load(os.path.join( 'scones', self.config.compatibility.log_path, 'checkpoint.pth'), map_location=self.config.device) else: cpat_states = torch.load(os.path.join( 'scones', self.config.compatibility.log_path, f'checkpoint_{self.config.compatibility.ckpt_id}.pth'), map_location=self.config.device) cpat = get_compatibility(self.config) cpat.load_state_dict(cpat_states[0]) sigmas_th = get_sigmas(self.config.ncsn) sigmas = sigmas_th.cpu().numpy() fids = {} for ckpt in tqdm.tqdm(range(self.config.ncsn.fast_fid.begin_ckpt, self.config.ncsn.fast_fid.end_ckpt + 1, 5000), desc="processing ckpt"): states = torch.load(os.path.join(self.args.log_path, f'checkpoint_{ckpt}.pth'), map_location=self.config.device) if self.config.ncsn.model.ema: ema_helper = EMAHelper(mu=self.config.ncsn.model.ema_rate) ema_helper.register(score) ema_helper.load_state_dict(states[-1]) ema_helper.ema(score) else: score.load_state_dict(states[0]) score.eval() num_iters = self.config.ncsn.fast_fid.num_samples // self.config.ncsn.fast_fid.batch_size output_path = os.path.join(self.args.image_folder, 'ckpt_{}'.format(ckpt)) os.makedirs(output_path, exist_ok=True) for i in range(num_iters): try: (Xs, _) = next(source_iter) Xs_global = torch.cat( [Xs] * self.config.ncsn.sampling.samples_per_source, dim=0).to(self.config.device) except StopIteration: source_iter = iter(source_dataloader) (Xs, _) = next(source_iter) Xs_global = torch.cat( [Xs] * self.config.ncsn.sampling.samples_per_source, dim=0).to(self.config.device) init_samples = torch.rand(self.config.ncsn.fast_fid.batch_size, self.config.target.data.channels, self.config.target.data.image_size, self.config.target.data.image_size, device=self.config.device) init_samples = data_transform(self.config.target, init_samples) init_samples.requires_grad = True init_samples = init_samples.to(self.config.device) all_samples = anneal_Langevin_dynamics( init_samples, Xs_global, score, cpat, sigmas, self.config.ncsn.fast_fid.n_steps_each, self.config.ncsn.fast_fid.step_lr, verbose=self.config.ncsn.fast_fid.verbose, final_only=self.config.ncsn.sampling.final_only, denoise=self.config.ncsn.sampling.denoise) final_samples = all_samples[-1] for id, sample in enumerate(final_samples): sample = sample.view(self.config.target.data.channels, self.config.target.data.image_size, self.config.target.data.image_size) sample = inverse_data_transform(self.config.target, sample) save_image( sample, os.path.join(output_path, 'sample_{}.png'.format(id))) stat_path = get_fid_stats_path(self.args, self.config.ncsn, download=True) fid = get_fid(stat_path, output_path) fids[ckpt] = fid print("ckpt: {}, fid: {}".format(ckpt, fid)) with open(os.path.join(self.args.image_folder, 'fids.pickle'), 'wb') as handle: pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def sample(self): source_dataset, _ = get_dataset(self.args, self.config.source) baryproj = get_bary(self.config) baryproj.eval() if self.config.sampling.ckpt_id is None: bp_states = torch.load(os.path.join(self.args.log_path, 'checkpoint.pth'), map_location=self.config.device) else: bp_states = torch.load(os.path.join(self.args.log_path, f'checkpoint_{self.config.compatibility.ckpt_id}.pth'), map_location=self.config.device) baryproj.load_state_dict(bp_states[0]) if(not self.config.sampling.fid): dataloader = DataLoader(source_dataset, batch_size=self.config.sampling.batch_size, shuffle=True, num_workers=self.config.source.data.num_workers) batch_samples = [] for i in range(self.config.sampling.n_batches): (Xs, _) = next(iter(dataloader)) Xs = data_transform(self.config.source, Xs) transport = baryproj(Xs) batch_samples.append(inverse_data_transform(self.config, transport)) sample = torch.cat(batch_samples, dim=0) image_grid = make_grid(sample[:min(64, len(sample))], nrow=8) save_image(image_grid, os.path.join(self.args.image_folder, 'sample_grid.png')) source_grid = make_grid(Xs[:min(64, len(Xs))], nrow=8) save_image(source_grid, os.path.join(self.args.image_folder, 'source_grid.png')) np.save(os.path.join(self.args.image_folder, 'sample.npy'), sample.detach().cpu().numpy()) np.save(os.path.join(self.args.image_folder, 'sources.npy'), Xs.detach().cpu().numpy()) else: batch_size = self.config.sampling.samples_per_batch total_n_samples = self.config.sampling.num_samples4fid n_rounds = total_n_samples // batch_size dataloader = DataLoader(source_dataset, batch_size=self.config.sampling.samples_per_batch, shuffle=True, num_workers=self.config.source.data.num_workers) data_iter = iter(dataloader) img_id = 0 for _ in tqdm(range(n_rounds), desc='Generating image samples for FID/inception score evaluation'): with torch.no_grad(): (Xs, _) = next(data_iter) Xs = data_transform(self.config.source, Xs).to(self.config.device) transport = baryproj(Xs) for img in transport: img = inverse_data_transform(self.config.target, img) save_image(img, os.path.join(self.args.image_folder, 'image_{}.png'.format(img_id))) img_id += 1 del Xs del transport
def train(self): source_dataset, source_test_dataset = get_dataset(self.args, self.config.source) source_loader = DataLoader(source_dataset, batch_size=self.config.training.batch_size, shuffle=True, num_workers=self.config.source.data.num_workers, drop_last=True) source_batches = iter(source_loader) target_dataset, target_test_dataset = get_dataset(self.args, self.config.target) target_loader = DataLoader(target_dataset, batch_size=self.config.training.batch_size, shuffle=True, num_workers=self.config.target.data.num_workers, drop_last=True) target_batches = iter(target_loader) cpat = get_compatibility(self.config) cpat_opt = get_optimizer(self.config, cpat.parameters()) if(self.args.resume_training): states = torch.load(os.path.join(self.args.log_path, 'checkpoint.pt')) cpat.load_state_dict(states[0]) cpat_opt.load_state_dict(states[1]) logging.info(f"Resuming training after {states[2]} steps.") logging.info("Optimizing the compatibility function.") with tqdm(total=self.config.training.n_iters) as progress: for d_step in range(self.config.training.n_iters): try: (Xs, ys) = next(source_batches) (Xt, yt) = next(target_batches) except StopIteration: # Refresh after one epoch source_batches = iter(source_loader) target_batches = iter(target_loader) (Xs, ys) = next(source_batches) (Xt, yt) = next(target_batches) Xs = data_transform(self.config.source, Xs) Xs = Xs.to(self.config.device) Xt = data_transform(self.config.target, Xt) Xt = Xt.to(self.config.device) obj = cpat_opt.step(lambda: self._cpat_closure(Xs, Xt, cpat, cpat_opt)) avg_density = torch.mean(cpat.forward(Xs, Xt)) obj_val = round(obj.item(), 5) avg_density_val = round(avg_density.item(), 5) progress.update(1) progress.set_description_str(f"Average Density: {avg_density_val}") self.config.tb_logger.add_scalars('Optimization', { 'Objective': obj_val, 'Average Density': avg_density_val }, d_step) if(d_step % self.config.training.snapshot_freq == 0): states = [ cpat.state_dict(), cpat_opt.state_dict(), d_step ] torch.save(states, os.path.join(self.args.log_path, f'checkpoint_{d_step}.pth')) torch.save(states, os.path.join(self.args.log_path, f'checkpoint.pth'))