def get_initsamples(self,
dataloader,
sigma_begin=None,
inpainting=False,
data_iter=None,
bs=None):
if inpainting:
data_iter = iter(dataloader)
refer_images, _ = next(data_iter)
refer_images = refer_images.to(self.args.device)
width = int(np.sqrt(self.config.sampling.batch_size))
init_samples = torch.rand(width,
width,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.args.device)
init_samples = data_transform(self.config.data, init_samples)
return init_samples, refer_images
elif self.config.sampling.data_init:
_return_iter = data_iter is not None
if data_iter is None:
data_iter = iter(dataloader)
try:
samples, _ = next(data_iter)
except StopIteration:
data_iter = iter(dataloader)
samples, _ = next(data_iter)
if bs is not None:
samples = samples[:bs]
samples = samples.to(self.args.device)
samples = data_transform(self.config.data, samples)
init_samples = samples
if not self.config.sampling.noise_first:
init_samples += sigma_begin * torch.randn_like(samples)
if _return_iter:
return init_samples, data_iter
return init_samples
else:
bs = self.config.sampling.batch_size if bs is None else bs # fid else config.fast_fid.batch_size
init_samples = torch.rand(bs,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.args.device)
init_samples = data_transform(self.config.data, init_samples)
return init_samples
def sample_interpolation(self, S):
dataset, test_dataset = get_dataset(self.args, self.config)
train_loader = data.DataLoader(
dataset,
batch_size=2,
shuffle=True,
num_workers=self.config.data.num_workers,
)
for i, (x, y) in enumerate(train_loader):
images = tvu.make_grid(x, nrow=2, padding=1, pad_value=1, normalize=False)
tvu.save_image(images, os.path.join(self.args.image_folder, "reals.png"))
break
noise = torch.randn(
1,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.device,
).repeat(2, 1, 1, 1) * np.sqrt(0.1)
x = noise + data_transform(self.config, x.to(self.device))
coef = torch.linspace(0, 1, 10).view(-1, 1, 1, 1)
coef = coef.to(self.device)
x = x[[1]] * coef + x[[0]] * (1 - coef)
x = sbp_stage2_interpolation(x, S, self.config, sigma_sq=0.1, record=True)
images = tvu.make_grid(inverse_data_transform(self.config, x), nrow=10, padding=1, pad_value=1, normalize=False)
tvu.save_image(images, os.path.join(self.args.image_folder, "2-final.png"))
def test(self):
score = get_model(self.config)
score = torch.nn.DataParallel(score)
sigmas = get_sigmas(self.config)
dataset, test_dataset = get_dataset(self.args, self.config)
test_dataloader = DataLoader(
test_dataset,
batch_size=self.config.test.batch_size,
shuffle=True,
num_workers=self.config.data.num_workers,
drop_last=True,
)
verbose = False
for ckpt in tqdm.tqdm(
range(self.config.test.begin_ckpt, self.config.test.end_ckpt + 1, 5000),
desc="processing ckpt:",
):
states = torch.load(
os.path.join(self.args.log_path, f"checkpoint_{ckpt}.pth"),
map_location=self.config.device,
)
if self.config.model.ema:
ema_helper = EMAHelper(mu=self.config.model.ema_rate)
ema_helper.register(score)
ema_helper.load_state_dict(states[-1])
ema_helper.ema(score)
else:
score.load_state_dict(states[0])
score.eval()
step = 0
mean_loss = 0.0
mean_grad_norm = 0.0
average_grad_scale = 0.0
for x, y in test_dataloader:
step += 1
x = x.to(self.config.device)
x = data_transform(self.config, x)
test_loss = anneal_sliced_score_estimation_vr(
score, x, sigmas, None, self.config.training.anneal_power
)
if verbose:
logging.info(
"step: {}, test_loss: {}".format(step, test_loss.item())
)
mean_loss += test_loss.item()
mean_loss /= step
mean_grad_norm /= step
average_grad_scale /= step
logging.info("ckpt: {}, average test loss: {}".format(ckpt, mean_loss))
def sample_inpainting(self, S):
dataset, test_dataset = get_dataset(self.args, self.config)
train_loader = data.DataLoader(
dataset,
batch_size=4,
shuffle=True,
num_workers=self.config.data.num_workers,
)
for i, (x, y) in enumerate(train_loader):
images = tvu.make_grid(x, nrow=1, padding=1, pad_value=1, normalize=False)
tvu.save_image(images, os.path.join(self.args.image_folder, "reals.png"))
break
mask = torch.zeros(
4,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.device,
)
mask[:, :, :, :16] += 1 # 0 for missing pixels
x_occluded = x.to(self.device) * mask
images = tvu.make_grid(x_occluded, nrow=1, padding=1, pad_value=1, normalize=False)
tvu.save_image(images, os.path.join(self.args.image_folder, "2-occluded.png"))
torch.manual_seed(1)
x = data_transform(self.config, x.to(self.device))
x = sbp_stage2_inpainting(x, mask, S, self.config, sigma_sq=self.args.sigma_sq, record=True)
images = tvu.make_grid(inverse_data_transform(self.config, x), nrow=1, padding=1, pad_value=1, normalize=False)
tvu.save_image(images, os.path.join(self.args.image_folder, "2-final.png"))
def fast_ensemble_fid(self):
from evaluation.fid_score import get_fid, get_fid_stats_path
import pickle
num_ensembles = 5
scores = [NCSN(self.config).to(self.config.device) for _ in range(num_ensembles)]
scores = [torch.nn.DataParallel(score) for score in scores]
sigmas_th = get_sigmas(self.config)
sigmas = sigmas_th.cpu().numpy()
fids = {}
for ckpt in tqdm.tqdm(range(self.config.fast_fid.begin_ckpt, self.config.fast_fid.end_ckpt + 1, 5000),
desc="processing ckpt"):
begin_ckpt = max(self.config.fast_fid.begin_ckpt, ckpt - (num_ensembles - 1) * 5000)
index = 0
for i in range(begin_ckpt, ckpt + 5000, 5000):
states = torch.load(os.path.join(self.args.log_path, f'checkpoint_{i}.pth'),
map_location=self.config.device)
scores[index].load_state_dict(states[0])
scores[index].eval()
index += 1
def scorenet(x, labels):
num_ckpts = (ckpt - begin_ckpt) // 5000 + 1
return sum([scores[i](x, labels) for i in range(num_ckpts)]) / num_ckpts
num_iters = self.config.fast_fid.num_samples // self.config.fast_fid.batch_size
output_path = os.path.join(self.args.image_folder, 'ckpt_{}'.format(ckpt))
os.makedirs(output_path, exist_ok=True)
for i in range(num_iters):
init_samples = torch.rand(self.config.fast_fid.batch_size, self.config.data.channels,
self.config.data.image_size, self.config.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config, init_samples)
all_samples = anneal_Langevin_dynamics(init_samples, scorenet, sigmas,
self.config.fast_fid.n_steps_each,
self.config.fast_fid.step_lr,
verbose=self.config.fast_fid.verbose,
denoise=self.config.sampling.denoise)
final_samples = all_samples[-1]
for id, sample in enumerate(final_samples):
sample = sample.view(self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
save_image(sample, os.path.join(output_path, 'sample_{}.png'.format(id)))
stat_path = get_fid_stats_path(self.args, self.config, download=True)
fid = get_fid(stat_path, output_path)
fids[ckpt] = fid
print("ckpt: {}, fid: {}".format(ckpt, fid))
with open(os.path.join(self.args.image_folder, 'fids.pickle'), 'wb') as handle:
pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def train_d(self):
D = DensityRatioEstNet(
self.config.model.ngf_d, self.config.data.image_size, self.config.data.channels
).to(self.device)
optimizerD = optim.Adam(
D.parameters(),
lr=self.config.optim_d.lr,
weight_decay=self.config.optim_d.weight_decay,
betas=(self.config.optim_d.beta1, self.config.optim_d.beta2)
)
D.train()
dataset, test_dataset = get_dataset(self.args, self.config)
train_loader = data.DataLoader(
dataset,
batch_size=self.config.training.batch_size,
shuffle=True,
num_workers=self.config.data.num_workers,
)
start_epoch, step = 0, 0
for epoch in range(start_epoch, self.config.training.n_epochs_d):
for i, (x, y) in enumerate(train_loader):
step += 1
x = x.to(self.device)
x = data_transform(self.config, x)
e = torch.randn_like(x)
x_real = x + e * np.sqrt(self.args.sigma_sq)
z = torch.randn_like(x).to(self.device) * np.sqrt(self.args.tau)
real_score = D(x_real)
fake_score = D(z)
optimizerD.zero_grad()
loss_d_real = torch.log(1 + torch.exp(-real_score)).mean()
loss_d_fake = torch.log(1 + torch.exp(fake_score)).mean()
loss_d = loss_d_real + loss_d_fake
loss_d.backward()
optimizerD.step()
if step >= self.config.training.n_iters_d:
break
if not step % 100:
logging.info(f"step: {step}, loss: {loss_d}")
if step >= self.config.training.n_iters_d:
break
state = {'D': D.state_dict()}
torch.save(state, os.path.join(self.args.log_path, f"ckpt_DRE_{self.args.sigma_sq}_{self.args.tau}.pth"))
def forward(self, x):
if not self.source_logit_transform and not self.source_rescaled:
h = 2 * x - 1.
else:
h = x
output = self.begin_conv(h)
layer1 = self._compute_cond_module(self.res1, output)
layer2 = self._compute_cond_module(self.res2, layer1)
layer3 = self._compute_cond_module(self.res3, layer2)
layer4 = self._compute_cond_module(self.res4, layer3)
ref1 = self.refine1([layer4], layer4.shape[2:])
ref2 = self.refine2([layer3, ref1], layer3.shape[2:])
ref3 = self.refine3([layer2, ref2], layer2.shape[2:])
output = self.refine4([layer1, ref3], layer1.shape[2:])
output = self.normalizer(output)
output = self.act(output)
output = self.end_conv(output)
output = torch.sigmoid(output)
return data_transform(self.config.target, output)
def fast_fid(self):
### Test the fids of ensembled checkpoints.
### Shouldn't be used for pretrained with ema
if self.config.fast_fid.ensemble:
if self.config.model.ema:
raise RuntimeError(
"Cannot apply ensembling to pretrained with EMA.")
self.fast_ensemble_fid()
return
from ncsn.evaluation.fid_score import get_fid, get_fid_stats_path
import pickle
score = get_model(self.config)
score = torch.nn.DataParallel(score)
sigmas_th = get_sigmas(self.config)
sigmas = sigmas_th.cpu().numpy()
fids = {}
for ckpt in tqdm.tqdm(range(self.config.fast_fid.begin_ckpt,
self.config.fast_fid.end_ckpt + 1, 5000),
desc="processing ckpt"):
states = torch.load(os.path.join(self.args.log_path,
f'checkpoint_{ckpt}.pth'),
map_location=self.config.device)
if self.config.model.ema:
ema_helper = EMAHelper(mu=self.config.model.ema_rate)
ema_helper.register(score)
ema_helper.load_state_dict(states[-1])
ema_helper.ema(score)
else:
score.load_state_dict(states[0])
score.eval()
num_iters = self.config.fast_fid.num_samples // self.config.fast_fid.batch_size
output_path = os.path.join(self.args.image_folder,
'ckpt_{}'.format(ckpt))
os.makedirs(output_path, exist_ok=True)
for i in range(num_iters):
init_samples = torch.rand(self.config.fast_fid.batch_size,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config, init_samples)
all_samples = anneal_Langevin_dynamics(
init_samples,
score,
sigmas,
self.config.fast_fid.n_steps_each,
self.config.fast_fid.step_lr,
verbose=self.config.fast_fid.verbose,
denoise=self.config.sampling.denoise)
final_samples = all_samples[-1]
for id, sample in enumerate(final_samples):
sample = sample.view(self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
save_image(
sample,
os.path.join(output_path, 'sample_{}.png'.format(id)))
stat_path = get_fid_stats_path(self.args,
self.config,
download=True)
fid = get_fid(stat_path, output_path)
fids[ckpt] = fid
print("ckpt: {}, fid: {}".format(ckpt, fid))
with open(os.path.join(self.args.image_folder, 'fids.pickle'),
'wb') as handle:
pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def train(self):
dataset, test_dataset = get_dataset(self.args, self.config)
dataloader = DataLoader(dataset,
batch_size=self.config.training.batch_size,
shuffle=True,
num_workers=self.config.data.num_workers)
test_loader = DataLoader(test_dataset,
batch_size=self.config.training.batch_size,
shuffle=True,
num_workers=self.config.data.num_workers,
drop_last=True)
test_iter = iter(test_loader)
self.config.input_dim = self.config.data.image_size**2 * self.config.data.channels
tb_logger = self.config.tb_logger
score = get_model(self.config)
score = torch.nn.DataParallel(score)
optimizer = get_optimizer(self.config, score.parameters())
start_epoch = 0
step = 0
if self.config.model.ema:
ema_helper = EMAHelper(mu=self.config.model.ema_rate)
ema_helper.register(score)
if self.args.resume_training:
states = torch.load(
os.path.join(self.args.log_path, 'checkpoint.pth'))
score.load_state_dict(states[0])
### Make sure we can resume with different eps
states[1]['param_groups'][0]['eps'] = self.config.optim.eps
optimizer.load_state_dict(states[1])
start_epoch = states[2]
step = states[3]
if self.config.model.ema:
ema_helper.load_state_dict(states[4])
sigmas = get_sigmas(self.config)
if self.config.training.log_all_sigmas:
### Commented out training time logging to save time.
test_loss_per_sigma = [None for _ in range(len(sigmas))]
def hook(loss, labels):
# for i in range(len(sigmas)):
# if torch.any(labels == i):
# test_loss_per_sigma[i] = torch.mean(loss[labels == i])
pass
def tb_hook():
# for i in range(len(sigmas)):
# if test_loss_per_sigma[i] is not None:
# tb_logger.add_scalar('test_loss_sigma_{}'.format(i), test_loss_per_sigma[i],
# global_step=step)
pass
def test_hook(loss, labels):
for i in range(len(sigmas)):
if torch.any(labels == i):
test_loss_per_sigma[i] = torch.mean(loss[labels == i])
def test_tb_hook():
for i in range(len(sigmas)):
if test_loss_per_sigma[i] is not None:
tb_logger.add_scalar('test_loss_sigma_{}'.format(i),
test_loss_per_sigma[i],
global_step=step)
else:
hook = test_hook = None
def tb_hook():
pass
def test_tb_hook():
pass
for epoch in range(start_epoch, self.config.training.n_epochs):
for i, (X, y) in enumerate(dataloader):
score.train()
step += 1
X = X.to(self.config.device)
X = data_transform(self.config, X)
loss = anneal_dsm_score_estimation(
score, X, sigmas, None, self.config.training.anneal_power,
hook)
tb_logger.add_scalar('loss', loss, global_step=step)
tb_hook()
logging.info("step: {}, loss: {}".format(step, loss.item()))
optimizer.zero_grad()
loss.backward()
optimizer.step()
if self.config.model.ema:
ema_helper.update(score)
if step >= self.config.training.n_iters:
return 0
if step % 100 == 0:
if self.config.model.ema:
test_score = ema_helper.ema_copy(score)
else:
test_score = score
test_score.eval()
try:
test_X, test_y = next(test_iter)
except StopIteration:
test_iter = iter(test_loader)
test_X, test_y = next(test_iter)
test_X = test_X.to(self.config.device)
test_X = data_transform(self.config, test_X)
with torch.no_grad():
test_dsm_loss = anneal_dsm_score_estimation(
test_score,
test_X,
sigmas,
None,
self.config.training.anneal_power,
hook=test_hook)
tb_logger.add_scalar('test_loss',
test_dsm_loss,
global_step=step)
test_tb_hook()
logging.info("step: {}, test_loss: {}".format(
step, test_dsm_loss.item()))
del test_score
if step % self.config.training.snapshot_freq == 0:
states = [
score.state_dict(),
optimizer.state_dict(),
epoch,
step,
]
if self.config.model.ema:
states.append(ema_helper.state_dict())
torch.save(
states,
os.path.join(self.args.log_path,
'checkpoint_{}.pth'.format(step)))
torch.save(
states,
os.path.join(self.args.log_path, 'checkpoint.pth'))
if self.config.training.snapshot_sampling:
if self.config.model.ema:
test_score = ema_helper.ema_copy(score)
else:
test_score = score
test_score.eval()
## Different part from NeurIPS 2019.
## Random state will be affected because of sampling during training time.
init_samples = torch.rand(36,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config,
init_samples)
all_samples = anneal_Langevin_dynamics(
init_samples,
test_score,
sigmas.cpu().numpy(),
self.config.sampling.n_steps_each,
self.config.sampling.step_lr,
final_only=True,
verbose=True,
denoise=self.config.sampling.denoise)
sample = all_samples[-1].view(
all_samples[-1].shape[0],
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(sample, 6)
save_image(
image_grid,
os.path.join(self.args.log_sample_path,
'image_grid_{}.png'.format(step)))
torch.save(
sample,
os.path.join(self.args.log_sample_path,
'samples_{}.pth'.format(step)))
del test_score
del all_samples
def sample(self, return_NCSN=False):
if self.config.sampling.ckpt_id is None:
states = torch.load(os.path.join(self.args.log_path,
'checkpoint.pth'),
map_location=self.config.device)
else:
states = torch.load(os.path.join(
self.args.log_path,
f'checkpoint_{self.config.sampling.ckpt_id}.pth'),
map_location=self.config.device)
score = get_model(self.config)
score = torch.nn.DataParallel(score)
score.load_state_dict(states[0], strict=True)
if self.config.model.ema:
ema_helper = EMAHelper(mu=self.config.model.ema_rate)
ema_helper.register(score)
ema_helper.load_state_dict(states[-1])
ema_helper.ema(score)
sigmas_th = get_sigmas(self.config)
sigmas = sigmas_th.cpu().numpy()
dataset, _ = get_dataset(self.args, self.config)
dataloader = DataLoader(dataset,
batch_size=self.config.sampling.batch_size,
shuffle=True,
num_workers=4)
score.eval()
if (return_NCSN):
return score
if not self.config.sampling.fid:
if self.config.sampling.inpainting:
data_iter = iter(dataloader)
refer_images, _ = next(data_iter)
refer_images = refer_images.to(self.config.device)
width = int(np.sqrt(self.config.sampling.batch_size))
init_samples = torch.rand(width,
width,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config, init_samples)
all_samples = anneal_Langevin_dynamics_inpainting(
init_samples, refer_images[:width, ...], score, sigmas,
self.config.data.image_size,
self.config.sampling.n_steps_each,
self.config.sampling.step_lr)
torch.save(
refer_images[:width, ...],
os.path.join(self.args.image_folder, 'refer_image.pth'))
refer_images = refer_images[:width, None, ...].expand(
-1, width, -1, -1, -1).reshape(-1, *refer_images.shape[1:])
save_image(refer_images,
os.path.join(self.args.image_folder,
'refer_image.png'),
nrow=width)
if not self.config.sampling.final_only:
for i, sample in enumerate(tqdm.tqdm(all_samples)):
sample = sample.view(self.config.sampling.batch_size,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(
sample,
int(np.sqrt(self.config.sampling.batch_size)))
save_image(
image_grid,
os.path.join(self.args.image_folder,
'image_grid_{}.png'.format(i)))
torch.save(
sample,
os.path.join(self.args.image_folder,
'completion_{}.pth'.format(i)))
else:
sample = all_samples[-1].view(
self.config.sampling.batch_size,
self.config.data.channels, self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(
sample, int(np.sqrt(self.config.sampling.batch_size)))
save_image(
image_grid,
os.path.join(
self.args.image_folder, 'image_grid_{}.png'.format(
self.config.sampling.ckpt_id)))
torch.save(
sample,
os.path.join(
self.args.image_folder, 'completion_{}.pth'.format(
self.config.sampling.ckpt_id)))
elif self.config.sampling.interpolation:
if self.config.sampling.data_init:
data_iter = iter(dataloader)
samples, _ = next(data_iter)
samples = samples.to(self.config.device)
samples = data_transform(self.config, samples)
init_samples = samples + sigmas_th[0] * torch.randn_like(
samples)
else:
init_samples = torch.rand(self.config.sampling.batch_size,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config, init_samples)
all_samples = anneal_Langevin_dynamics_interpolation(
init_samples,
score,
sigmas,
self.config.sampling.n_interpolations,
self.config.sampling.n_steps_each,
self.config.sampling.step_lr,
verbose=True,
final_only=self.config.sampling.final_only)
if not self.config.sampling.final_only:
for i, sample in tqdm.tqdm(enumerate(all_samples),
total=len(all_samples),
desc="saving image samples"):
sample = sample.view(sample.shape[0],
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(
sample, nrow=self.config.sampling.n_interpolations)
save_image(
image_grid,
os.path.join(self.args.image_folder,
'image_grid_{}.png'.format(i)))
torch.save(
sample,
os.path.join(self.args.image_folder,
'samples_{}.pth'.format(i)))
else:
sample = all_samples[-1].view(all_samples[-1].shape[0],
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(
sample, self.config.sampling.n_interpolations)
save_image(
image_grid,
os.path.join(
self.args.image_folder, 'image_grid_{}.png'.format(
self.config.sampling.ckpt_id)))
torch.save(
sample,
os.path.join(
self.args.image_folder, 'samples_{}.pth'.format(
self.config.sampling.ckpt_id)))
else:
if self.config.sampling.data_init:
data_iter = iter(dataloader)
samples, _ = next(data_iter)
samples = samples.to(self.config.device)
samples = data_transform(self.config, samples)
init_samples = samples + sigmas_th[0] * torch.randn_like(
samples)
else:
init_samples = torch.rand(self.config.sampling.batch_size,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config, init_samples)
all_samples = anneal_Langevin_dynamics(
init_samples,
score,
sigmas,
self.config.sampling.n_steps_each,
self.config.sampling.step_lr,
verbose=True,
final_only=self.config.sampling.final_only,
denoise=self.config.sampling.denoise)
if not self.config.sampling.final_only:
for i, sample in tqdm.tqdm(enumerate(all_samples),
total=len(all_samples),
desc="saving image samples"):
sample = sample.view(sample.shape[0],
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(
sample,
int(np.sqrt(self.config.sampling.batch_size)))
save_image(
image_grid,
os.path.join(self.args.image_folder,
'image_grid_{}.png'.format(i)))
torch.save(
sample,
os.path.join(self.args.image_folder,
'samples_{}.pth'.format(i)))
else:
sample = all_samples[-1].view(all_samples[-1].shape[0],
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(
sample, int(np.sqrt(self.config.sampling.batch_size)))
save_image(
image_grid,
os.path.join(
self.args.image_folder, 'image_grid_{}.png'.format(
self.config.sampling.ckpt_id)))
torch.save(
sample,
os.path.join(
self.args.image_folder, 'samples_{}.pth'.format(
self.config.sampling.ckpt_id)))
else:
total_n_samples = self.config.sampling.num_samples4fid
n_rounds = total_n_samples // self.config.sampling.batch_size
if self.config.sampling.data_init:
dataloader = DataLoader(
dataset,
batch_size=self.config.sampling.batch_size,
shuffle=True,
num_workers=4)
data_iter = iter(dataloader)
img_id = 0
for _ in tqdm.tqdm(
range(n_rounds),
desc=
'Generating image samples for FID/inception score evaluation'
):
if self.config.sampling.data_init:
try:
samples, _ = next(data_iter)
except StopIteration:
data_iter = iter(dataloader)
samples, _ = next(data_iter)
samples = samples.to(self.config.device)
samples = data_transform(self.config, samples)
samples = samples + sigmas_th[0] * torch.randn_like(
samples)
else:
samples = torch.rand(self.config.sampling.batch_size,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.config.device)
samples = data_transform(self.config, samples)
all_samples = anneal_Langevin_dynamics(
samples,
score,
sigmas,
self.config.sampling.n_steps_each,
self.config.sampling.step_lr,
verbose=False,
denoise=self.config.sampling.denoise)
samples = all_samples[-1]
for img in samples:
img = inverse_data_transform(self.config, img)
save_image(
img,
os.path.join(self.args.image_folder,
'image_{}.png'.format(img_id)))
img_id += 1
def train(self):
source_dataset, source_test_dataset = get_dataset(self.args, self.config.source)
source_loader = DataLoader(source_dataset, batch_size=self.config.training.batch_size,
shuffle=True, num_workers=self.config.source.data.num_workers, drop_last=True)
source_batches = iter(source_loader)
target_dataset, target_test_dataset = get_dataset(self.args, self.config.target)
target_loader = DataLoader(target_dataset, batch_size=self.config.training.batch_size,
shuffle=True, num_workers=self.config.target.data.num_workers, drop_last=True)
target_batches = iter(target_loader)
if self.config.compatibility.ckpt_id is None:
states = torch.load(os.path.join('baryproj', self.config.compatibility.log_path, 'checkpoint.pth'), map_location=self.config.device)
else:
states = torch.load(os.path.join('baryproj', self.config.compatibility.log_path, f'checkpoint_{self.config.compatibility.ckpt_id}.pth'),
map_location=self.config.device)
cpat = get_compatibility(self.config)
cpat.load_state_dict(states[0])
cpat.eval()
baryproj = get_bary(self.config)
bp_opt = get_optimizer(self.config, baryproj.parameters())
if(self.args.resume_training):
states = torch.load(os.path.join(self.args.log_path, 'checkpoint.pt'))
baryproj.load_state_dict(states[0])
bp_opt.load_state_dict(states[1])
logging.info(f"Resuming training after {states[2]} steps.")
logging.info("Optimizing the barycentric projection of the OT map.")
with tqdm(total=self.config.training.n_iters) as progress:
for d_step in range(self.config.training.n_iters):
try:
(Xs, ys) = next(source_batches)
(Xt, yt) = next(target_batches)
except StopIteration:
# Refresh after one epoch
source_batches = iter(source_loader)
target_batches = iter(target_loader)
(Xs, ys) = next(source_batches)
(Xt, yt) = next(target_batches)
Xs = data_transform(self.config.source, Xs)
Xs = Xs.to(self.config.device)
Xt = data_transform(self.config.target, Xt)
Xt = Xt.to(self.config.device)
obj = bp_opt.step(lambda: self._bp_closure(Xs, Xt, cpat, baryproj, bp_opt))
obj_val = round(obj.item(), 5)
progress.update(1)
progress.set_description_str(f"L2 Error: {obj_val}")
self.config.tb_logger.add_scalars('Optimization', {
'Objective': obj_val
}, d_step)
if(d_step % self.config.training.sample_freq == 0):
with torch.no_grad():
samples = baryproj(Xs)
img_grid1 = torchvision.utils.make_grid(torch.clamp(samples, 0, 1))
img_grid2 = torchvision.utils.make_grid(torch.clamp(Xs, 0, 1))
self.config.tb_logger.add_image('Samples', img_grid1, d_step)
self.config.tb_logger.add_image('Sources', img_grid2, d_step)
if(d_step % self.config.training.snapshot_freq == 0):
states = [
baryproj.state_dict(),
d_step
]
torch.save(states, os.path.join(self.args.log_path, f'checkpoint_{d_step}.pth'))
torch.save(states, os.path.join(self.args.log_path, f'checkpoint.pth'))
def train(self):
args, config = self.args, self.config
vdl_logger = self.config.vdl_logger
dataset, test_dataset = get_dataset(args, config)
train_loader = data.DataLoader(
dataset,
batch_size=config.training.batch_size,
shuffle=True,
num_workers=config.data.num_workers,
use_shared_memory=False,
)
model = Model(config)
model = model
model = paddle.DataParallel(model)
optimizer = get_optimizer(self.config, model.parameters())
if self.config.model.ema:
ema_helper = EMAHelper(mu=self.config.model.ema_rate)
ema_helper.register(model)
else:
ema_helper = None
start_epoch, step = 0, 0
if self.args.resume_training:
states = paddle.load(os.path.join(self.args.log_path, "ckpt.pdl"))
model.set_state_dict({
k.split("$model_")[-1]: v
for k, v in states.items() if "$model_" in k
})
optimizer.set_state_dict({
k.split("$optimizer_")[-1]: v
for k, v in states.items() if "$optimizer_" in k
})
optimizer._epsilon = self.config.optim.eps
start_epoch = states["$epoch"]
step = states["$step"]
if self.config.model.ema:
ema_helper.set_state_dict({
k.split("$ema_")[-1]: v
for k, v in states.items() if "$ema_" in k
})
for epoch in range(start_epoch, self.config.training.n_epochs):
data_start = time.time()
data_time = 0
for i, (x, y) in enumerate(train_loader):
n = x.shape[0]
data_time += time.time() - data_start
model.train()
step += 1
x = data_transform(self.config, x)
e = paddle.randn(x.shape)
b = self.betas
# antithetic sampling
t = paddle.randint(low=0,
high=self.num_timesteps,
shape=(n // 2 + 1, ))
t = paddle.concat([t, self.num_timesteps - t - 1], 0)[:n]
loss = loss_registry[config.model.type](model, x, t, e, b)
vdl_logger.add_scalar("loss", loss, step=step)
logging.info(
f"step: {step}, loss: {loss.numpy()[0]}, data time: {data_time / (i+1)}"
)
optimizer.clear_grad()
loss.backward()
optimizer.step()
if self.config.model.ema:
ema_helper.update(model)
if step % self.config.training.snapshot_freq == 0 or step == 1:
states = dict(
**{
"$model_" + k: v
for k, v in model.state_dict().items()
},
**{
"$optimizer_" + k: v
for k, v in optimizer.state_dict().items()
},
**{"$epoch": epoch},
**{"$step": step},
)
if self.config.model.ema:
states.update({
"$ema_" + k: v
for k, v in ema_helper.state_dict().items()
})
paddle.save(
states,
os.path.join(self.args.log_path,
"ckpt_{}.pdl".format(step)),
)
paddle.save(states,
os.path.join(self.args.log_path, "ckpt.pdl"))
data_start = time.time()
def calculate_fid(self):
import fid, pickle
import tensorflow as tf
stats_path = "fid_stats_cifar10_train.npz" # training set statistics
inception_path = fid.check_or_download_inception(
"./tmp/"
) # download inception network
score = get_model(self.config)
score = torch.nn.DataParallel(score)
sigmas_th = get_sigmas(self.config)
sigmas = sigmas_th.cpu().numpy()
fids = {}
for ckpt in tqdm.tqdm(
range(
self.config.fast_fid.begin_ckpt, self.config.fast_fid.end_ckpt + 1, 5000
),
desc="processing ckpt",
):
states = torch.load(
os.path.join(self.args.log_path, f"checkpoint_{ckpt}.pth"),
map_location=self.config.device,
)
if self.config.model.ema:
ema_helper = EMAHelper(mu=self.config.model.ema_rate)
ema_helper.register(score)
ema_helper.load_state_dict(states[-1])
ema_helper.ema(score)
else:
score.load_state_dict(states[0])
score.eval()
num_iters = (
self.config.fast_fid.num_samples // self.config.fast_fid.batch_size
)
output_path = os.path.join(self.args.image_folder, "ckpt_{}".format(ckpt))
os.makedirs(output_path, exist_ok=True)
for i in range(num_iters):
init_samples = torch.rand(
self.config.fast_fid.batch_size,
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.config.device,
)
init_samples = data_transform(self.config, init_samples)
all_samples = anneal_Langevin_dynamics(
init_samples,
score,
sigmas,
self.config.fast_fid.n_steps_each,
self.config.fast_fid.step_lr,
verbose=self.config.fast_fid.verbose,
)
final_samples = all_samples[-1]
for id, sample in enumerate(final_samples):
sample = sample.view(
self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
)
sample = inverse_data_transform(self.config, sample)
save_image(
sample, os.path.join(output_path, "sample_{}.png".format(id))
)
# load precalculated training set statistics
f = np.load(stats_path)
mu_real, sigma_real = f["mu"][:], f["sigma"][:]
f.close()
fid.create_inception_graph(
inception_path
) # load the graph into the current TF graph
final_samples = (
(final_samples - final_samples.min())
/ (final_samples.max() - final_samples.min()).data.cpu().numpy()
* 255
)
final_samples = np.transpose(final_samples, [0, 2, 3, 1])
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
mu_gen, sigma_gen = fid.calculate_activation_statistics(
final_samples, sess, batch_size=100
)
fid_value = fid.calculate_frechet_distance(
mu_gen, sigma_gen, mu_real, sigma_real
)
print("FID: %s" % fid_value)
with open(os.path.join(self.args.image_folder, "fids.pickle"), "wb") as handle:
pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def sample(self):
if self.config.ncsn.sampling.ckpt_id is None:
ncsn_states = torch.load(os.path.join(
'scones', self.config.ncsn.sampling.log_path,
'checkpoint.pth'),
map_location=self.config.device)
else:
ncsn_states = torch.load(os.path.join(
'scones', self.config.ncsn.sampling.log_path,
f'checkpoint_{self.config.ncsn.sampling.ckpt_id}.pth'),
map_location=self.config.device)
score = get_scorenet(self.config)
score = torch.nn.DataParallel(score)
sigmas_th = get_sigmas(self.config.ncsn)
sigmas = sigmas_th.cpu().numpy()
if ("module.sigmas" in ncsn_states[0].keys()):
ncsn_states[0]["module.sigmas"] = sigmas_th
score.load_state_dict(ncsn_states[0], strict=True)
score.eval()
baryproj_data_init = (hasattr(self.config, "baryproj")
and self.config.ncsn.sampling.data_init)
if (baryproj_data_init):
if (self.config.baryproj.ckpt_id is None):
bproj_states = torch.load(os.path.join(
'scones', self.config.baryproj.log_path, 'checkpoint.pth'),
map_location=self.config.device)
else:
bproj_states = torch.load(os.path.join(
'scones', self.config.baryproj.log_path,
f'checkpoint_{self.config.baryproj.ckpt_id}.pth'),
map_location=self.config.device)
bproj = get_bary(self.config)
bproj.load_state_dict(bproj_states[0])
bproj = torch.nn.DataParallel(bproj)
bproj.eval()
if self.config.compatibility.ckpt_id is None:
cpat_states = torch.load(os.path.join(
'scones', self.config.compatibility.log_path,
'checkpoint.pth'),
map_location=self.config.device)
else:
cpat_states = torch.load(os.path.join(
'scones', self.config.compatibility.log_path,
f'checkpoint_{self.config.compatibility.ckpt_id}.pth'),
map_location=self.config.device)
cpat = get_compatibility(self.config)
cpat.load_state_dict(cpat_states[0])
if self.config.ncsn.model.ema:
ema_helper = EMAHelper(mu=self.config.ncsn.model.ema_rate)
ema_helper.register(score)
ema_helper.load_state_dict(ncsn_states[-1])
ema_helper.ema(score)
source_dataset, _ = get_dataset(self.args, self.config.source)
dataloader = DataLoader(
source_dataset,
batch_size=self.config.ncsn.sampling.sources_per_batch,
shuffle=True,
num_workers=self.config.source.data.num_workers)
data_iter = iter(dataloader)
(Xs, labels) = next(data_iter)
Xs_global = torch.cat([Xs] *
self.config.ncsn.sampling.samples_per_source,
dim=0).to(self.config.device)
Xs_global = data_transform(self.config.source, Xs_global)
if (hasattr(self.config.ncsn.sampling, "n_sigmas_skip")):
n_sigmas_skip = self.config.ncsn.sampling.n_sigmas_skip
else:
n_sigmas_skip = 0
if not self.config.ncsn.sampling.fid:
if self.config.ncsn.sampling.inpainting:
''' NCSN INPAINTING CODE. EITHER PATCH THIS FOR SCONES OR REMOVE IT.
data_iter = iter(dataloader)
refer_images, _ = next(data_iter)
refer_images = refer_images.to(self.config.device)
width = int(np.sqrt(self.config.sampling.batch_size))
init_samples = torch.rand(width, width, self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config, init_samples)
all_samples = anneal_Langevin_dynamics_inpainting(init_samples, refer_images[:width, ...], score,
sigmas,
self.config.data.image_size,
self.config.sampling.n_steps_each,
self.config.sampling.step_lr)
torch.save(refer_images[:width, ...], os.path.join(self.args.image_folder, 'refer_image.pth'))
refer_images = refer_images[:width, None, ...].expand(-1, width, -1, -1, -1).reshape(-1,
*refer_images.shape[
1:])
save_image(refer_images, os.path.join(self.args.image_folder, 'refer_image.png'), nrow=width)
if not self.config.sampling.final_only:
for i, sample in enumerate(tqdm.tqdm(all_samples)):
sample = sample.view(self.config.sampling.batch_size, self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(sample, int(np.sqrt(self.config.sampling.batch_size)))
save_image(image_grid, os.path.join(self.args.image_folder, 'image_grid_{}.png'.format(i)))
torch.save(sample, os.path.join(self.args.image_folder, 'completion_{}.pth'.format(i)))
else:
sample = all_samples[-1].view(self.config.sampling.batch_size, self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(sample, int(np.sqrt(self.config.sampling.batch_size)))
save_image(image_grid, os.path.join(self.args.image_folder,
'image_grid_{}.png'.format(self.config.ncsn.sampling.ckpt_id)))
torch.save(sample, os.path.join(self.args.image_folder,
'completion_{}.pth'.format(self.config.sampling.ckpt_id)))
'''
raise NotImplementedError(
"Inpainting with SCONES is not currently implemented.")
elif self.config.ncsn.sampling.interpolation:
''' NCSN INTERPOLATION CODE. EITHER PATCH THIS FOR SCONES OR REMOVE IT.
if self.config.sampling.data_init:
data_iter = iter(dataloader)
samples, _ = next(data_iter)
samples = samples.to(self.config.device)
samples = data_transform(self.config, samples)
init_samples = samples + sigmas_th[0] * torch.randn_like(samples)
else:
init_samples = torch.rand(self.config.sampling.batch_size, self.config.data.channels,
self.config.data.image_size, self.config.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config, init_samples)
all_samples = anneal_Langevin_dynamics_interpolation(init_samples, score, sigmas,
self.config.sampling.n_interpolations,
self.config.sampling.n_steps_each,
self.config.sampling.step_lr, verbose=True,
final_only=self.config.sampling.final_only)
if not self.config.sampling.final_only:
for i, sample in tqdm.tqdm(enumerate(all_samples), total=len(all_samples),
desc="saving image samples"):
sample = sample.view(sample.shape[0], self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(sample, nrow=self.config.sampling.n_interpolations)
save_image(image_grid, os.path.join(self.args.image_folder, 'image_grid_{}.png'.format(i)))
torch.save(sample, os.path.join(self.args.image_folder, 'samples_{}.pth'.format(i)))
else:
sample = all_samples[-1].view(all_samples[-1].shape[0], self.config.data.channels,
self.config.data.image_size,
self.config.data.image_size)
sample = inverse_data_transform(self.config, sample)
image_grid = make_grid(sample, self.config.sampling.n_interpolations)
save_image(image_grid, os.path.join(self.args.image_folder,
'image_grid_{}.png'.format(self.config.sampling.ckpt_id)))
torch.save(sample, os.path.join(self.args.image_folder,
'samples_{}.pth'.format(self.config.sampling.ckpt_id)))
'''
raise NotImplementedError(
"Interpolation with SCONES is not currently implemented.")
else:
if self.config.ncsn.sampling.data_init:
if (baryproj_data_init):
with torch.no_grad():
init_Xt = (bproj(Xs_global) +
sigmas_th[n_sigmas_skip] *
torch.randn_like(Xs_global)).detach()
else:
init_Xt = Xs_global + sigmas_th[
n_sigmas_skip] * torch.randn_like(Xs_global)
init_Xt.requires_grad = True
init_Xt = init_Xt.to(self.config.device)
else:
init_Xt = torch.rand(
self.config.ncsn.sampling.sources_per_batch *
self.config.ncsn.sampling.samples_per_source,
self.config.target.data.channels,
self.config.target.data.image_size,
self.config.target.data.image_size,
device=self.config.device)
init_Xt = data_transform(self.config.target, init_Xt)
init_Xt.requires_grad = True
init_Xt = init_Xt.to(self.config.device)
all_samples = anneal_Langevin_dynamics(
init_Xt,
Xs_global,
score,
cpat,
sigmas,
self.config.ncsn.sampling.n_steps_each,
self.config.ncsn.sampling.step_lr,
verbose=True,
final_only=self.config.ncsn.sampling.final_only,
denoise=self.config.ncsn.sampling.denoise,
n_sigmas_skip=n_sigmas_skip)
all_samples = torch.stack(all_samples, dim=0)
if not self.config.ncsn.sampling.final_only:
all_samples = all_samples.view(
(-1, self.config.ncsn.sampling.sources_per_batch,
self.config.ncsn.sampling.samples_per_source,
self.config.target.data.channels,
self.config.target.data.image_size,
self.config.target.data.image_size))
np.save(
os.path.join(self.args.image_folder,
'all_samples.npy'),
all_samples.detach().cpu().numpy())
sample = all_samples[-1].view(
self.config.ncsn.sampling.sources_per_batch *
self.config.ncsn.sampling.samples_per_source,
self.config.target.data.channels,
self.config.target.data.image_size,
self.config.target.data.image_size)
sample = inverse_data_transform(self.config.target, sample)
image_grid = make_grid(
sample, nrow=self.config.ncsn.sampling.sources_per_batch)
save_image(
image_grid,
os.path.join(self.args.image_folder, 'sample_grid.png'))
source_grid = make_grid(
Xs, nrow=self.config.ncsn.sampling.sources_per_batch)
save_image(
source_grid,
os.path.join(self.args.image_folder, 'source_grid.png'))
bproj_of_source = make_grid(
bproj(Xs),
nrow=self.config.ncsn.sampling.sources_per_batch)
save_image(
bproj_of_source,
os.path.join(self.args.image_folder, 'bproj_sources.png'))
np.save(os.path.join(self.args.image_folder, 'sources.npy'),
Xs.detach().cpu().numpy())
np.save(
os.path.join(self.args.image_folder, 'source_labels.npy'),
labels.detach().cpu().numpy())
np.save(os.path.join(self.args.image_folder, 'bproj.npy'),
bproj(Xs).detach().cpu().numpy())
np.save(os.path.join(self.args.image_folder, 'samples.npy'),
sample.detach().cpu().numpy())
else:
batch_size = self.config.ncsn.sampling.sources_per_batch * self.config.ncsn.sampling.samples_per_source
total_n_samples = self.config.ncsn.sampling.num_samples4fid
n_rounds = total_n_samples // batch_size
if self.config.ncsn.sampling.data_init:
dataloader = DataLoader(
source_dataset,
batch_size=self.config.ncsn.sampling.sources_per_batch,
shuffle=True,
num_workers=self.config.source.data.num_workers)
data_iter = iter(dataloader)
img_id = 0
for r in tqdm.tqdm(
range(n_rounds),
desc=
'Generating image samples for FID/inception score evaluation'
):
if self.config.ncsn.sampling.data_init:
try:
init_samples, labels = next(data_iter)
init_samples = torch.cat(
[init_samples] *
self.config.ncsn.sampling.samples_per_source,
dim=0)
labels = torch.cat(
[labels] *
self.config.ncsn.sampling.samples_per_source,
dim=0)
except StopIteration:
data_iter = iter(dataloader)
init_samples, labels = next(data_iter)
init_samples = torch.cat(
[init_samples] *
self.config.ncsn.sampling.samples_per_source,
dim=0)
labels = torch.cat(
[labels] *
self.config.ncsn.sampling.samples_per_source,
dim=0)
init_samples = init_samples.to(self.config.device)
init_samples = data_transform(self.config.target,
init_samples)
if (baryproj_data_init):
with torch.no_grad():
bproj_samples = bproj(init_samples).detach()
else:
bproj_samples = torch.clone(init_samples).detach()
samples = bproj_samples + sigmas_th[
n_sigmas_skip] * torch.randn_like(bproj_samples)
samples.requires_grad = True
samples = samples.to(self.config.device)
else:
samples = torch.rand(batch_size,
self.config.target.data.channels,
self.config.target.data.image_size,
self.config.target.data.image_size,
device=self.config.device)
init_samples = torch.clone(samples)
samples = data_transform(self.config.target, samples)
samples.requires_grad = True
samples = samples.to(self.config.device)
all_samples = anneal_Langevin_dynamics(
samples,
Xs_global,
score,
cpat,
sigmas,
self.config.ncsn.sampling.n_steps_each,
self.config.ncsn.sampling.step_lr,
verbose=True,
final_only=self.config.ncsn.sampling.final_only,
denoise=self.config.ncsn.sampling.denoise,
n_sigmas_skip=n_sigmas_skip)
samples = all_samples[-1]
for img in samples:
img = inverse_data_transform(self.config.target, img)
save_image(
img,
os.path.join(self.args.image_folder,
'image_{}.png'.format(img_id)))
img_id += 1
if (self.args.save_labels):
save_path = os.path.join(self.args.image_folder, 'labels')
np.save(os.path.join(save_path, f'sources_{r}.npy'),
init_samples.detach().cpu().numpy())
np.save(os.path.join(save_path, f'source_labels_{r}.npy'),
labels.detach().cpu().numpy())
np.save(os.path.join(save_path, f"bproj_{r}.npy"),
bproj_samples.detach().cpu().numpy())
np.save(os.path.join(save_path, f"samples_{r}.npy"),
samples.detach().cpu().numpy())
def fast_ensemble_fid(self):
from ncsn.evaluation.fid_score import get_fid, get_fid_stats_path
import pickle
num_ensembles = 5
scores = [
NCSN(self.config.ncsn).to(self.config.device)
for _ in range(num_ensembles)
]
scores = [torch.nn.DataParallel(score) for score in scores]
sigmas_th = get_sigmas(self.config.ncsn)
sigmas = sigmas_th.cpu().numpy()
if self.config.compatibility.ckpt_id is None:
cpat_states = torch.load(os.path.join(
'scones', self.config.compatibility.log_path,
'checkpoint.pth'),
map_location=self.config.device)
else:
cpat_states = torch.load(os.path.join(
'scones', self.config.compatibility.log_path,
f'checkpoint_{self.config.compatibility.ckpt_id}.pth'),
map_location=self.config.device)
cpat = get_compatibility(self.config)
cpat.load_state_dict(cpat_states[0])
source_dataset, _ = get_dataset(self.args, self.config.source)
source_dataloader = DataLoader(
source_dataset,
batch_size=self.config.ncsn.sampling.sources_per_batch,
shuffle=True,
num_workers=self.config.source.data.num_workers)
source_iter = iter(source_dataloader)
fids = {}
for ckpt in tqdm.tqdm(range(self.config.ncsn.fast_fid.begin_ckpt,
self.config.ncsn.fast_fid.end_ckpt + 1,
5000),
desc="processing ckpt"):
begin_ckpt = max(self.config.ncsn.fast_fid.begin_ckpt,
ckpt - (num_ensembles - 1) * 5000)
index = 0
for i in range(begin_ckpt, ckpt + 5000, 5000):
states = torch.load(os.path.join(self.args.log_path,
f'checkpoint_{i}.pth'),
map_location=self.config.device)
scores[index].load_state_dict(states[0])
scores[index].eval()
index += 1
def scorenet(x, labels):
num_ckpts = (ckpt - begin_ckpt) // 5000 + 1
return sum([scores[i](x, labels)
for i in range(num_ckpts)]) / num_ckpts
num_iters = self.config.ncsn.fast_fid.num_samples // self.config.ncsn.fast_fid.batch_size
output_path = os.path.join(self.args.image_folder,
'ckpt_{}'.format(ckpt))
os.makedirs(output_path, exist_ok=True)
for i in range(num_iters):
try:
(Xs, _) = next(source_iter)
Xs_global = torch.cat(
[Xs] * self.config.ncsn.sampling.samples_per_source,
dim=0).to(self.config.device)
except StopIteration:
source_iter = iter(source_dataloader)
(Xs, _) = next(source_iter)
Xs_global = torch.cat(
[Xs] * self.config.ncsn.sampling.samples_per_source,
dim=0).to(self.config.device)
init_samples = torch.rand(self.config.ncsn.fast_fid.batch_size,
self.config.target.data.channels,
self.config.target.data.image_size,
self.config.target.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config.target, init_samples)
init_samples.requires_grad = True
init_samples = init_samples.to(self.config.device)
all_samples = anneal_Langevin_dynamics(
init_samples,
Xs_global,
scorenet,
cpat,
sigmas,
self.config.ncsn.fast_fid.n_steps_each,
self.config.ncsn.fast_fid.step_lr,
verbose=self.config.ncsn.fast_fid.verbose,
final_only=self.config.ncsn.sampling.final_only,
denoise=self.config.ncsn.sampling.denoise)
final_samples = all_samples[-1]
for id, sample in enumerate(final_samples):
sample = sample.view(self.config.target.data.channels,
self.config.target.data.image_size,
self.config.target.data.image_size)
sample = inverse_data_transform(self.config.target, sample)
save_image(
sample,
os.path.join(output_path, 'sample_{}.png'.format(id)))
stat_path = get_fid_stats_path(self.args,
self.config.ncsn,
download=True)
fid = get_fid(stat_path, output_path)
fids[ckpt] = fid
print("ckpt: {}, fid: {}".format(ckpt, fid))
with open(os.path.join(self.args.image_folder, 'fids.pickle'),
'wb') as handle:
pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def fast_fid(self):
### Test the fids of ensembled checkpoints.
### Shouldn't be used for pretrained with ema
if self.config.ncsn.fast_fid.ensemble:
if self.config.ncsn.model.ema:
raise RuntimeError(
"Cannot apply ensembling to pretrained with EMA.")
self.fast_ensemble_fid()
return
from ncsn.evaluation.fid_score import get_fid, get_fid_stats_path
import pickle
source_dataset, _ = get_dataset(self.args, self.config.source)
source_dataloader = DataLoader(
source_dataset,
batch_size=self.config.ncsn.sampling.sources_per_batch,
shuffle=True,
num_workers=self.config.source.data.num_workers)
source_iter = iter(source_dataloader)
score = get_scorenet(self.config.ncsn)
score = torch.nn.DataParallel(score)
if self.config.compatibility.ckpt_id is None:
cpat_states = torch.load(os.path.join(
'scones', self.config.compatibility.log_path,
'checkpoint.pth'),
map_location=self.config.device)
else:
cpat_states = torch.load(os.path.join(
'scones', self.config.compatibility.log_path,
f'checkpoint_{self.config.compatibility.ckpt_id}.pth'),
map_location=self.config.device)
cpat = get_compatibility(self.config)
cpat.load_state_dict(cpat_states[0])
sigmas_th = get_sigmas(self.config.ncsn)
sigmas = sigmas_th.cpu().numpy()
fids = {}
for ckpt in tqdm.tqdm(range(self.config.ncsn.fast_fid.begin_ckpt,
self.config.ncsn.fast_fid.end_ckpt + 1,
5000),
desc="processing ckpt"):
states = torch.load(os.path.join(self.args.log_path,
f'checkpoint_{ckpt}.pth'),
map_location=self.config.device)
if self.config.ncsn.model.ema:
ema_helper = EMAHelper(mu=self.config.ncsn.model.ema_rate)
ema_helper.register(score)
ema_helper.load_state_dict(states[-1])
ema_helper.ema(score)
else:
score.load_state_dict(states[0])
score.eval()
num_iters = self.config.ncsn.fast_fid.num_samples // self.config.ncsn.fast_fid.batch_size
output_path = os.path.join(self.args.image_folder,
'ckpt_{}'.format(ckpt))
os.makedirs(output_path, exist_ok=True)
for i in range(num_iters):
try:
(Xs, _) = next(source_iter)
Xs_global = torch.cat(
[Xs] * self.config.ncsn.sampling.samples_per_source,
dim=0).to(self.config.device)
except StopIteration:
source_iter = iter(source_dataloader)
(Xs, _) = next(source_iter)
Xs_global = torch.cat(
[Xs] * self.config.ncsn.sampling.samples_per_source,
dim=0).to(self.config.device)
init_samples = torch.rand(self.config.ncsn.fast_fid.batch_size,
self.config.target.data.channels,
self.config.target.data.image_size,
self.config.target.data.image_size,
device=self.config.device)
init_samples = data_transform(self.config.target, init_samples)
init_samples.requires_grad = True
init_samples = init_samples.to(self.config.device)
all_samples = anneal_Langevin_dynamics(
init_samples,
Xs_global,
score,
cpat,
sigmas,
self.config.ncsn.fast_fid.n_steps_each,
self.config.ncsn.fast_fid.step_lr,
verbose=self.config.ncsn.fast_fid.verbose,
final_only=self.config.ncsn.sampling.final_only,
denoise=self.config.ncsn.sampling.denoise)
final_samples = all_samples[-1]
for id, sample in enumerate(final_samples):
sample = sample.view(self.config.target.data.channels,
self.config.target.data.image_size,
self.config.target.data.image_size)
sample = inverse_data_transform(self.config.target, sample)
save_image(
sample,
os.path.join(output_path, 'sample_{}.png'.format(id)))
stat_path = get_fid_stats_path(self.args,
self.config.ncsn,
download=True)
fid = get_fid(stat_path, output_path)
fids[ckpt] = fid
print("ckpt: {}, fid: {}".format(ckpt, fid))
with open(os.path.join(self.args.image_folder, 'fids.pickle'),
'wb') as handle:
pickle.dump(fids, handle, protocol=pickle.HIGHEST_PROTOCOL)
def sample(self):
source_dataset, _ = get_dataset(self.args, self.config.source)
baryproj = get_bary(self.config)
baryproj.eval()
if self.config.sampling.ckpt_id is None:
bp_states = torch.load(os.path.join(self.args.log_path, 'checkpoint.pth'), map_location=self.config.device)
else:
bp_states = torch.load(os.path.join(self.args.log_path, f'checkpoint_{self.config.compatibility.ckpt_id}.pth'),
map_location=self.config.device)
baryproj.load_state_dict(bp_states[0])
if(not self.config.sampling.fid):
dataloader = DataLoader(source_dataset,
batch_size=self.config.sampling.batch_size,
shuffle=True,
num_workers=self.config.source.data.num_workers)
batch_samples = []
for i in range(self.config.sampling.n_batches):
(Xs, _) = next(iter(dataloader))
Xs = data_transform(self.config.source, Xs)
transport = baryproj(Xs)
batch_samples.append(inverse_data_transform(self.config, transport))
sample = torch.cat(batch_samples, dim=0)
image_grid = make_grid(sample[:min(64, len(sample))], nrow=8)
save_image(image_grid, os.path.join(self.args.image_folder, 'sample_grid.png'))
source_grid = make_grid(Xs[:min(64, len(Xs))], nrow=8)
save_image(source_grid, os.path.join(self.args.image_folder, 'source_grid.png'))
np.save(os.path.join(self.args.image_folder, 'sample.npy'), sample.detach().cpu().numpy())
np.save(os.path.join(self.args.image_folder, 'sources.npy'), Xs.detach().cpu().numpy())
else:
batch_size = self.config.sampling.samples_per_batch
total_n_samples = self.config.sampling.num_samples4fid
n_rounds = total_n_samples // batch_size
dataloader = DataLoader(source_dataset,
batch_size=self.config.sampling.samples_per_batch,
shuffle=True,
num_workers=self.config.source.data.num_workers)
data_iter = iter(dataloader)
img_id = 0
for _ in tqdm(range(n_rounds), desc='Generating image samples for FID/inception score evaluation'):
with torch.no_grad():
(Xs, _) = next(data_iter)
Xs = data_transform(self.config.source, Xs).to(self.config.device)
transport = baryproj(Xs)
for img in transport:
img = inverse_data_transform(self.config.target, img)
save_image(img, os.path.join(self.args.image_folder, 'image_{}.png'.format(img_id)))
img_id += 1
del Xs
del transport
def train(self):
source_dataset, source_test_dataset = get_dataset(self.args, self.config.source)
source_loader = DataLoader(source_dataset, batch_size=self.config.training.batch_size,
shuffle=True, num_workers=self.config.source.data.num_workers, drop_last=True)
source_batches = iter(source_loader)
target_dataset, target_test_dataset = get_dataset(self.args, self.config.target)
target_loader = DataLoader(target_dataset, batch_size=self.config.training.batch_size,
shuffle=True, num_workers=self.config.target.data.num_workers, drop_last=True)
target_batches = iter(target_loader)
cpat = get_compatibility(self.config)
cpat_opt = get_optimizer(self.config, cpat.parameters())
if(self.args.resume_training):
states = torch.load(os.path.join(self.args.log_path, 'checkpoint.pt'))
cpat.load_state_dict(states[0])
cpat_opt.load_state_dict(states[1])
logging.info(f"Resuming training after {states[2]} steps.")
logging.info("Optimizing the compatibility function.")
with tqdm(total=self.config.training.n_iters) as progress:
for d_step in range(self.config.training.n_iters):
try:
(Xs, ys) = next(source_batches)
(Xt, yt) = next(target_batches)
except StopIteration:
# Refresh after one epoch
source_batches = iter(source_loader)
target_batches = iter(target_loader)
(Xs, ys) = next(source_batches)
(Xt, yt) = next(target_batches)
Xs = data_transform(self.config.source, Xs)
Xs = Xs.to(self.config.device)
Xt = data_transform(self.config.target, Xt)
Xt = Xt.to(self.config.device)
obj = cpat_opt.step(lambda: self._cpat_closure(Xs, Xt, cpat, cpat_opt))
avg_density = torch.mean(cpat.forward(Xs, Xt))
obj_val = round(obj.item(), 5)
avg_density_val = round(avg_density.item(), 5)
progress.update(1)
progress.set_description_str(f"Average Density: {avg_density_val}")
self.config.tb_logger.add_scalars('Optimization', {
'Objective': obj_val,
'Average Density': avg_density_val
}, d_step)
if(d_step % self.config.training.snapshot_freq == 0):
states = [
cpat.state_dict(),
cpat_opt.state_dict(),
d_step
]
torch.save(states, os.path.join(self.args.log_path, f'checkpoint_{d_step}.pth'))
torch.save(states, os.path.join(self.args.log_path, f'checkpoint.pth'))
