def train_step_P(net, x, y, optimizerP, args):
alpha = args['alpha']
batch_size = x.shape[0]
# zero the gradient
net['P'].zero_grad()
# raal data
real_data = torch.cat([x, y], 1)
real_loss = net['P'](real_data).mean()
# generator fake data
with torch.autograd.no_grad():
fake_y = sample_generator(net['G'], x)
fake_y_data = torch.cat([x, fake_y], 1)
fake_y_loss = net['P'](fake_y_data.data).mean()
grad_y_loss = gradient_penalty(real_data, fake_y_data, net['P'],
args['lambda_gp'])
loss_y = alpha * (fake_y_loss - real_loss)
loss_yg = alpha * grad_y_loss
# Denoiser fake data
with torch.autograd.no_grad():
fake_x = y - net['D'](y)
fake_x_data = torch.cat([fake_x, y], 1)
fake_x_loss = net['P'](fake_x_data.data).mean()
grad_x_loss = gradient_penalty(real_data, fake_x_data, net['P'],
args['lambda_gp'])
loss_x = (1 - alpha) * (fake_x_loss - real_loss)
loss_xg = (1 - alpha) * grad_x_loss
loss = loss_x + loss_xg + loss_y + loss_yg
# backward
loss.backward()
optimizerP.step()
return loss, loss_x, loss_xg, loss_y, loss_yg
def train_step_G(net, x, y, optimizerG, args):
alpha = args['alpha']
batch_size = x.shape[0]
# zero the gradient
net['G'].zero_grad()
fake_y = sample_generator(net['G'], x)
loss_mean = args['tau_G'] * mean_match(x, y, fake_y, kernel.to(x.device),
_C)
fake_y_data = torch.cat([x, fake_y], 1)
fake_y_loss = net['P'](fake_y_data).mean()
loss_y = -alpha * fake_y_loss
loss = loss_y + loss_mean
# backward
loss.backward()
optimizerG.step()
return loss, loss_y, loss_mean, fake_y.data
def train_step_G(net, x, y, optimizerG, args): # Noise generator - residual only
alpha = args['alpha']
batch_size = x.shape[0]
# zero the gradient
net['G'].zero_grad()
fake_y = sample_generator(net['G'], x)
##################
x_ = x[:, 1, :, :].unsqueeze(1)
y_ = y[:, 1, :, :].unsqueeze(1)
fake_y_ = fake_y[:, 1, :, :].unsqueeze(1)
##################
loss_mean = args['tau_G'] * mean_match(x_.repeat(1, 3, 1, 1), y_.repeat(1, 3, 1, 1), fake_y_.repeat(1, 3, 1, 1),
kernel.to(x_.repeat(1, 3, 1, 1).device), _C)
fake_y_data = torch.cat([x_, fake_y_], 1)
fake_y_loss = net['P'](fake_y_data).mean()
loss_y = -alpha * fake_y_loss
loss = loss_y + loss_mean
# backward
loss.backward()
optimizerG.step()
return loss, loss_y, loss_mean, fake_y.data
def train_step_P(net, x, y, optimizerP, args): # Discriminator
##################
x_ = x[:, 1, :, :].unsqueeze(1)
y_ = y[:, 1, :, :].unsqueeze(1)
##################
alpha = args['alpha']
batch_size =x.shape[0]
# zero the gradient
net['P'].zero_grad()
# raal data
real_data = torch.cat([x_,y_], 1) ### x<-1, y<-1
real_loss = net['P'](real_data).mean()
# generator fake data
with torch.autograd.no_grad():
fake_y = sample_generator(net['G'], x) ### x<-3
fake_y = fake_y[:, 1, :, :].unsqueeze(1)
fake_y_data = torch.cat([x_, fake_y], 1)
fake_y_loss = net['P'](fake_y_data.data).mean() ### <<<<<<<<<<
grad_y_loss = gradient_penalty(real_data, fake_y_data, net['P'], args['lambda_gp'])
loss_y = alpha * (fake_y_loss - real_loss)
loss_yg = alpha * grad_y_loss
# Denoiser fake data
with torch.autograd.no_grad():
fake_x = y - net['D'](y) ### <<<<<<<<<<
fake_x = fake_x[:, 1, :, :].unsqueeze(1)
fake_x_data = torch.cat([fake_x, y_], 1)
fake_x_loss = net['P'](fake_x_data.data).mean() ### <<<<<<<<<<
grad_x_loss = gradient_penalty(real_data, fake_x_data, net['P'], args['lambda_gp'])
loss_x = (1-alpha) * (fake_x_loss - real_loss)
loss_xg = (1-alpha) * grad_x_loss
loss = loss_x + loss_xg + loss_y + loss_yg
# backward
loss.backward()
optimizerP.step()
return loss, loss_x, loss_xg, loss_y, loss_yg
# load the pretrained model
net.load_state_dict(
torch.load('./model_states/DANet.pt', map_location='cpu')['G'])
# read the images
im_noisy_real = loadmat('./test_data/SIDD/noisy.mat')['im_noisy']
im_gt = loadmat('./test_data/SIDD/gt.mat')['im_gt']
# denoising
inputs = torch.from_numpy(img_as_float32(im_gt).transpose(
[2, 0, 1])).unsqueeze(0).cuda()
with torch.autograd.no_grad():
padunet = PadUNet(inputs, dep_U=5)
inputs_pad = padunet.pad()
outputs_pad = sample_generator(net, inputs_pad)
outputs = padunet.pad_inverse(outputs_pad)
outputs.clamp_(0.0, 1.0)
im_noisy_fake = img_as_ubyte(outputs.cpu().numpy()[0, ].transpose([1, 2, 0]))
plt.subplot(1, 3, 1)
plt.imshow(im_gt)
plt.title('Gt Image')
plt.axis('off')
plt.subplot(1, 3, 2)
plt.imshow(im_noisy_real)
plt.title('Real Noisy Image')
plt.axis('off')
plt.subplot(1, 3, 3)
plt.imshow(im_noisy_fake)
def train_model(net, netG, datasets, optimizer, lr_scheduler, args):
NReal = ceil(args['batch_size'] / (1 + args['fake_ratio']))
batch_size = {'train': args['batch_size'], 'val': 4}
data_loader = {
phase: uData.DataLoader(datasets[phase],
batch_size=batch_size[phase],
shuffle=True,
num_workers=args['num_workers'],
pin_memory=True)
for phase in _modes
}
num_data = {phase: len(datasets[phase]) for phase in _modes}
num_iter_epoch = {
phase: ceil(num_data[phase] / batch_size[phase])
for phase in _modes
}
step = args['step'] if args['resume'] else 0
step_img = args['step_img'] if args['resume'] else {x: 0 for x in _modes}
writer = SummaryWriter(str(Path(args['log_dir'])))
clip_grad = args['clip_normD']
for epoch in range(args['epoch_start'], args['epochs']):
mae_per_epoch = {x: 0 for x in _modes}
tic = time.time()
# train stage
net.train()
lr = optimizer.param_groups[0]['lr']
grad_mean = 0
phase = 'train'
for ii, data in enumerate(data_loader[phase]):
im_noisy, im_gt = [x.cuda() for x in data]
with torch.autograd.no_grad():
im_noisy[NReal:, ] = sample_generator(netG, im_gt[NReal:, ])
im_noisy[NReal:, ].clamp_(0.0, 1.0)
optimizer.zero_grad()
im_denoise = im_noisy - net(im_noisy)
loss = F.l1_loss(im_denoise, im_gt,
reduction='sum') / im_gt.shape[0]
# backpropagation
loss.backward()
# clip the grad
total_grad = nn.utils.clip_grad_norm_(net.parameters(), clip_grad)
grad_mean = grad_mean * ii / (ii + 1) + total_grad / (ii + 1)
optimizer.step()
mae_iter = loss.item() / (im_gt.shape[1] * im_gt.shape[2] *
im_gt.shape[3])
mae_per_epoch[phase] += mae_iter
if (ii + 1) % args['print_freq'] == 0:
template = '[Epoch:{:>2d}/{:<2d}] {:s}:{:0>5d}/{:0>5d}, Loss={:5.2e}, ' + \
'Grad:{:.2e}/{:.2e}, lr={:.2e}'
print(
template.format(epoch + 1, args['epochs'], phase, ii + 1,
num_iter_epoch[phase], mae_iter, clip_grad,
total_grad, lr))
writer.add_scalar('Train Loss Iter', mae_iter, step)
step += 1
if (ii + 1) % (20 * args['print_freq']) == 0:
x1 = vutils.make_grid(im_denoise,
normalize=True,
scale_each=True)
writer.add_image(phase + ' Denoised images', x1,
step_img[phase])
x2 = vutils.make_grid(im_gt, normalize=True, scale_each=True)
writer.add_image(phase + ' GroundTruth', x2, step_img[phase])
x3 = vutils.make_grid(im_noisy,
normalize=True,
scale_each=True)
writer.add_image(phase + ' Noisy Image', x3, step_img[phase])
step_img[phase] += 1
mae_per_epoch[phase] /= (ii + 1)
clip_grad = min(grad_mean, clip_grad)
print('{:s}: Loss={:+.2e}, grad_mean={:.2e}'.format(
phase, mae_per_epoch[phase], grad_mean))
print('-' * 100)
# test stage
net.eval()
psnr_per_epoch = ssim_per_epoch = 0
phase = 'val'
for ii, data in enumerate(data_loader[phase]):
im_noisy, im_gt = [x.cuda() for x in data]
with torch.set_grad_enabled(False):
im_denoise = im_noisy - net(im_noisy)
im_denoise.clamp_(0.0, 1.0)
mae_iter = F.l1_loss(im_denoise, im_gt)
mae_per_epoch[phase] += mae_iter
psnr_iter = batch_PSNR(im_denoise, im_gt)
psnr_per_epoch += psnr_iter
ssim_iter = batch_SSIM(im_denoise, im_gt)
ssim_per_epoch += ssim_iter
# print statistics every log_interval mini_batches
if (ii + 1) % 50 == 0:
log_str = '[Epoch:{:>2d}/{:<2d}] {:s}:{:0>3d}/{:0>3d}, mae={:.2e}, ' + \
'psnr={:4.2f}, ssim={:5.4f}'
print(
log_str.format(epoch + 1, args['epochs'], phase, ii + 1,
num_iter_epoch[phase], mae_iter, psnr_iter,
ssim_iter))
# tensorboard summary
x1 = vutils.make_grid(im_denoise,
normalize=True,
scale_each=True)
writer.add_image(phase + ' Denoised images', x1,
step_img[phase])
x2 = vutils.make_grid(im_gt, normalize=True, scale_each=True)
writer.add_image(phase + ' GroundTruth', x2, step_img[phase])
x5 = vutils.make_grid(im_noisy,
normalize=True,
scale_each=True)
writer.add_image(phase + ' Noisy Image', x5, step_img[phase])
step_img[phase] += 1
psnr_per_epoch /= (ii + 1)
ssim_per_epoch /= (ii + 1)
mae_per_epoch[phase] /= (ii + 1)
print('{:s}: mse={:.3e}, PSNR={:4.2f}, SSIM={:5.4f}'.format(
phase, mae_per_epoch[phase], psnr_per_epoch, ssim_per_epoch))
print('-' * 100)
# adjust the learning rate
lr_scheduler.step()
# save model
save_path_model = str(
Path(args['model_dir']) / ('model_' + str(epoch + 1)))
torch.save(
{
'epoch': epoch + 1,
'step': step + 1,
'step_img': {x: step_img[x] + 1
for x in _modes},
'clip_grad': clip_grad,
'model_state_dict': net.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'lr_scheduler_state_dict': lr_scheduler.state_dict()
}, save_path_model)
save_path_model = str(
Path(args['model_dir']) /
('model_state_' + str(epoch + 1) + '.pt'))
torch.save(net.state_dict(), save_path_model)
writer.add_scalars('MAE_epoch', mae_per_epoch, epoch)
writer.add_scalar('Val PSNR epoch', psnr_per_epoch, epoch)
writer.add_scalar('Val SSIM epoch', ssim_per_epoch, epoch)
toc = time.time()
print('This epoch take time {:.2f}'.format(toc - tic))
writer.close()
print('Reach the maximal epochs! Finish training')
net = UNetG(3, wf=32, depth=5).cuda()
# load the pretrained model
net.load_state_dict(
torch.load('./model_states/DANet.pt', map_location='cpu')['G'])
# read the images
im_noisy_real = loadmat('./test_data/SIDD/noisy.mat')['im_noisy']
im_gt = loadmat('./test_data/SIDD/gt.mat')['im_gt']
L = 50
AKLD = 0
im_noisy_real = torch.from_numpy(
img_as_float32(im_noisy_real).transpose([2, 0, 1])).unsqueeze(0).cuda()
im_gt = torch.from_numpy(img_as_float32(im_gt).transpose(
[2, 0, 1])).unsqueeze(0).cuda()
sigma_real = estimate_sigma_gauss(im_noisy_real, im_gt)
with torch.autograd.no_grad():
padunet = PadUNet(im_gt, dep_U=5)
im_gt_pad = padunet.pad()
for _ in range(L):
outputs_pad = sample_generator(net, im_gt_pad)
im_noisy_fake = padunet.pad_inverse(outputs_pad)
im_noisy_fake.clamp_(0.0, 1.0)
sigma_fake = estimate_sigma_gauss(im_noisy_fake, im_gt)
kl_dis = kl_gauss_zero_center(sigma_fake, sigma_real)
AKLD += kl_dis
AKLD /= L
print("AKLD value: {:.3f}".format(AKLD))