def one_scale(cam_flow_fwd, cam_flow_bwd, flow_fwd, flow_bwd, tgt_img, ref_img_fwd, ref_img_bwd, ws):
b, _, h, w = cam_flow_fwd.size()
tgt_img_scaled = nn.functional.adaptive_avg_pool2d(tgt_img, (h, w))
ref_img_scaled_fwd = nn.functional.adaptive_avg_pool2d(ref_img_fwd, (h, w))
ref_img_scaled_bwd = nn.functional.adaptive_avg_pool2d(ref_img_bwd, (h, w))
cam_warped_im_fwd = flow_warp(ref_img_scaled_fwd, cam_flow_fwd)
cam_warped_im_bwd = flow_warp(ref_img_scaled_bwd, cam_flow_bwd)
flow_warped_im_fwd = flow_warp(ref_img_scaled_fwd, flow_fwd)
flow_warped_im_bwd = flow_warp(ref_img_scaled_bwd, flow_bwd)
valid_pixels_cam_fwd = 1 - (cam_warped_im_fwd == 0).prod(1, keepdim=True).type_as(cam_warped_im_fwd)
valid_pixels_cam_bwd = 1 - (cam_warped_im_bwd == 0).prod(1, keepdim=True).type_as(cam_warped_im_bwd)
valid_pixels_cam = logical_or(valid_pixels_cam_fwd, valid_pixels_cam_bwd) # if one of them is valid, then valid
valid_pixels_flow_fwd = 1 - (flow_warped_im_fwd == 0).prod(1, keepdim=True).type_as(flow_warped_im_fwd)
valid_pixels_flow_bwd = 1 - (flow_warped_im_bwd == 0).prod(1, keepdim=True).type_as(flow_warped_im_bwd)
valid_pixels_flow = logical_or(valid_pixels_flow_fwd, valid_pixels_flow_bwd) # if one of them is valid, then valid
cam_err_fwd = ((1-wssim)*robust_l1_per_pix(tgt_img_scaled - cam_warped_im_fwd).mean(1,keepdim=True) \
+ wssim*(1 - ssim(tgt_img_scaled, cam_warped_im_fwd)).mean(1, keepdim=True))
cam_err_bwd = ((1-wssim)*robust_l1_per_pix(tgt_img_scaled - cam_warped_im_bwd).mean(1,keepdim=True) \
+ wssim*(1 - ssim(tgt_img_scaled, cam_warped_im_bwd)).mean(1, keepdim=True))
cam_err = torch.min(cam_err_fwd, cam_err_bwd) * valid_pixels_cam
flow_err = (1-wssim)*robust_l1_per_pix(tgt_img_scaled - flow_warped_im_fwd).mean(1, keepdim=True) \
+ wssim*(1 - ssim(tgt_img_scaled, flow_warped_im_fwd)).mean(1, keepdim=True)
# flow_err_bwd = (1-wssim)*robust_l1_per_pix(tgt_img_scaled - flow_warped_im_bwd).mean(1, keepdim=True) \
# + wssim*(1 - ssim(tgt_img_scaled, flow_warped_im_bwd)).mean(1, keepdim=True)
# flow_err = torch.min(flow_err_fwd, flow_err_bwd)
exp_target = (wrig*cam_err <= (flow_err+epsilon)).type_as(cam_err)
return exp_target
def one_scale(flows):
#assert(explainability_mask is None or flows[0].size()[2:] == explainability_mask.size()[2:])
assert (len(flows) == len(ref_imgs))
reconstruction_loss = 0
b, _, h, w = flows[0].size()
tgt_img_scaled = nn.functional.adaptive_avg_pool2d(tgt_img, (h, w))
ref_imgs_scaled = [
nn.functional.adaptive_avg_pool2d(ref_img, (h, w))
for ref_img in ref_imgs
]
loss = 0.0
for i, ref_img in enumerate(ref_imgs_scaled):
current_flow = flows[i]
ref_img_warped = flow_warp(ref_img, current_flow)
valid_pixels = 1 - (ref_img_warped == 0).prod(
1, keepdim=True).type_as(ref_img_warped)
diff = (tgt_img_scaled - ref_img_warped)
if wssim:
ssim_loss = 1 - ssim(tgt_img_scaled, ref_img_warped)
reconstruction_loss = (1 - wssim) * robust_l1_per_pix(
diff.mean(1, True),
q=qch) * valid_pixels + wssim * ssim_loss.mean(1, True)
else:
reconstruction_loss = robust_l1_per_pix(diff.mean(1, True),
q=qch) * valid_pixels
loss += reconstruction_loss.sum() / valid_pixels.sum()
return loss
def SSIM(pred, gt, shave_border=0):
height, width = pred.shape[:2]
pred = pred[shave_border:height - shave_border,
shave_border:width - shave_border]
gt = gt[shave_border:height - shave_border,
shave_border:width - shave_border]
return np.mean(ssim(pred, gt))
def simpleTest(device,
dataloader,
generator,
MSE_Loss,
step,
alpha,
resultpath='result.jpg'):
for i, (x2_target_image, x4_target_image, target_image,
input_image) in enumerate(dataloader):
if step == 1:
target_image = x2_target_image.to(device)
elif step == 2:
target_image = x4_target_image.to(device)
else:
target_image = target_image.to(device)
input_image = input_image.to(device)
predicted_image = generator(input_image, step, alpha)
mse_loss = MSE_Loss(0.5 * predicted_image + 0.5,
0.5 * target_image + 0.5)
psnr = 10 * log10(1. / mse_loss.item())
_ssim = ssim(0.5 * predicted_image + 0.5, 0.5 * target_image + 0.5)
ms_ssim = msssim(0.5 * predicted_image + 0.5, 0.5 * target_image + 0.5)
sys.stdout.write('\r [%d/%d] Test progress... PSNR: %6.4f' %
(i, len(dataloader), psnr))
utils.save_image(0.5 * predicted_image + 0.5, resultpath)
print('Image generated!')
def one_scale(explainability_mask, occ_masks, flows):
assert(explainability_mask is None or flows[0].size()[2:] == explainability_mask.size()[2:])
assert(len(flows) == len(ref_imgs))
reconstruction_loss = 0
b, _, h, w = flows[0].size()
downscale = tgt_img.size(2)/h
tgt_img_scaled = nn.functional.adaptive_avg_pool2d(tgt_img, (h, w))
ref_imgs_scaled = [nn.functional.adaptive_avg_pool2d(ref_img, (h, w)) for ref_img in ref_imgs]
weight = 1.
for i, ref_img in enumerate(ref_imgs_scaled):
current_flow = flows[i]
ref_img_warped = flow_warp(ref_img, current_flow)#fomulate 48 w_c
valid_pixels = 1 - (ref_img_warped == 0).prod(1, keepdim=True).type_as(ref_img_warped)
diff = (tgt_img_scaled - ref_img_warped) * valid_pixels
ssim_loss = 1 - ssim(tgt_img_scaled, ref_img_warped) * valid_pixels
oob_normalization_const = valid_pixels.nelement()/valid_pixels.sum()
if explainability_mask is not None:
diff = diff * explainability_mask[:,i:i+1].expand_as(diff)
ssim_loss = ssim_loss * explainability_mask[:,i:i+1].expand_as(ssim_loss)
if occ_masks is not None:
diff = diff *(1-occ_masks[:,i:i+1]).expand_as(diff)
ssim_loss = ssim_loss*(1-occ_masks[:,i:i+1]).expand_as(ssim_loss)
reconstruction_loss += (1- wssim)*weight*oob_normalization_const*(robust_l1(diff, q=qch) + wssim*ssim_loss.mean()) + lambda_oob*robust_l1(1 - valid_pixels, q=qch)
#weight /= 2.83
assert((reconstruction_loss == reconstruction_loss).item() == 1)
return reconstruction_loss
def test(dataloader, generator, MSE_Loss, step, alpha):
avg_psnr = 0
avg_ssim = 0
avg_msssim = 0
for i, (x2_target_image, x4_target_image, target_image,
input_image) in enumerate(dataloader):
if step == 1:
target_image = x2_target_image.to(device)
elif step == 2:
target_image = x4_target_image.to(device)
else:
target_image = target_image.to(device)
input_image = input_image.to(device)
predicted_image = generator(input_image, step, alpha)
mse_loss = MSE_Loss(0.5 * predicted_image + 0.5,
0.5 * target_image + 0.5)
psnr = 10 * log10(1. / mse_loss.item())
avg_psnr += psnr
_ssim = ssim(0.5 * predicted_image + 0.5, 0.5 * target_image + 0.5)
avg_ssim += _ssim.item()
ms_ssim = msssim(0.5 * predicted_image + 0.5, 0.5 * target_image + 0.5)
avg_msssim += ms_ssim.item()
sys.stdout.write('\r [%d/%d] Test progress... PSNR: %6.4f' %
(i, len(dataloader), psnr))
utils.save_image(0.5 * predicted_image + 0.5,
os.path.join(args.result_path, '%d_results.jpg' % i))
print(
'Test done, Average PSNR:%6.4f, Average SSIM:%6.4f, Average MS-SSIM:%6.4f '
% (avg_psnr / len(dataloader), avg_ssim / len(dataloader),
avg_msssim / len(dataloader)))
def one_scale(depth, explainability_mask, occ_masks):
assert (explainability_mask is None
or depth.size()[2:] == explainability_mask.size()[2:])
assert (pose.size(1) == len(ref_imgs))
reconstruction_loss = 0
b, _, h, w = depth.size()
downscale = tgt_img.size(2) / h
tgt_img_scaled = nn.functional.adaptive_avg_pool2d(tgt_img, (h, w))
ref_imgs_scaled = [
nn.functional.adaptive_avg_pool2d(ref_img, (h, w))
for ref_img in ref_imgs
]
intrinsics_scaled = torch.cat(
(intrinsics[:, 0:2] / downscale, intrinsics[:, 2:]), dim=1)
intrinsics_scaled_inv = torch.cat(
(intrinsics_inv[:, :, 0:2] * downscale, intrinsics_inv[:, :, 2:]),
dim=2)
weight = 1.
for i, ref_img in enumerate(ref_imgs_scaled):
current_pose = pose[:, i]
ref_img_warped = inverse_warp(ref_img, depth[:, 0], current_pose,
intrinsics_scaled,
intrinsics_scaled_inv, rotation_mode,
padding_mode)
valid_pixels = 1 - (ref_img_warped == 0).prod(
1, keepdim=True).type_as(ref_img_warped)
diff = (tgt_img_scaled - ref_img_warped) * valid_pixels
ssim_loss = 1 - ssim(tgt_img_scaled, ref_img_warped) * valid_pixels
oob_normalization_const = valid_pixels.nelement(
) / valid_pixels.sum()
assert ((oob_normalization_const == oob_normalization_const
).item() == 1)
if explainability_mask is not None:
diff = diff * (1 - occ_masks[:, i:i + 1]
) * explainability_mask[:,
i:i + 1].expand_as(diff)
ssim_loss = ssim_loss * (
1 - occ_masks[:, i:i + 1]
) * explainability_mask[:, i:i + 1].expand_as(ssim_loss)
else:
diff = diff * (1 - occ_masks[:, i:i + 1]).expand_as(diff)
ssim_loss = ssim_loss * (
1 - occ_masks[:, i:i + 1]).expand_as(ssim_loss)
reconstruction_loss += (
1 - wssim) * weight * oob_normalization_const * (
robust_l1(diff, q=qch) + wssim * ssim_loss.mean()
) + lambda_oob * robust_l1(1 - valid_pixels, q=qch)
assert ((reconstruction_loss == reconstruction_loss).item() == 1)
#weight /= 2.83
return reconstruction_loss
def compare_images(a, b, metric='psnr'):
if metric == 'mse':
return ((a - b)**2).mean()
elif metric == 'psnr':
mse = ((a - b)**2).mean()
return 10 * np.log10(1. / mse.item())
elif metric == 'ssim':
return ssim(a, b, data_range=1.)
else:
raise ValueError('invalid error metric')
def main():
parser = ArgumentParser('Calculate SSIM, MSSSIM')
parser.add_argument('--meta_path', type=str)
parser.add_argument('--img_dir', type=str)
parser.add_argument('--pred_dir', type=str)
args = parser.parse_args()
meta_path = args.meta_path
img_dir = args.img_dir
pred_dir = args.pred_dir
meta = json.load(open(meta_path, encoding="utf-8"))
target = meta['target_chars']
target = [x for x in target]
target_uni = [hex(ord(x))[2:].upper() for x in target]
ssims = []
msssims = []
fonts = os.listdir(img_dir)
for font in fonts:
imgs = []
preds = []
for char in target_uni:
img = Image.open(img_dir + '/' + font + '/uni' + char + '.png')
img = transforms.ToTensor()(img)
imgs.append(img)
pred = Image.open(pred_dir + '/' + font + '/inferred_' + char + '.png')
pred = transforms.ToTensor()(pred)
preds.append(pred)
# print(len(imgs), len(preds))
img_tensor = torch.stack(imgs).to(torch.device("cuda"))
pred_tensor = torch.stack(preds).to(torch.device("cuda"))
SSIM = ssim(img_tensor, pred_tensor)
MSSSIM = msssim(img_tensor, pred_tensor)
ssims.append(SSIM.item())
msssims.append(MSSSIM.item())
print(font, "SSIM:", SSIM.item(), "MSSSIM", MSSSIM.item())
print("AVERAGE SSIM:", sum(ssims)/len(ssims),
"MSSSIM:", sum(msssims)/len(msssims))
def reconstruction_quality(self, reconstructed_movies, actual_movies):
"""
reconstructed_movies : output from self.predict
actual_movies : ground truth
"""
ssims = []
for i in range(len(actual_movies)):
x0 = actual_movies[i]
T = x0.shape[2]
sims = []
for i_tr in range(len(reconstructed_movies[i])):
x = reconstructed_movies[i][i_tr]
sims.append([ssim(x0[:,:,i_t], x[:,:,i_t])
for i_t in range(T)])
ssims.append(sims)
return ssims
def reconstruction_quality(self, reconstructed_movies, actual_movies):
"""
reconstructed_movies : output from self.predict
actual_movies : ground truth
"""
ssims = []
for i in range(len(actual_movies)):
x0 = actual_movies[i]
T = x0.shape[2]
sims = []
for i_tr in range(len(reconstructed_movies[i])):
x = reconstructed_movies[i][i_tr]
sims.append(
[ssim(x0[:, :, i_t], x[:, :, i_t]) for i_t in range(T)])
ssims.append(sims)
return ssims
def one_scale(depth):
assert (len(pose) == len(ref_imgs))
mask_list = []
b, _, h, w = depth.size()
downscale = tgt_img.size(2) / h
tgt_img_scaled = nn.functional.adaptive_avg_pool2d(tgt_img, (h, w))
ref_imgs_scaled = [
nn.functional.adaptive_avg_pool2d(ref_img, (h, w))
for ref_img in ref_imgs
]
intrinsics_scaled = torch.cat(
(intrinsics[:, 0:2] / downscale, intrinsics[:, 2:]), dim=1)
intrinsics_scaled_inv = torch.cat(
(intrinsics_inv[:, :, 0:2] * downscale, intrinsics_inv[:, :, 2:]),
dim=2)
# print(tgt_img_scaled.size()) [4, 3, 384, 512])
for i, ref_img in enumerate(ref_imgs_scaled): # i=0, i=1
current_pose = pose[i]
# print(current_pose.size())
ref_img_warped = inverse_warp(ref_img, depth[:, 0], current_pose,
intrinsics_scaled,
intrinsics_scaled_inv, rotation_mode,
padding_mode)
diff = (tgt_img_scaled - ref_img_warped).abs()
ssim_loss = 1 - ssim(tgt_img_scaled, ref_img_warped)
mean_reconstruction_loss = (
(1 - wssim) * robust_l1_per_pix(diff, q=qch) +
wssim * ssim_loss).sum(1).sum(1).sum(1) / (3 * h * w) # [B]
Res_mask = torch.where(
diff.mean(1).unsqueeze(1) > mean_reconstruction_loss.unsqueeze(
1).unsqueeze(1).unsqueeze(1),
torch.ones_like(diff.mean(1).unsqueeze(1)),
torch.zeros_like(diff.mean(1).unsqueeze(1)))
# print(diff.mean(1).size(),mean_reconstruction_loss.unsqueeze(1).unsqueeze(1).size())
mask_list.append(Res_mask)
return mask_list, ref_img_warped, diff
def one_scale(depth):
# assert(len(pose) == len(ref_imgs))
mask_list = []
b, _, h, w = depth.size()
downscale = tgt_img.size(2)/h
tgt_img_scaled = nn.functional.adaptive_avg_pool2d(tgt_img, (h, w))
ref_imgs_scaled = [nn.functional.adaptive_avg_pool2d(ref_img, (h, w)) for ref_img in ref_imgs]
intrinsics_scaled = torch.cat((intrinsics[:, 0:2]/downscale, intrinsics[:, 2:]), dim=1)
intrinsics_scaled_inv = torch.cat((intrinsics_inv[:, :, 0:2]*downscale, intrinsics_inv[:, :, 2:]), dim=2)
# print(tgt_img_scaled.size()) [4, 3, 384, 512])
for i, ref_img in enumerate(ref_imgs_scaled): # i=0, i=1
# current_pose = pose[i]
# print(current_pose.size())
current_pose = pose[:, i]
# print(current_pose.size())
ref_img_warped = inverse_warp(ref_img, depth[:,0], current_pose, intrinsics_scaled, intrinsics_scaled_inv, rotation_mode, padding_mode)
diff = (tgt_img_scaled - ref_img_warped).abs()
ssim_loss = 1 - ssim(tgt_img_scaled, ref_img_warped)
# mean_reconstruction_loss = ((1- wssim)*robust_l1_per_pix(diff, q=qch) + wssim*ssim_loss) .mean(1).mean(1).mean(1) # [B]
# Res_mask = torch.where(diff.mean(1).unsqueeze(1)>(mean_reconstruction_loss.unsqueeze(1).unsqueeze(1).unsqueeze(1)), torch.ones_like(diff.mean(1).unsqueeze(1)),torch.zeros_like(diff.mean(1).unsqueeze(1)))
# print(diff.mean(1).size(),mean_reconstruction_loss.unsqueeze(1).unsqueeze(1).size())
# mask_list.append(Res_mask)
with torch.no_grad():
final_mask = torch.ones_like(explainability_mask[:,i:i+1])
threshold =( (((1- wssim) * robust_l1_per_pix(diff, q=qch) + wssim * ssim_loss )).sum(1).unsqueeze(1).sum(2).unsqueeze(2).sum(3).unsqueeze(3) ) / ( final_mask.sum(1).unsqueeze(1).sum(2).unsqueeze(2).sum(3).unsqueeze(3) + 1 )
#threshold_max =( (((1- wssim) * robust_l1_per_pix(diff, q=qch) + wssim * ssim_loss )* final_mask).sum(1).unsqueeze(1)).max(2)[0].unsqueeze(2).max(3)[0].unsqueeze(3)
treshould_matrix = ((1- wssim) * robust_l1_per_pix(diff, q=qch) + wssim * ssim_loss).mean(1).unsqueeze(1)
#treshould_matrix = ((1- wssim) * robust_l1_per_pix(diff, q=qch) + wssim * ssim_loss).sum(1).unsqueeze(1)
threshold_mask = torch.where(treshould_matrix < threshold, torch.ones_like(treshould_matrix), torch.zeros_like(treshould_matrix))
#threshold_mask = 1-treshould_matrix/threshold_max
# final_mask = threshold_mask * final_mask
mask_list.append(threshold_mask)
return mask_list, ref_img_warped, diff
def main():
# Parse the command line arguments
cfg = parse_args(
description=
'Compares two feature images using the specified quality metrics.')
# Load the images
image1 = load_image(cfg.input[0], num_channels=3)
image2 = load_image(cfg.input[1], num_channels=3)
feature1 = get_image_feature(cfg.input[0])
feature2 = get_image_feature(cfg.input[1])
if feature1 != feature2:
error('cannot compare different features')
# Load metadata for the images if it exists
tonemap_exposure = cfg.exposure
if os.path.dirname(cfg.input[0]) == os.path.dirname(cfg.input[1]):
metadata = load_image_metadata(os.path.commonprefix(cfg.input))
if metadata:
tonemap_exposure = metadata['exposure']
# Convert the images to tensors
image1 = to_tensor(image1).unsqueeze(0)
image2 = to_tensor(image2).unsqueeze(0)
# Transform the images to sRGB
image1 = transform_image(image1, feature1, None, tonemap_exposure)
image2 = transform_image(image2, feature2, None, tonemap_exposure)
# Compute the metrics
metric_str = ''
for metric in cfg.metric:
if metric == 'mse':
value = ((image1 - image2)**2).mean()
elif metric == 'ssim':
value = ssim(image1, image2, data_range=1.)
if metric_str:
metric_str += ', '
metric_str += '%s = %.4f' % (metric, value)
if metric_str:
print(metric_str)
def calc_ssim(sr, hr, scale, rgb_range, dataset=None):
if hr.nelement() == 1: return 0
sr = sr / rgb_range
hr = hr / rgb_range
if dataset and dataset.dataset.benchmark:
shave = scale
if hr.size(1) > 1:
gray_coeffs = [65.738, 129.057, 25.064]
convert = sr.new_tensor(gray_coeffs).view(1, 3, 1, 1) / 256
sr = sr.mul(convert).sum(dim=1)
convert = hr.new_tensor(gray_coeffs).view(1, 3, 1, 1) / 256
hr = hr.mul(convert).sum(dim=1)
else:
shave = scale + 6
sr = sr.unsqueeze(1)
hr = hr.unsqueeze(1)
sr = sr[..., shave:-shave, shave:-shave]
hr = hr[..., shave:-shave, shave:-shave]
return ssim.ssim(sr, hr).cpu()
def test(dataloader, generator, MSE_Loss, step, alpha):
avg_psnr = 0
avg_ssim = 0
avg_msssim = 0
# https://stackoverflow.com/a/24658101
for i, (x2_target_image, x4_target_image, target_image, input_image) in enumerate(dataloader):
input_image = input_image.to(device)
if step==1:
target_image = x2_target_image.to(device)
elif step==2:
target_image = x4_target_image.to(device)
else:
target_image = target_image.to(device)
# define input image
input_image = input_image.to(device)
# make a prediction
predicted_image = generator(input_image, step, alpha)
predicted_image = predicted_image.double()
# retrieve the original image and compute losses
target_image = target_image.double()
mse_loss = MSE_Loss(0.5*predicted_image+0.5, 0.5*target_image+0.5)
psnr = 10*log10(1./mse_loss.item())
avg_psnr += psnr
_ssim = ssim(0.5*predicted_image+0.5, 0.5*target_image+0.5)
avg_ssim += _ssim.item()
ms_ssim = msssim(0.5*predicted_image+0.5, 0.5*target_image+0.5)
avg_msssim += ms_ssim.item()
sys.stdout.write('\r [%d/%d] Test progress... PSNR: %6.4f'%(i, len(dataloader), psnr))
save_image = torch.cat([predicted_image, target_image], dim=0)
if args.local_rank==0:
utils.save_image(0.5*save_image+0.5, os.path.join(args.result_path, '%d_results.jpg'%i))
print('Test done, Average PSNR:%6.4f, Average SSIM:%6.4f, Average MS-SSIM:%6.4f '%(avg_psnr/len(dataloader),avg_ssim/len(dataloader), avg_msssim/len(dataloader)))
def main():
#watermark('lena.png','lena_out.png','Copyright by Luca','#ffffff','chintzy.ttf',60,(0.5,0.98),0.7)
#testdwt('lena.png')
img1 = Image.open('lena.png')#.convert('L')
[img2, lengthCode] = mineWatermark.insert(img1,[1,0]*100000,3)
print "Code inserted = " , lengthCode
img2.save('out.jpg','JPEG',quality=90)
img2 = Image.open('out.jpg')
img1G = numpy.array(img1.convert('L').getdata()).reshape(img1.size)
img2G = numpy.array(img2.convert('L').getdata()).reshape(img2.size)
res = ssim.ssim(img1G,img2G)
print res
code=mineWatermark.extract(img2,3)
print len(code), " ",code
def build(self, is_train=True):
c = int(self.c / self.num_input)
rescale = 1 if self.dataset_type == 'scene' else 1.5
# Input {{{
# =========
source_image, source_pose = \
self.image[:, :, :, -c:], self.camera_pose[:, :, -1]
target_image, target_pose = \
self.image[:, :, :, :c], self.camera_pose[:, :, 0]
# }}}
# Graph {{{
# =========
synthesizer = Synthesizer(self.config, is_train=is_train)
target_image_pred = synthesizer(source_image, source_pose, target_pose)
# }}}
# Build Losses {{{
# ==============
self.loss = tf.reduce_mean(
tf.abs(target_image - target_image_pred)) * rescale
# renormalize images from [-1, 1] to [0, 1] before computing ssim
self.ssim = ssim(
self.renormalize_image(target_image_pred),
self.renormalize_image(target_image),
)
# }}}
# Tensorboard Summary {{{
# ==============
# scalar
train_test_summary("loss/loss", self.loss)
train_test_summary("loss/ssim", self.ssim)
# image
dummy_grid = tf.ones_like(source_image)
dummy_grid *= 1 if self.dataset_type == 'scene' else -1
tb_image = tf.clip_by_value(
tf.concat([source_image, target_image, target_image_pred], axis=1),
-1, 1)
train_test_summary("image",
tb_image if self.dataset_type == 'scene' else 1 -
tb_image,
summary_type='image')
# }}}
# Output {{{
# ==============
self.output = {}
# }}}
# Evaluation {{{
# ==============
self.eval_loss = {
'l1_loss': self.loss,
'ssim': self.ssim,
}
self.eval_img = {
'display': self.renormalize_image(tb_image),
}
# }}}
log.warn('Successfully loaded the model.')
img_HR_net = img_HR_net.astype(np.uint8)
img_LR_Bic = img_LR_Bic.cpu()
img_LR_Bic = img_LR_Bic.data[0].numpy()
img_LR_Bic *= 255.0
img_LR_Bic = img_LR_Bic.clip(0, 255)
img_LR_Bic = img_LR_Bic.astype(np.uint8)
img_HR = img_HR.reshape((1, img_HR.shape[0], img_HR.shape[1]))
model_PSNR_cur[idx] = psnr(
(img_HR).reshape(img_HR.shape[1], img_HR.shape[2]).astype(int),
(img_HR_net).reshape(img_HR_net.shape[1],
img_HR_net.shape[2]).astype(int))
model_SSIM_cur[idx] = ssim(
(img_HR).reshape(img_HR.shape[1], img_HR.shape[2]).astype(int),
(img_HR_net).reshape(img_HR_net.shape[1],
img_HR_net.shape[2]).astype(int))
bicubic_PSNR_cur[idx] = psnr(
(img_HR).reshape(img_HR.shape[1], img_HR.shape[2]).astype(int),
(img_LR_Bic).reshape(img_LR_Bic.shape[1],
img_LR_Bic.shape[2]).astype(int))
bicubic_SSIM_cur[idx] = ssim(
(img_HR).reshape(img_HR.shape[1], img_HR.shape[2]).astype(int),
(img_LR_Bic).reshape(img_LR_Bic.shape[1],
img_LR_Bic.shape[2]).astype(int))
model_time_cur[idx] = (end - start)
# Repeat to 3 channels to save and display
img_HR_net = np.repeat(img_HR_net, 3, axis=0)
img_HR_net = np.transpose(img_HR_net, (1, 2, 0))
imgToTensor = ToTensor()
criterionL1 = torch.nn.L1Loss()
criterionMSE = torch.nn.MSELoss()
test_size = len(os.listdir(path))
for i, filename in enumerate(os.listdir(path)):
if (i % 1 == 0):
print("%i/%i" % (i, test_size))
fullPath = os.path.join(path, filename)
img = Image.open(fullPath).convert('RGB')
# #img = image.imread(fullPath)
# height = len(img)
# width = len(img[0])
# middle = int(width/2)
w, h = img.size
w2 = int(w / 2)
shortExpImg = img.crop((0, 0, w2, h))
longExpImg = img.crop((w2, 0, w, h))
shortExpImg = resizeTr(shortExpImg)
longExpImg = resizeTr(longExpImg)
shortExpImg = torch.unsqueeze(imgToTensor(shortExpImg), 0)
longExpImg = torch.unsqueeze(imgToTensor(longExpImg), 0)
L1_total += criterionL1(shortExpImg, longExpImg).item()
PSNR_total += 10 * log10(1 / criterionMSE(shortExpImg, longExpImg).item())
SSIM_total += ssim.ssim(shortExpImg, longExpImg).item()
matching_total += get_Matching(shortExpImg, longExpImg)
print("L1: %.4f, PSNR %.4f, SSIM: %.4f, matching score: %.4f" %
(L1_total / test_size, PSNR_total / test_size, SSIM_total / test_size,
matching_total / test_size))
if not m:
print "Could not find resolution in file name: %s" % (ref_file)
exit(1)
width, height = int(m.group(1)), int(m.group(2))
print "Comparing %s to %s, resolution %d x %d" % (ref_file, dist_file, width, height)
ref_fh = open(ref_file, "rb")
dist_fh = open(dist_file, "rb")
frame_num = 0
while True:
ref, _, _ = img_read_yuv(ref_fh, width, height)
dist, _, _ = img_read_yuv(dist_fh, width, height)
vifp_value = vifp.vifp_mscale(ref.astype(float), dist.astype(float))
ssim_value = ssim.ssim(ref, dist)
print "Frame=%d VIFP=%f SSIM=%f" % (frame_num, vifp_value, ssim_value)
frame_num += 1
else:
# Inputs are image files
ref = scipy.misc.imread(ref_file, flatten=True).astype(numpy.float32)
dist = scipy.misc.imread(dist_file, flatten=True).astype(numpy.float32)
width, height = ref.shape[1], ref.shape[0]
print "Comparing %s to %s, resolution %d x %d" % (ref_file, dist_file, width, height)
vifp_value = vifp.vifp_mscale(ref, dist)
print "VIFP=%f" % (vifp_value)
ssim_value = ssim.ssim_exact(ref / 255, dist / 255)
def test_unet(root,\
psf_path,
method,\
scale, \
model_path,\
visual, \
use_gpu,
b_size=1):
'''
Model UNet
'''
model_name = method + '_poisson'
save_images_path = './Results/' + model_name + '_peak_' + str(
int(scale)) + '/'
test_dataset = CellDataset(root, psf_path, 'poisson', scale, 0.0)
test_loader = DataLoader(test_dataset,
batch_size=b_size,
shuffle=False,
num_workers=1)
model = UNet(mode='batch')
state_dict = torch.load(os.path.join(model_path, model_name))
state_dict = state_dict['model_state_dict']
new_state_dict = OrderedDict()
for k, v in state_dict.items():
new_state_dict[k] = v
model.load_state_dict(new_state_dict)
if use_gpu == 1:
model.cuda()
model.eval()
psnr_values_test = []
ssim_values_test = []
distorted_psnr_test = []
distorted_ssim_test = []
with torch.no_grad():
for i_batch, ((gt, image), psf, index, image_name, peak,
_) in enumerate(tqdm(test_loader)):
image = image.reshape(
(b_size, 1, image.shape[-2], image.shape[-1]))
gt = gt.reshape((b_size, 1, gt.shape[-2], gt.shape[-1]))
if use_gpu == 1:
image = image.cuda()
gt = gt.cuda()
for l in range(gt.shape[0]):
image[l] = image[l] / gt[l].max()
gt[l] /= gt[l].max()
output = model(image)
distorted_psnr = calc_psnr(image.clamp(0, 1), gt)
distorted_ssim = ssim(image.clamp(0, 1), gt)
psnr_test = calc_psnr(output.clamp(0, 1), gt)
s_sim_test = ssim(output.clamp(0, 1), gt)
psnr_values_test.append(psnr_test.item())
ssim_values_test.append(s_sim_test.item())
distorted_psnr_test.append(distorted_psnr.item())
distorted_ssim_test.append(distorted_ssim.item())
#Save image
if visual == 1:
if not os.path.exists(save_images_path):
os.makedirs(save_images_path, exist_ok=True)
io.imsave(os.path.join(save_images_path, 'output_' + str(image_name[0][:-4]) + '_' + \
str(model_name) + '_' + str(int(scale)) + '.png'), np.uint8(output[0][0].detach().cpu().numpy().clip(0,1) * 255.))
print('Test on Poisson noise with peak %d: PSNR %.2f, SSIM %.4f, distorted PSNR %.2f, distorted SSIM %.4f' % (peak, np.array(psnr_values_test).mean(), \
np.array(ssim_values_test).mean(), \
np.array(distorted_psnr_test).mean(), \
np.array(distorted_ssim_test).mean()))
return
def test_unet(root,\
psf_path,
method,\
std, \
model_path,\
visual, \
use_gpu,\
b_size=1):
"""
Model UNet
"""
model_name = method + '_gaussian'
save_images_path = './Results/' + model_name + '_std_' + str(std).replace(
'.', '') + '/'
test_dataset = CellDataset(root, psf_path, 'gaussian', 1.0, std)
test_loader = DataLoader(test_dataset,
batch_size=b_size,
shuffle=False,
num_workers=1)
model = UNet(mode='batch')
state_dict = torch.load(os.path.join(model_path, model_name))
state_dict = state_dict['model_state_dict']
new_state_dict = OrderedDict()
for k, v in state_dict.items():
new_state_dict[k] = v
model.load_state_dict(new_state_dict)
model.eval()
if use_gpu == 1:
model.cuda()
psnr_values_test = []
ssim_values_test = []
distorted_psnr_test = []
distorted_ssim_test = []
for i_batch, ((gt, image), psf, index, image_name, _,
std) in enumerate(tqdm(test_loader)):
image = image.reshape((b_size, 1, image.shape[-2], image.shape[-1]))
gt = gt.reshape((b_size, 1, gt.shape[-2], gt.shape[-1]))
if use_gpu == 1:
image = image.cuda()
gt = gt.cuda()
distorted_psnr = calc_psnr(image.clamp(gt.min(), gt.max()), gt)
distorted_ssim = ssim(image.clamp(gt.min(), gt.max()), gt)
output = model(image)
psnr_test = calc_psnr(output.clamp(gt.min(), gt.max()), gt)
s_sim_test = ssim(output.clamp(gt.min(), gt.max()), gt)
psnr_values_test.append(psnr_test.item())
ssim_values_test.append(s_sim_test.item())
distorted_psnr_test.append(distorted_psnr.item())
distorted_ssim_test.append(distorted_ssim.item())
#Save image
if visual == 1:
if not os.path.exists(save_images_path):
os.makedirs(save_images_path, exist_ok=True)
io.imsave(os.path.join(save_images_path, 'output_' + str(image_name[0][:-4]) + '_' + \
str(model_name) + '_' + str(std.item()).replace('.', '') + '.png'), np.uint8(output[0][0].detach().cpu().numpy().clip(0,1) * 255.))
print('Test on Gaussian noise with %.3f std: PSNR %.2f, SSIM %.4f, distorted PSNR %.2f, distorted SSIM %.4f' % (std, np.array(psnr_values_test).mean(), \
np.array(ssim_values_test).mean(), \
np.array(distorted_psnr_test).mean(), \
np.array(distorted_ssim_test).mean()))
return
def __init__(self, opt):
"""Initialize the CycleGAN class.
Parameters:
opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions
"""
BaseModel.__init__(self, opt)
# specify the training losses you want to print out. The training/test scripts will call <BaseModel.get_current_losses>
self.loss_names = [
'D_A', 'G_A', 'cycle_A', 'idt_A', 'D_B', 'G_B', 'cycle_B', 'idt_B'
]
# specify the images you want to save/display. The training/test scripts will call <BaseModel.get_current_visuals>
visual_names_A = ['real_A', 'fake_B', 'rec_A']
visual_names_B = ['real_B', 'fake_A', 'rec_B']
if self.isTrain and self.opt.lambda_identity > 0.0: # if identity loss is used, we also visualize idt_B=G_A(B) ad idt_A=G_A(B)
visual_names_A.append('idt_B')
visual_names_B.append('idt_A')
self.visual_names = visual_names_A + visual_names_B # combine visualizations for A and B
# specify the models you want to save to the disk. The training/test scripts will call <BaseModel.save_networks> and <BaseModel.load_networks>.
if self.isTrain:
self.model_names = ['G_A', 'G_B', 'D_A', 'D_B']
else: # during test time, only load Gs
self.model_names = ['G_A', 'G_B']
# define networks (both Generators and discriminators)
# The naming is different from those used in the paper.
# Code (vs. paper): G_A (G), G_B (F), D_A (D_Y), D_B (D_X)
self.netG_A = networks.define_G(opt.input_nc, opt.output_nc, opt.ngf,
opt.netG, opt.norm, not opt.no_dropout,
opt.init_type, opt.init_gain,
self.gpu_ids)
self.netG_B = networks.define_G(opt.output_nc, opt.input_nc, opt.ngf,
opt.netG, opt.norm, not opt.no_dropout,
opt.init_type, opt.init_gain,
self.gpu_ids)
if self.isTrain: # define discriminators
self.netD_A = networks.define_D(opt.output_nc, opt.ndf, opt.netD,
opt.n_layers_D, opt.norm,
opt.init_type, opt.init_gain,
self.gpu_ids)
self.netD_B = networks.define_D(opt.input_nc, opt.ndf, opt.netD,
opt.n_layers_D, opt.norm,
opt.init_type, opt.init_gain,
self.gpu_ids)
if self.isTrain:
if opt.lambda_identity > 0.0: # only works when input and output images have the same number of channels
assert (opt.input_nc == opt.output_nc)
self.fake_A_pool = ImagePool(
opt.pool_size
) # create image buffer to store previously generated images
self.fake_B_pool = ImagePool(
opt.pool_size
) # create image buffer to store previously generated images
# define loss functions
self.criterionGAN = networks.GANLoss(opt.gan_mode).to(
self.device) # define GAN loss.
self.criterionCycle = torch.nn.L1Loss()
self.criterionIdt = torch.nn.L1Loss()
self.criterionSSIM = ssim.ssim()
# initialize optimizers; schedulers will be automatically created by function <BaseModel.setup>.
self.optimizer_G = torch.optim.Adam(itertools.chain(
self.netG_A.parameters(), self.netG_B.parameters()),
lr=opt.lr,
betas=(opt.beta1, 0.999))
self.optimizer_D = torch.optim.Adam(itertools.chain(
self.netD_A.parameters(), self.netD_B.parameters()),
lr=opt.lr,
betas=(opt.beta1, 0.999))
self.optimizers.append(self.optimizer_G)
self.optimizers.append(self.optimizer_D)
def one_scale(flows):
assert (len(flows) == len(ref_imgs))
# reconstruction_loss = 0
b, _, h, w = flows[0].size()
tgt_img_scaled = nn.functional.adaptive_avg_pool2d(tgt_img, (h, w))
ref_imgs_scaled = [
nn.functional.adaptive_avg_pool2d(ref_img, (h, w))
for ref_img in ref_imgs
]
reconstruction_loss_all = []
reconstruction_weight_all = []
# consistancy_loss_all = []
ssim_loss = 0.0
for i, ref_img in enumerate(ref_imgs_scaled):
current_flow = flows[i]
ref_img_warped = flow_warp(ref_img, current_flow)
diff = (tgt_img_scaled - ref_img_warped)
if wssim:
ssim_loss += wssim * (
1 - ssim(tgt_img_scaled, ref_img_warped)).mean()
# reconstruction_loss = gradient_photometric_loss(tgt_img_scaled, ref_img_warped, qch)
reconstruction_loss = gradient_photometric_all_direction_loss(
tgt_img_scaled, ref_img_warped, qch)
reconstruction_weight = robust_l1_per_pix(diff.mean(1, True),
q=qch)
# reconstruction_weight = reconstruction_loss
reconstruction_loss_all.append(reconstruction_loss)
reconstruction_weight_all.append(reconstruction_weight)
# consistancy_loss_all.append(reconstruction_loss)
reconstruction_loss = torch.cat(reconstruction_loss_all, 1)
reconstruction_weight = torch.cat(reconstruction_weight_all, 1)
# consistancy_loss = torch.cat(consistancy_loss_all,1)
# reconstruction_weight_min,_ = reconstruction_weight.min(1,keepdim=True)
# reconstruction_weight_min = reconstruction_weight_min.repeat(1,2,1,1)
# reconstruction_weight_sum = reconstruction_weight.sum(1,keepdim=True)
# reconstruction_weight_sum = reconstruction_weight_sum.repeat(1,2,1,1)
# consistancy_loss = consistancy_loss[:,0,:,:]-consistancy_loss[:,1,:,:]
# consistancy_loss = wconsis*torch.mean(torch.abs(consistancy_loss))
# loss_weight = reconstruction_weight_min/(reconstruction_weight)
# loss_weight = reconstruction_weight/reconstruction_weight_sum
loss_weight = 1 - torch.nn.functional.softmax(reconstruction_weight, 1)
# loss_weight = (loss_weight >= 0.4).type_as(reconstruction_loss)
# print(loss_weight.size())
# loss_weight = loss_weight[:,:,:-1,:-1]
loss_weight = loss_weight[:, :, 1:-1, 1:-1]
# loss_weight = scale_weight(loss_weight,0.3,10)
# # loss_weight = torch.pow(loss_weight,4)
loss_weight = Variable(loss_weight.data, requires_grad=False)
loss = reconstruction_loss * loss_weight
# loss, _ = torch.min(reconstruction_loss, dim=1)
# # loss = torch.mean(loss,3)
# # loss = torch.mean(loss,2)
# # loss = torch.mean(loss,0)
# loss, _ = torch.min(reconstruction_loss, dim=1)
loss = loss.sum() / loss_weight.sum()
return loss + ssim_loss, loss_weight
str(L3dval)
])
rec = cv2.imread(denoised_loc)
rec = rec[:, :, 0].astype(np.float32)
im_clean = cv2.imread(path_in_str)
im_clean = im_clean[:, :, 0].astype(np.float32)
this_mse = mean_squared_error(im_clean, rec)
total_mse += this_mse
this_psnr = 20. * np.log10(255.) - 10. * np.log10(this_mse)
total_psnr += this_psnr
this_ssim = ssim(im_clean, rec)
total_ssim += this_ssim
pred_idx += 1
print('\nCurrent image predicted L3d value is %.4f' % L3dval)
print('Current image PSNR = %.2f' % this_psnr)
print('Current image SSIM = %.2f' % this_ssim)
print('Current image MSE = %.2f\n' % this_mse)
print('\nAverage PSNR = %.2f' % (total_psnr / pred_count))
print('Average SSIM = %.4f' % (total_ssim / pred_count))
print('Average MSE = %.2f\n' % (total_mse / pred_count))
print('Total time taken for processing = %.2fs.\n' % (time() - total_time))
print "Could not find resolution in file name: %s" % (ref_file)
exit(1)
width, height = int(m.group(1)), int(m.group(2))
print "Comparing %s to %s, resolution %d x %d" % (ref_file, dist_file,
width, height)
ref_fh = open(ref_file, "rb")
dist_fh = open(dist_file, "rb")
frame_num = 0
while True:
ref, _, _ = img_read_yuv(ref_fh, width, height)
dist, _, _ = img_read_yuv(dist_fh, width, height)
vifp_value = vifp.vifp_mscale(ref.astype(float), dist.astype(float))
ssim_value = ssim.ssim(ref, dist)
print "Frame=%d VIFP=%f SSIM=%f" % (frame_num, vifp_value, ssim_value)
frame_num += 1
else:
# Inputs are image files
ref = scipy.misc.imread(ref_file, flatten=True).astype(numpy.float32)
dist = scipy.misc.imread(dist_file, flatten=True).astype(numpy.float32)
width, height = ref.shape[1], ref.shape[0]
print "Comparing %s to %s, resolution %d x %d" % (ref_file, dist_file,
width, height)
vifp_value = vifp.vifp_mscale(ref, dist)
print "VIFP=%f" % (vifp_value)
def test_wienerUnet(data_path,\
psf_path,
method,\
model_path,\
std, \
visual, \
use_gpu,\
b_size=1):
'''
Gradient descent scheme + predicted with UNet per-image gradient of a regularizer
'''
model_name = method + '_gaussian'
save_images_path = './Results/' + model_name + '_std_' + str(std).replace(
'.', '') + '/'
test_dataset = CellDataset(data_path, psf_path, 'gaussian', 1.0, std)
test_loader = DataLoader(test_dataset, batch_size=b_size, shuffle=False)
model = model_load('gaussian', method, model_path)
model.eval()
if use_gpu == 1:
model.cuda()
psnr_values_test = []
ssim_values_test = []
distorted_psnr_test = []
distorted_ssim_test = []
for i_batch, ((gt, image), psf, index, image_name, _,
std) in enumerate(tqdm(test_loader)):
image = image.reshape((b_size, 1, image.shape[-2], image.shape[-1]))
gt = gt.reshape((b_size, 1, gt.shape[-2], gt.shape[-1]))
psf = psf.reshape([b_size, 1, psf.shape[-2], psf.shape[-1]]).float()
if use_gpu == 1:
image = image.cuda()
gt = gt.cuda()
psf = psf.cuda()
distorted_psnr = calc_psnr(image.clamp(gt.min(), gt.max()), gt)
distorted_ssim = ssim(image.clamp(gt.min(), gt.max()), gt)
image = EdgeTaper.apply(pad_psf_shape(image, psf), psf[0][0])
out = model(image, psf)
out = crop_psf_shape(out, psf)
psnr_test = calc_psnr(out.clamp(gt.min(), gt.max()), gt)
s_sim_test = ssim(out.clamp(gt.min(), gt.max()), gt)
psnr_values_test.append(psnr_test.item())
ssim_values_test.append(s_sim_test.item())
distorted_psnr_test.append(distorted_psnr.item())
distorted_ssim_test.append(distorted_ssim.item())
#Save image
if visual == 1:
if not os.path.exists(save_images_path):
os.makedirs(save_images_path, exist_ok=True)
io.imsave(os.path.join(save_images_path, 'output_' + str(image_name[0][:-4]) + '_' + \
str(model_name) + '_' + str(std.item()).replace('.', '') + '.png'), np.uint8(out[0][0].detach().cpu().numpy().clip(0,1) * 255.))
print('Tested on Gaussian noise with %.3f std: PSNR %.2f, SSIM %.4f, distorted PSNR %.2f, distorted SSIM %.4f' % (std,\
np.array(psnr_values_test).mean(), \
np.array(ssim_values_test).mean(), \
np.array(distorted_psnr_test).mean(), \
np.array(distorted_ssim_test).mean()))
return
def cal_ssim(img1, img2):
img1 = img1 / 255.0
img2 = img2 / 255.0
img1 = torch.from_numpy(img1).permute(2, 0, 1).unsqueeze(0).cuda()
img2 = torch.from_numpy(img2).permute(2, 0, 1).unsqueeze(0).cuda()
return ssim.ssim(img1, img2, window_size=16)
def test_wienerUnet(data_path,\
psf_path,
method,\
scale, \
model_path,\
visual, \
use_gpu,\
n_iter=10,\
b_size=1):
'''
Gradient descent scheme + predicted with UNet per-image gradient of a regularizer
'''
model_name = method + '_poisson'
save_images_path = './Results/' + model_name + '_peak_' + str(
int(scale)) + '/'
test_dataset = CellDataset(data_path, psf_path, 'poisson', scale, 0.0)
test_loader = DataLoader(test_dataset, batch_size=b_size, shuffle=False)
model = model_load('poisson', method, model_path)
model.eval()
if use_gpu == 1:
model.cuda()
psnr_values_test = []
ssim_values_test = []
distorted_psnr_test = []
distorted_ssim_test = []
with torch.no_grad():
for i_batch, ((gt, image), psf, index, image_name, peak,
_) in enumerate(tqdm(test_loader)):
image = image.reshape(
(b_size, 1, image.shape[-2], image.shape[-1]))
gt = gt.reshape((b_size, 1, gt.shape[-2], gt.shape[-1]))
psf = psf.reshape([b_size, 1, psf.shape[-2],
psf.shape[-1]]).float()
if use_gpu == 1:
image = image.cuda()
gt = gt.cuda()
psf = psf.cuda()
image_batch_tmp = image.clone()
image = anscombe(image)
image = EdgeTaper.apply(pad_psf_shape(image, psf), psf[0][0])
out = model(image, psf, n_iter)
out = exact_unbiased(out)
out = crop_psf_shape(out, psf)
for j in range(out.shape[0]):
out[j] = out[j] / gt[j].max()
image_batch_tmp[j] = image_batch_tmp[j] / gt[j].max()
gt[j] /= gt[j].max()
distorted_psnr = calc_psnr(image_batch_tmp.clamp(0, 1), gt)
distorted_ssim = ssim(image_batch_tmp.clamp(0, 1), gt)
psnr_test = calc_psnr(out.clamp(0, 1), gt)
s_sim_test = ssim(out.clamp(0, 1), gt)
psnr_values_test.append(psnr_test.item())
ssim_values_test.append(s_sim_test.item())
distorted_psnr_test.append(distorted_psnr.item())
distorted_ssim_test.append(distorted_ssim.item())
#Save image
if visual == 1:
if not os.path.exists(save_images_path):
os.makedirs(save_images_path, exist_ok=True)
io.imsave(os.path.join(save_images_path, 'output_' + str(image_name[0][:-4]) + '_' + \
str(model_name) + '_' + str(int(scale)) + '.png'), np.uint8(out[0][0].detach().cpu().numpy().clip(0,1) * 255.))
print('Test on Poisson noise with peak %d: PSNR %.2f, SSIM %.4f, distorted PSNR %.2f, distorted SSIM %.4f' % (peak, np.array(psnr_values_test).mean(), \
np.array(ssim_values_test).mean(), \
np.array(distorted_psnr_test).mean(), \
np.array(distorted_ssim_test).mean()))
return
def test_wienerKPN_SA(data_path,\
psf_path,
method,\
model_path,\
std, \
visual, \
use_gpu,\
b_size=1):
'''
Wiener filter + predictable per-pixel kernels for each image
'''
model_name = method + '_gaussian'
save_images_path = './Results/' + model_name + '_std_' + str(std).replace(
'.', '') + '/'
test_dataset = CellDataset(data_path, psf_path, 'gaussian', 1.0, std)
test_loader = DataLoader(test_dataset, batch_size=b_size, shuffle=False)
model = model_load('gaussian', method, model_path)
model.eval()
# model = Wiener_KPN_SA()
# state_dict = torch.load(os.path.join(model_path, model_name))
# state_dict = state_dict['model_state_dict']
# # create new OrderedDict that does not contain `module.`
# from collections import OrderedDict
# new_state_dict = OrderedDict()
# for k, v in state_dict.items():
# name = k[7:] # remove `module.`
# new_state_dict[name] = v
# # load params
# model.load_state_dict(new_state_dict)
# model.eval()
# model = nn.DataParallel(model).cuda()
# loss_values_test = []
# psnr_values_test = []
# ssim_values_test = []
# ground_truths = []
# restored_imgs = []
# image_names = []
# baseline_psnr_test = []
# baseline_ssim_test = []
# # model.train(False)
# with torch.no_grad():
# for i_batch, ((gt_batch_test, image_batch_test), psf_test, index_test, image_name_test, peak, std) in enumerate(test_loader):
# image_batch_test = image_batch_test.reshape((b_size, 1, image_batch_test.shape[-2],
# image_batch_test.shape[-1])).cuda()
# gt_batch_test = gt_batch_test.reshape((b_size, 1, gt_batch_test.shape[-2],
# gt_batch_test.shape[-1])).cuda()
# psf_test = psf_test.reshape([b_size, 1, psf_test.shape[-2], psf_test.shape[-1]]).float().cuda()
# print(psf_test.norm())
# baseline_psnr = calc_psnr(image_batch_test.clamp(0, 1), gt_batch_test)
# baseline_ssim = ssim(image_batch_test.clamp(0, 1), gt_batch_test)
# image_batch_test = EdgeTaper.apply(pad_psf_shape(image_batch_test, psf_test),
# psf_test[0][0])
# # print(psf_test.shape)
# out_test = model(image_batch_test, psf_test)
# out_test = crop_psf_shape(out_test, psf_test)
# loss_test = nn.functional.l1_loss(out_test.clamp(0,1), gt_batch_test) + \
# nn.functional.l1_loss(get_image_grad(out_test.clamp(0,1))[0], get_image_grad(gt_batch_test)[0]) + \
# nn.functional.l1_loss(get_image_grad(out_test.clamp(0,1))[1], get_image_grad(gt_batch_test)[1])
# psnr_test = calc_psnr(out_test.clamp(0,1), gt_batch_test)
# s_sim_test = ssim(out_test.clamp(0,1), gt_batch_test)
# loss_values_test.append(loss_test.item())
# psnr_values_test.append(psnr_test.item())
# ssim_values_test.append(s_sim_test.item())
# baseline_psnr_test.append(baseline_psnr.item())
# baseline_ssim_test.append(baseline_ssim.item())
# print('Test: {}, {}, loss {}, PSNR {}, SSIM {}, baseline {}'.format(image_name_test[0],\
# index_test.item(),\
# loss_test.item(),
# psnr_test.item(),
# s_sim_test.item(),
# baseline_psnr.item()))
# ground_truths.append(gt_batch_test[0][0].detach().cpu().numpy())
# restored_imgs.append(out_test[0][0].detach().cpu().numpy())
# image_names.append(image_name_test)
# if visual==1:
# if not os.path.exists(save_images_path):
# os.makedirs(save_images_path, exist_ok=True)
# io.imsave(os.path.join(save_images_path, 'output_' + str(image_name_test[0][:-4]) + '_' + \
# str(model_name) + '_' + str(std.item()).replace('.', '') + '.png'), np.uint8(out_test[0][0].detach().cpu().numpy().clip(0,1) * 255.))
# return
if use_gpu == 1:
model.cuda()
psnr_values_test = []
ssim_values_test = []
distorted_psnr_test = []
distorted_ssim_test = []
with torch.no_grad():
for i_batch, ((gt, image), psf, index, image_name, _,
std) in enumerate(tqdm(test_loader)):
image = image.reshape(
(b_size, 1, image.shape[-2], image.shape[-1]))
gt = gt.reshape((b_size, 1, gt.shape[-2], gt.shape[-1]))
psf = psf.reshape([b_size, 1, psf.shape[-2],
psf.shape[-1]]).float()
if use_gpu == 1:
image = image.cuda()
gt = gt.cuda()
psf = psf.cuda()
distorted_psnr = calc_psnr(image.clamp(gt.min(), gt.max()), gt)
distorted_ssim = ssim(image.clamp(gt.min(), gt.max()), gt)
image = EdgeTaper.apply(pad_psf_shape(image, psf), psf[0][0])
out = model(image, psf)
# out = image
out = crop_psf_shape(out, psf)
psnr_test = calc_psnr(out.clamp(gt.min(), gt.max()), gt)
s_sim_test = ssim(out.clamp(gt.min(), gt.max()), gt)
psnr_values_test.append(psnr_test.item())
ssim_values_test.append(s_sim_test.item())
distorted_psnr_test.append(distorted_psnr.item())
distorted_ssim_test.append(distorted_ssim.item())
#Save image
if visual == 1:
if not os.path.exists(save_images_path):
os.makedirs(save_images_path, exist_ok=True)
io.imsave(os.path.join(save_images_path, 'output_' + str(image_name[0][:-4]) + '_' + \
str(model_name) + '_' + str(std.item()).replace('.', '') + '.png'), np.uint8(out[0][0].detach().cpu().numpy().clip(0,1) * 255.))
print('Tested on Gaussian noise with %.3f std: PSNR %.2f, SSIM %.4f, distorted PSNR %.2f, distorted SSIM %.4f' % (std,\
np.array(psnr_values_test).mean(), \
np.array(ssim_values_test).mean(), \
np.array(distorted_psnr_test).mean(), \
np.array(distorted_ssim_test).mean()))
return
def testFromH5Folder(input_dataset_dir, output_dataset_dir, upscale_factor, input_field_name, model_name, cuda_flag, if_save):
h5_file_list = [x for x in listdir(input_dataset_dir) if is_h5_file(x)]
h5_file_list.sort()
h5_len = len(h5_file_list)
model_PSNR = np.zeros(h5_len)
model_SSIM = np.zeros(h5_len)
bicubic_PSNR = np.zeros(h5_len)
bicubic_SSIM = np.zeros(h5_len)
model_time = np.zeros(h5_len)
# Load SR Model
model = Net_SRCNN(upscale_factor=UPSCALE_FACTOR)
model = torch.load(MODEL_NAME)
if torch.cuda.is_available() & cuda_flag:
model = model.cuda()
else:
model = model.cpu()
if not os.path.exists(output_dataset_dir):
os.makedirs(output_dataset_dir)
for idx in tqdm(range(h5_len), desc='SR of upscale factor = ' + str(upscale_factor)):
h5_file_name = h5_file_list[idx]
h5_file = h5py.File(input_dataset_dir + '/' + h5_file_name, 'r')
img_HR_y = h5_file['HR'].value
img_LR_y = h5_file['LR'].value
img_LR_bic_y = h5_file['LR_bic_y'].value
img_LR_bic_cb = h5_file['LR_bic_cb'].value
img_LR_bic_cr = h5_file['LR_bic_cr'].value
img_HR_RGB = h5_file['HR_RGB'].value
img_HR_y = img_HR_y.astype(np.float32)
img_LR_y = img_LR_y.astype(np.float32)
img_LR_bic_y = img_LR_bic_y.astype(np.float32)
img_LR_bic_cb = img_LR_bic_cb.astype(np.float32)
img_LR_bic_cr = img_LR_bic_cr.astype(np.float32)
img_HR_RGB = img_HR_RGB.astype(np.float32)
img_HR_y = img_HR_y / 255.0
img_LR_y = img_LR_y / 255.0
img_HR_RGB = img_HR_RGB / 255.0
if input_field_name == 'LR':
img_LR_4d = img_LR_y.reshape(1, 1, img_LR_y.shape[0], img_LR_y.shape[1])
elif input_field_name == 'LR_bic_y':
img_LR_4d = img_LR_bic_y.reshape(1, 1, img_LR_bic_y.shape[0], img_LR_bic_y.shape[1])
image = Variable(torch.from_numpy(img_LR_4d))
if torch.cuda.is_available() & cuda_flag:
image = image.cuda()
start = time.time()
target = model(image)
end = time.time()
target = target.cpu()
img_HR_y_net = target.data[0][0].numpy()
if if_save:
img_HR_ycbcr_net = np.zeros(img_HR_RGB.shape)
img_HR_ycbcr_net[:,:,0] = img_HR_y_net
img_HR_ycbcr_net[:,:,1] = img_LR_bic_cb
img_HR_ycbcr_net[:,:,2] = img_LR_bic_cr
img_HR_ycbcr_net = img_HR_ycbcr_net.clip(0.0, 1.0)
img_HR_RGB_net = ycbcr2rgb(img_HR_ycbcr_net)
img_HR_RGB_net = img_HR_RGB_net.clip(0.0, 1.0)
img_HR_RGB_net *= 255.0
img_HR_BGR_net = rgb2bgr(img_HR_RGB_net)
image_name = 'img_' + str(idx) + '_net.png'
cv2.imwrite(output_dataset_dir + '/' + image_name, img_HR_BGR_net.astype(np.uint8))
# Compute Stat
model_PSNR[idx] = psnr((img_HR_y*255.0).astype(int), (img_HR_y_net*255.0).astype(int))
model_SSIM[idx] = ssim((img_HR_y*255.0).astype(int), (img_HR_y_net*255.0).astype(int))
bicubic_PSNR[idx] = psnr((img_HR_y*255.0).astype(int), (img_LR_bic_y*255.0).astype(int))
bicubic_SSIM[idx] = ssim((img_HR_y*255.0).astype(int), (img_LR_bic_y*255.0).astype(int))
model_time[idx] = (end-start)
print("===> Test on" + input_dataset_dir +" Complete: Model PSNR: {:.4f} dB, Model SSIM: {:.4f} , Bicubic PSNR: {:.4f} dB, Bicubic SSIM: {:.4f} , Average time: {:.4f}"
.format(np.average(model_PSNR), np.average(model_SSIM), np.average(bicubic_PSNR), np.average(bicubic_SSIM), np.average(model_time)*1000))
def main():
# Parse the command line arguments
cfg = parse_args(description='Performs inference on a dataset using the specified training result.')
# Initialize the PyTorch device
device = init_device(cfg)
# Open the result
result_dir = get_result_dir(cfg)
if not os.path.isdir(result_dir):
error('result does not exist')
print('Result:', cfg.result)
# Load the result config
result_cfg = load_config(result_dir)
cfg.features = result_cfg.features
cfg.transfer = result_cfg.transfer
is_hdr = 'hdr' in cfg.features
# Inference function
def infer(model, transfer, input, exposure):
x = input.clone()
# Apply the transfer function
color = x[:, 0:3, ...]
if is_hdr:
color *= exposure
color = transfer.forward(color)
x[:, 0:3, ...] = color
# Pad the output
shape = x.shape
x = F.pad(x, (0, pad(shape[3])-shape[3], 0, pad(shape[2])-shape[2]))
# Run the inference
x = model(x)
# Unpad the output
x = x[:, :, :shape[2], :shape[3]]
# Sanitize the output
x = torch.clamp(x, min=0.)
# Apply the inverse transfer function
x = transfer.inverse(x)
if is_hdr:
x /= exposure
else:
x = torch.clamp(x, max=1.)
return x
# Converts image to sRGB (tonemapping if HDR + gamma correction)
def to_srgb(image, exposure):
return srgb_forward(tonemap(image * exposure) if is_hdr else image)
# Saves an image in different formats
def save_images(path, image, image_srgb):
image = to_numpy(image)
image_srgb = to_numpy(image_srgb)
suffix = '.hdr' if is_hdr else '.ldr'
if 'exr' in cfg.format:
save_image(path + suffix + '.exr', image)
if 'pfm' in cfg.format:
save_image(path + suffix + '.pfm', image)
if 'png' in cfg.format:
save_image(path + suffix + '.png', image_srgb)
# Initialize the dataset
data_dir = get_data_dir(cfg, cfg.input_data)
image_sample_groups = get_image_sample_groups(data_dir, cfg.features)
# Initialize the model
model = Autoencoder(get_num_channels(cfg.features))
model.to(device)
# Load the checkpoint
checkpoint = load_checkpoint(cfg, device, cfg.checkpoint, model)
epoch = checkpoint['epoch']
# Initialize the transfer function
transfer = get_transfer_function(cfg.transfer)
# Iterate over the images
output_dir = os.path.join(cfg.output_dir, cfg.input_data)
model.eval()
with torch.no_grad():
for group, input_names, target_name in image_sample_groups:
# Create the output directory if it does not exist
output_group_dir = os.path.join(output_dir, os.path.dirname(group))
if not os.path.isdir(output_group_dir):
os.makedirs(output_group_dir)
# Load metadata for the images if it exists
tonemap_exposure = 1.
metadata = load_image_metadata(os.path.join(data_dir, group))
if metadata:
tonemap_exposure = metadata['exposure']
save_image_metadata(os.path.join(output_dir, group), metadata)
# Load the target image if it exists
if target_name:
target = load_target_image(os.path.join(data_dir, target_name), cfg.features)
target = to_tensor(target).unsqueeze(0).to(device)
target_srgb = to_srgb(target, tonemap_exposure)
# Iterate over the input images
for input_name in input_names:
progress_str = input_name
# Load the input image
input = load_input_image(os.path.join(data_dir, input_name), cfg.features)
# Compute the autoexposure value
exposure = autoexposure(input) if is_hdr else 1.
# Infer
input = to_tensor(input).unsqueeze(0).to(device)
output = infer(model, transfer, input, exposure)
input = input[:, 0:3, ...] # keep only the color
input_srgb = to_srgb(input, tonemap_exposure)
output_srgb = to_srgb(output, tonemap_exposure)
# Compute metrics
if target_name and cfg.metric:
metric_str = ''
for metric in cfg.metric:
if metric == 'mse':
value = ((output_srgb - target_srgb) ** 2).mean()
elif metric == 'ssim':
value = ssim(output_srgb, target_srgb, data_range=1.)
if metric_str:
metric_str += ', '
metric_str += '%s = %.4f' % (metric, value)
progress_str += ' (' + metric_str + ')'
# Save the input and output images
output_name = input_name + '.' + cfg.result
if cfg.checkpoint:
output_name += '_%d' % epoch
if cfg.save_all:
save_images(os.path.join(output_dir, input_name), input, input_srgb)
save_images(os.path.join(output_dir, output_name), output, output_srgb)
# Print progress
print(progress_str)
# Save the target image if it exists
if cfg.save_all and target_name:
save_images(os.path.join(output_dir, target_name), target, target_srgb)