def do_one_demo(args, config, hdf5_file, model, sample_num, cuda):
depth_grp = hdf5_file['val']['disparity']
# Create output directory
if not args.no_save:
base_dir = os.path.join(config['PATH']['output_dir'], 'warped')
if not os.path.isdir(base_dir):
pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True)
save_dir = get_sub_dir_for_saving(base_dir)
model.eval()
SNUM = sample_num
start_time = time.time()
print("Working on image", SNUM)
depth_images = torch.squeeze(
torch.tensor(depth_grp['images'][SNUM], dtype=torch.float32))
colour_grp = hdf5_file['val']['colour']
colour_images = torch.tensor(colour_grp['images'][SNUM],
dtype=torch.float32)
sample = {
'depth': depth_images,
'colour': colour_images,
'grid_size': depth_images.shape[0]
}
warped = data_transform.transform_to_warped(sample)
im_input = warped['inputs'].unsqueeze_(0)
if cuda:
im_input = im_input.cuda()
output = model(im_input)
output += im_input[:, :-1]
output = torch.clamp(output, 0.0, 1.0)
time_taken = time.time() - start_time
print("Time taken was {:.0f}s".format(time_taken))
grid_size = 64
psnr_accumulator = (0, 0, 0)
ssim_accumulator = (0, 0, 0)
if not args.no_save:
print("Saving output to", save_dir)
no_cnn_dir = os.path.join(save_dir, "no_cnn")
cnn_dir = os.path.join(save_dir, "cnn")
os.mkdir(cnn_dir)
os.mkdir(no_cnn_dir)
output = torch.squeeze(denormalise_lf(output))
output = data_transform.torch_unstack(output)
cpu_output = np.around(output.cpu().detach().numpy()).astype(np.uint8)
if (not args.no_eval) or args.get_diff:
ground_truth = np.around(denormalise_lf(colour_images).numpy()).astype(
np.uint8)
grid_len = int(math.sqrt(grid_size))
for i in range(grid_size):
row, col = i // grid_len, i % grid_len
if not args.no_save:
file_name = 'Colour{}{}.png'.format(row, col)
save_location = os.path.join(cnn_dir, file_name)
if i == 0:
print("Saving images of size ", cpu_output[i].shape)
image_warping.save_array_as_image(cpu_output[i], save_location)
if args.get_diff and not args.no_save:
colour = ground_truth[i]
diff = image_warping.get_diff_image(colour, cpu_output[i])
#diff = get_diff_image_floatint(res, colour)
file_name = 'Diff{}{}.png'.format(row, col)
save_location = os.path.join(cnn_dir, file_name)
image_warping.save_array_as_image(diff, save_location)
if not args.no_eval:
img = ground_truth[i]
if not args.no_save:
file_name = 'GT_Colour{}{}.png'.format(row, col)
save_location = os.path.join(save_dir, file_name)
image_warping.save_array_as_image(img, save_location)
psnr = evaluate.my_psnr(cpu_output[i], img)
ssim = evaluate.ssim(cpu_output[i], img)
psnr_accumulator = welford.update(psnr_accumulator, psnr)
ssim_accumulator = welford.update(ssim_accumulator, ssim)
psnr_mean, psnr_var, _ = welford.finalize(psnr_accumulator)
ssim_mean, ssim_var, _ = welford.finalize(ssim_accumulator)
print("For cnn, psnr average {:5f}, stddev {:5f}".format(
psnr_mean, math.sqrt(psnr_var)))
print("For cnn, ssim average {:5f}, stddev {:5f}".format(
ssim_mean, math.sqrt(ssim_var)))
psnr1 = psnr_mean
ssim1 = ssim_mean
psnr_accumulator = (0, 0, 0)
ssim_accumulator = (0, 0, 0)
psnr2, ssim2 = 0, 0
if args.no_cnn:
squeeze_input = torch.squeeze(denormalise_lf(im_input[:, :-1]))
squeeze_input = data_transform.torch_unstack(squeeze_input)
cpu_input = np.around(squeeze_input.cpu().detach().numpy()).astype(
np.uint8)
for i in range(grid_size):
row, col = i // grid_len, i % grid_len
if not args.no_save:
file_name = 'Colour{}{}.png'.format(row, col)
save_location = os.path.join(no_cnn_dir, file_name)
if i == 0:
print("Saving images of size ", cpu_input[i].shape)
image_warping.save_array_as_image(cpu_input[i], save_location)
if args.get_diff and not args.no_save:
colour = ground_truth[i]
diff = image_warping.get_diff_image(colour, cpu_output[i])
#diff = get_diff_image_floatint(res, colour)
file_name = 'Diff{}{}.png'.format(row, col)
save_location = os.path.join(no_cnn_dir, file_name)
image_warping.save_array_as_image(diff, save_location)
if not args.no_eval:
img = ground_truth[i]
psnr = evaluate.my_psnr(cpu_input[i], img)
ssim = evaluate.ssim(cpu_input[i], img)
psnr_accumulator = welford.update(psnr_accumulator, psnr)
ssim_accumulator = welford.update(ssim_accumulator, ssim)
psnr_mean, psnr_var, _ = welford.finalize(psnr_accumulator)
ssim_mean, ssim_var, _ = welford.finalize(ssim_accumulator)
print("For no cnn, psnr average {:5f}, stddev {:5f}".format(
psnr_mean, math.sqrt(psnr_var)))
print("For no cnn, ssim average {:5f}, stddev {:5f}".format(
ssim_mean, math.sqrt(ssim_var)))
psnr2, ssim2 = psnr_mean, ssim_mean
return psnr1, ssim1, psnr2, ssim2
def main(args, config, sample_index):
cuda = cnn_utils.check_cuda(config)
model = cnn_utils.load_model_and_weights(args, config)
if cuda:
model = model.cuda()
model.eval()
# Create output directory
base_dir = os.path.join(config['PATH']['output_dir'], 'warped')
if not os.path.isdir(base_dir):
pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True)
save_dir = get_sub_dir_for_saving(base_dir)
# TODO if GT is available, can get diff images
start_time = time.time()
file_path = os.path.join(config['PATH']['hdf5_dir'],
config['PATH']['hdf5_name'])
with h5py.File(file_path, mode='r', libver='latest') as hdf5_file:
depth_grp = hdf5_file['val']['disparity']
SNUM = sample_index
depth_images = torch.squeeze(
torch.tensor(depth_grp['images'][SNUM], dtype=torch.float32))
colour_grp = hdf5_file['val']['colour']
colour_images = torch.tensor(colour_grp['images'][SNUM],
dtype=torch.float32)
sample = {
'depth': depth_images,
'colour': colour_images,
'grid_size': depth_images.shape[0]
}
warped = data_transform.transform_to_warped(sample)
im_input = warped['inputs'].unsqueeze_(0)
if cuda:
im_input = im_input.cuda()
output = model(im_input)
output += im_input[:, :-1]
output = torch.clamp(output, 0.0, 1.0)
time_taken = time.time() - start_time
print("Time taken was {:.0f}s".format(time_taken))
grid_size = 64
psnr_accumulator = (0, 0, 0)
ssim_accumulator = (0, 0, 0)
print("Saving output to", save_dir)
if args.no_cnn:
no_cnn_dir = os.path.join(save_dir, "no_cnn")
cnn_dir = os.path.join(save_dir, "cnn")
os.mkdir(cnn_dir)
os.mkdir(no_cnn_dir)
output = torch.squeeze(denormalise_lf(output))
output = data_transform.torch_unstack(output)
cpu_output = np.around(output.cpu().detach().numpy()).astype(np.uint8)
if (not args.no_eval) or args.get_diff:
ground_truth = np.around(
denormalise_lf(colour_images).numpy()).astype(np.uint8)
grid_len = int(math.sqrt(grid_size))
for i in range(grid_size):
row, col = i // grid_len, i % grid_len
file_name = 'Colour{}{}.png'.format(row, col)
save_location = os.path.join(cnn_dir, file_name)
if i == 0:
print("Saving images of size ", cpu_output[i].shape)
image_warping.save_array_as_image(cpu_output[i], save_location)
if args.get_diff:
colour = ground_truth[i]
diff = image_warping.get_diff_image(colour, cpu_output[i])
#diff = get_diff_image_floatint(res, colour)
file_name = 'Diff{}{}.png'.format(row, col)
save_location = os.path.join(cnn_dir, file_name)
image_warping.save_array_as_image(diff, save_location)
if not args.no_eval:
img = ground_truth[i]
file_name = 'GT_Colour{}{}.png'.format(row, col)
save_location = os.path.join(save_dir, file_name)
image_warping.save_array_as_image(img, save_location)
psnr = evaluate.my_psnr(cpu_output[i], img)
ssim = evaluate.ssim(cpu_output[i], img)
psnr_accumulator = welford.update(psnr_accumulator, psnr)
ssim_accumulator = welford.update(ssim_accumulator, ssim)
psnr_mean, psnr_var, _ = welford.finalize(psnr_accumulator)
ssim_mean, ssim_var, _ = welford.finalize(ssim_accumulator)
print("For cnn, psnr average {:5f}, stddev {:5f}".format(
psnr_mean, math.sqrt(psnr_var)))
print("For cnn, ssim average {:5f}, stddev {:5f}".format(
ssim_mean, math.sqrt(ssim_var)))
psnr_accumulator = (0, 0, 0)
ssim_accumulator = (0, 0, 0)
if args.no_cnn:
squeeze_input = torch.squeeze(denormalise_lf(im_input[:, :-1]))
squeeze_input = data_transform.torch_unstack(squeeze_input)
cpu_input = np.around(squeeze_input.cpu().detach().numpy()).astype(
np.uint8)
for i in range(grid_size):
row, col = i // grid_len, i % grid_len
file_name = 'Colour{}{}.png'.format(row, col)
save_location = os.path.join(no_cnn_dir, file_name)
if i == 0:
print("Saving images of size ", cpu_input[i].shape)
image_warping.save_array_as_image(cpu_input[i], save_location)
if args.get_diff:
colour = ground_truth[i]
diff = image_warping.get_diff_image(colour, cpu_output[i])
#diff = get_diff_image_floatint(res, colour)
file_name = 'Diff{}{}.png'.format(row, col)
save_location = os.path.join(no_cnn_dir, file_name)
image_warping.save_array_as_image(diff, save_location)
if not args.no_eval:
img = ground_truth[i]
psnr = evaluate.my_psnr(cpu_input[i], img)
ssim = evaluate.ssim(cpu_input[i], img)
psnr_accumulator = welford.update(psnr_accumulator, psnr)
ssim_accumulator = welford.update(ssim_accumulator, ssim)
psnr_mean, psnr_var, _ = welford.finalize(psnr_accumulator)
ssim_mean, ssim_var, _ = welford.finalize(ssim_accumulator)
print("For no cnn, psnr average {:5f}, stddev {:5f}".format(
psnr_mean, math.sqrt(psnr_var)))
print("For no cnn, ssim average {:5f}, stddev {:5f}".format(
ssim_mean, math.sqrt(ssim_var)))
#Ground truth possible
"""
def main(args, config):
hdf5_path = os.path.join(config['PATH']['output_dir'],
config['PATH']['hdf5_name'])
#warp_type = WARP_TYPE.TORCH_GPU
warp_type = args.warp_type
print("Performing image warping using {}".format(warp_type))
with h5py.File(hdf5_path, mode='r', libver='latest') as hdf5_file:
grid_size = 64
grid_one_way = 8
sample_index = grid_size // 2 + (grid_one_way // 2)
depth_grp = hdf5_file['val']['disparity']
overall_psnr_accum = (0, 0, 0)
overall_ssim_accum = (0, 0, 0)
for sample_num in range(args.nSamples):
SNUM = sample_num
print("Working on image", SNUM)
depth_image = np.squeeze(depth_grp['images'][SNUM, sample_index])
#Hardcoded some values for now
colour_grp = hdf5_file['val']['colour']
colour_image = colour_grp['images'][SNUM, sample_index]
#Can later expand like 0000 if needed
base_dir = os.path.join(config['PATH']['output_dir'], 'warped')
get_diff = (config['DEFAULT']['should_get_diff'] == 'True')
if not args.no_save:
save_dir = get_sub_dir_for_saving(base_dir)
print("Saving images to {}".format(save_dir))
else:
print("Not saving images, only evaluating output")
psnr_accumulator = (0, 0, 0)
ssim_accumulator = (0, 0, 0)
start_time = time()
if warp_type == WARP_TYPE.TORCH_ALL:
final = depth_rendering(colour_image, depth_image)
print("Time taken was {:4f}".format(time() - start_time))
if warp_type == WARP_TYPE.TORCH_GPU:
final = depth_rendering_gpu(colour_image, depth_image).cpu()
print("Time taken was {:4f}".format(time() - start_time))
ref_pos = np.asarray([4, 4])
print("Reference position is ({}, {})".format(*ref_pos))
for i in range(8):
for j in range(8):
if warp_type == WARP_TYPE.FW:
res = fw_warp_image(colour_image, depth_image, ref_pos,
np.asarray([i, j]))
elif warp_type == WARP_TYPE.SK:
res = sk_warp(colour_image,
depth_image,
ref_pos,
np.asarray([i, j]),
preserve_range=True)
elif warp_type == WARP_TYPE.SLOW:
res = slow_fw_warp_image(colour_image, depth_image,
ref_pos, np.asarray([i, j]))
elif warp_type == WARP_TYPE.TORCH:
res = np.around(
torch_warp(colour_image, depth_image, ref_pos,
np.asarray([i, j])).numpy()).astype(
np.uint8)
elif (warp_type == WARP_TYPE.TORCH_ALL
or warp_type == WARP_TYPE.TORCH_GPU):
res = np.around(final[i * 8 + j].numpy()).astype(
np.uint8)
if not args.no_save:
file_name = 'Warped_Colour{}{}.png'.format(i, j)
save_location = os.path.join(save_dir, file_name)
save_array_as_image(res, save_location)
idx = i * 8 + j
file_name = 'GT_Colour{}{}.png'.format(i, j)
save_location = os.path.join(save_dir, file_name)
save_array_as_image(colour_grp['images'][SNUM][idx],
save_location)
if get_diff:
colour = colour_grp['images'][SNUM, i * 8 + j]
diff = get_diff_image(colour, res)
#diff = get_diff_image_floatint(res, colour)
file_name = 'Diff{}{}.png'.format(i, j)
save_location = os.path.join(save_dir, file_name)
save_array_as_image(diff, save_location)
psnr = evaluate.my_psnr(
res, colour_grp['images'][SNUM, i * 8 + j])
ssim = evaluate.ssim(res, colour_grp['images'][SNUM,
i * 8 + j])
print("Position ({}, {}): PSNR {:4f}, SSIM {:4f}".format(
i, j, psnr, ssim))
psnr_accumulator = welford.update(psnr_accumulator, psnr)
ssim_accumulator = welford.update(ssim_accumulator, ssim)
psnr_mean, psnr_var, _ = welford.finalize(psnr_accumulator)
ssim_mean, ssim_var, _ = welford.finalize(ssim_accumulator)
print("\npsnr average {:5f}, stddev {:5f}".format(
psnr_mean, math.sqrt(psnr_var)))
print("ssim average {:5f}, stddev {:5f}".format(
ssim_mean, math.sqrt(ssim_var)))
overall_psnr_accum = welford.update(overall_psnr_accum, psnr_mean)
overall_ssim_accum = welford.update(overall_ssim_accum, ssim_mean)
if args.nSamples > 1:
psnr_mean, psnr_var, _ = welford.finalize(overall_psnr_accum)
ssim_mean, ssim_var, _ = welford.finalize(overall_ssim_accum)
print("\nOverall psnr average {:5f}, stddev {:5f}".format(
psnr_mean, math.sqrt(psnr_var)))
print("Overall ssim average {:5f}, stddev {:5f}".format(
ssim_mean, math.sqrt(ssim_var)))