def test_warp_tensor_offset_x1y1(self, batch_size, device, dtype):
channels, height, width = 3, 3, 5 # output shape
pinhole_src, pinhole_dst = self._create_pinhole_pair(
batch_size, device, dtype)
pinhole_dst.tx += 1.0 # apply offset to tx
pinhole_dst.ty += 1.0 # apply offset to ty
# initialize depth to one
depth_src = torch.ones(batch_size,
1,
height,
width,
device=device,
dtype=dtype)
# create warper, initialize projection matrices and warp grid
warper = kornia.DepthWarper(pinhole_dst, height, width)
warper.compute_projection_matrix(pinhole_src)
# create patch to warp
patch_dst = (torch.arange(float(height * width),
device=device,
dtype=dtype).view(1, 1, height,
width).expand(
batch_size, channels,
-1, -1))
# warpd source patch by depth
patch_src = warper(depth_src, patch_dst)
# compare patches
assert_allclose(patch_dst[..., 1:, 1:],
patch_src[..., :2, :4],
atol=1e-4,
rtol=1e-4)
def test_warp_grid_offset_x1_depth1(self, batch_size, device, dtype):
height, width = 3, 5 # output shape
pinhole_src, pinhole_dst = self._create_pinhole_pair(
batch_size, device, dtype)
pinhole_dst.tx += 1.0 # apply offset to tx
# initialize depth to one
depth_src = torch.ones(batch_size,
1,
height,
width,
device=device,
dtype=dtype)
# create warper, initialize projection matrices and warp grid
warper = kornia.DepthWarper(pinhole_dst, height, width)
warper.compute_projection_matrix(pinhole_src)
grid_warped = warper.warp_grid(depth_src)
assert grid_warped.shape == (batch_size, height, width, 2)
# normalize base meshgrid
grid = warper.grid[..., :2].to(device=device, dtype=dtype)
grid_norm = normalize_pixel_coordinates(grid, height, width)
# check offset in x-axis
assert_allclose(grid_warped[..., -2, 0],
grid_norm[..., -1, 0].repeat(batch_size, 1),
atol=1e-4,
rtol=1e-4)
# check that y-axis remain the same
assert_allclose(grid_warped[..., -1, 1],
grid_norm[..., -1, 1].repeat(batch_size, 1),
rtol=1e-4,
atol=1e-4)
def warp(self, depth_src, rgb_dst, cam_intrinsic, pose_src, pose_dest):
cam_intrinsic = cam_intrinsic.squeeze(1)
pose_src = pose_src.squeeze(1)
pose_dest = pose_dest.squeeze(1)
height = rgb_dst.size()[2]
width = rgb_dst.size()[3]
height_tensor = torch.tensor([height])
width_tensor = torch.tensor([width])
# pinholes camera models
pinhole_dst = kornia.PinholeCamera(cam_intrinsic, pose_dest,
height_tensor, width_tensor)
pinhole_src = kornia.PinholeCamera(cam_intrinsic, pose_src,
height_tensor, width_tensor)
# create the depth warper, compute the projection matrix
warper = kornia.DepthWarper(pinhole_dst, height, width)
warper.compute_projection_matrix(pinhole_src)
# warp the destionation frame to reference by depth
recon_rgb_src = warper(depth_src, rgb_dst) # NxCxHxW
return recon_rgb_src
def test_warp_grid_offset_x1y1_depth1(self, batch_size):
height, width = 3, 5 # output shape
pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size)
pinhole_dst.tx += 1. # apply offset to tx
pinhole_dst.ty += 1. # apply offset to ty
# initialize depth to one
depth_src = torch.ones(batch_size, 1, height, width)
# create warper, initialize projection matrices and warp grid
warper = kornia.DepthWarper(pinhole_dst, height, width)
warper.compute_projection_matrix(pinhole_src)
grid_warped = warper.warp_grid(depth_src)
assert grid_warped.shape == (batch_size, height, width, 2)
# normalize base meshgrid
grid = warper.grid[..., :2]
grid_norm = normalize_pixel_coordinates(grid, height, width)
# check offset in x-axis
assert utils.check_equal_torch(grid_norm[..., -1, 0],
grid_warped[..., -2, 0])
# check that y-axis remain the same
assert utils.check_equal_torch(grid_norm[..., -1, :, 1],
grid_warped[..., -2, :, 1])
def test_compute_subpixel_step(self, batch_size):
height, width = 3, 5 # output shape
pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size)
# create warper, initialize projection matrices and warp grid
warper = kornia.DepthWarper(pinhole_dst, height, width)
warper.compute_projection_matrix(pinhole_src)
# test compute_subpixel_step
subpixel_step = warper.compute_subpixel_step()
assert pytest.approx(subpixel_step.item(), 0.3536)
def test_compute_projection(self, batch_size):
height, width = 3, 5 # output shape
pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size)
# create warper, initialize projection matrices and warp grid
warper = kornia.DepthWarper(pinhole_dst, height, width)
warper.compute_projection_matrix(pinhole_src)
# test compute_projection
xy_projected = warper._compute_projection(0.0, 0.0, 1.0)
assert xy_projected.shape == (batch_size, 2)
def test_compute_projection_matrix(self, batch_size):
height, width = 3, 5 # output shape
pinhole_src, pinhole_dst = self._create_pinhole_pair(batch_size)
pinhole_dst.tx += 1. # apply offset to tx
# create warper
warper = kornia.DepthWarper(pinhole_dst, height, width)
assert warper._dst_proj_src is None
# initialize projection matrices
warper.compute_projection_matrix(pinhole_src)
assert warper._dst_proj_src is not None
# retreive computed projection matrix and compare to expected
dst_proj_src = warper._dst_proj_src
dst_proj_src_expected = torch.eye(4)[None].repeat(batch_size, 1,
1) # Bx4x4
dst_proj_src_expected[..., 0, -2] += pinhole_src.cx
dst_proj_src_expected[..., 1, -2] += pinhole_src.cy
dst_proj_src_expected[..., 0, -1] += 1. # offset to x-axis
assert utils.check_equal_torch(dst_proj_src, dst_proj_src_expected)
def DepthWarperApp():
parser = argparse.ArgumentParser(
description='Warp images by depth application.')
# data parameters
parser.add_argument('--input-dir',
type=str,
required=True,
help='the path to the directory with the input data.')
parser.add_argument('--output-dir',
type=str,
required=True,
help='the path to output the results.')
parser.add_argument('--sequence-name',
type=str,
default='alley_1',
help='the name of the sequence.')
parser.add_argument('--frame-ref-id',
type=int,
default=1,
help='the id for the reference image in the sequence.')
parser.add_argument('--frame-i-id',
type=int,
default=2,
help='the id for the image i in the sequence.')
# device parameters
parser.add_argument('--cuda',
action='store_true',
default=False,
help='enables CUDA training')
parser.add_argument('--seed',
type=int,
default=666,
metavar='S',
help='random seed (default: 666)')
args = parser.parse_args()
torch.manual_seed(args.seed)
# configure syntel SDK path
root_path = os.path.abspath(args.input_dir)
sys.path.append(os.path.join(root_path, 'sdk/python'))
# load the data
root_dir = os.path.join(root_path, 'training')
img_ref, depth_ref, cam_ref = load_data(root_dir, args.sequence_name,
args.frame_ref_id)
img_i, _, cam_i = load_data(root_dir, args.sequence_name, args.frame_i_id)
# instantiate the homography warper from `kornia`
warper = dgm.DepthWarper(cam_i)
warper.compute_homographies(cam_ref)
# compute the inverse depth and warp the source image
inv_depth_ref = 1.0 / depth_ref
img_i_to_ref = warper(inv_depth_ref, img_i)
# generate occlusion mask
mask = ((img_ref - img_i_to_ref).mean(1) < 1e-1).float()
img_vis_warped = 0.5 * img_i_to_ref + img_ref
img_vis_warped_masked = mask * (0.5 * img_i_to_ref + img_ref)
# save warped image to disk
file_name = os.path.join(
args.output_dir,
f'warped_{args.frame_i_id}_to_{args.frame_ref_id}.png')
cv2.imwrite(file_name, dgm.utils.tensor_to_image(255.0 * img_vis_warped))
cv2.imwrite(file_name + 'mask.png',
dgm.utils.tensor_to_image(255.0 * mask))
cv2.imwrite(file_name + 'warpedmask.png',
dgm.utils.tensor_to_image(255.0 * img_vis_warped_masked))
def DepthRegressionApp():
# data settings
parser = argparse.ArgumentParser(
description='Depth Regression with photometric loss.')
parser.add_argument('--input-dir',
type=str,
required=True,
help='the path to the directory with the input data.')
parser.add_argument('--output-dir',
type=str,
required=True,
help='the path to output the results.')
parser.add_argument('--num-iterations',
type=int,
default=1000,
metavar='N',
help='number of training iterations (default: 1000)')
parser.add_argument('--sequence-name',
type=str,
default='alley_1',
help='the name of the sequence.')
parser.add_argument('--frame-ref-id',
type=int,
default=1,
help='the id for the reference image in the sequence.')
parser.add_argument('--frame-i-id',
type=int,
default=2,
help='the id for the image i in the sequence.')
# optimization parameters
parser.add_argument('--lr',
type=float,
default=1e-3,
metavar='LR',
help='learning rate (default: 1e-3)')
# device parameters
parser.add_argument('--cuda',
action='store_true',
default=False,
help='enables CUDA training')
parser.add_argument('--seed',
type=int,
default=666,
metavar='S',
help='random seed (default: 666)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument(
'--log-interval-vis',
type=int,
default=100,
metavar='N',
help='how many batches to wait before visual logging training status')
args = parser.parse_args()
# define the device to use for inference
use_cuda = args.cuda and torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
torch.manual_seed(args.seed)
# configure sintel SDK path
root_path = os.path.abspath(args.input_dir)
sys.path.append(os.path.join(root_path, 'sdk/python'))
# load the data
root_dir = os.path.join(root_path, 'training')
img_ref, depth_ref, cam_ref = load_data(root_dir, args.sequence_name,
args.frame_ref_id)
img_i, _, cam_i = load_data(root_dir, args.sequence_name, args.frame_i_id)
# instantiate the depth warper from `kornia`
warper = tgm.DepthWarper(cam_i)
warper.compute_homographies(cam_ref)
# create the inverse depth as a parameter to be optimized
height, width = img_ref.shape[-2:]
inv_depth_ref = InvDepth(height, width).to(device)
# create optimizer
optimizer = optim.Adam(inv_depth_ref.parameters(), lr=args.lr)
# send data to device
img_ref, img_i = img_ref.to(device), img_i.to(device)
# main training loop
for iter_idx in range(args.num_iterations):
# compute the inverse depth and warp the source image
img_i_to_ref = warper(inv_depth_ref(), img_i)
# compute the photometric loss
loss = F.l1_loss(img_i_to_ref, img_ref, reduction='none')
loss = torch.mean(loss)
# compute gradient and update optimizer parameters
optimizer.zero_grad()
loss.backward()
optimizer.step()
if iter_idx % args.log_interval == 0 or \
iter_idx == args.num_iterations - 1:
print('Train iteration: {}/{}\tLoss: {:.6}'.format(
iter_idx, args.num_iterations, loss.item()))
if iter_idx % args.log_interval_vis == 0:
# merge warped and target image for visualization
img_i_to_ref = warper(inv_depth_ref(), img_i)
img_both_vis = 0.5 * (img_i_to_ref + img_ref)
img_both_vis = clip_and_convert_tensor(img_both_vis)
img_i_to_ref_vis = clip_and_convert_tensor(img_i_to_ref)
inv_depth_ref_vis = tgm.utils.tensor_to_image(
inv_depth_ref() /
(inv_depth_ref().max() + 1e-6)).squeeze()
inv_depth_ref_vis = np.clip(255. * inv_depth_ref_vis, 0, 255)
inv_depth_ref_vis = inv_depth_ref_vis.astype('uint8')
# save warped image and depth to disk
def file_name(x):
return os.path.join(args.output_dir,
"{0}_{1}.png".format(x, iter_idx))
cv2.imwrite(file_name("warped"), img_i_to_ref_vis)
cv2.imwrite(file_name("warped_both"), img_both_vis)
cv2.imwrite(file_name("inv_depth_ref"), inv_depth_ref_vis)
