def world_xyz_images_from_dodeca(self):
"""The world-space XYZ image derived from the 20 Dodeca images."""
if not hasattr(self, '_world_xyz_images_from_dodeca'):
world_images = []
for i in range(20):
im_i = geom_util_np.apply_4x4(
self.cam_xyz_images_from_dodeca[i, ...],
self.dodeca_cam2world[i, ...],
are_points=True)
mask = np_util.make_mask(self.depth_images[0, i, ...])
world_images.append(np_util.zero_by_mask(mask, im_i).astype(np.float32))
self._world_xyz_images_from_dodeca = np.stack(world_images)
return self._world_xyz_images_from_dodeca
def r2n2_xyz_images(self):
"""The GT R2N2 XYZ images in world space."""
if not hasattr(self, '_r2n2_xyz_images'):
xyz_images = []
for i in range(24):
im_i = geom_util_np.apply_4x4(
self.r2n2_cam_images[i, ...],
self.r2n2_cam2world[i, ...],
are_points=True)
mask = np_util.make_mask(self.r2n2_depth_images[i, ...])
xyz_images.append(np_util.zero_by_mask(mask, im_i).astype(np.float32))
self._r2n2_xyz_images = np.stack(xyz_images)
return self._r2n2_xyz_images
def transform_r2n2_depth_image_to_gaps_frame(depth_im, idx, e):
"""Transforms a depth image predicted in the r2n2 space to the GAPS space."""
# depth_im = self.r2n2_depth_images[idx, ...]
# TODO(kgenova) Add support for predicted extrinsics.
is_valid = depth_im != 0.0
is_valid = np.reshape(is_valid, [224, 224])
# Depth2cam:
cam_im = gaps_depth_image_to_cam_image(depth_im, e.r2n2_xfov[idx])
# Cam2world
xyz_im = geom_util_np.apply_4x4(
cam_im, e.r2n2_cam2world[idx, ...], are_points=True)
mask = np_util.make_mask(depth_im)
xyz_im = np_util.zero_by_mask(mask, xyz_im).astype(np.float32)
return xyz_im
def r2n2_normal_cam_images(self):
"""The from-depth GAPS-predicted R2N2 normal images in camera space."""
if not hasattr(self, '_r2n2_normal_cam_images'):
nrm_world = self.r2n2_normal_world_images
cam2world = self.r2n2_cam2world.copy()
cam_images = []
for i in range(24):
# Use the inverse-transpose of the needed matrix:
im_i = geom_util_np.apply_4x4(
nrm_world[i, ...], cam2world[i, :, :].T, are_points=False)
nrm = np.linalg.norm(im_i, axis=-1, keepdims=True) + 1e-10
im_i /= nrm
mask = np_util.make_mask(self.r2n2_depth_images[i, ...])
cam_images.append(np_util.zero_by_mask(mask, im_i).astype(np.float32))
self._r2n2_normal_cam_images = np.stack(cam_images)
return self._r2n2_normal_cam_images
def world_normal_images(self):
"""Normal images transformed to world space."""
if not hasattr(self, '_world_normal_images'):
cam_normals = self.cam_normal_images.copy()
cam2world = self.dodeca_cam2world.copy()
world_normals = []
for i in range(20):
im_i = cam_normals[i, ...]
# Normals are transformed by the inverse transpose
cam2world_invt = np.linalg.inv(cam2world[i, ...]).T
im_i = geom_util_np.apply_4x4(im_i, cam2world_invt, are_points=False)
nrm = np.linalg.norm(im_i, axis=-1, keepdims=True) + 1e-10
im_i /= nrm
mask = np_util.make_mask(self.depth_images[0, i, ...])
world_normals.append(
np_util.zero_by_mask(mask, im_i).astype(np.float32))
self._world_normal_images = np.stack(world_normals)
return self._world_normal_images