def world_xyz_images_from_dodeca(self):
"""The world-space XYZ image derived from the 20 Dodeca images."""
if not hasattr(self, '_world_xyz_images_from_dodeca'):
world_images = []
for i in range(20):
im_i = geom_util_np.apply_4x4(
self.cam_xyz_images_from_dodeca[i, ...],
self.dodeca_cam2world[i, ...],
are_points=True)
mask = np_util.make_mask(self.depth_images[0, i, ...])
world_images.append(np_util.zero_by_mask(mask, im_i).astype(np.float32))
self._world_xyz_images_from_dodeca = np.stack(world_images)
return self._world_xyz_images_from_dodeca
def r2n2_xyz_images(self):
"""The GT R2N2 XYZ images in world space."""
if not hasattr(self, '_r2n2_xyz_images'):
xyz_images = []
for i in range(24):
im_i = geom_util_np.apply_4x4(
self.r2n2_cam_images[i, ...],
self.r2n2_cam2world[i, ...],
are_points=True)
mask = np_util.make_mask(self.r2n2_depth_images[i, ...])
xyz_images.append(np_util.zero_by_mask(mask, im_i).astype(np.float32))
self._r2n2_xyz_images = np.stack(xyz_images)
return self._r2n2_xyz_images
def transform_r2n2_depth_image_to_gaps_frame(depth_im, idx, e):
"""Transforms a depth image predicted in the r2n2 space to the GAPS space."""
# depth_im = self.r2n2_depth_images[idx, ...]
# TODO(kgenova) Add support for predicted extrinsics.
is_valid = depth_im != 0.0
is_valid = np.reshape(is_valid, [224, 224])
# Depth2cam:
cam_im = gaps_depth_image_to_cam_image(depth_im, e.r2n2_xfov[idx])
# Cam2world
xyz_im = geom_util_np.apply_4x4(
cam_im, e.r2n2_cam2world[idx, ...], are_points=True)
mask = np_util.make_mask(depth_im)
xyz_im = np_util.zero_by_mask(mask, xyz_im).astype(np.float32)
return xyz_im
def r2n2_normal_cam_images(self):
"""The from-depth GAPS-predicted R2N2 normal images in camera space."""
if not hasattr(self, '_r2n2_normal_cam_images'):
nrm_world = self.r2n2_normal_world_images
cam2world = self.r2n2_cam2world.copy()
cam_images = []
for i in range(24):
# Use the inverse-transpose of the needed matrix:
im_i = geom_util_np.apply_4x4(
nrm_world[i, ...], cam2world[i, :, :].T, are_points=False)
nrm = np.linalg.norm(im_i, axis=-1, keepdims=True) + 1e-10
im_i /= nrm
mask = np_util.make_mask(self.r2n2_depth_images[i, ...])
cam_images.append(np_util.zero_by_mask(mask, im_i).astype(np.float32))
self._r2n2_normal_cam_images = np.stack(cam_images)
return self._r2n2_normal_cam_images
def compute_argmax_image(xyz_image, decoder, embedding, k=1): """Uses the world space XYZ image to compute the maxblob influence image.""" mask = np_util.make_pixel_mask(xyz_image) # TODO(kgenova) Figure this out... assert len(mask.shape) == 2 flat_xyz = np.reshape(xyz_image, [-1, 3]) influences = decoder.rbf_influence_at_samples(embedding, flat_xyz) assert len(influences.shape) == 2 rbf_image = np.reshape(influences, list(mask.shape) + [-1]) # argmax_image = np.expand_dims(np.argmax(rbf_image, axis=-1), axis=-1) argmax_image = np.flip(np.argsort(rbf_image, axis=-1), axis=-1) argmax_image = argmax_image[..., :k] # TODO(kgenova) Insert an equivalence class map here. log.info(mask.shape) log.info(argmax_image.shape) argmax_image = np_util.zero_by_mask(mask, argmax_image, replace_with=-1) log.info(argmax_image.shape) return argmax_image.astype(np.int32)
def world_normal_images(self):
"""Normal images transformed to world space."""
if not hasattr(self, '_world_normal_images'):
cam_normals = self.cam_normal_images.copy()
cam2world = self.dodeca_cam2world.copy()
world_normals = []
for i in range(20):
im_i = cam_normals[i, ...]
# Normals are transformed by the inverse transpose
cam2world_invt = np.linalg.inv(cam2world[i, ...]).T
im_i = geom_util_np.apply_4x4(im_i, cam2world_invt, are_points=False)
nrm = np.linalg.norm(im_i, axis=-1, keepdims=True) + 1e-10
im_i /= nrm
mask = np_util.make_mask(self.depth_images[0, i, ...])
world_normals.append(
np_util.zero_by_mask(mask, im_i).astype(np.float32))
self._world_normal_images = np.stack(world_normals)
return self._world_normal_images