def test_mat34_pose_inverse(self):
identity = [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0],
[0.0, 0.0, 1.0, 0.0]]
translate = [[1.0, 0.0, 0.0, 1.0], [0.0, 1.0, 0.0, 2.0],
[0.0, 0.0, 1.0, 3.0]]
untranslate = [[1.0, 0.0, 0.0, -1.0], [0.0, 1.0, 0.0, -2.0],
[0.0, 0.0, 1.0, -3.0]]
# Rotate around Y axis and translate along X axis.
c = math.cos(1.0)
s = math.sin(1.0)
rotate = [[c, 0.0, -s, 10.0], [0.0, 1.0, 0.0, 0.0], [s, 0.0, c, 0.0]]
unrotate = [[c, 0.0, s, -10.0 * c], [0.0, 1.0, 0.0, 0.0],
[-s, 0.0, c, 10 * s]]
data = [
# A series of (matrix, inverse) pairs to check. We check them both ways.
(identity, identity),
(translate, untranslate),
(rotate, unrotate),
# Batched examples
([identity, translate, rotate], [identity, untranslate, unrotate]),
([[identity, translate], [rotate,
untranslate]], [[identity, untranslate],
[unrotate, translate]])
]
for (matrix, inverse) in data:
result = geometry.mat34_pose_inverse(tf.constant(matrix))
self.assertAllClose(inverse, result)
reverse = geometry.mat34_pose_inverse(tf.constant(inverse))
self.assertAllClose(matrix, reverse)
# Additionally, check that the product with the inverse is the identity
product = geometry.mat34_product(
tf.constant(matrix), geometry.mat34_pose_inverse(tf.constant(matrix)))
(_, identities) = utils.broadcast_to_match(product, tf.constant(identity))
self.assertAllClose(product, identities)
def test_mat34_transform_planes(self):
# Some planes...
planes = tf.convert_to_tensor([[1.0, 2.0, 3.0, 4.0],
[0.5, 0.3, 0.1, 0.0],
[0.0, 1.0, 0.0, 100.0],
[-1.0, 0.0, -1.0, 0.25]])
# ...and some points
points = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]
# (Since we know it's all linear, four planes and three
# points are enough to span all possibilities!)
i = [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]]
m = [[1.0, 0.0, 0.0, 4.0], [0.0, 0.0, -1.0, 5.0], [0.0, 1.0, 0.0, 6.0]]
# Identity transform doesn't change the planes:
input_planes = tf.constant(planes)
identity_planes = geometry.mat34_transform_planes(
tf.constant(i), input_planes)
self.assertAllEqual(identity_planes, planes)
# Transform and inverse transform is the identity:
self.assertAllEqual(
geometry.mat34_transform_planes(
tf.constant(m),
geometry.mat34_transform_planes(
geometry.mat34_pose_inverse(tf.constant(m)),
input_planes)), planes)
# Dot products between planes and points are preserved (since
# m doesn't change scale):
input_points = tf.constant(points)
dot_products = tf.matmul(geometry.homogenize(input_points),
planes,
transpose_b=True)
transformed_planes = geometry.mat34_transform_planes(
tf.constant(m), input_planes)
transformed_points = geometry.mat34_transform(tf.constant(m),
input_points)
transformed_dot_products = tf.matmul(
geometry.homogenize(transformed_points),
transformed_planes,
transpose_b=True)
self.assertAllClose(dot_products, transformed_dot_products)
def render_layers(layers,
depths,
pose,
intrinsics,
target_pose,
target_intrinsics,
height=None,
width=None,
clamp=True):
"""Render target layers from MPI representation.
Args:
layers: [..., L, H, W, C] MPI layers, back to front.
depths: [..., L] MPI plane depths, back to front.
pose: [..., 3, 4] reference camera pose.
intrinsics: [..., 4] reference intrinsics.
target_pose: [..., 3, 4] target camera pose.
target_intrinsics: [..., 4] target intrinsics.
height: height to render to in pixels (or None for input height).
width: width to render to in pixels (or None for input width).
clamp: whether to clamp image coordinates (see geometry.sample_image doc),
i.e. extending the image beyond its size or not.
Returns:
[..., L, height, width, C] The layers warped to the target view by applying
an appropriate homography to each one.
"""
source_to_target_pose = geometry.mat34_product(
target_pose, geometry.mat34_pose_inverse(pose))
# Add a dimension to correspond to L in the poses and intrinsics.
pose = pose[Ellipsis, tf.newaxis, :, :] # [..., 1, 3, 4]
target_pose = target_pose[Ellipsis, tf.newaxis, :, :] # [..., 1, 3, 4]
intrinsics = intrinsics[Ellipsis, tf.newaxis, :] # [..., 1, 4]
target_intrinsics = target_intrinsics[Ellipsis,
tf.newaxis, :] # [..., 1, 4]
# Fronto-parallel plane equations at the given depths, in the reference
# camera's frame.
normals = tf.constant([0.0, 0.0, 1.0], shape=[1, 3])
depths = -depths[Ellipsis, tf.newaxis] # [..., L, 1]
normals, depths = utils.broadcast_to_match(normals, depths, ignore_axes=1)
planes = tf.concat([normals, depths], axis=-1) # [..., L, 4]
homographies = geometry.inverse_homography(pose, intrinsics, target_pose,
target_intrinsics,
planes) # [..., L, 3, 3]
# Each of the resulting [..., L] homographies knows how to inverse-warp one
# of the [..., (H,W), L] images into a new [... (H',W')] target images.
target_layers = geometry.homography_warp(layers,
homographies,
height=height,
width=width,
clamp=clamp)
# The next few lines implement back-face culling.
#
# We don't want to render content that is behind the camera. (If we did, we
# might see upside-down images of the layers.) A typical graphics approach
# would be to test each pixel of each layer against a near-plane and discard
# those that are in front of it. Here we implement something cheaper:
# back-face culling. If the target camera sees the "back" of a layer then we
# set that layer's alpha to zero. This is simple and sufficient in practice
# to avoid nasty artefacts.
# Convert planes to target camera space. target_planes is [..., L, 4]
target_planes = geometry.mat34_transform_planes(source_to_target_pose,
planes)
# Fourth coordinate of plane is negative distance in front of the camera.
# target_visible is [..., L]
target_visible = tf.cast(target_planes[Ellipsis, -1] < 0.0,
dtype=tf.float32)
# per_layer_alpha is [..., L, 1, 1, 1]
per_layer_alpha = target_visible[Ellipsis, tf.newaxis, tf.newaxis,
tf.newaxis]
# Multiply alpha channel by per_layer_alpha:
non_alpha_channels = target_layers[Ellipsis, :-1]
alpha = target_layers[Ellipsis, -1:] * per_layer_alpha
target_layers = tf.concat([non_alpha_channels, alpha], axis=-1)
return target_layers