def test_perspective_correct_interpolation_preset(self):
"""Tests that perspective_correct_interpolation generates expected results."""
camera_origin = np.array((0.0, 0.0, 0.0))
camera_up = np.array((0.0, 1.0, 0.0))
look_at_point = np.array((0.0, 0.0, 1.0))
fov = np.array((90.0 * np.math.pi / 180.0,))
bottom_left = np.array((0.0, 0.0))
image_size = np.array((501.0, 501.0))
near_plane = np.array((0.01,))
far_plane = np.array((10.0,))
batch_size = np.random.randint(1, 5)
triangle_x_y = np.random.uniform(-10.0, 10.0, (batch_size, 3, 2))
triangle_z = np.random.uniform(2.0, 10.0, (batch_size, 3, 1))
triangles = np.concatenate((triangle_x_y, triangle_z), axis=-1)
# Builds barycentric weights.
barycentric_weights = np.random.uniform(size=(batch_size, 3))
barycentric_weights = barycentric_weights / np.sum(
barycentric_weights, axis=-1, keepdims=True)
# Barycentric interpolation of vertex positions.
convex_combination = np.einsum("ba, bac -> bc", barycentric_weights,
triangles)
# Build matrices.
model_to_eye_matrix = look_at.right_handed(camera_origin, look_at_point,
camera_up)
perspective_matrix = perspective.right_handed(
fov, (image_size[0:1] / image_size[1:2]), near_plane, far_plane)
# Computes where those points project in screen coordinates.
pixel_position, _ = glm.model_to_screen(convex_combination,
model_to_eye_matrix,
perspective_matrix, image_size,
bottom_left)
# Builds attributes.
num_pixels = pixel_position.shape[0]
attribute_size = np.random.randint(10)
attributes = np.random.uniform(size=(num_pixels, 3, attribute_size))
prediction = glm.perspective_correct_interpolation(triangles, attributes,
pixel_position[..., 0:2],
model_to_eye_matrix,
perspective_matrix,
image_size, bottom_left)
groundtruth = np.einsum("ba, bac -> bc", barycentric_weights, attributes)
self.assertAllClose(prediction, groundtruth)
def test_rasterize_preset(self):
camera_origin = (0.0, 0.0, 0.0)
camera_up = (0.0, 1.0, 0.0)
look_at_point = (0.0, 0.0, 1.0)
field_of_view = (60 * np.math.pi / 180, )
near_plane = (0.01, )
far_plane = (400.0, )
# Construct the view projection matrix.
model_to_eye_matrix = look_at.right_handed(camera_origin,
look_at_point, camera_up)
perspective_matrix = perspective.right_handed(
field_of_view, (float(_IMAGE_WIDTH) / float(_IMAGE_HEIGHT), ),
near_plane, far_plane)
view_projection_matrix = tf.linalg.matmul(perspective_matrix,
model_to_eye_matrix)
view_projection_matrix = tf.expand_dims(view_projection_matrix, axis=0)
depth = 1.0
vertices = np.array([[(-2.0 * _TRIANGLE_SIZE, 0.0, depth),
(0.0, _TRIANGLE_SIZE, depth), (0.0, 0.0, depth),
(0.0, -_TRIANGLE_SIZE, depth)]],
dtype=np.float32)
triangles = np.array(((1, 2, 0), (0, 2, 3)), np.int32)
predicted_fb = rasterization_backend.rasterize(
vertices, triangles, view_projection_matrix,
(_IMAGE_WIDTH, _IMAGE_HEIGHT))
with self.subTest(name="triangle_index"):
groundtruth_triangle_index = np.zeros(
(1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1), dtype=np.int32)
groundtruth_triangle_index[..., :_IMAGE_WIDTH // 2, 0] = 0
groundtruth_triangle_index[..., :_IMAGE_HEIGHT // 2,
_IMAGE_WIDTH // 2:, 0] = 1
self.assertAllEqual(groundtruth_triangle_index,
predicted_fb.triangle_id)
with self.subTest(name="mask"):
groundtruth_mask = np.ones((1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1),
dtype=np.int32)
groundtruth_mask[..., :_IMAGE_WIDTH // 2, 0] = 0
self.assertAllEqual(groundtruth_mask, predicted_fb.foreground_mask)
attributes = np.array(((1.0, 0.0, 0.0), (0.0, 1.0, 0.0),
(0.0, 0.0, 1.0))).astype(np.float32)
perspective_correct_interpolation = lambda geometry, pixels: glm.perspective_correct_interpolation( # pylint: disable=g-long-lambda,line-too-long
geometry, attributes, pixels, model_to_eye_matrix,
perspective_matrix,
np.array((_IMAGE_WIDTH, _IMAGE_HEIGHT)).astype(np.float32),
np.array((0.0, 0.0)).astype(np.float32))
with self.subTest(name="barycentric_coordinates_triangle_0"):
geometry_0 = tf.gather(vertices, triangles[0, :], axis=1)
pixels_0 = tf.transpose(grid.generate((3.5, 2.5), (6.5, 4.5),
(4, 3)),
perm=(1, 0, 2))
barycentrics_gt_0 = perspective_correct_interpolation(
geometry_0, pixels_0)
self.assertAllClose(barycentrics_gt_0,
predicted_fb.barycentrics.value[0, 2:, 3:, :],
atol=1e-3)
with self.subTest(name="barycentric_coordinates_triangle_1"):
geometry_1 = tf.gather(vertices, triangles[1, :], axis=1)
pixels_1 = tf.transpose(grid.generate((3.5, 0.5), (6.5, 1.5),
(4, 2)),
perm=(1, 0, 2))
barycentrics_gt_1 = perspective_correct_interpolation(
geometry_1, pixels_1)
self.assertAllClose(barycentrics_gt_1,
predicted_fb.barycentrics.value[0, 0:2, 3:, :],
atol=1e-3)
def rasterize(self,
scene_vertices=None,
scene_attributes=None,
scene_triangles=None,
name=None):
"""Rasterizes the scene.
This rasterizer estimates which triangle is associated with each pixel using
OpenGL. Then the value of attributes are estimated using Tensorflow,
allowing to get gradients flowing through the attributes. Attributes can be
depth, appearance, or more generally, any K-dimensional representation. Note
that similarly to algorithms like Iterative Closest Point (ICP), not having
gradients through correspondence does not prevent from optimizing the scene
geometry. Custom gradients can be defined to alleviate this property.
Note:
In the following, A1 to An are optional batch dimensions.
Args:
scene_vertices: A tensor of shape `[A1, ..., An, V, 3]` containing batches
of `V` vertices, each defined by a 3D point.
scene_attributes: A tensor of shape `[A1, ..., An, V, K]` containing
batches of `V` vertices, each associated with K-dimensional attributes.
scene_triangles: A tensor of shape `[T, 3]` containing `T` triangles, each
associated with 3 vertices from `scene_vertices`
name: A name for this op. Defaults to 'triangle_rasterizer_rasterize'.
Returns:
A tensor of shape `[A1, ..., An, H, W, K]` containing batches of images of
height `H` and width `W`, where each pixel contains attributes rasterized
from the scene.
"""
with tf.compat.v1.name_scope(
name, "triangle_rasterizer_rasterize",
(scene_vertices, scene_attributes, scene_triangles)):
scene_vertices = tf.convert_to_tensor(value=scene_vertices)
scene_attributes = tf.convert_to_tensor(value=scene_attributes)
scene_triangles = tf.convert_to_tensor(value=scene_triangles)
shape.check_static(
tensor=scene_vertices,
tensor_name="scene_vertices",
has_rank_greater_than=1,
has_dim_equals=((-1, 3)))
shape.compare_batch_dimensions(
tensors=(scene_vertices, scene_attributes),
last_axes=-2,
tensor_names=("vertex_positions", "vertex_attributes"),
broadcast_compatible=False)
shape.check_static(
tensor=scene_triangles,
tensor_name="scene_triangles",
has_dim_equals=((-1, 3)))
batch_dims_triangles = len(scene_triangles.shape[:-2])
scene_attributes = tf.gather(
scene_attributes,
scene_triangles,
axis=-2,
batch_dims=batch_dims_triangles)
scene_geometry = tf.gather(
scene_vertices,
scene_triangles,
axis=-2,
batch_dims=batch_dims_triangles)
batch_shape = scene_geometry.shape[:-3]
batch_shape = [_dim_value(dim) for dim in batch_shape]
background_geometry = tf.broadcast_to(
self._background_geometry,
batch_shape + self._background_geometry.shape)
background_attribute = tf.broadcast_to(
self._background_attribute,
batch_shape + self._background_attribute.shape)
geometry = tf.concat((background_geometry, scene_geometry), axis=-3)
attributes = tf.concat((background_attribute, scene_attributes), axis=-3)
view_projection_matrix = tf.broadcast_to(
input=self._view_projection_matrix,
shape=batch_shape + self._view_projection_matrix.shape)
rasterized_face = render_ops.rasterize(
num_points=geometry.shape[-3],
variable_names=("view_projection_matrix", "triangular_mesh"),
variable_kinds=("mat", "buffer"),
variable_values=(view_projection_matrix,
tf.reshape(geometry, shape=batch_shape + [-1])),
output_resolution=self._image_size_int,
vertex_shader=vertex_shader,
geometry_shader=geometry_shader,
fragment_shader=fragment_shader)
triangle_index = tf.cast(rasterized_face[..., 0], tf.int32)
vertices_per_pixel = tf.gather(
geometry, triangle_index, axis=-3, batch_dims=len(batch_shape))
attributes_per_pixel = tf.gather(
attributes, triangle_index, axis=-3, batch_dims=len(batch_shape))
return glm.perspective_correct_interpolation(
vertices_per_pixel, attributes_per_pixel, self._pixel_position,
self._camera_origin, self._look_at, self._camera_up,
self._field_of_view, self._image_size_glm, self._near_plane,
self._far_plane, self._bottom_left)
def test_rasterize_preset(self):
model_to_eye_matrix = rasterization_test_utils.make_look_at_matrix(
look_at_point=(0.0, 0.0, 1.0))
perspective_matrix = rasterization_test_utils.make_perspective_matrix(
_IMAGE_WIDTH, _IMAGE_HEIGHT)
view_projection_matrix = tf.linalg.matmul(perspective_matrix,
model_to_eye_matrix)
view_projection_matrix = tf.expand_dims(view_projection_matrix, axis=0)
depth = 1.0
vertices = np.array([[(-2.0 * _TRIANGLE_SIZE, 0.0, depth),
(0.0, _TRIANGLE_SIZE, depth), (0.0, 0.0, depth),
(0.0, -_TRIANGLE_SIZE, depth)]],
dtype=np.float32)
triangles = np.array(((1, 2, 0), (0, 2, 3)), np.int32)
predicted_fb = _proxy_rasterize(vertices, triangles,
view_projection_matrix)
with self.subTest(name="triangle_index"):
groundtruth_triangle_index = np.zeros(
(1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1), dtype=np.int32)
groundtruth_triangle_index[..., :_IMAGE_WIDTH // 2, 0] = 0
groundtruth_triangle_index[..., :_IMAGE_HEIGHT // 2,
_IMAGE_WIDTH // 2:, 0] = 1
self.assertAllEqual(groundtruth_triangle_index,
predicted_fb.triangle_id)
with self.subTest(name="mask"):
groundtruth_mask = np.ones((1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1),
dtype=np.int32)
groundtruth_mask[..., :_IMAGE_WIDTH // 2, 0] = 0
self.assertAllEqual(groundtruth_mask, predicted_fb.foreground_mask)
attributes = np.array(((1.0, 0.0, 0.0), (0.0, 1.0, 0.0),
(0.0, 0.0, 1.0))).astype(np.float32)
perspective_correct_interpolation = lambda geometry, pixels: glm.perspective_correct_interpolation( # pylint: disable=g-long-lambda,line-too-long
geometry, attributes, pixels, model_to_eye_matrix,
perspective_matrix,
np.array((_IMAGE_WIDTH, _IMAGE_HEIGHT)).astype(np.float32),
np.array((0.0, 0.0)).astype(np.float32))
with self.subTest(name="barycentric_coordinates_triangle_0"):
geometry_0 = tf.gather(vertices, triangles[0, :], axis=1)
pixels_0 = tf.transpose(grid.generate((3.5, 2.5), (6.5, 4.5),
(4, 3)),
perm=(1, 0, 2))
barycentrics_gt_0 = perspective_correct_interpolation(
geometry_0, pixels_0)
self.assertAllClose(barycentrics_gt_0,
predicted_fb.barycentrics.value[0, 2:, 3:, :],
atol=1e-3)
with self.subTest(name="barycentric_coordinates_triangle_1"):
geometry_1 = tf.gather(vertices, triangles[1, :], axis=1)
pixels_1 = tf.transpose(grid.generate((3.5, 0.5), (6.5, 1.5),
(4, 2)),
perm=(1, 0, 2))
barycentrics_gt_1 = perspective_correct_interpolation(
geometry_1, pixels_1)
self.assertAllClose(barycentrics_gt_1,
predicted_fb.barycentrics.value[0, 0:2, 3:, :],
atol=1e-3)
