def test_perspective_correct_interpolation_preset(self): """Tests that perspective_correct_interpolation generates expected results.""" camera_origin = np.array((0.0, 0.0, 0.0)) camera_up = np.array((0.0, 1.0, 0.0)) look_at_point = np.array((0.0, 0.0, 1.0)) fov = np.array((90.0 * np.math.pi / 180.0,)) bottom_left = np.array((0.0, 0.0)) image_size = np.array((501.0, 501.0)) near_plane = np.array((0.01,)) far_plane = np.array((10.0,)) batch_size = np.random.randint(1, 5) triangle_x_y = np.random.uniform(-10.0, 10.0, (batch_size, 3, 2)) triangle_z = np.random.uniform(2.0, 10.0, (batch_size, 3, 1)) triangles = np.concatenate((triangle_x_y, triangle_z), axis=-1) # Builds barycentric weights. barycentric_weights = np.random.uniform(size=(batch_size, 3)) barycentric_weights = barycentric_weights / np.sum( barycentric_weights, axis=-1, keepdims=True) # Barycentric interpolation of vertex positions. convex_combination = np.einsum("ba, bac -> bc", barycentric_weights, triangles) # Build matrices. model_to_eye_matrix = look_at.right_handed(camera_origin, look_at_point, camera_up) perspective_matrix = perspective.right_handed( fov, (image_size[0:1] / image_size[1:2]), near_plane, far_plane) # Computes where those points project in screen coordinates. pixel_position, _ = glm.model_to_screen(convex_combination, model_to_eye_matrix, perspective_matrix, image_size, bottom_left) # Builds attributes. num_pixels = pixel_position.shape[0] attribute_size = np.random.randint(10) attributes = np.random.uniform(size=(num_pixels, 3, attribute_size)) prediction = glm.perspective_correct_interpolation(triangles, attributes, pixel_position[..., 0:2], model_to_eye_matrix, perspective_matrix, image_size, bottom_left) groundtruth = np.einsum("ba, bac -> bc", barycentric_weights, attributes) self.assertAllClose(prediction, groundtruth)
def test_rasterize_preset(self): camera_origin = (0.0, 0.0, 0.0) camera_up = (0.0, 1.0, 0.0) look_at_point = (0.0, 0.0, 1.0) field_of_view = (60 * np.math.pi / 180, ) near_plane = (0.01, ) far_plane = (400.0, ) # Construct the view projection matrix. model_to_eye_matrix = look_at.right_handed(camera_origin, look_at_point, camera_up) perspective_matrix = perspective.right_handed( field_of_view, (float(_IMAGE_WIDTH) / float(_IMAGE_HEIGHT), ), near_plane, far_plane) view_projection_matrix = tf.linalg.matmul(perspective_matrix, model_to_eye_matrix) view_projection_matrix = tf.expand_dims(view_projection_matrix, axis=0) depth = 1.0 vertices = np.array([[(-2.0 * _TRIANGLE_SIZE, 0.0, depth), (0.0, _TRIANGLE_SIZE, depth), (0.0, 0.0, depth), (0.0, -_TRIANGLE_SIZE, depth)]], dtype=np.float32) triangles = np.array(((1, 2, 0), (0, 2, 3)), np.int32) predicted_fb = rasterization_backend.rasterize( vertices, triangles, view_projection_matrix, (_IMAGE_WIDTH, _IMAGE_HEIGHT)) with self.subTest(name="triangle_index"): groundtruth_triangle_index = np.zeros( (1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1), dtype=np.int32) groundtruth_triangle_index[..., :_IMAGE_WIDTH // 2, 0] = 0 groundtruth_triangle_index[..., :_IMAGE_HEIGHT // 2, _IMAGE_WIDTH // 2:, 0] = 1 self.assertAllEqual(groundtruth_triangle_index, predicted_fb.triangle_id) with self.subTest(name="mask"): groundtruth_mask = np.ones((1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1), dtype=np.int32) groundtruth_mask[..., :_IMAGE_WIDTH // 2, 0] = 0 self.assertAllEqual(groundtruth_mask, predicted_fb.foreground_mask) attributes = np.array(((1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0))).astype(np.float32) perspective_correct_interpolation = lambda geometry, pixels: glm.perspective_correct_interpolation( # pylint: disable=g-long-lambda,line-too-long geometry, attributes, pixels, model_to_eye_matrix, perspective_matrix, np.array((_IMAGE_WIDTH, _IMAGE_HEIGHT)).astype(np.float32), np.array((0.0, 0.0)).astype(np.float32)) with self.subTest(name="barycentric_coordinates_triangle_0"): geometry_0 = tf.gather(vertices, triangles[0, :], axis=1) pixels_0 = tf.transpose(grid.generate((3.5, 2.5), (6.5, 4.5), (4, 3)), perm=(1, 0, 2)) barycentrics_gt_0 = perspective_correct_interpolation( geometry_0, pixels_0) self.assertAllClose(barycentrics_gt_0, predicted_fb.barycentrics.value[0, 2:, 3:, :], atol=1e-3) with self.subTest(name="barycentric_coordinates_triangle_1"): geometry_1 = tf.gather(vertices, triangles[1, :], axis=1) pixels_1 = tf.transpose(grid.generate((3.5, 0.5), (6.5, 1.5), (4, 2)), perm=(1, 0, 2)) barycentrics_gt_1 = perspective_correct_interpolation( geometry_1, pixels_1) self.assertAllClose(barycentrics_gt_1, predicted_fb.barycentrics.value[0, 0:2, 3:, :], atol=1e-3)
def rasterize(self, scene_vertices=None, scene_attributes=None, scene_triangles=None, name=None): """Rasterizes the scene. This rasterizer estimates which triangle is associated with each pixel using OpenGL. Then the value of attributes are estimated using Tensorflow, allowing to get gradients flowing through the attributes. Attributes can be depth, appearance, or more generally, any K-dimensional representation. Note that similarly to algorithms like Iterative Closest Point (ICP), not having gradients through correspondence does not prevent from optimizing the scene geometry. Custom gradients can be defined to alleviate this property. Note: In the following, A1 to An are optional batch dimensions. Args: scene_vertices: A tensor of shape `[A1, ..., An, V, 3]` containing batches of `V` vertices, each defined by a 3D point. scene_attributes: A tensor of shape `[A1, ..., An, V, K]` containing batches of `V` vertices, each associated with K-dimensional attributes. scene_triangles: A tensor of shape `[T, 3]` containing `T` triangles, each associated with 3 vertices from `scene_vertices` name: A name for this op. Defaults to 'triangle_rasterizer_rasterize'. Returns: A tensor of shape `[A1, ..., An, H, W, K]` containing batches of images of height `H` and width `W`, where each pixel contains attributes rasterized from the scene. """ with tf.compat.v1.name_scope( name, "triangle_rasterizer_rasterize", (scene_vertices, scene_attributes, scene_triangles)): scene_vertices = tf.convert_to_tensor(value=scene_vertices) scene_attributes = tf.convert_to_tensor(value=scene_attributes) scene_triangles = tf.convert_to_tensor(value=scene_triangles) shape.check_static( tensor=scene_vertices, tensor_name="scene_vertices", has_rank_greater_than=1, has_dim_equals=((-1, 3))) shape.compare_batch_dimensions( tensors=(scene_vertices, scene_attributes), last_axes=-2, tensor_names=("vertex_positions", "vertex_attributes"), broadcast_compatible=False) shape.check_static( tensor=scene_triangles, tensor_name="scene_triangles", has_dim_equals=((-1, 3))) batch_dims_triangles = len(scene_triangles.shape[:-2]) scene_attributes = tf.gather( scene_attributes, scene_triangles, axis=-2, batch_dims=batch_dims_triangles) scene_geometry = tf.gather( scene_vertices, scene_triangles, axis=-2, batch_dims=batch_dims_triangles) batch_shape = scene_geometry.shape[:-3] batch_shape = [_dim_value(dim) for dim in batch_shape] background_geometry = tf.broadcast_to( self._background_geometry, batch_shape + self._background_geometry.shape) background_attribute = tf.broadcast_to( self._background_attribute, batch_shape + self._background_attribute.shape) geometry = tf.concat((background_geometry, scene_geometry), axis=-3) attributes = tf.concat((background_attribute, scene_attributes), axis=-3) view_projection_matrix = tf.broadcast_to( input=self._view_projection_matrix, shape=batch_shape + self._view_projection_matrix.shape) rasterized_face = render_ops.rasterize( num_points=geometry.shape[-3], variable_names=("view_projection_matrix", "triangular_mesh"), variable_kinds=("mat", "buffer"), variable_values=(view_projection_matrix, tf.reshape(geometry, shape=batch_shape + [-1])), output_resolution=self._image_size_int, vertex_shader=vertex_shader, geometry_shader=geometry_shader, fragment_shader=fragment_shader) triangle_index = tf.cast(rasterized_face[..., 0], tf.int32) vertices_per_pixel = tf.gather( geometry, triangle_index, axis=-3, batch_dims=len(batch_shape)) attributes_per_pixel = tf.gather( attributes, triangle_index, axis=-3, batch_dims=len(batch_shape)) return glm.perspective_correct_interpolation( vertices_per_pixel, attributes_per_pixel, self._pixel_position, self._camera_origin, self._look_at, self._camera_up, self._field_of_view, self._image_size_glm, self._near_plane, self._far_plane, self._bottom_left)
def test_rasterize_preset(self): model_to_eye_matrix = rasterization_test_utils.make_look_at_matrix( look_at_point=(0.0, 0.0, 1.0)) perspective_matrix = rasterization_test_utils.make_perspective_matrix( _IMAGE_WIDTH, _IMAGE_HEIGHT) view_projection_matrix = tf.linalg.matmul(perspective_matrix, model_to_eye_matrix) view_projection_matrix = tf.expand_dims(view_projection_matrix, axis=0) depth = 1.0 vertices = np.array([[(-2.0 * _TRIANGLE_SIZE, 0.0, depth), (0.0, _TRIANGLE_SIZE, depth), (0.0, 0.0, depth), (0.0, -_TRIANGLE_SIZE, depth)]], dtype=np.float32) triangles = np.array(((1, 2, 0), (0, 2, 3)), np.int32) predicted_fb = _proxy_rasterize(vertices, triangles, view_projection_matrix) with self.subTest(name="triangle_index"): groundtruth_triangle_index = np.zeros( (1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1), dtype=np.int32) groundtruth_triangle_index[..., :_IMAGE_WIDTH // 2, 0] = 0 groundtruth_triangle_index[..., :_IMAGE_HEIGHT // 2, _IMAGE_WIDTH // 2:, 0] = 1 self.assertAllEqual(groundtruth_triangle_index, predicted_fb.triangle_id) with self.subTest(name="mask"): groundtruth_mask = np.ones((1, _IMAGE_HEIGHT, _IMAGE_WIDTH, 1), dtype=np.int32) groundtruth_mask[..., :_IMAGE_WIDTH // 2, 0] = 0 self.assertAllEqual(groundtruth_mask, predicted_fb.foreground_mask) attributes = np.array(((1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0))).astype(np.float32) perspective_correct_interpolation = lambda geometry, pixels: glm.perspective_correct_interpolation( # pylint: disable=g-long-lambda,line-too-long geometry, attributes, pixels, model_to_eye_matrix, perspective_matrix, np.array((_IMAGE_WIDTH, _IMAGE_HEIGHT)).astype(np.float32), np.array((0.0, 0.0)).astype(np.float32)) with self.subTest(name="barycentric_coordinates_triangle_0"): geometry_0 = tf.gather(vertices, triangles[0, :], axis=1) pixels_0 = tf.transpose(grid.generate((3.5, 2.5), (6.5, 4.5), (4, 3)), perm=(1, 0, 2)) barycentrics_gt_0 = perspective_correct_interpolation( geometry_0, pixels_0) self.assertAllClose(barycentrics_gt_0, predicted_fb.barycentrics.value[0, 2:, 3:, :], atol=1e-3) with self.subTest(name="barycentric_coordinates_triangle_1"): geometry_1 = tf.gather(vertices, triangles[1, :], axis=1) pixels_1 = tf.transpose(grid.generate((3.5, 0.5), (6.5, 1.5), (4, 2)), perm=(1, 0, 2)) barycentrics_gt_1 = perspective_correct_interpolation( geometry_1, pixels_1) self.assertAllClose(barycentrics_gt_1, predicted_fb.barycentrics.value[0, 0:2, 3:, :], atol=1e-3)