def rasterize(): return rasterizer.rasterize( num_points=3, variable_names=variable_names, variable_kinds=variable_kinds, variable_values=variable_values, output_resolution=(width, height), vertex_shader=test_vertex_shader, geometry_shader=test_geometry_shader, fragment_shader=test_fragment_shader, )
def test_rasterize(self): height = 500 width = 500 camera_origin = (0.0, 0.0, 0.0) camera_up = (0.0, 1.0, 0.0) look_at = (0.0, 0.0, 1.0) fov = (60.0 * np.math.pi / 180, ) near_plane = (1.0, ) far_plane = (10.0, ) world_to_camera = glm.look_at_right_handed(camera_origin, look_at, camera_up) perspective_matrix = glm.perspective_right_handed( fov, (float(width) / float(height), ), near_plane, far_plane) view_projection_matrix = tf.matmul(perspective_matrix, world_to_camera) view_projection_matrix = tf.squeeze(view_projection_matrix) for depth in range(2, 5): tris = np.array([[-10.0, 10.0, depth], [10.0, 10.0, depth], [0.0, -10.0, depth]], dtype=np.float32) tris_tf = tf.reshape(tris, [-1]) render_parameters = { "view_projection_matrix": ("mat", view_projection_matrix), "triangular_mesh": ("buffer", tris_tf) } render_parameters = list(six.iteritems(render_parameters)) variable_names = [v[0] for v in render_parameters] variable_kinds = [v[1][0] for v in render_parameters] variable_values = [v[1][1] for v in render_parameters] result = rasterizer.rasterize( num_points=tris.shape[0], variable_names=variable_names, variable_kinds=variable_kinds, variable_values=variable_values, output_resolution=(width, height), vertex_shader=test_vertex_shader, geometry_shader=test_geometry_shader, fragment_shader=test_fragment_shader, ) gt = np.tile((0, depth), (width, height, 1)) self.assertAllClose(result[..., 2:4], gt)
def test_invalid_variable_inputs(self, error_msg, variable_names, variable_kinds, variable_values, error_eager, error_graph_mode): height = 1 width = 1 empty_shader_code = "#version 460\n void main() { }\n" if tf.executing_eagerly(): error = error_eager else: error = error_graph_mode with self.assertRaisesRegexp(error, error_msg): self.evaluate( rasterizer.rasterize(num_points=0, variable_names=variable_names, variable_kinds=variable_kinds, variable_values=variable_values, output_resolution=(width, height), vertex_shader=empty_shader_code, geometry_shader=empty_shader_code, fragment_shader=empty_shader_code))
def rasterize(vertices, triangles, view_projection_matrices, image_size, name=None): """Rasterizes the scene. This rasterizer estimates which triangle is associated with each pixel using OpenGL. Note: In the following, A1 to An are optional batch dimensions which must be broadcast compatible for inputs `vertices` and `view_projection_matrices`. Args: vertices: A tensor of shape `[A1, ..., An, V, 3]` containing batches of `V` vertices, each defined by a 3D point. triangles: A tensor of shape `[T, 3]` containing `T` triangles, each associated with 3 vertices from `scene_vertices` view_projection_matrices: A tensor of shape `[A1, ..., An, 4, 4]` containing batches of view projection matrices image_size: An tuple of integers (height, width) containing the dimensions in pixels of the rasterized image. name: A name for this op. Defaults to 'rasterization_backend_rasterize'. Returns: A tuple of 3 elements. The first one of shape `[A1, ..., An, H, W, 1]` representing the triangle index associated with each pixel. If no triangle is associated to a pixel, the index is set to -1. The second element in the tuple is of shape `[A1, ..., An, H, W, 3]` and correspond to barycentric coordinates per pixel. The last element in the tuple is of shape `[A1, ..., An, H, W, 1]` and stores a value of `0` of the pixel is assciated with the background, and `1` with the foreground """ with tf.compat.v1.name_scope(name, "rasterization_backend_rasterize", (vertices, triangles, view_projection_matrices)): vertices = tf.convert_to_tensor(value=vertices) triangles = tf.convert_to_tensor(value=triangles) view_projection_matrices = tf.convert_to_tensor( value=view_projection_matrices) shape.check_static( tensor=vertices, tensor_name="vertices", has_rank_greater_than=1, has_dim_equals=((-1, 3))) shape.check_static( tensor=triangles, tensor_name="triangles", has_rank=2, has_dim_equals=((-1, 3))) shape.check_static( tensor=view_projection_matrices, tensor_name="view_projection_matrices", has_rank_greater_than=1, has_dim_equals=((-1, 4), (-2, 4))) shape.compare_batch_dimensions( tensors=(vertices, view_projection_matrices), tensor_names=("vertices", "view_projection_matrices"), last_axes=(-3, -3), broadcast_compatible=True) common_batch_shape = shape.get_broadcasted_shape( vertices.shape[:-2], view_projection_matrices.shape[:-2]) common_batch_shape = [_dim_value(dim) for dim in common_batch_shape] vertices = tf.broadcast_to(vertices, common_batch_shape + vertices.shape[-2:]) view_projection_matrices = tf.broadcast_to(view_projection_matrices, common_batch_shape + [4, 4]) geometry = tf.gather(vertices, triangles, axis=-2) rasterized = render_ops.rasterize( num_points=geometry.shape[-3], variable_names=("view_projection_matrix", "triangular_mesh"), variable_kinds=("mat", "buffer"), variable_values=(view_projection_matrices, tf.reshape(geometry, shape=common_batch_shape + [-1])), output_resolution=image_size, vertex_shader=vertex_shader, geometry_shader=geometry_shader, fragment_shader=fragment_shader) triangle_index = tf.cast(rasterized[..., 0] - 1, tf.int32) barycentric_coordinates = rasterized[..., 1:3] barycentric_coordinates = tf.concat( (barycentric_coordinates, 1.0 - barycentric_coordinates[..., 0:1] - barycentric_coordinates[..., 1:2]), axis=-1) mask = tf.cast(triangle_index >= 0, tf.int32) return triangle_index, barycentric_coordinates, mask
def test_rasterize(self): max_depth = 10 min_depth = 2 height = 480 width = 640 camera_origin = (0.0, 0.0, 0.0) camera_up = (0.0, 1.0, 0.0) look_at = (0.0, 0.0, 1.0) fov = (60.0 * np.math.pi / 180, ) near_plane = (1.0, ) far_plane = (10.0, ) batch_shape = tf.convert_to_tensor(value=(2, (max_depth - min_depth) // 2), dtype=tf.int32) world_to_camera = glm.look_at_right_handed(camera_origin, look_at, camera_up) perspective_matrix = glm.perspective_right_handed( fov, (float(width) / float(height), ), near_plane, far_plane) view_projection_matrix = tf.matmul(perspective_matrix, world_to_camera) view_projection_matrix = tf.squeeze(view_projection_matrix) # Generate triangles at different depths and associated ground truth. tris = np.zeros((max_depth - min_depth, 9), dtype=np.float32) gt = np.zeros((max_depth - min_depth, width, height, 2), dtype=np.float32) for idx in range(max_depth - min_depth): tris[idx, :] = (-100.0, 100.0, idx + min_depth, 100.0, 100.0, idx + min_depth, 0.0, -100.0, idx + min_depth) gt[idx, :, :, :] = (0, idx + min_depth) # Broadcast the variables. render_parameters = { "view_projection_matrix": ("mat", tf.broadcast_to( input=view_projection_matrix, shape=tf.concat( values=(batch_shape, tf.shape(input=view_projection_matrix)[-2:]), axis=0))), "triangular_mesh": ("buffer", tf.reshape(tris, shape=tf.concat(values=(batch_shape, (9, )), axis=0))) } # Reshape the ground truth. gt = tf.reshape(gt, shape=tf.concat(values=(batch_shape, (height, width, 2)), axis=0)) render_parameters = list(six.iteritems(render_parameters)) variable_names = [v[0] for v in render_parameters] variable_kinds = [v[1][0] for v in render_parameters] variable_values = [v[1][1] for v in render_parameters] result = rasterizer.rasterize( num_points=3, variable_names=variable_names, variable_kinds=variable_kinds, variable_values=variable_values, output_resolution=(width, height), vertex_shader=test_vertex_shader, geometry_shader=test_geometry_shader, fragment_shader=test_fragment_shader, ) self.assertAllClose(result[..., 2:4], gt)
def rasterize(self, scene_vertices=None, scene_attributes=None, scene_triangles=None, name=None): """Rasterizes the scene. This rasterizer estimates which triangle is associated with each pixel using OpenGL. Then the value of attributes are estimated using Tensorflow, allowing to get gradients flowing through the attributes. Attributes can be depth, appearance, or more generally, any K-dimensional representation. Note that similarly to algorithms like Iterative Closest Point (ICP), not having gradients through correspondence does not prevent from optimizing the scene geometry. Custom gradients can be defined to alleviate this property. Note: In the following, A1 to An are optional batch dimensions. Args: scene_vertices: A tensor of shape `[A1, ..., An, V, 3]` containing batches of `V` vertices, each defined by a 3D point. scene_attributes: A tensor of shape `[A1, ..., An, V, K]` containing batches of `V` vertices, each associated with K-dimensional attributes. scene_triangles: A tensor of shape `[T, 3]` containing `T` triangles, each associated with 3 vertices from `scene_vertices` name: A name for this op. Defaults to 'triangle_rasterizer_rasterize'. Returns: A tensor of shape `[A1, ..., An, H, W, K]` containing batches of images of height `H` and width `W`, where each pixel contains attributes rasterized from the scene. """ with tf.compat.v1.name_scope( name, "triangle_rasterizer_rasterize", (scene_vertices, scene_attributes, scene_triangles)): scene_vertices = tf.convert_to_tensor(value=scene_vertices) scene_attributes = tf.convert_to_tensor(value=scene_attributes) scene_triangles = tf.convert_to_tensor(value=scene_triangles) shape.check_static( tensor=scene_vertices, tensor_name="scene_vertices", has_rank_greater_than=1, has_dim_equals=((-1, 3))) shape.compare_batch_dimensions( tensors=(scene_vertices, scene_attributes), last_axes=-2, tensor_names=("vertex_positions", "vertex_attributes"), broadcast_compatible=False) shape.check_static( tensor=scene_triangles, tensor_name="scene_triangles", has_dim_equals=((-1, 3))) batch_dims_triangles = len(scene_triangles.shape[:-2]) scene_attributes = tf.gather( scene_attributes, scene_triangles, axis=-2, batch_dims=batch_dims_triangles) scene_geometry = tf.gather( scene_vertices, scene_triangles, axis=-2, batch_dims=batch_dims_triangles) batch_shape = scene_geometry.shape[:-3] batch_shape = [_dim_value(dim) for dim in batch_shape] background_geometry = tf.broadcast_to( self._background_geometry, batch_shape + self._background_geometry.shape) background_attribute = tf.broadcast_to( self._background_attribute, batch_shape + self._background_attribute.shape) geometry = tf.concat((background_geometry, scene_geometry), axis=-3) attributes = tf.concat((background_attribute, scene_attributes), axis=-3) view_projection_matrix = tf.broadcast_to( input=self._view_projection_matrix, shape=batch_shape + self._view_projection_matrix.shape) rasterized_face = render_ops.rasterize( num_points=geometry.shape[-3], variable_names=("view_projection_matrix", "triangular_mesh"), variable_kinds=("mat", "buffer"), variable_values=(view_projection_matrix, tf.reshape(geometry, shape=batch_shape + [-1])), output_resolution=self._image_size_int, vertex_shader=vertex_shader, geometry_shader=geometry_shader, fragment_shader=fragment_shader) triangle_index = tf.cast(rasterized_face[..., 0], tf.int32) vertices_per_pixel = tf.gather( geometry, triangle_index, axis=-3, batch_dims=len(batch_shape)) attributes_per_pixel = tf.gather( attributes, triangle_index, axis=-3, batch_dims=len(batch_shape)) return glm.perspective_correct_interpolation( vertices_per_pixel, attributes_per_pixel, self._pixel_position, self._camera_origin, self._look_at, self._camera_up, self._field_of_view, self._image_size_glm, self._near_plane, self._far_plane, self._bottom_left)
def rasterize(vertices, triangles, view_projection_matrices, image_size, name=None): """Rasterizes the scene. This rasterizer estimates which triangle is associated with each pixel using OpenGL. Note: In the following, A1 to An are optional batch dimensions which must be broadcast compatible for inputs `vertices` and `view_projection_matrices`. Args: vertices: A tensor of shape `[A1, ..., An, V, 3]` containing batches of `V` vertices, each defined by a 3D point. triangles: A tensor of shape `[T, 3]` containing `T` triangles, each associated with 3 vertices from `scene_vertices` view_projection_matrices: A tensor of shape `[A1, ..., An, 4, 4]` containing batches of view projection matrices image_size: An tuple of integers (width, height) containing the dimensions in pixels of the rasterized image. name: A name for this op. Defaults to 'rasterization_backend_rasterize'. Returns: A Framebuffer containing the rasterized values: barycentrics, triangle_id, foreground_mask, vertex_ids. Returned Tensors have shape [batch, num_layers, height, width, channels] Note: triangle_id contains the triangle id value for each pixel in the output image. For pixels within the mesh, this is the integer value in the range [0, num_vertices] from triangles. For vertices outside the mesh this is 0; 0 can either indicate belonging to triangle 0, or being outside the mesh. This ensures all returned triangle ids will validly index into the vertex array, enabling the use of tf.gather with indices from this tensor. The barycentric coordinates can be used to determine pixel validity instead. See framebuffer.py for a description of the Framebuffer fields. """ with tf.compat.v1.name_scope( name, "rasterization_backend_rasterize", (vertices, triangles, view_projection_matrices)): vertices = tf.convert_to_tensor(value=vertices) triangles = tf.convert_to_tensor(value=triangles) view_projection_matrices = tf.convert_to_tensor( value=view_projection_matrices) shape.check_static(tensor=vertices, tensor_name="vertices", has_rank_greater_than=1, has_dim_equals=((-1, 3))) shape.check_static(tensor=triangles, tensor_name="triangles", has_rank=2, has_dim_equals=((-1, 3))) shape.check_static(tensor=view_projection_matrices, tensor_name="view_projection_matrices", has_rank_greater_than=1, has_dim_equals=((-1, 4), (-2, 4))) shape.compare_batch_dimensions( tensors=(vertices, view_projection_matrices), tensor_names=("vertices", "view_projection_matrices"), last_axes=(-3, -3), broadcast_compatible=True) common_batch_shape = shape.get_broadcasted_shape( vertices.shape[:-2], view_projection_matrices.shape[:-2]) common_batch_shape = [_dim_value(dim) for dim in common_batch_shape] vertices = tf.broadcast_to(vertices, common_batch_shape + vertices.shape[-2:]) view_projection_matrices = tf.broadcast_to(view_projection_matrices, common_batch_shape + [4, 4]) geometry = tf.gather(vertices, triangles, axis=-2) rasterized = render_ops.rasterize( num_points=geometry.shape[-3], alpha_clear=0.0, enable_cull_face=True, variable_names=("view_projection_matrix", "triangular_mesh"), variable_kinds=("mat", "buffer"), variable_values=(view_projection_matrices, tf.reshape(geometry, shape=common_batch_shape + [-1])), output_resolution=image_size, vertex_shader=vertex_shader, geometry_shader=geometry_shader, fragment_shader=fragment_shader) triangle_index = tf.cast(rasterized[..., 0], tf.int32) # Slicing of the tensor will result in all batch dimensions being # `None` for tensorflow graph mode, therefore we have to fix it in order to # have explicit shape. width, height = image_size triangle_index = tf.reshape(triangle_index, common_batch_shape + [height, width, 1]) barycentric_coordinates = rasterized[..., 1:3] barycentric_coordinates = tf.concat( (barycentric_coordinates, 1.0 - barycentric_coordinates[..., 0:1] - barycentric_coordinates[..., 1:2]), axis=-1) mask = tf.cast(rasterized[..., 3], tf.int32) mask = tf.reshape(mask, common_batch_shape + [height, width, 1]) triangles_batch = tf.broadcast_to(triangles, common_batch_shape + triangles.shape) vertex_ids = tf.gather(triangles_batch, triangle_index[..., 0], batch_dims=len(common_batch_shape)) return fb.Framebuffer(foreground_mask=mask, triangle_id=triangle_index, vertex_ids=vertex_ids, barycentrics=fb.RasterizedAttribute( value=barycentric_coordinates, d_dx=None, d_dy=None))
def rasterize(vertices, triangles, view_projection_matrices, image_size, enable_cull_face, num_layers, name=None): """Rasterizes the scene. This rasterizer estimates which triangle is associated with each pixel using OpenGL. Note: In the following, A1 to An are optional batch dimensions which must be broadcast compatible for inputs `vertices` and `view_projection_matrices`. Args: vertices: A tensor of shape `[batch, num_vertices, 3]` containing batches vertices, each defined by a 3D point. triangles: A tensor of shape `[num_triangles, 3]` each associated with 3 vertices from `scene_vertices` view_projection_matrices: A tensor of shape `[batch, 4, 4]` containing batches of view projection matrices image_size: An tuple of integers (width, height) containing the dimensions in pixels of the rasterized image. enable_cull_face: A boolean, which will enable BACK face culling when True and no face culling when False. Default is True. num_layers: Number of depth layers to render. Not supported by current backend yet, but exists for interface compatibility. name: A name for this op. Defaults to 'rasterization_backend_rasterize'. Returns: A Framebuffer containing the rasterized values: barycentrics, triangle_id, foreground_mask, vertex_ids. Returned Tensors have shape [batch, num_layers, height, width, channels] Note: triangle_id contains the triangle id value for each pixel in the output image. For pixels within the mesh, this is the integer value in the range [0, num_vertices] from triangles. For vertices outside the mesh this is 0; 0 can either indicate belonging to triangle 0, or being outside the mesh. This ensures all returned triangle ids will validly index into the vertex array, enabling the use of tf.gather with indices from this tensor. The barycentric coordinates can be used to determine pixel validity instead. See framebuffer.py for a description of the Framebuffer fields. """ with tf.compat.v1.name_scope( name, "rasterization_backend_rasterize", (vertices, triangles, view_projection_matrices)): if num_layers != 1: raise ValueError("OpenGL rasterizer only supports single layer.") vertices = tf.convert_to_tensor(value=vertices) triangles = tf.convert_to_tensor(value=triangles) view_projection_matrices = tf.convert_to_tensor( value=view_projection_matrices) shape.check_static(tensor=vertices, tensor_name="vertices", has_rank=3, has_dim_equals=((-1, 3))) shape.check_static(tensor=triangles, tensor_name="triangles", has_rank=2, has_dim_equals=((-1, 3))) shape.check_static(tensor=view_projection_matrices, tensor_name="view_projection_matrices", has_rank=3, has_dim_equals=((-1, 4), (-2, 4))) shape.compare_batch_dimensions( tensors=(vertices, view_projection_matrices), tensor_names=("vertices", "view_projection_matrices"), last_axes=(-3, -3), broadcast_compatible=True) geometry = tf.gather(vertices, triangles, axis=-2) # Extract batch size in order to make sure it is preserved after `gather` # operation. batch_size = _dim_value(vertices.shape[0]) rasterized = render_ops.rasterize( num_points=geometry.shape[-3], alpha_clear=0.0, enable_cull_face=enable_cull_face, variable_names=("view_projection_matrix", "triangular_mesh"), variable_kinds=("mat", "buffer"), variable_values=(view_projection_matrices, tf.reshape(geometry, shape=[batch_size, -1])), output_resolution=image_size, vertex_shader=vertex_shader, geometry_shader=geometry_shader, fragment_shader=fragment_shader) triangle_index = tf.cast(rasterized[..., 0], tf.int32) # Slicing of the tensor will result in all batch dimensions being # `None` for tensorflow graph mode, therefore we have to fix it in order to # have explicit shape. width, height = image_size triangle_index = tf.reshape(triangle_index, [batch_size, height, width, 1]) barycentric_coordinates = rasterized[..., 1:3] barycentric_coordinates = tf.concat( (barycentric_coordinates, 1.0 - barycentric_coordinates[..., 0:1] - barycentric_coordinates[..., 1:2]), axis=-1) mask = rasterized[..., 3] mask = tf.reshape(mask, [batch_size, height, width, 1]) barycentric_coordinates = mask * barycentric_coordinates vertex_ids = tf.gather(triangles, triangle_index[..., 0], batch_dims=0) # Stop gradient for tensors coming out of custom op in order to avoid # confusing Tensorflow that they are differentiable. barycentric_coordinates = tf.stop_gradient(barycentric_coordinates) mask = tf.stop_gradient(mask) return fb.Framebuffer(foreground_mask=mask, triangle_id=triangle_index, vertex_ids=vertex_ids, barycentrics=fb.RasterizedAttribute( value=barycentric_coordinates, d_dx=None, d_dy=None))