def rasterize():
return rasterizer.rasterize(
num_points=3,
variable_names=variable_names,
variable_kinds=variable_kinds,
variable_values=variable_values,
output_resolution=(width, height),
vertex_shader=test_vertex_shader,
geometry_shader=test_geometry_shader,
fragment_shader=test_fragment_shader,
)
def test_rasterize(self):
height = 500
width = 500
camera_origin = (0.0, 0.0, 0.0)
camera_up = (0.0, 1.0, 0.0)
look_at = (0.0, 0.0, 1.0)
fov = (60.0 * np.math.pi / 180, )
near_plane = (1.0, )
far_plane = (10.0, )
world_to_camera = glm.look_at_right_handed(camera_origin, look_at,
camera_up)
perspective_matrix = glm.perspective_right_handed(
fov, (float(width) / float(height), ), near_plane, far_plane)
view_projection_matrix = tf.matmul(perspective_matrix, world_to_camera)
view_projection_matrix = tf.squeeze(view_projection_matrix)
for depth in range(2, 5):
tris = np.array([[-10.0, 10.0, depth], [10.0, 10.0, depth],
[0.0, -10.0, depth]],
dtype=np.float32)
tris_tf = tf.reshape(tris, [-1])
render_parameters = {
"view_projection_matrix": ("mat", view_projection_matrix),
"triangular_mesh": ("buffer", tris_tf)
}
render_parameters = list(six.iteritems(render_parameters))
variable_names = [v[0] for v in render_parameters]
variable_kinds = [v[1][0] for v in render_parameters]
variable_values = [v[1][1] for v in render_parameters]
result = rasterizer.rasterize(
num_points=tris.shape[0],
variable_names=variable_names,
variable_kinds=variable_kinds,
variable_values=variable_values,
output_resolution=(width, height),
vertex_shader=test_vertex_shader,
geometry_shader=test_geometry_shader,
fragment_shader=test_fragment_shader,
)
gt = np.tile((0, depth), (width, height, 1))
self.assertAllClose(result[..., 2:4], gt)
def test_invalid_variable_inputs(self, error_msg, variable_names,
variable_kinds, variable_values,
error_eager, error_graph_mode):
height = 1
width = 1
empty_shader_code = "#version 460\n void main() { }\n"
if tf.executing_eagerly():
error = error_eager
else:
error = error_graph_mode
with self.assertRaisesRegexp(error, error_msg):
self.evaluate(
rasterizer.rasterize(num_points=0,
variable_names=variable_names,
variable_kinds=variable_kinds,
variable_values=variable_values,
output_resolution=(width, height),
vertex_shader=empty_shader_code,
geometry_shader=empty_shader_code,
fragment_shader=empty_shader_code))
def rasterize(vertices,
triangles,
view_projection_matrices,
image_size,
name=None):
"""Rasterizes the scene.
This rasterizer estimates which triangle is associated with each pixel using
OpenGL.
Note:
In the following, A1 to An are optional batch dimensions which must be
broadcast compatible for inputs `vertices` and `view_projection_matrices`.
Args:
vertices: A tensor of shape `[A1, ..., An, V, 3]` containing batches of `V`
vertices, each defined by a 3D point.
triangles: A tensor of shape `[T, 3]` containing `T` triangles, each
associated with 3 vertices from `scene_vertices`
view_projection_matrices: A tensor of shape `[A1, ..., An, 4, 4]` containing
batches of view projection matrices
image_size: An tuple of integers (height, width) containing the dimensions
in pixels of the rasterized image.
name: A name for this op. Defaults to 'rasterization_backend_rasterize'.
Returns:
A tuple of 3 elements. The first one of shape `[A1, ..., An, H, W, 1]`
representing the triangle index associated with each pixel. If no triangle
is associated to a pixel, the index is set to -1.
The second element in the tuple is of shape `[A1, ..., An, H, W, 3]` and
correspond to barycentric coordinates per pixel. The last element in the
tuple is of shape `[A1, ..., An, H, W, 1]` and stores a value of `0` of the
pixel is assciated with the background, and `1` with the foreground
"""
with tf.compat.v1.name_scope(name, "rasterization_backend_rasterize",
(vertices, triangles, view_projection_matrices)):
vertices = tf.convert_to_tensor(value=vertices)
triangles = tf.convert_to_tensor(value=triangles)
view_projection_matrices = tf.convert_to_tensor(
value=view_projection_matrices)
shape.check_static(
tensor=vertices,
tensor_name="vertices",
has_rank_greater_than=1,
has_dim_equals=((-1, 3)))
shape.check_static(
tensor=triangles,
tensor_name="triangles",
has_rank=2,
has_dim_equals=((-1, 3)))
shape.check_static(
tensor=view_projection_matrices,
tensor_name="view_projection_matrices",
has_rank_greater_than=1,
has_dim_equals=((-1, 4), (-2, 4)))
shape.compare_batch_dimensions(
tensors=(vertices, view_projection_matrices),
tensor_names=("vertices", "view_projection_matrices"),
last_axes=(-3, -3),
broadcast_compatible=True)
common_batch_shape = shape.get_broadcasted_shape(
vertices.shape[:-2], view_projection_matrices.shape[:-2])
common_batch_shape = [_dim_value(dim) for dim in common_batch_shape]
vertices = tf.broadcast_to(vertices,
common_batch_shape + vertices.shape[-2:])
view_projection_matrices = tf.broadcast_to(view_projection_matrices,
common_batch_shape + [4, 4])
geometry = tf.gather(vertices, triangles, axis=-2)
rasterized = render_ops.rasterize(
num_points=geometry.shape[-3],
variable_names=("view_projection_matrix", "triangular_mesh"),
variable_kinds=("mat", "buffer"),
variable_values=(view_projection_matrices,
tf.reshape(geometry, shape=common_batch_shape + [-1])),
output_resolution=image_size,
vertex_shader=vertex_shader,
geometry_shader=geometry_shader,
fragment_shader=fragment_shader)
triangle_index = tf.cast(rasterized[..., 0] - 1, tf.int32)
barycentric_coordinates = rasterized[..., 1:3]
barycentric_coordinates = tf.concat(
(barycentric_coordinates, 1.0 - barycentric_coordinates[..., 0:1] -
barycentric_coordinates[..., 1:2]),
axis=-1)
mask = tf.cast(triangle_index >= 0, tf.int32)
return triangle_index, barycentric_coordinates, mask
def test_rasterize(self):
max_depth = 10
min_depth = 2
height = 480
width = 640
camera_origin = (0.0, 0.0, 0.0)
camera_up = (0.0, 1.0, 0.0)
look_at = (0.0, 0.0, 1.0)
fov = (60.0 * np.math.pi / 180, )
near_plane = (1.0, )
far_plane = (10.0, )
batch_shape = tf.convert_to_tensor(value=(2, (max_depth - min_depth) //
2),
dtype=tf.int32)
world_to_camera = glm.look_at_right_handed(camera_origin, look_at,
camera_up)
perspective_matrix = glm.perspective_right_handed(
fov, (float(width) / float(height), ), near_plane, far_plane)
view_projection_matrix = tf.matmul(perspective_matrix, world_to_camera)
view_projection_matrix = tf.squeeze(view_projection_matrix)
# Generate triangles at different depths and associated ground truth.
tris = np.zeros((max_depth - min_depth, 9), dtype=np.float32)
gt = np.zeros((max_depth - min_depth, width, height, 2),
dtype=np.float32)
for idx in range(max_depth - min_depth):
tris[idx, :] = (-100.0, 100.0, idx + min_depth, 100.0, 100.0,
idx + min_depth, 0.0, -100.0, idx + min_depth)
gt[idx, :, :, :] = (0, idx + min_depth)
# Broadcast the variables.
render_parameters = {
"view_projection_matrix":
("mat",
tf.broadcast_to(
input=view_projection_matrix,
shape=tf.concat(
values=(batch_shape,
tf.shape(input=view_projection_matrix)[-2:]),
axis=0))),
"triangular_mesh":
("buffer",
tf.reshape(tris,
shape=tf.concat(values=(batch_shape, (9, )), axis=0)))
}
# Reshape the ground truth.
gt = tf.reshape(gt,
shape=tf.concat(values=(batch_shape, (height, width,
2)),
axis=0))
render_parameters = list(six.iteritems(render_parameters))
variable_names = [v[0] for v in render_parameters]
variable_kinds = [v[1][0] for v in render_parameters]
variable_values = [v[1][1] for v in render_parameters]
result = rasterizer.rasterize(
num_points=3,
variable_names=variable_names,
variable_kinds=variable_kinds,
variable_values=variable_values,
output_resolution=(width, height),
vertex_shader=test_vertex_shader,
geometry_shader=test_geometry_shader,
fragment_shader=test_fragment_shader,
)
self.assertAllClose(result[..., 2:4], gt)
def rasterize(self,
scene_vertices=None,
scene_attributes=None,
scene_triangles=None,
name=None):
"""Rasterizes the scene.
This rasterizer estimates which triangle is associated with each pixel using
OpenGL. Then the value of attributes are estimated using Tensorflow,
allowing to get gradients flowing through the attributes. Attributes can be
depth, appearance, or more generally, any K-dimensional representation. Note
that similarly to algorithms like Iterative Closest Point (ICP), not having
gradients through correspondence does not prevent from optimizing the scene
geometry. Custom gradients can be defined to alleviate this property.
Note:
In the following, A1 to An are optional batch dimensions.
Args:
scene_vertices: A tensor of shape `[A1, ..., An, V, 3]` containing batches
of `V` vertices, each defined by a 3D point.
scene_attributes: A tensor of shape `[A1, ..., An, V, K]` containing
batches of `V` vertices, each associated with K-dimensional attributes.
scene_triangles: A tensor of shape `[T, 3]` containing `T` triangles, each
associated with 3 vertices from `scene_vertices`
name: A name for this op. Defaults to 'triangle_rasterizer_rasterize'.
Returns:
A tensor of shape `[A1, ..., An, H, W, K]` containing batches of images of
height `H` and width `W`, where each pixel contains attributes rasterized
from the scene.
"""
with tf.compat.v1.name_scope(
name, "triangle_rasterizer_rasterize",
(scene_vertices, scene_attributes, scene_triangles)):
scene_vertices = tf.convert_to_tensor(value=scene_vertices)
scene_attributes = tf.convert_to_tensor(value=scene_attributes)
scene_triangles = tf.convert_to_tensor(value=scene_triangles)
shape.check_static(
tensor=scene_vertices,
tensor_name="scene_vertices",
has_rank_greater_than=1,
has_dim_equals=((-1, 3)))
shape.compare_batch_dimensions(
tensors=(scene_vertices, scene_attributes),
last_axes=-2,
tensor_names=("vertex_positions", "vertex_attributes"),
broadcast_compatible=False)
shape.check_static(
tensor=scene_triangles,
tensor_name="scene_triangles",
has_dim_equals=((-1, 3)))
batch_dims_triangles = len(scene_triangles.shape[:-2])
scene_attributes = tf.gather(
scene_attributes,
scene_triangles,
axis=-2,
batch_dims=batch_dims_triangles)
scene_geometry = tf.gather(
scene_vertices,
scene_triangles,
axis=-2,
batch_dims=batch_dims_triangles)
batch_shape = scene_geometry.shape[:-3]
batch_shape = [_dim_value(dim) for dim in batch_shape]
background_geometry = tf.broadcast_to(
self._background_geometry,
batch_shape + self._background_geometry.shape)
background_attribute = tf.broadcast_to(
self._background_attribute,
batch_shape + self._background_attribute.shape)
geometry = tf.concat((background_geometry, scene_geometry), axis=-3)
attributes = tf.concat((background_attribute, scene_attributes), axis=-3)
view_projection_matrix = tf.broadcast_to(
input=self._view_projection_matrix,
shape=batch_shape + self._view_projection_matrix.shape)
rasterized_face = render_ops.rasterize(
num_points=geometry.shape[-3],
variable_names=("view_projection_matrix", "triangular_mesh"),
variable_kinds=("mat", "buffer"),
variable_values=(view_projection_matrix,
tf.reshape(geometry, shape=batch_shape + [-1])),
output_resolution=self._image_size_int,
vertex_shader=vertex_shader,
geometry_shader=geometry_shader,
fragment_shader=fragment_shader)
triangle_index = tf.cast(rasterized_face[..., 0], tf.int32)
vertices_per_pixel = tf.gather(
geometry, triangle_index, axis=-3, batch_dims=len(batch_shape))
attributes_per_pixel = tf.gather(
attributes, triangle_index, axis=-3, batch_dims=len(batch_shape))
return glm.perspective_correct_interpolation(
vertices_per_pixel, attributes_per_pixel, self._pixel_position,
self._camera_origin, self._look_at, self._camera_up,
self._field_of_view, self._image_size_glm, self._near_plane,
self._far_plane, self._bottom_left)
def rasterize(vertices,
triangles,
view_projection_matrices,
image_size,
name=None):
"""Rasterizes the scene.
This rasterizer estimates which triangle is associated with each pixel using
OpenGL.
Note:
In the following, A1 to An are optional batch dimensions which must be
broadcast compatible for inputs `vertices` and `view_projection_matrices`.
Args:
vertices: A tensor of shape `[A1, ..., An, V, 3]` containing batches of `V`
vertices, each defined by a 3D point.
triangles: A tensor of shape `[T, 3]` containing `T` triangles, each
associated with 3 vertices from `scene_vertices`
view_projection_matrices: A tensor of shape `[A1, ..., An, 4, 4]` containing
batches of view projection matrices
image_size: An tuple of integers (width, height) containing the dimensions
in pixels of the rasterized image.
name: A name for this op. Defaults to 'rasterization_backend_rasterize'.
Returns:
A Framebuffer containing the rasterized values: barycentrics, triangle_id,
foreground_mask, vertex_ids. Returned Tensors have shape
[batch, num_layers, height, width, channels]
Note: triangle_id contains the triangle id value for each pixel in the
output image. For pixels within the mesh, this is the integer value in the
range [0, num_vertices] from triangles. For vertices outside the mesh this
is 0; 0 can either indicate belonging to triangle 0, or being outside the
mesh. This ensures all returned triangle ids will validly index into the
vertex array, enabling the use of tf.gather with indices from this tensor.
The barycentric coordinates can be used to determine pixel validity instead.
See framebuffer.py for a description of the Framebuffer fields.
"""
with tf.compat.v1.name_scope(
name, "rasterization_backend_rasterize",
(vertices, triangles, view_projection_matrices)):
vertices = tf.convert_to_tensor(value=vertices)
triangles = tf.convert_to_tensor(value=triangles)
view_projection_matrices = tf.convert_to_tensor(
value=view_projection_matrices)
shape.check_static(tensor=vertices,
tensor_name="vertices",
has_rank_greater_than=1,
has_dim_equals=((-1, 3)))
shape.check_static(tensor=triangles,
tensor_name="triangles",
has_rank=2,
has_dim_equals=((-1, 3)))
shape.check_static(tensor=view_projection_matrices,
tensor_name="view_projection_matrices",
has_rank_greater_than=1,
has_dim_equals=((-1, 4), (-2, 4)))
shape.compare_batch_dimensions(
tensors=(vertices, view_projection_matrices),
tensor_names=("vertices", "view_projection_matrices"),
last_axes=(-3, -3),
broadcast_compatible=True)
common_batch_shape = shape.get_broadcasted_shape(
vertices.shape[:-2], view_projection_matrices.shape[:-2])
common_batch_shape = [_dim_value(dim) for dim in common_batch_shape]
vertices = tf.broadcast_to(vertices,
common_batch_shape + vertices.shape[-2:])
view_projection_matrices = tf.broadcast_to(view_projection_matrices,
common_batch_shape + [4, 4])
geometry = tf.gather(vertices, triangles, axis=-2)
rasterized = render_ops.rasterize(
num_points=geometry.shape[-3],
alpha_clear=0.0,
enable_cull_face=True,
variable_names=("view_projection_matrix", "triangular_mesh"),
variable_kinds=("mat", "buffer"),
variable_values=(view_projection_matrices,
tf.reshape(geometry,
shape=common_batch_shape + [-1])),
output_resolution=image_size,
vertex_shader=vertex_shader,
geometry_shader=geometry_shader,
fragment_shader=fragment_shader)
triangle_index = tf.cast(rasterized[..., 0], tf.int32)
# Slicing of the tensor will result in all batch dimensions being
# `None` for tensorflow graph mode, therefore we have to fix it in order to
# have explicit shape.
width, height = image_size
triangle_index = tf.reshape(triangle_index,
common_batch_shape + [height, width, 1])
barycentric_coordinates = rasterized[..., 1:3]
barycentric_coordinates = tf.concat(
(barycentric_coordinates, 1.0 - barycentric_coordinates[..., 0:1] -
barycentric_coordinates[..., 1:2]),
axis=-1)
mask = tf.cast(rasterized[..., 3], tf.int32)
mask = tf.reshape(mask, common_batch_shape + [height, width, 1])
triangles_batch = tf.broadcast_to(triangles,
common_batch_shape + triangles.shape)
vertex_ids = tf.gather(triangles_batch,
triangle_index[..., 0],
batch_dims=len(common_batch_shape))
return fb.Framebuffer(foreground_mask=mask,
triangle_id=triangle_index,
vertex_ids=vertex_ids,
barycentrics=fb.RasterizedAttribute(
value=barycentric_coordinates,
d_dx=None,
d_dy=None))
def rasterize(vertices,
triangles,
view_projection_matrices,
image_size,
enable_cull_face,
num_layers,
name=None):
"""Rasterizes the scene.
This rasterizer estimates which triangle is associated with each pixel using
OpenGL.
Note:
In the following, A1 to An are optional batch dimensions which must be
broadcast compatible for inputs `vertices` and `view_projection_matrices`.
Args:
vertices: A tensor of shape `[batch, num_vertices, 3]` containing batches
vertices, each defined by a 3D point.
triangles: A tensor of shape `[num_triangles, 3]` each associated with 3
vertices from `scene_vertices`
view_projection_matrices: A tensor of shape `[batch, 4, 4]` containing
batches of view projection matrices
image_size: An tuple of integers (width, height) containing the dimensions
in pixels of the rasterized image.
enable_cull_face: A boolean, which will enable BACK face culling when True
and no face culling when False. Default is True.
num_layers: Number of depth layers to render. Not supported by current
backend yet, but exists for interface compatibility.
name: A name for this op. Defaults to 'rasterization_backend_rasterize'.
Returns:
A Framebuffer containing the rasterized values: barycentrics, triangle_id,
foreground_mask, vertex_ids. Returned Tensors have shape
[batch, num_layers, height, width, channels]
Note: triangle_id contains the triangle id value for each pixel in the
output image. For pixels within the mesh, this is the integer value in the
range [0, num_vertices] from triangles. For vertices outside the mesh this
is 0; 0 can either indicate belonging to triangle 0, or being outside the
mesh. This ensures all returned triangle ids will validly index into the
vertex array, enabling the use of tf.gather with indices from this tensor.
The barycentric coordinates can be used to determine pixel validity instead.
See framebuffer.py for a description of the Framebuffer fields.
"""
with tf.compat.v1.name_scope(
name, "rasterization_backend_rasterize",
(vertices, triangles, view_projection_matrices)):
if num_layers != 1:
raise ValueError("OpenGL rasterizer only supports single layer.")
vertices = tf.convert_to_tensor(value=vertices)
triangles = tf.convert_to_tensor(value=triangles)
view_projection_matrices = tf.convert_to_tensor(
value=view_projection_matrices)
shape.check_static(tensor=vertices,
tensor_name="vertices",
has_rank=3,
has_dim_equals=((-1, 3)))
shape.check_static(tensor=triangles,
tensor_name="triangles",
has_rank=2,
has_dim_equals=((-1, 3)))
shape.check_static(tensor=view_projection_matrices,
tensor_name="view_projection_matrices",
has_rank=3,
has_dim_equals=((-1, 4), (-2, 4)))
shape.compare_batch_dimensions(
tensors=(vertices, view_projection_matrices),
tensor_names=("vertices", "view_projection_matrices"),
last_axes=(-3, -3),
broadcast_compatible=True)
geometry = tf.gather(vertices, triangles, axis=-2)
# Extract batch size in order to make sure it is preserved after `gather`
# operation.
batch_size = _dim_value(vertices.shape[0])
rasterized = render_ops.rasterize(
num_points=geometry.shape[-3],
alpha_clear=0.0,
enable_cull_face=enable_cull_face,
variable_names=("view_projection_matrix", "triangular_mesh"),
variable_kinds=("mat", "buffer"),
variable_values=(view_projection_matrices,
tf.reshape(geometry, shape=[batch_size, -1])),
output_resolution=image_size,
vertex_shader=vertex_shader,
geometry_shader=geometry_shader,
fragment_shader=fragment_shader)
triangle_index = tf.cast(rasterized[..., 0], tf.int32)
# Slicing of the tensor will result in all batch dimensions being
# `None` for tensorflow graph mode, therefore we have to fix it in order to
# have explicit shape.
width, height = image_size
triangle_index = tf.reshape(triangle_index,
[batch_size, height, width, 1])
barycentric_coordinates = rasterized[..., 1:3]
barycentric_coordinates = tf.concat(
(barycentric_coordinates, 1.0 - barycentric_coordinates[..., 0:1] -
barycentric_coordinates[..., 1:2]),
axis=-1)
mask = rasterized[..., 3]
mask = tf.reshape(mask, [batch_size, height, width, 1])
barycentric_coordinates = mask * barycentric_coordinates
vertex_ids = tf.gather(triangles, triangle_index[..., 0], batch_dims=0)
# Stop gradient for tensors coming out of custom op in order to avoid
# confusing Tensorflow that they are differentiable.
barycentric_coordinates = tf.stop_gradient(barycentric_coordinates)
mask = tf.stop_gradient(mask)
return fb.Framebuffer(foreground_mask=mask,
triangle_id=triangle_index,
vertex_ids=vertex_ids,
barycentrics=fb.RasterizedAttribute(
value=barycentric_coordinates,
d_dx=None,
d_dy=None))
