def testFullRenderGradientComputation(self): """Verify the Jacobian matrix for the entire renderer. This ensures correct gradients are propagated backwards through the entire process, not just through the rasterization kernel. Uses the simple cube forward pass. """ image_height = 21 image_width = 28 # rotate the cube for the test: model_transforms = camera_utils.euler_matrices( [[-20.0, 0.0, 60.0], [45.0, 60.0, 0.0]])[:, :3, :3] normals_world_space = torch.matmul( torch.stack([self.cube_normals, self.cube_normals]), model_transforms.transpose()) # camera position: eye = torch.tensor([0.0, 0.0, 6.0], dtype=torch.float32) center = torch.tensor([0.0, 0.0, 0.0], dtype=torch.float32) world_up = torch.tensor([0.0, 1.0, 0.0], dtype=torch.float32) # Scene has a single light from the viewer's eye. light_positions = torch.unsqueeze(torch.stack([eye, eye], dim=0), 1) light_intensities = torch.ones([2, 1, 3], dtype=torch.float32) def render_complex_cube(cube_vertices): vertices_world_space = torch.matmul( torch.stack([cube_vertices, cube_vertices]), model_transforms.transpose()) vertex_diffuse_colors = torch.ones_like(vertices_world_space, dtype=torch.float32) return mesh_renderer.mesh_renderer( vertices_world_space, self.cube_triangles, normals_world_space, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height ) jacobians_match = torch.autograd.gradcheck( render_complex_cube, self.cube_vertices, eps=1e-3, atol=0.1, rtol=0.01) self.assertTrue( jacobians_match, "Analytical and numerical jacobians have too many relative or absolute outliers")
def testFullRenderGradientComputation(self): """Verifies the Jacobian matrix for the entire renderer. This ensures correct gradients are propagated backwards through the entire process, not just through the rasterization kernel. Uses the simple cube forward pass. """ image_height = 21 image_width = 28 # rotate the cube for the test: model_transforms = camera_utils.euler_matrices([[-20.0, 0.0, 60.0], [45.0, 60.0, 0.0]])[:, :3, :3] vertices_world_space = tf.matmul(tf.stack( [self.cube_vertices, self.cube_vertices]), model_transforms, transpose_b=True) normals_world_space = tf.matmul(tf.stack( [self.cube_normals, self.cube_normals]), model_transforms, transpose_b=True) # camera position: eye = tf.constant([0.0, 0.0, 6.0], dtype=tf.float32) center = tf.constant([0.0, 0.0, 0.0], dtype=tf.float32) world_up = tf.constant([0.0, 1.0, 0.0], dtype=tf.float32) # Scene has a single light from the viewer's eye. light_positions = tf.expand_dims(tf.stack([eye, eye], axis=0), axis=1) light_intensities = tf.ones([2, 1, 3], dtype=tf.float32) vertex_diffuse_colors = tf.ones_like(vertices_world_space, dtype=tf.float32) rendered = mesh_renderer.mesh_renderer( vertices_world_space, self.cube_triangles, normals_world_space, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height) with self.test_session(): theoretical, numerical = tf.test.compute_gradient( self.cube_vertices, (8, 3), rendered, (2, image_height, image_width, 4), x_init_value=self.cube_vertices.eval(), delta=1e-3) jacobians_match, message = ( test_utils.check_jacobians_are_nearly_equal( theoretical, numerical, 0.01, 0.01)) self.assertTrue(jacobians_match, message)
def testRendersSimpleCube(self): """Renders a simple cube to test the full forward pass. Verifies the functionality of both the custom kernel and the python wrapper. """ model_transforms = camera_utils.euler_matrices([[-20.0, 0.0, 60.0], [45.0, 60.0, 0.0]])[:, :3, :3] vertices_world_space = tf.matmul(tf.stack( [self.cube_vertices, self.cube_vertices]), model_transforms, transpose_b=True) normals_world_space = tf.matmul(tf.stack( [self.cube_normals, self.cube_normals]), model_transforms, transpose_b=True) # camera position: eye = tf.constant(2 * [[0.0, 0.0, 6.0]], dtype=tf.float32) center = tf.constant(2 * [[0.0, 0.0, 0.0]], dtype=tf.float32) world_up = tf.constant(2 * [[0.0, 1.0, 0.0]], dtype=tf.float32) image_width = 640 image_height = 480 light_positions = tf.constant([[[0.0, 0.0, 6.0]], [[0.0, 0.0, 6.0]]]) light_intensities = tf.ones([2, 1, 3], dtype=tf.float32) vertex_diffuse_colors = tf.ones_like(vertices_world_space, dtype=tf.float32) rendered = mesh_renderer.mesh_renderer( vertices_world_space, self.cube_triangles, normals_world_space, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height) with self.test_session() as sess: images = sess.run(rendered, feed_dict={}) for image_id in range(images.shape[0]): target_image_name = 'Gray_Cube_%i.png' % image_id baseline_image_path = os.path.join(self.test_data_directory, target_image_name) test_utils.expect_image_file_and_render_are_near( self, sess, baseline_image_path, images[image_id, :, :, :])
def testComplexShading(self): """Tests specular highlights, colors, and multiple lights per image.""" # rotate the cube for the test: model_transforms = camera_utils.euler_matrices( [[-20.0, 0.0, 60.0], [45.0, 60.0, 0.0]])[:, :3, :3] vertices_world_space = tf.matmul( tf.stack([self.cube_vertices, self.cube_vertices]), model_transforms, transpose_b=True) normals_world_space = tf.matmul( tf.stack([self.cube_normals, self.cube_normals]), model_transforms, transpose_b=True) # camera position: eye = tf.constant([[0.0, 0.0, 6.0], [0., 0.2, 18.0]], dtype=tf.float32) center = tf.constant([[0.0, 0.0, 0.0], [0.1, -0.1, 0.1]], dtype=tf.float32) world_up = tf.constant( [[0.0, 1.0, 0.0], [0.1, 1.0, 0.15]], dtype=tf.float32) fov_y = tf.constant([40., 13.3], dtype=tf.float32) near_clip = tf.constant(0.1, dtype=tf.float32) far_clip = tf.constant(25.0, dtype=tf.float32) image_width = 640 image_height = 480 light_positions = tf.constant([[[0.0, 0.0, 6.0], [1.0, 2.0, 6.0]], [[0.0, -2.0, 4.0], [1.0, 3.0, 4.0]]]) light_intensities = tf.constant( [[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], [[2.0, 0.0, 1.0], [0.0, 2.0, 1.0]]], dtype=tf.float32) # pyformat: disable vertex_diffuse_colors = tf.constant(2*[[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 0.0], [1.0, 0.0, 1.0], [0.0, 1.0, 1.0], [0.5, 0.5, 0.5]]], dtype=tf.float32) vertex_specular_colors = tf.constant(2*[[[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 0.0], [1.0, 0.0, 1.0], [0.0, 1.0, 1.0], [0.5, 0.5, 0.5], [1.0, 0.0, 0.0]]], dtype=tf.float32) # pyformat: enable shininess_coefficients = 6.0 * tf.ones([2, 8], dtype=tf.float32) ambient_color = tf.constant( [[0., 0., 0.], [0.1, 0.1, 0.2]], dtype=tf.float32) renders = mesh_renderer.mesh_renderer( vertices_world_space, self.cube_triangles, normals_world_space, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height, vertex_specular_colors, shininess_coefficients, ambient_color, fov_y, near_clip, far_clip) tonemapped_renders = tf.concat( [ mesh_renderer.tone_mapper(renders[:, :, :, 0:3], 0.7), renders[:, :, :, 3:4] ], axis=3) # Check that shininess coefficient broadcasting works by also rendering # with a scalar shininess coefficient, and ensuring the result is identical: broadcasted_renders = mesh_renderer.mesh_renderer( vertices_world_space, self.cube_triangles, normals_world_space, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height, vertex_specular_colors, 6.0, ambient_color, fov_y, near_clip, far_clip) tonemapped_broadcasted_renders = tf.concat( [ mesh_renderer.tone_mapper(broadcasted_renders[:, :, :, 0:3], 0.7), broadcasted_renders[:, :, :, 3:4] ], axis=3) with self.test_session() as sess: images, broadcasted_images = sess.run( [tonemapped_renders, tonemapped_broadcasted_renders], feed_dict={}) for image_id in range(images.shape[0]): target_image_name = 'Colored_Cube_%i.png' % image_id baseline_image_path = os.path.join(self.test_data_directory, target_image_name) test_utils.expect_image_file_and_render_are_near( self, sess, baseline_image_path, images[image_id, :, :, :]) test_utils.expect_image_file_and_render_are_near( self, sess, baseline_image_path, broadcasted_images[image_id, :, :, :])
def testThatCubeRotates(self): """Optimize a simple cube's rotation using pixel loss. The rotation is represented as static-basis euler angles. This test checks that the computed gradients are useful. """ image_height = 480 image_width = 640 initial_euler_angles = [[0.0, 0.0, 0.0]] euler_angles = tf.Variable(initial_euler_angles) model_rotation = camera_utils.euler_matrices(euler_angles)[0, :3, :3] vertices_world_space = tf.reshape( tf.matmul(self.cube_vertices, model_rotation, transpose_b=True), [1, 8, 3]) normals_world_space = tf.reshape( tf.matmul(self.cube_normals, model_rotation, transpose_b=True), [1, 8, 3]) # camera position: eye = tf.constant([[0.0, 0.0, 6.0]], dtype=tf.float32) center = tf.constant([[0.0, 0.0, 0.0]], dtype=tf.float32) world_up = tf.constant([[0.0, 1.0, 0.0]], dtype=tf.float32) vertex_diffuse_colors = tf.ones_like(vertices_world_space, dtype=tf.float32) light_positions = tf.reshape(eye, [1, 1, 3]) light_intensities = tf.ones([1, 1, 3], dtype=tf.float32) render = mesh_renderer.mesh_renderer( vertices_world_space, self.cube_triangles, normals_world_space, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height) render = tf.reshape(render, [image_height, image_width, 4]) # Pick the desired cube rotation for the test: test_model_rotation = camera_utils.euler_matrices([[-20.0, 0.0, 60.0]])[0, :3, :3] desired_vertex_positions = tf.reshape( tf.matmul(self.cube_vertices, test_model_rotation, transpose_b=True), [1, 8, 3]) desired_normals = tf.reshape( tf.matmul(self.cube_normals, test_model_rotation, transpose_b=True), [1, 8, 3]) desired_render = mesh_renderer.mesh_renderer( desired_vertex_positions, self.cube_triangles, desired_normals, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height) desired_render = tf.reshape(desired_render, [image_height, image_width, 4]) loss = tf.reduce_mean(tf.abs(render - desired_render)) optimizer = tf.train.MomentumOptimizer(0.7, 0.1) grad = tf.gradients(loss, [euler_angles]) grad, _ = tf.clip_by_global_norm(grad, 1.0) opt_func = optimizer.apply_gradients([(grad[0], euler_angles)]) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for _ in range(35): sess.run([loss, opt_func]) final_image, desired_image = sess.run([render, desired_render]) target_image_name = 'Gray_Cube_0.png' baseline_image_path = os.path.join(self.test_data_directory, target_image_name) test_utils.expect_image_file_and_render_are_near( self, sess, baseline_image_path, desired_image) test_utils.expect_image_file_and_render_are_near( self, sess, baseline_image_path, final_image, max_outlier_fraction=0.01, pixel_error_threshold=0.04)
import camera_utils import test_utils tf.reset_default_graph() # Set up a basic cube centered at the origin, with vertex normals pointing # outwards along the line from the origin to the cube vertices: cube_vertices = tf.constant( [[-1, -1, 1], [-1, -1, -1], [-1, 1, -1], [-1, 1, 1], [1, -1, 1], [1, -1, -1], [1, 1, -1], [1, 1, 1]], dtype=tf.float32) cube_normals = tf.nn.l2_normalize(cube_vertices, dim=1) cube_triangles = tf.constant( [[0, 1, 2], [2, 3, 0], [3, 2, 6], [6, 7, 3], [7, 6, 5], [5, 4, 7], [4, 5, 1], [1, 0, 4], [5, 6, 2], [2, 1, 5], [7, 4, 0], [0, 3, 7]], dtype=tf.int32) model_transforms = camera_utils.euler_matrices([[-20.0, 0.0, 60.0], [45.0, 60.0, 0.0]])[:, :3, :3] vertices_world_space = tf.matmul(tf.stack([cube_vertices, cube_vertices]), model_transforms, transpose_b=True) normals_world_space = tf.matmul(tf.stack([cube_normals, cube_normals]), model_transforms, transpose_b=True) # camera position: eye = tf.constant(2 * [[0.0, 0.0, 6.0]], dtype=tf.float32) center = tf.constant(2 * [[0.0, 0.0, 0.0]], dtype=tf.float32) world_up = tf.constant(2 * [[0.0, 1.0, 0.0]], dtype=tf.float32) image_width = 1920 image_height = 1080
def main(): test_data_directory = ('./test_data_face/render') # load obj face_mesh = m3io.import_mesh('./test_data_face/mesh.obj') texture_index = (face_mesh.tcoords.points[:, ::-1] * face_mesh.texture.shape).astype(np.int32) vertex_color = face_mesh.texture.pixels[:, 1 - texture_index[:, 0], texture_index[:, 1]].T tf.reset_default_graph() # Set up a basic cube centered at the origin, with vertex normals pointing # outwards along the line from the origin to the cube vertices: face_vertices = tf.constant(face_mesh.points, dtype=tf.float32) face_normals = tf.nn.l2_normalize(face_vertices, dim=1) face_triangles = tf.constant(face_mesh.trilist, dtype=tf.int32) # testRendersSimpleCube: """Renders a simple cube to test the full forward pass. Verifies the functionality of both the custom kernel and the python wrapper. """ n_randering = 16 model_transforms = camera_utils.euler_matrices( tf.random_uniform([n_randering, 3]) * np.pi / 2 - np.pi / 4.)[:, :3, :3] vertices_world_space = tf.matmul(tf.stack( [face_vertices for _ in range(n_randering)]), model_transforms, transpose_b=True) normals_world_space = tf.matmul(tf.stack( [face_normals for _ in range(n_randering)]), model_transforms, transpose_b=True) # camera position: eye = tf.constant(n_randering * [[0.0, 0.0, 6.0]], dtype=tf.float32) center = tf.constant(n_randering * [[0.0, 0.0, 0.0]], dtype=tf.float32) world_up = tf.constant(n_randering * [[0.0, 1.0, 0.0]], dtype=tf.float32) ambient_colors = tf.constant(n_randering * [[0.2, 0.2, 0.2]], dtype=tf.float32) image_width = 256 image_height = 256 light_positions = tf.constant(n_randering * [[[6.0, 6.0, 6.0], [-6.0, -6.0, 6.0]]]) light_intensities = tf.ones([n_randering, 1, 3], dtype=tf.float32) vertex_diffuse_colors = tf.constant(np.stack( [vertex_color for _ in range(n_randering)]), dtype=tf.float32) rendered = mesh_renderer.mesh_renderer( vertices_world_space, triangles=face_triangles, normals=normals_world_space, diffuse_colors=vertex_diffuse_colors, camera_position=eye, camera_lookat=center, camera_up=world_up, light_positions=light_positions, light_intensities=light_intensities, image_width=image_width, image_height=image_height, ambient_color=ambient_colors) image_id = 0 with tf.Session() as sess: fps_list = [] while (image_id < 100): start_time = time.time() images = sess.run(rendered, feed_dict={}) for image in images: target_image_name = 'Gray_face_%i.png' % image_id image_id += 1 baseline_image_path = os.path.join(test_data_directory, target_image_name) mio.export_image(Image.init_from_channels_at_back( image[..., :3].clip(0, 1)), baseline_image_path, overwrite=True) end_time = time.time() fps = n_randering / (end_time - start_time) fps_list.append(fps) if len(fps_list) > 5: fps_list.pop(0) print(np.mean(fps_list))
def testComplexShading(self): """Test specular highlights, colors, and multiple lights per image.""" model_transforms = camera_utils.euler_matrices( [[-20.0, 0.0, 60.0], [45.0, 60.0, 0.0]])[:, :3, :3] vertices_world_space = torch.matmul( torch.stack([self.cube_vertices, self.cube_vertices]), model_transforms.transpose()) normals_world_space = torch.matmul( torch.stack([self.cube_normals, self.cube_normals]), model_transforms.transpose()) # camera position: eye = torch.tensor([[0.0, 0.0, 6.0], [0.0, 0.2, 18.0]], dtype=torch.float32) center = torch.tensor([[0.0, 0.0, 0.0], [0.1, -0.1, 0.1]], dtype=torch.float32) world_up = torch.constant([[0.0, 1.0, 0.0], [0.1, 1.0, 0.15]], dtype=torch.float32) fov_y = torch.tensor([40.0, 13.3], dtype=torch.float32) near_clip = 0.1 far_clip = 25.0 image_width = 640 image_height = 480 light_positions = torch.tensor([[[0.0, 0.0, 6.0], [1.0, 2.0, 6.0]], [[0.0, -2.0, 4.0], [1.0, 3.0, 4.0]]]) light_intensities = torch.tensor( [[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], [[2.0, 0.0, 1.0], [0.0, 2.0, 1.0]]], dtype=torch.float32) vertex_diffuse_colors = torch.tensor(2*[[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 0.0], [1.0, 0.0, 1.0], [0.0, 1.0, 1.0], [0.5, 0.5, 0.5]]], dtype=torch.float32) vertex_specular_colors = torch.tensor(2*[[[0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 0.0], [1.0, 0.0, 1.0], [0.0, 1.0, 1.0], [0.5, 0.5, 0.5], [1.0, 0.0, 0.0]]], dtype=torch.float32) shininess_coefficients = 6.0 * torch.ones([2, 8], dtype=torch.float32) ambient_color = torch.tensor([[0.0, 0.0, 0.0], [0.1, 0.1, 0.2]], dtype=torch.float32) renders = mesh_renderer.mesh_renderer( vertices_world_space, self.cube_triangles, normals_world_space, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height, vertex_specular_colors, shininess_coefficients, ambient_color, fov_y, near_clip, far_clip) tonemapped_renders = torch.cat([ mesh_renderer.tone_mapper(renders[:, :, :, 0:3], 0.7), renders[:, :, :, 3:4] ], dim=3) # Check that shininess coefficient broadcasting works by also rendering # with a scalar shininess coefficient, and ensuring the result is identical: broadcasted_renders = mesh_renderer.mesh_renderer( vertices_world_space, self.cube_triangles, normals_world_space, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height, vertex_specular_colors, 6.0, ambient_color, fov_y, near_clip, far_clip) tonemapped_broadcasted_renders = torch.cat([ mesh_renderer.tone_mapper(broadcasted_renders[:, :, :, 0:3], 0.7), broadcasted_renders[:, :, :, 3:4] ], dim=3) for image_id in range(renders.shape[0]): target_image_name = "Colored_Cube_%i.png" % image_id baseline_image_path = os.path.join(self.test_data_directory, target_image_name) test_utils.expect_image_file_and_render_are_near( self, baseline_image_path, tonemapped_renders[image_id, :, :, :]) test_utils.expect_image_file_and_render_are_near( self, baseline_image_path, tonemapped_broadcasted_renders[image_id, :, :, :])
def testThatCubeRotates(self): """Optimize a simple cube's rotation using pixel loss. The rotation is represented as static-basis euler angles. This test checks that the computed gradients are useful. """ image_height = 480 image_width = 640 initial_euler_angles = [[0.0, 0.0, 0.0]] euler_angles = torch.tensor(initial_euler_angles, requires_grad=True) model_rotation = camera_utils.euler_matrices(euler_angles)[0, :3, :3] model_rotation.requires_grad = True vertices_world_space = torch.reshape( torch.matmul(self.cube_vertices, model_rotation.transpose()), [1, 8, 3]) normals_world_space = torch.reshape( torch.matmul(self.cube_normals, model_rotation.transpose()), [1, 8, 3]) # camera position: eye = torch.tensor([[0.0, 0.0, 6.0]], dtype=torch.float32) center = torch.tensor([[0.0, 0.0, 0.0]], dtype=torch.float32) world_up = torch.tensor([[0.0, 1.0, 0.0]], dtype=torch.float32) vertex_diffuse_colors = torch.ones_like(vertices_world_space) light_positions = torch.reshape(eye, [1, 1, 3]) light_intensities = torch.ones([1, 1, 3], dtype=torch.float32) # Pick the desired cube rotation for the test: test_model_rotation = camera_utils.euler_matrices([[-20.0, 0.0, 60.0]])[0, :3, :3] desired_vertex_positions = torch.reshape( torch.matmul(self.cube_vertices, test_model_rotation.transpose()) [1, 8, 3]) desired_normals = torch.reshape( torch.matmul(self.cube_normals, test_model_rotation.transpose()), [1, 8, 3]) optimizer = torch.optim.SGD([euler_angles], lr=0.7, momentum=0.1) for _ in range(35): optimizer.zero_grad() render = mesh_renderer.mesh_renderer( vertices_world_space, self.cube_triangles, normals_world_space, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height) desired_render = mesh_renderer.mesh_renderer( desired_vertex_positions, self.cube_triangles, desired_normals, vertex_diffuse_colors, eye, center, world_up, light_positions, light_intensities, image_width, image_height) loss = torch.mean(torch.abs(render - desired_render)) loss.backward() optimizer.step() render = torch.reshape(render, [image_height, image_width, 4]) desired_render = torch.reshape(desired_render, [image_height, image_width, 4]) target_image_name = "Gray_Cube_0.png" baseline_image_path = os.path.join(self.test_data_directory, target_image_name) test_utils.expect_image_file_and_render_are_near( self, baseline_image_path, desired_render) test_utils.expect_image_file_and_render_are_near( self, baseline_image_path, render, max_outlier_fraction=0.01, pixel_error_threshold=0.04)