def test_texture_map(self): """ Test a mesh with a texture map is loaded and rendered correctly. The pupils in the eyes of the cow should always be looking to the left. """ device = torch.device("cuda:0") obj_filename = TUTORIAL_DATA_DIR / "cow_mesh/cow.obj" # Load mesh + texture verts, faces, aux = load_obj(obj_filename, device=device, load_textures=True, texture_wrap=None) tex_map = list(aux.texture_images.values())[0] tex_map = tex_map[None, ...].to(faces.textures_idx.device) textures = TexturesUV(maps=tex_map, faces_uvs=[faces.textures_idx], verts_uvs=[aux.verts_uvs]) mesh = Meshes(verts=[verts], faces=[faces.verts_idx], textures=textures) # Init rasterizer settings R, T = look_at_view_transform(2.7, 0, 0) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) raster_settings = RasterizationSettings(image_size=512, blur_radius=0.0, faces_per_pixel=1) # Init shader settings materials = Materials(device=device) lights = PointLights(device=device) # Place light behind the cow in world space. The front of # the cow is facing the -z direction. lights.location = torch.tensor([0.0, 0.0, 2.0], device=device)[None] blend_params = BlendParams( sigma=1e-1, gamma=1e-4, background_color=torch.tensor([1.0, 1.0, 1.0], device=device), ) # Init renderer renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=TexturedSoftPhongShader( lights=lights, cameras=cameras, materials=materials, blend_params=blend_params, ), ) # Load reference image image_ref = load_rgb_image("test_texture_map_back.png", DATA_DIR) for bin_size in [0, None]: # Check both naive and coarse to fine produce the same output. renderer.rasterizer.raster_settings.bin_size = bin_size images = renderer(mesh) rgb = images[0, ..., :3].squeeze().cpu() if DEBUG: Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save( DATA_DIR / "DEBUG_texture_map_back.png") # NOTE some pixels can be flaky and will not lead to # `cond1` being true. Add `cond2` and check `cond1 or cond2` cond1 = torch.allclose(rgb, image_ref, atol=0.05) cond2 = ((rgb - image_ref).abs() > 0.05).sum() < 5 self.assertTrue(cond1 or cond2) # Check grad exists [verts] = mesh.verts_list() verts.requires_grad = True mesh2 = Meshes(verts=[verts], faces=mesh.faces_list(), textures=mesh.textures) images = renderer(mesh2) images[0, ...].sum().backward() self.assertIsNotNone(verts.grad) ########################################## # Check rendering of the front of the cow ########################################## R, T = look_at_view_transform(2.7, 0, 180) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) # Move light to the front of the cow in world space lights.location = torch.tensor([0.0, 0.0, -2.0], device=device)[None] # Load reference image image_ref = load_rgb_image("test_texture_map_front.png", DATA_DIR) for bin_size in [0, None]: # Check both naive and coarse to fine produce the same output. renderer.rasterizer.raster_settings.bin_size = bin_size images = renderer(mesh, cameras=cameras, lights=lights) rgb = images[0, ..., :3].squeeze().cpu() if DEBUG: Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save( DATA_DIR / "DEBUG_texture_map_front.png") # NOTE some pixels can be flaky and will not lead to # `cond1` being true. Add `cond2` and check `cond1 or cond2` cond1 = torch.allclose(rgb, image_ref, atol=0.05) cond2 = ((rgb - image_ref).abs() > 0.05).sum() < 5 self.assertTrue(cond1 or cond2) ################################# # Add blurring to rasterization ################################# R, T = look_at_view_transform(2.7, 0, 180) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) blend_params = BlendParams(sigma=5e-4, gamma=1e-4) raster_settings = RasterizationSettings( image_size=512, blur_radius=np.log(1.0 / 1e-4 - 1.0) * blend_params.sigma, faces_per_pixel=100, clip_barycentric_coords=True, perspective_correct=False, ) # Load reference image image_ref = load_rgb_image("test_blurry_textured_rendering.png", DATA_DIR) for bin_size in [0, None]: # Check both naive and coarse to fine produce the same output. renderer.rasterizer.raster_settings.bin_size = bin_size images = renderer( mesh.clone(), cameras=cameras, raster_settings=raster_settings, blend_params=blend_params, ) rgb = images[0, ..., :3].squeeze().cpu() if DEBUG: Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save( DATA_DIR / "DEBUG_blurry_textured_rendering.png") self.assertClose(rgb, image_ref, atol=0.05)
def test_batch_uvs(self): """Test that two random tori with TexturesUV render the same as each individually.""" torch.manual_seed(1) device = torch.device("cuda:0") plain_torus = torus(r=1, R=4, sides=10, rings=10, device=device) [verts] = plain_torus.verts_list() [faces] = plain_torus.faces_list() nocolor = torch.zeros((100, 100), device=device) color_gradient = torch.linspace(0, 1, steps=100, device=device) color_gradient1 = color_gradient[None].expand_as(nocolor) color_gradient2 = color_gradient[:, None].expand_as(nocolor) colors1 = torch.stack([nocolor, color_gradient1, color_gradient2], dim=2) colors2 = torch.stack([color_gradient1, color_gradient2, nocolor], dim=2) verts_uvs1 = torch.rand(size=(verts.shape[0], 2), device=device) verts_uvs2 = torch.rand(size=(verts.shape[0], 2), device=device) textures1 = TexturesUV(maps=[colors1], faces_uvs=[faces], verts_uvs=[verts_uvs1]) textures2 = TexturesUV(maps=[colors2], faces_uvs=[faces], verts_uvs=[verts_uvs2]) mesh1 = Meshes(verts=[verts], faces=[faces], textures=textures1) mesh2 = Meshes(verts=[verts], faces=[faces], textures=textures2) mesh_both = join_meshes_as_batch([mesh1, mesh2]) R, T = look_at_view_transform(10, 10, 0) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) raster_settings = RasterizationSettings(image_size=128, blur_radius=0.0, faces_per_pixel=1) # Init shader settings lights = PointLights(device=device) lights.location = torch.tensor([0.0, 0.0, 2.0], device=device)[None] blend_params = BlendParams( sigma=1e-1, gamma=1e-4, background_color=torch.tensor([1.0, 1.0, 1.0], device=device), ) # Init renderer renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, lights=lights, cameras=cameras, blend_params=blend_params), ) outputs = [] for meshes in [mesh_both, mesh1, mesh2]: outputs.append(renderer(meshes)) if DEBUG: Image.fromarray( (outputs[0][0, ..., :3].cpu().numpy() * 255).astype( np.uint8)).save(DATA_DIR / "test_batch_uvs0.png") Image.fromarray( (outputs[1][0, ..., :3].cpu().numpy() * 255).astype( np.uint8)).save(DATA_DIR / "test_batch_uvs1.png") Image.fromarray( (outputs[0][1, ..., :3].cpu().numpy() * 255).astype( np.uint8)).save(DATA_DIR / "test_batch_uvs2.png") Image.fromarray( (outputs[2][0, ..., :3].cpu().numpy() * 255).astype( np.uint8)).save(DATA_DIR / "test_batch_uvs3.png") diff = torch.abs(outputs[0][0, ..., :3] - outputs[1][0, ..., :3]) Image.fromarray(((diff > 1e-5).cpu().numpy().astype(np.uint8) * 255)).save(DATA_DIR / "test_batch_uvs01.png") diff = torch.abs(outputs[0][1, ..., :3] - outputs[2][0, ..., :3]) Image.fromarray(((diff > 1e-5).cpu().numpy().astype(np.uint8) * 255)).save(DATA_DIR / "test_batch_uvs23.png") self.assertClose(outputs[0][0, ..., :3], outputs[1][0, ..., :3], atol=1e-5) self.assertClose(outputs[0][1, ..., :3], outputs[2][0, ..., :3], atol=1e-5)
def test_join_uvs(self): """Meshes with TexturesUV joined into a scene""" # Test the result of rendering three tori with separate textures. # The expected result is consistent with rendering them each alone. # This tests TexturesUV.join_scene with rectangle flipping, # and we check the form of the merged map as well. torch.manual_seed(1) device = torch.device("cuda:0") R, T = look_at_view_transform(18, 0, 0) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) raster_settings = RasterizationSettings( image_size=256, blur_radius=0.0, faces_per_pixel=1 ) lights = AmbientLights(device=device) blend_params = BlendParams( sigma=1e-1, gamma=1e-4, background_color=torch.tensor([1.0, 1.0, 1.0], device=device), ) renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader( device=device, blend_params=blend_params, cameras=cameras, lights=lights ), ) plain_torus = torus(r=1, R=4, sides=5, rings=6, device=device) [verts] = plain_torus.verts_list() verts_shifted1 = verts.clone() verts_shifted1 *= 0.5 verts_shifted1[:, 1] += 7 verts_shifted2 = verts.clone() verts_shifted2 *= 0.5 verts_shifted2[:, 1] -= 7 verts_shifted3 = verts.clone() verts_shifted3 *= 0.5 verts_shifted3[:, 1] -= 700 [faces] = plain_torus.faces_list() nocolor = torch.zeros((100, 100), device=device) color_gradient = torch.linspace(0, 1, steps=100, device=device) color_gradient1 = color_gradient[None].expand_as(nocolor) color_gradient2 = color_gradient[:, None].expand_as(nocolor) colors1 = torch.stack([nocolor, color_gradient1, color_gradient2], dim=2) colors2 = torch.stack([color_gradient1, color_gradient2, nocolor], dim=2) verts_uvs1 = torch.rand(size=(verts.shape[0], 2), device=device) verts_uvs2 = torch.rand(size=(verts.shape[0], 2), device=device) for i, align_corners, padding_mode in [ (0, True, "border"), (1, False, "border"), (2, False, "zeros"), ]: textures1 = TexturesUV( maps=[colors1], faces_uvs=[faces], verts_uvs=[verts_uvs1], align_corners=align_corners, padding_mode=padding_mode, ) # These downsamplings of colors2 are chosen to ensure a flip and a non flip # when the maps are merged. # We have maps of size (100, 100), (50, 99) and (99, 50). textures2 = TexturesUV( maps=[colors2[::2, :-1]], faces_uvs=[faces], verts_uvs=[verts_uvs2], align_corners=align_corners, padding_mode=padding_mode, ) offset = torch.tensor([0, 0, 0.5], device=device) textures3 = TexturesUV( maps=[colors2[:-1, ::2] + offset], faces_uvs=[faces], verts_uvs=[verts_uvs2], align_corners=align_corners, padding_mode=padding_mode, ) mesh1 = Meshes(verts=[verts], faces=[faces], textures=textures1) mesh2 = Meshes(verts=[verts_shifted1], faces=[faces], textures=textures2) mesh3 = Meshes(verts=[verts_shifted2], faces=[faces], textures=textures3) # mesh4 is like mesh1 but outside the field of view. It is here to test # that having another texture with the same map doesn't produce # two copies in the joined map. mesh4 = Meshes(verts=[verts_shifted3], faces=[faces], textures=textures1) mesh = join_meshes_as_scene([mesh1, mesh2, mesh3, mesh4]) output = renderer(mesh)[0, ..., :3].cpu() output1 = renderer(mesh1)[0, ..., :3].cpu() output2 = renderer(mesh2)[0, ..., :3].cpu() output3 = renderer(mesh3)[0, ..., :3].cpu() # The background color is white and the objects do not overlap, so we can # predict the merged image by taking the minimum over every channel merged = torch.min(torch.min(output1, output2), output3) image_ref = load_rgb_image(f"test_joinuvs{i}_final.png", DATA_DIR) map_ref = load_rgb_image(f"test_joinuvs{i}_map.png", DATA_DIR) if DEBUG: Image.fromarray((output.numpy() * 255).astype(np.uint8)).save( DATA_DIR / f"test_joinuvs{i}_final_.png" ) Image.fromarray((merged.numpy() * 255).astype(np.uint8)).save( DATA_DIR / f"test_joinuvs{i}_merged.png" ) Image.fromarray((output1.numpy() * 255).astype(np.uint8)).save( DATA_DIR / f"test_joinuvs{i}_1.png" ) Image.fromarray((output2.numpy() * 255).astype(np.uint8)).save( DATA_DIR / f"test_joinuvs{i}_2.png" ) Image.fromarray((output3.numpy() * 255).astype(np.uint8)).save( DATA_DIR / f"test_joinuvs{i}_3.png" ) Image.fromarray( (mesh.textures.maps_padded()[0].cpu().numpy() * 255).astype( np.uint8 ) ).save(DATA_DIR / f"test_joinuvs{i}_map_.png") Image.fromarray( (mesh2.textures.maps_padded()[0].cpu().numpy() * 255).astype( np.uint8 ) ).save(DATA_DIR / f"test_joinuvs{i}_map2.png") Image.fromarray( (mesh3.textures.maps_padded()[0].cpu().numpy() * 255).astype( np.uint8 ) ).save(DATA_DIR / f"test_joinuvs{i}_map3.png") self.assertClose(output, merged) self.assertClose(output, image_ref, atol=0.005) self.assertClose(mesh.textures.maps_padded()[0].cpu(), map_ref, atol=0.05)