def test_texture_map(self):
"""
Test a mesh with a texture map is loaded and rendered correctly.
The pupils in the eyes of the cow should always be looking to the left.
"""
device = torch.device("cuda:0")
obj_filename = TUTORIAL_DATA_DIR / "cow_mesh/cow.obj"
# Load mesh + texture
verts, faces, aux = load_obj(obj_filename,
device=device,
load_textures=True,
texture_wrap=None)
tex_map = list(aux.texture_images.values())[0]
tex_map = tex_map[None, ...].to(faces.textures_idx.device)
textures = TexturesUV(maps=tex_map,
faces_uvs=[faces.textures_idx],
verts_uvs=[aux.verts_uvs])
mesh = Meshes(verts=[verts],
faces=[faces.verts_idx],
textures=textures)
# Init rasterizer settings
R, T = look_at_view_transform(2.7, 0, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(image_size=512,
blur_radius=0.0,
faces_per_pixel=1)
# Init shader settings
materials = Materials(device=device)
lights = PointLights(device=device)
# Place light behind the cow in world space. The front of
# the cow is facing the -z direction.
lights.location = torch.tensor([0.0, 0.0, 2.0], device=device)[None]
blend_params = BlendParams(
sigma=1e-1,
gamma=1e-4,
background_color=torch.tensor([1.0, 1.0, 1.0], device=device),
)
# Init renderer
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=TexturedSoftPhongShader(
lights=lights,
cameras=cameras,
materials=materials,
blend_params=blend_params,
),
)
# Load reference image
image_ref = load_rgb_image("test_texture_map_back.png", DATA_DIR)
for bin_size in [0, None]:
# Check both naive and coarse to fine produce the same output.
renderer.rasterizer.raster_settings.bin_size = bin_size
images = renderer(mesh)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / "DEBUG_texture_map_back.png")
# NOTE some pixels can be flaky and will not lead to
# `cond1` being true. Add `cond2` and check `cond1 or cond2`
cond1 = torch.allclose(rgb, image_ref, atol=0.05)
cond2 = ((rgb - image_ref).abs() > 0.05).sum() < 5
self.assertTrue(cond1 or cond2)
# Check grad exists
[verts] = mesh.verts_list()
verts.requires_grad = True
mesh2 = Meshes(verts=[verts],
faces=mesh.faces_list(),
textures=mesh.textures)
images = renderer(mesh2)
images[0, ...].sum().backward()
self.assertIsNotNone(verts.grad)
##########################################
# Check rendering of the front of the cow
##########################################
R, T = look_at_view_transform(2.7, 0, 180)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
# Move light to the front of the cow in world space
lights.location = torch.tensor([0.0, 0.0, -2.0], device=device)[None]
# Load reference image
image_ref = load_rgb_image("test_texture_map_front.png", DATA_DIR)
for bin_size in [0, None]:
# Check both naive and coarse to fine produce the same output.
renderer.rasterizer.raster_settings.bin_size = bin_size
images = renderer(mesh, cameras=cameras, lights=lights)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / "DEBUG_texture_map_front.png")
# NOTE some pixels can be flaky and will not lead to
# `cond1` being true. Add `cond2` and check `cond1 or cond2`
cond1 = torch.allclose(rgb, image_ref, atol=0.05)
cond2 = ((rgb - image_ref).abs() > 0.05).sum() < 5
self.assertTrue(cond1 or cond2)
#################################
# Add blurring to rasterization
#################################
R, T = look_at_view_transform(2.7, 0, 180)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
blend_params = BlendParams(sigma=5e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=np.log(1.0 / 1e-4 - 1.0) * blend_params.sigma,
faces_per_pixel=100,
clip_barycentric_coords=True,
perspective_correct=False,
)
# Load reference image
image_ref = load_rgb_image("test_blurry_textured_rendering.png",
DATA_DIR)
for bin_size in [0, None]:
# Check both naive and coarse to fine produce the same output.
renderer.rasterizer.raster_settings.bin_size = bin_size
images = renderer(
mesh.clone(),
cameras=cameras,
raster_settings=raster_settings,
blend_params=blend_params,
)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / "DEBUG_blurry_textured_rendering.png")
self.assertClose(rgb, image_ref, atol=0.05)
def test_batch_uvs(self):
"""Test that two random tori with TexturesUV render the same as each individually."""
torch.manual_seed(1)
device = torch.device("cuda:0")
plain_torus = torus(r=1, R=4, sides=10, rings=10, device=device)
[verts] = plain_torus.verts_list()
[faces] = plain_torus.faces_list()
nocolor = torch.zeros((100, 100), device=device)
color_gradient = torch.linspace(0, 1, steps=100, device=device)
color_gradient1 = color_gradient[None].expand_as(nocolor)
color_gradient2 = color_gradient[:, None].expand_as(nocolor)
colors1 = torch.stack([nocolor, color_gradient1, color_gradient2],
dim=2)
colors2 = torch.stack([color_gradient1, color_gradient2, nocolor],
dim=2)
verts_uvs1 = torch.rand(size=(verts.shape[0], 2), device=device)
verts_uvs2 = torch.rand(size=(verts.shape[0], 2), device=device)
textures1 = TexturesUV(maps=[colors1],
faces_uvs=[faces],
verts_uvs=[verts_uvs1])
textures2 = TexturesUV(maps=[colors2],
faces_uvs=[faces],
verts_uvs=[verts_uvs2])
mesh1 = Meshes(verts=[verts], faces=[faces], textures=textures1)
mesh2 = Meshes(verts=[verts], faces=[faces], textures=textures2)
mesh_both = join_meshes_as_batch([mesh1, mesh2])
R, T = look_at_view_transform(10, 10, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(image_size=128,
blur_radius=0.0,
faces_per_pixel=1)
# Init shader settings
lights = PointLights(device=device)
lights.location = torch.tensor([0.0, 0.0, 2.0], device=device)[None]
blend_params = BlendParams(
sigma=1e-1,
gamma=1e-4,
background_color=torch.tensor([1.0, 1.0, 1.0], device=device),
)
# Init renderer
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=HardPhongShader(device=device,
lights=lights,
cameras=cameras,
blend_params=blend_params),
)
outputs = []
for meshes in [mesh_both, mesh1, mesh2]:
outputs.append(renderer(meshes))
if DEBUG:
Image.fromarray(
(outputs[0][0, ..., :3].cpu().numpy() * 255).astype(
np.uint8)).save(DATA_DIR / "test_batch_uvs0.png")
Image.fromarray(
(outputs[1][0, ..., :3].cpu().numpy() * 255).astype(
np.uint8)).save(DATA_DIR / "test_batch_uvs1.png")
Image.fromarray(
(outputs[0][1, ..., :3].cpu().numpy() * 255).astype(
np.uint8)).save(DATA_DIR / "test_batch_uvs2.png")
Image.fromarray(
(outputs[2][0, ..., :3].cpu().numpy() * 255).astype(
np.uint8)).save(DATA_DIR / "test_batch_uvs3.png")
diff = torch.abs(outputs[0][0, ..., :3] - outputs[1][0, ..., :3])
Image.fromarray(((diff > 1e-5).cpu().numpy().astype(np.uint8) *
255)).save(DATA_DIR / "test_batch_uvs01.png")
diff = torch.abs(outputs[0][1, ..., :3] - outputs[2][0, ..., :3])
Image.fromarray(((diff > 1e-5).cpu().numpy().astype(np.uint8) *
255)).save(DATA_DIR / "test_batch_uvs23.png")
self.assertClose(outputs[0][0, ..., :3],
outputs[1][0, ..., :3],
atol=1e-5)
self.assertClose(outputs[0][1, ..., :3],
outputs[2][0, ..., :3],
atol=1e-5)
def test_join_uvs(self):
"""Meshes with TexturesUV joined into a scene"""
# Test the result of rendering three tori with separate textures.
# The expected result is consistent with rendering them each alone.
# This tests TexturesUV.join_scene with rectangle flipping,
# and we check the form of the merged map as well.
torch.manual_seed(1)
device = torch.device("cuda:0")
R, T = look_at_view_transform(18, 0, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=256, blur_radius=0.0, faces_per_pixel=1
)
lights = AmbientLights(device=device)
blend_params = BlendParams(
sigma=1e-1,
gamma=1e-4,
background_color=torch.tensor([1.0, 1.0, 1.0], device=device),
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(
device=device, blend_params=blend_params, cameras=cameras, lights=lights
),
)
plain_torus = torus(r=1, R=4, sides=5, rings=6, device=device)
[verts] = plain_torus.verts_list()
verts_shifted1 = verts.clone()
verts_shifted1 *= 0.5
verts_shifted1[:, 1] += 7
verts_shifted2 = verts.clone()
verts_shifted2 *= 0.5
verts_shifted2[:, 1] -= 7
verts_shifted3 = verts.clone()
verts_shifted3 *= 0.5
verts_shifted3[:, 1] -= 700
[faces] = plain_torus.faces_list()
nocolor = torch.zeros((100, 100), device=device)
color_gradient = torch.linspace(0, 1, steps=100, device=device)
color_gradient1 = color_gradient[None].expand_as(nocolor)
color_gradient2 = color_gradient[:, None].expand_as(nocolor)
colors1 = torch.stack([nocolor, color_gradient1, color_gradient2], dim=2)
colors2 = torch.stack([color_gradient1, color_gradient2, nocolor], dim=2)
verts_uvs1 = torch.rand(size=(verts.shape[0], 2), device=device)
verts_uvs2 = torch.rand(size=(verts.shape[0], 2), device=device)
for i, align_corners, padding_mode in [
(0, True, "border"),
(1, False, "border"),
(2, False, "zeros"),
]:
textures1 = TexturesUV(
maps=[colors1],
faces_uvs=[faces],
verts_uvs=[verts_uvs1],
align_corners=align_corners,
padding_mode=padding_mode,
)
# These downsamplings of colors2 are chosen to ensure a flip and a non flip
# when the maps are merged.
# We have maps of size (100, 100), (50, 99) and (99, 50).
textures2 = TexturesUV(
maps=[colors2[::2, :-1]],
faces_uvs=[faces],
verts_uvs=[verts_uvs2],
align_corners=align_corners,
padding_mode=padding_mode,
)
offset = torch.tensor([0, 0, 0.5], device=device)
textures3 = TexturesUV(
maps=[colors2[:-1, ::2] + offset],
faces_uvs=[faces],
verts_uvs=[verts_uvs2],
align_corners=align_corners,
padding_mode=padding_mode,
)
mesh1 = Meshes(verts=[verts], faces=[faces], textures=textures1)
mesh2 = Meshes(verts=[verts_shifted1], faces=[faces], textures=textures2)
mesh3 = Meshes(verts=[verts_shifted2], faces=[faces], textures=textures3)
# mesh4 is like mesh1 but outside the field of view. It is here to test
# that having another texture with the same map doesn't produce
# two copies in the joined map.
mesh4 = Meshes(verts=[verts_shifted3], faces=[faces], textures=textures1)
mesh = join_meshes_as_scene([mesh1, mesh2, mesh3, mesh4])
output = renderer(mesh)[0, ..., :3].cpu()
output1 = renderer(mesh1)[0, ..., :3].cpu()
output2 = renderer(mesh2)[0, ..., :3].cpu()
output3 = renderer(mesh3)[0, ..., :3].cpu()
# The background color is white and the objects do not overlap, so we can
# predict the merged image by taking the minimum over every channel
merged = torch.min(torch.min(output1, output2), output3)
image_ref = load_rgb_image(f"test_joinuvs{i}_final.png", DATA_DIR)
map_ref = load_rgb_image(f"test_joinuvs{i}_map.png", DATA_DIR)
if DEBUG:
Image.fromarray((output.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_final_.png"
)
Image.fromarray((merged.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_merged.png"
)
Image.fromarray((output1.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_1.png"
)
Image.fromarray((output2.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_2.png"
)
Image.fromarray((output3.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_3.png"
)
Image.fromarray(
(mesh.textures.maps_padded()[0].cpu().numpy() * 255).astype(
np.uint8
)
).save(DATA_DIR / f"test_joinuvs{i}_map_.png")
Image.fromarray(
(mesh2.textures.maps_padded()[0].cpu().numpy() * 255).astype(
np.uint8
)
).save(DATA_DIR / f"test_joinuvs{i}_map2.png")
Image.fromarray(
(mesh3.textures.maps_padded()[0].cpu().numpy() * 255).astype(
np.uint8
)
).save(DATA_DIR / f"test_joinuvs{i}_map3.png")
self.assertClose(output, merged)
self.assertClose(output, image_ref, atol=0.005)
self.assertClose(mesh.textures.maps_padded()[0].cpu(), map_ref, atol=0.05)
