def test_perspective(self):
far = 10.0
near = 1.0
cameras = FoVPerspectiveCameras(znear=near, zfar=far, fov=60.0)
P = cameras.get_projection_transform()
# vertices are at the far clipping plane so z gets mapped to 1.
vertices = torch.tensor([1, 2, far], dtype=torch.float32)
projected_verts = torch.tensor(
[np.sqrt(3) / far, 2 * np.sqrt(3) / far, 1.0], dtype=torch.float32)
vertices = vertices[None, None, :]
v1 = P.transform_points(vertices)
v2 = perspective_project_naive(vertices, fov=60.0)
self.assertClose(v1[..., :2], v2[..., :2])
self.assertClose(far * v1[..., 2], v2[..., 2])
self.assertClose(v1.squeeze(), projected_verts)
# vertices are at the near clipping plane so z gets mapped to 0.0.
vertices[..., 2] = near
projected_verts = torch.tensor(
[np.sqrt(3) / near, 2 * np.sqrt(3) / near, 0.0],
dtype=torch.float32)
v1 = P.transform_points(vertices)
v2 = perspective_project_naive(vertices, fov=60.0)
self.assertClose(v1[..., :2], v2[..., :2])
self.assertClose(v1.squeeze(), projected_verts)
def test_get_full_transform(self):
cam = FoVPerspectiveCameras()
T = torch.tensor([0.0, 0.0, 1.0]).view(1, -1)
R = look_at_rotation(T)
P = cam.get_full_projection_transform(R=R, T=T)
self.assertTrue(isinstance(P, Transform3d))
self.assertClose(cam.R, R)
self.assertClose(cam.T, T)
def test_perspective_kwargs(self):
cameras = FoVPerspectiveCameras(znear=5.0, zfar=100.0, fov=0.0)
# Override defaults by passing in values to get_projection_transform
far = 10.0
P = cameras.get_projection_transform(znear=1.0, zfar=far, fov=60.0)
vertices = torch.tensor([1, 2, far], dtype=torch.float32)
projected_verts = torch.tensor(
[np.sqrt(3) / far, 2 * np.sqrt(3) / far, 1.0], dtype=torch.float32)
vertices = vertices[None, None, :]
v1 = P.transform_points(vertices)
self.assertClose(v1.squeeze(), projected_verts)
def test_camera_class_init(self):
device = torch.device("cuda:0")
cam = FoVPerspectiveCameras(znear=10.0, zfar=(100.0, 200.0))
# Check broadcasting
self.assertTrue(cam.znear.shape == (2,))
self.assertTrue(cam.zfar.shape == (2,))
# Test to
new_cam = cam.to(device=device)
self.assertTrue(new_cam.device == device)
def test_transform_points(self):
# Check transform_points methods works with default settings for
# RT and P
far = 10.0
cam = FoVPerspectiveCameras(znear=1.0, zfar=far, fov=60.0)
points = torch.tensor([1, 2, far], dtype=torch.float32)
points = points.view(1, 1, 3).expand(5, 10, -1)
projected_points = torch.tensor(
[np.sqrt(3) / far, 2 * np.sqrt(3) / far, 1.0], dtype=torch.float32)
projected_points = projected_points.view(1, 1, 3).expand(5, 10, -1)
new_points = cam.transform_points(points)
self.assertClose(new_points, projected_points)
def test_simple_sphere_batched(self):
device = torch.device("cuda:0")
sphere_mesh = ico_sphere(1, device)
verts_padded = sphere_mesh.verts_padded()
verts_padded[..., 1] += 0.2
verts_padded[..., 0] += 0.2
pointclouds = Pointclouds(
points=verts_padded, features=torch.ones_like(verts_padded)
)
batch_size = 20
pointclouds = pointclouds.extend(batch_size)
R, T = look_at_view_transform(2.7, 0.0, 0.0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = PointsRasterizationSettings(
image_size=256, radius=5e-2, points_per_pixel=1
)
rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings)
compositor = NormWeightedCompositor()
renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor)
# Load reference image
filename = "simple_pointcloud_sphere.png"
image_ref = load_rgb_image("test_%s" % filename, DATA_DIR)
images = renderer(pointclouds)
for i in range(batch_size):
rgb = images[i, ..., :3].squeeze().cpu()
if i == 0 and DEBUG:
filename = "DEBUG_%s" % filename
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / filename
)
self.assertClose(rgb, image_ref)
def test_simple_sphere(self):
device = torch.device("cuda:0")
sphere_mesh = ico_sphere(1, device)
verts_padded = sphere_mesh.verts_padded()
# Shift vertices to check coordinate frames are correct.
verts_padded[..., 1] += 0.2
verts_padded[..., 0] += 0.2
pointclouds = Pointclouds(
points=verts_padded, features=torch.ones_like(verts_padded)
)
R, T = look_at_view_transform(2.7, 0.0, 0.0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = PointsRasterizationSettings(
image_size=256, radius=5e-2, points_per_pixel=1
)
rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings)
compositor = NormWeightedCompositor()
renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor)
# Load reference image
filename = "simple_pointcloud_sphere.png"
image_ref = load_rgb_image("test_%s" % filename, DATA_DIR)
for bin_size in [0, None]:
# Check both naive and coarse to fine produce the same output.
renderer.rasterizer.raster_settings.bin_size = bin_size
images = renderer(pointclouds)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
filename = "DEBUG_%s" % filename
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / filename
)
self.assertClose(rgb, image_ref)
def baryclip_cuda(
num_meshes: int = 8,
ico_level: int = 5,
image_size: int = 64,
faces_per_pixel: int = 50,
device="cuda",
):
# Init meshes
sphere_meshes = ico_sphere(ico_level, device).extend(num_meshes)
# Init transform
R, T = look_at_view_transform(1.0, 0.0, 0.0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
# Init rasterizer
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=1e-4,
faces_per_pixel=faces_per_pixel,
clip_barycentric_coords=True,
)
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
torch.cuda.synchronize()
def raster_fn():
rasterizer(sphere_meshes)
torch.cuda.synchronize()
return raster_fn
def test_grad(self):
"""
Check that gradient flow is unaffected when the camera is inside the mesh
"""
device = torch.device("cuda:0")
mesh, verts = self.load_cube_mesh_with_texture(device=device,
with_grad=True)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=1e-5,
faces_per_pixel=5,
z_clip_value=1e-2,
perspective_correct=True,
bin_size=0,
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(raster_settings=raster_settings),
shader=SoftPhongShader(device=device),
)
dist = 0.4 # Camera is inside the cube
R, T = look_at_view_transform(dist, 0, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T, fov=90)
images = renderer(mesh, cameras=cameras)
images.sum().backward()
# Check gradients exist
self.assertIsNotNone(verts.grad)
def test_render_pointcloud(self):
"""
Test a textured point cloud is rendered correctly in a non square image.
"""
device = torch.device("cuda:0")
pointclouds = Pointclouds(
points=[torus_points * 2.0],
features=torch.ones_like(torus_points[None, ...]),
).to(device)
R, T = look_at_view_transform(2.7, 0.0, 0.0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = PointsRasterizationSettings(
image_size=(512, 1024), radius=5e-2, points_per_pixel=1
)
rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings)
compositor = AlphaCompositor()
renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor)
# Load reference image
image_ref = load_rgb_image("test_pointcloud_rectangle_image.png", DATA_DIR)
for bin_size in [0, None]:
# Check both naive and coarse to fine produce the same output.
renderer.rasterizer.raster_settings.bin_size = bin_size
images = renderer(pointclouds)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / "DEBUG_pointcloud_rectangle_image.png"
)
# NOTE some pixels can be flaky
cond1 = torch.allclose(rgb, image_ref, atol=0.05)
self.assertTrue(cond1)
def test_simple_sphere_outside_zfar(self):
"""
Test output when rendering a sphere that is beyond zfar with a SoftPhongShader.
This renders a sphere of radius 500, with the camera at x=1500 for different
settings of zfar. This is intended to check 1) setting cameras.zfar propagates
to the blender and that the rendered sphere is (soft) clipped if it is beyond
zfar, 2) make sure there are no numerical precision/overflow errors associated
with larger world coordinates
"""
device = torch.device("cuda:0")
# Init mesh
sphere_mesh = ico_sphere(5, device)
verts_padded = sphere_mesh.verts_padded() * 500
faces_padded = sphere_mesh.faces_padded()
feats = torch.ones_like(verts_padded, device=device)
textures = TexturesVertex(verts_features=feats)
sphere_mesh = Meshes(verts=verts_padded, faces=faces_padded, textures=textures)
R, T = look_at_view_transform(1500, 0.0, 0.0)
# Init shader settings
materials = Materials(device=device)
lights = PointLights(device=device)
lights.location = torch.tensor([0.0, 0.0, +1000.0], device=device)[None]
raster_settings = RasterizationSettings(
image_size=256, blur_radius=0.0, faces_per_pixel=1
)
for zfar in (10000.0, 100.0):
cameras = FoVPerspectiveCameras(
device=device, R=R, T=T, aspect_ratio=1.0, fov=60.0, zfar=zfar
)
rasterizer = MeshRasterizer(
cameras=cameras, raster_settings=raster_settings
)
blend_params = BlendParams(1e-4, 1e-4, (0, 0, 1.0))
shader = SoftPhongShader(
lights=lights,
cameras=cameras,
materials=materials,
blend_params=blend_params,
)
renderer = MeshRenderer(rasterizer=rasterizer, shader=shader)
images = renderer(sphere_mesh)
rgb = images[0, ..., :3].squeeze().cpu()
filename = "test_simple_sphere_outside_zfar_%d.png" % int(zfar)
# Load reference image
image_ref = load_rgb_image(filename, DATA_DIR)
if DEBUG:
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / ("DEBUG_" + filename)
)
self.assertClose(rgb, image_ref, atol=0.05)
def test_join_verts(self):
"""Meshes with TexturesVertex joined into a scene"""
# Test the result of rendering two tori with separate textures.
# The expected result is consistent with rendering them each alone.
torch.manual_seed(1)
device = torch.device("cuda:0")
plain_torus = torus(r=1, R=4, sides=5, rings=6, device=device)
[verts] = plain_torus.verts_list()
verts_shifted1 = verts.clone()
verts_shifted1 *= 0.5
verts_shifted1[:, 1] += 7
faces = plain_torus.faces_list()
textures1 = TexturesVertex(verts_features=[torch.rand_like(verts)])
textures2 = TexturesVertex(verts_features=[torch.rand_like(verts)])
mesh1 = Meshes(verts=[verts], faces=faces, textures=textures1)
mesh2 = Meshes(verts=[verts_shifted1], faces=faces, textures=textures2)
mesh = join_meshes_as_scene([mesh1, mesh2])
R, T = look_at_view_transform(18, 0, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=256, blur_radius=0.0, faces_per_pixel=1
)
lights = AmbientLights(device=device)
blend_params = BlendParams(
sigma=1e-1,
gamma=1e-4,
background_color=torch.tensor([1.0, 1.0, 1.0], device=device),
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(
device=device, blend_params=blend_params, cameras=cameras, lights=lights
),
)
output = renderer(mesh)
image_ref = load_rgb_image("test_joinverts_final.png", DATA_DIR)
if DEBUG:
debugging_outputs = []
for mesh_ in [mesh1, mesh2]:
debugging_outputs.append(renderer(mesh_))
Image.fromarray(
(output[0, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_joinverts_final_.png")
Image.fromarray(
(debugging_outputs[0][0, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_joinverts_1.png")
Image.fromarray(
(debugging_outputs[1][0, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_joinverts_2.png")
result = output[0, ..., :3].cpu()
self.assertClose(result, image_ref, atol=0.05)
def test_perspective_mixed_inputs_grad(self):
far = torch.tensor([10.0])
near = 1.0
fov = torch.tensor(60.0, requires_grad=True)
cameras = FoVPerspectiveCameras(znear=near, zfar=far, fov=fov)
P = cameras.get_projection_transform()
vertices = torch.tensor([1, 2, 10], dtype=torch.float32)
vertices_batch = vertices[None, None, :]
v1 = P.transform_points(vertices_batch).squeeze()
v1.sum().backward()
self.assertTrue(hasattr(fov, "grad"))
fov_grad = fov.grad.clone()
half_fov_rad = (math.pi / 180.0) * fov.detach() / 2.0
grad_cotan = -(1.0 / (torch.sin(half_fov_rad)**2.0) * 1 / 2.0)
grad_fov = (math.pi / 180.0) * grad_cotan
grad_fov = (vertices[0] + vertices[1]) * grad_fov / 10.0
self.assertClose(fov_grad, grad_fov)
def test_texture_sampling_cow(self):
# test texture sampling for the cow example by converting
# the cow mesh and its texture uv to a pointcloud with texture
device = torch.device("cuda:0")
obj_dir = get_pytorch3d_dir() / "docs/tutorials/data"
obj_filename = obj_dir / "cow_mesh/cow.obj"
for text_type in ("uv", "atlas"):
# Load mesh + texture
if text_type == "uv":
mesh = load_objs_as_meshes(
[obj_filename], device=device, load_textures=True, texture_wrap=None
)
elif text_type == "atlas":
mesh = load_objs_as_meshes(
[obj_filename],
device=device,
load_textures=True,
create_texture_atlas=True,
texture_atlas_size=8,
texture_wrap=None,
)
points, normals, textures = sample_points_from_meshes(
mesh, num_samples=50000, return_normals=True, return_textures=True
)
pointclouds = Pointclouds(points, normals=normals, features=textures)
for pos in ("front", "back"):
# Init rasterizer settings
if pos == "back":
azim = 0.0
elif pos == "front":
azim = 180
R, T = look_at_view_transform(2.7, 0, azim)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = PointsRasterizationSettings(
image_size=512, radius=1e-2, points_per_pixel=1
)
rasterizer = PointsRasterizer(
cameras=cameras, raster_settings=raster_settings
)
compositor = NormWeightedCompositor()
renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor)
images = renderer(pointclouds)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
filename = "DEBUG_cow_mesh_to_pointcloud_%s_%s.png" % (
text_type,
pos,
)
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / filename
)
def test_mesh_renderer_to(self):
"""
Test moving all the tensors in the mesh renderer to a new device.
"""
device1 = torch.device("cpu")
R, T = look_at_view_transform(1500, 0.0, 0.0)
# Init shader settings
materials = Materials(device=device1)
lights = PointLights(device=device1)
lights.location = torch.tensor([0.0, 0.0, +1000.0], device=device1)[None]
raster_settings = RasterizationSettings(
image_size=256, blur_radius=0.0, faces_per_pixel=1
)
cameras = FoVPerspectiveCameras(
device=device1, R=R, T=T, aspect_ratio=1.0, fov=60.0, zfar=100
)
rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings)
blend_params = BlendParams(
1e-4,
1e-4,
background_color=torch.zeros(3, dtype=torch.float32, device=device1),
)
shader = SoftPhongShader(
lights=lights,
cameras=cameras,
materials=materials,
blend_params=blend_params,
)
renderer = MeshRenderer(rasterizer=rasterizer, shader=shader)
mesh = ico_sphere(2, device1)
verts_padded = mesh.verts_padded()
textures = TexturesVertex(
verts_features=torch.ones_like(verts_padded, device=device1)
)
mesh.textures = textures
self._check_mesh_renderer_props_on_device(renderer, device1)
# Test rendering on cpu
output_images = renderer(mesh)
self.assertEqual(output_images.device, device1)
# Move renderer and mesh to another device and re render
# This also tests that background_color is correctly moved to
# the new device
device2 = torch.device("cuda:0")
renderer = renderer.to(device2)
mesh = mesh.to(device2)
self._check_mesh_renderer_props_on_device(renderer, device2)
output_images = renderer(mesh)
self.assertEqual(output_images.device, device2)
def test_camera_class_init(self):
device = torch.device("cuda:0")
cam = FoVPerspectiveCameras(znear=10.0, zfar=(100.0, 200.0))
# Check broadcasting
self.assertTrue(cam.znear.shape == (2, ))
self.assertTrue(cam.zfar.shape == (2, ))
# update znear element 1
cam[1].znear = 20.0
self.assertTrue(cam.znear[1] == 20.0)
# Get item and get value
c0 = cam[0]
self.assertTrue(c0.zfar == 100.0)
# Test to
new_cam = cam.to(device=device)
self.assertTrue(new_cam.device == device)
def test_perspective_mixed_inputs_broadcast(self):
far = torch.tensor([10.0, 20.0], dtype=torch.float32)
near = 1.0
fov = torch.tensor(60.0)
cameras = FoVPerspectiveCameras(znear=near, zfar=far, fov=fov)
P = cameras.get_projection_transform()
vertices = torch.tensor([1, 2, 10], dtype=torch.float32)
z1 = 1.0 # vertices at far clipping plane so z = 1.0
z2 = (20.0 / (20.0 - 1.0) * 10.0 + -20.0 / (20.0 - 1.0)) / 10.0
projected_verts = torch.tensor(
[
[np.sqrt(3) / 10.0, 2 * np.sqrt(3) / 10.0, z1],
[np.sqrt(3) / 10.0, 2 * np.sqrt(3) / 10.0, z2],
],
dtype=torch.float32,
)
vertices = vertices[None, None, :]
v1 = P.transform_points(vertices)
v2 = perspective_project_naive(vertices, fov=60.0)
self.assertClose(v1[..., :2], torch.cat([v2, v2])[..., :2])
self.assertClose(v1.squeeze(), projected_verts)
def test_case_4_no_duplicates(self):
"""
In the case of an simple mesh with one face that is cut by the image
plane into a quadrilateral, there shouldn't be duplicates indices of
the face in the pix_to_face output of rasterization.
"""
for (device, bin_size) in [("cpu", 0), ("cuda:0", 0),
("cuda:0", None)]:
verts = torch.tensor(
[[0.0, -10.0, 1.0], [-1.0, 2.0, -2.0], [1.0, 5.0, -10.0]],
dtype=torch.float32,
device=device,
)
faces = torch.tensor(
[
[0, 1, 2],
],
dtype=torch.int64,
device=device,
)
meshes = Meshes(verts=[verts], faces=[faces])
k = 3
settings = RasterizationSettings(
image_size=10,
blur_radius=0.05,
faces_per_pixel=k,
z_clip_value=1e-2,
perspective_correct=True,
cull_to_frustum=True,
bin_size=bin_size,
)
# The camera is positioned so that the image plane cuts
# the mesh face into a quadrilateral.
R, T = look_at_view_transform(0.2, 0, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T, fov=90)
rasterizer = MeshRasterizer(raster_settings=settings,
cameras=cameras)
fragments = rasterizer(meshes)
p2f = fragments.pix_to_face.reshape(-1, k)
unique_vals, idx_counts = p2f.unique(dim=0, return_counts=True)
# There is only one face in this mesh so if it hits a pixel
# it can only be at position k = 0
# For any pixel, the values [0, 0, 1] for the top K faces cannot be possible
double_hit = torch.tensor([0, 0, -1], device=device)
check_double_hit = any(
torch.allclose(i, double_hit) for i in unique_vals)
self.assertFalse(check_double_hit)
def rasterize_transform_with_init(num_meshes: int, ico_level: int = 5, device="cuda"):
# Init meshes
sphere_meshes = ico_sphere(ico_level, device).extend(num_meshes)
# Init transform
R, T = look_at_view_transform(1.0, 0.0, 0.0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
# Init rasterizer
rasterizer = MeshRasterizer(cameras=cameras)
torch.cuda.synchronize()
def raster_fn():
rasterizer.transform(sphere_meshes)
torch.cuda.synchronize()
return raster_fn
def init_render(self):
cameras = FoVPerspectiveCameras()
raster_settings = RasterizationSettings(image_size=128,
blur_radius=0.0,
faces_per_pixel=1)
lights = PointLights(
ambient_color=((1.0, 1.0, 1.0), ),
diffuse_color=((0, 0.0, 0), ),
specular_color=((0.0, 0, 0), ),
location=((0.0, 0.0, 1e5), ),
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=HardGouraudShader(cameras=cameras, lights=lights),
)
return renderer
def test_simple_sphere_pulsar(self):
for device in [torch.device("cpu"), torch.device("cuda")]:
sphere_mesh = ico_sphere(1, device)
verts_padded = sphere_mesh.verts_padded()
# Shift vertices to check coordinate frames are correct.
verts_padded[..., 1] += 0.2
verts_padded[..., 0] += 0.2
pointclouds = Pointclouds(
points=verts_padded, features=torch.ones_like(verts_padded)
)
for azimuth in [0.0, 90.0]:
R, T = look_at_view_transform(2.7, 0.0, azimuth)
for camera_name, cameras in [
("fovperspective", FoVPerspectiveCameras(device=device, R=R, T=T)),
(
"fovorthographic",
FoVOrthographicCameras(device=device, R=R, T=T),
),
("perspective", PerspectiveCameras(device=device, R=R, T=T)),
("orthographic", OrthographicCameras(device=device, R=R, T=T)),
]:
raster_settings = PointsRasterizationSettings(
image_size=256, radius=5e-2, points_per_pixel=1
)
rasterizer = PointsRasterizer(
cameras=cameras, raster_settings=raster_settings
)
renderer = PulsarPointsRenderer(rasterizer=rasterizer).to(device)
# Load reference image
filename = (
"pulsar_simple_pointcloud_sphere_"
f"azimuth{azimuth}_{camera_name}.png"
)
image_ref = load_rgb_image("test_%s" % filename, DATA_DIR)
images = renderer(
pointclouds, gamma=(1e-3,), znear=(1.0,), zfar=(100.0,)
)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
filename = "DEBUG_%s" % filename
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / filename
)
self.assertClose(rgb, image_ref, rtol=7e-3, atol=5e-3)
def test_getitem(self):
R_matrix = torch.randn((6, 3, 3))
cam = FoVPerspectiveCameras(znear=10.0, zfar=100.0, R=R_matrix)
# Check get item returns an instance of the same class
# with all the same keys
c0 = cam[0]
self.assertTrue(isinstance(c0, FoVPerspectiveCameras))
self.assertEqual(cam.__dict__.keys(), c0.__dict__.keys())
# Check all fields correct in get item with int index
self.assertEqual(len(c0), 1)
self.assertClose(c0.zfar, torch.tensor([100.0]))
self.assertClose(c0.znear, torch.tensor([10.0]))
self.assertClose(c0.R, R_matrix[0:1, ...])
self.assertEqual(c0.device, torch.device("cpu"))
# Check list(int) index
c012 = cam[[0, 1, 2]]
self.assertEqual(len(c012), 3)
self.assertClose(c012.zfar, torch.tensor([100.0] * 3))
self.assertClose(c012.znear, torch.tensor([10.0] * 3))
self.assertClose(c012.R, R_matrix[0:3, ...])
# Check torch.LongTensor index
index = torch.tensor([1, 3, 5], dtype=torch.int64)
c135 = cam[index]
self.assertEqual(len(c135), 3)
self.assertClose(c135.zfar, torch.tensor([100.0] * 3))
self.assertClose(c135.znear, torch.tensor([10.0] * 3))
self.assertClose(c135.R, R_matrix[[1, 3, 5], ...])
# Check errors with get item
with self.assertRaisesRegex(ValueError, "out of bounds"):
cam[6]
with self.assertRaisesRegex(ValueError, "Invalid index type"):
cam[slice(0, 1)]
with self.assertRaisesRegex(ValueError, "Invalid index type"):
index = torch.tensor([1, 3, 5], dtype=torch.float32)
cam[index]
def baryclip_pytorch(
num_meshes: int = 8,
ico_level: int = 5,
image_size: int = 64,
faces_per_pixel: int = 50,
device="cuda",
):
# Init meshes
sphere_meshes = ico_sphere(ico_level, device).extend(num_meshes)
# Init transform
R, T = look_at_view_transform(1.0, 0.0, 0.0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
# Init rasterizer
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=1e-4,
faces_per_pixel=faces_per_pixel,
clip_barycentric_coords=False,
)
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
torch.cuda.synchronize()
def raster_fn():
fragments = rasterizer(sphere_meshes)
# Clip bary and reinterpolate
clipped_bary_coords = _clip_barycentric_coordinates(
fragments.bary_coords)
clipped_zbuf = _interpolate_zbuf(fragments.pix_to_face,
clipped_bary_coords, sphere_meshes)
fragments = Fragments(
bary_coords=clipped_bary_coords,
zbuf=clipped_zbuf,
dists=fragments.dists,
pix_to_face=fragments.pix_to_face,
)
torch.cuda.synchronize()
return raster_fn
def test_points_renderer_to(self):
"""
Test moving all the tensors in the points renderer to a new device.
"""
device1 = torch.device("cpu")
R, T = look_at_view_transform(1500, 0.0, 0.0)
raster_settings = PointsRasterizationSettings(image_size=256,
radius=0.001,
points_per_pixel=1)
cameras = FoVPerspectiveCameras(device=device1,
R=R,
T=T,
aspect_ratio=1.0,
fov=60.0,
zfar=100)
rasterizer = PointsRasterizer(cameras=cameras,
raster_settings=raster_settings)
renderer = PointsRenderer(rasterizer=rasterizer,
compositor=AlphaCompositor())
mesh = ico_sphere(2, device1)
verts_padded = mesh.verts_padded()
pointclouds = Pointclouds(points=verts_padded,
features=torch.randn_like(verts_padded))
self._check_points_renderer_props_on_device(renderer, device1)
# Test rendering on cpu
output_images = renderer(pointclouds)
self.assertEqual(output_images.device, device1)
# Move renderer and pointclouds to another device and re render
device2 = torch.device("cuda:0")
renderer = renderer.to(device2)
pointclouds = pointclouds.to(device2)
self._check_points_renderer_props_on_device(renderer, device2)
output_images = renderer(pointclouds)
self.assertEqual(output_images.device, device2)
def test_texture_map_atlas(self):
"""
Test a mesh with a texture map as a per face atlas is loaded and rendered correctly.
Also check that the backward pass for texture atlas rendering is differentiable.
"""
device = torch.device("cuda:0")
obj_dir = Path(__file__).resolve().parent.parent / "docs/tutorials/data"
obj_filename = obj_dir / "cow_mesh/cow.obj"
# Load mesh and texture as a per face texture atlas.
verts, faces, aux = load_obj(
obj_filename,
device=device,
load_textures=True,
create_texture_atlas=True,
texture_atlas_size=8,
texture_wrap=None,
)
atlas = aux.texture_atlas
mesh = Meshes(
verts=[verts],
faces=[faces.verts_idx],
textures=TexturesAtlas(atlas=[atlas]),
)
# Init rasterizer settings
R, T = look_at_view_transform(2.7, 0, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0,
faces_per_pixel=1,
cull_backfaces=True,
perspective_correct=False,
)
# Init shader settings
materials = Materials(device=device, specular_color=((0, 0, 0),), shininess=0.0)
lights = PointLights(device=device)
# Place light behind the cow in world space. The front of
# the cow is facing the -z direction.
lights.location = torch.tensor([0.0, 0.0, 2.0], device=device)[None]
# The HardPhongShader can be used directly with atlas textures.
rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings)
renderer = MeshRenderer(
rasterizer=rasterizer,
shader=HardPhongShader(lights=lights, cameras=cameras, materials=materials),
)
images = renderer(mesh)
rgb = images[0, ..., :3].squeeze()
# Load reference image
image_ref = load_rgb_image("test_texture_atlas_8x8_back.png", DATA_DIR)
if DEBUG:
Image.fromarray((rgb.detach().cpu().numpy() * 255).astype(np.uint8)).save(
DATA_DIR / "DEBUG_texture_atlas_8x8_back.png"
)
self.assertClose(rgb.cpu(), image_ref, atol=0.05)
# Check gradients are propagated
# correctly back to the texture atlas.
# Because of how texture sampling is implemented
# for the texture atlas it is not possible to get
# gradients back to the vertices.
atlas.requires_grad = True
mesh = Meshes(
verts=[verts],
faces=[faces.verts_idx],
textures=TexturesAtlas(atlas=[atlas]),
)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0001,
faces_per_pixel=5,
cull_backfaces=True,
clip_barycentric_coords=True,
)
images = renderer(mesh, raster_settings=raster_settings)
images[0, ...].sum().backward()
fragments = rasterizer(mesh, raster_settings=raster_settings)
# Some of the bary coordinates are outisde the
# [0, 1] range as expected because the blur is > 0
self.assertTrue(fragments.bary_coords.ge(1.0).any())
self.assertIsNotNone(atlas.grad)
self.assertTrue(atlas.grad.sum().abs() > 0.0)
def test_joined_spheres(self):
"""
Test a list of Meshes can be joined as a single mesh and
the single mesh is rendered correctly with Phong, Gouraud
and Flat Shaders.
"""
device = torch.device("cuda:0")
# Init mesh with vertex textures.
# Initialize a list containing two ico spheres of different sizes.
sphere_list = [ico_sphere(3, device), ico_sphere(4, device)]
# [(42 verts, 80 faces), (162 verts, 320 faces)]
# The scale the vertices need to be set at to resize the spheres
scales = [0.25, 1]
# The distance the spheres ought to be offset horizontally to prevent overlap.
offsets = [1.2, -0.3]
# Initialize a list containing the adjusted sphere meshes.
sphere_mesh_list = []
for i in range(len(sphere_list)):
verts = sphere_list[i].verts_padded() * scales[i]
verts[0, :, 0] += offsets[i]
sphere_mesh_list.append(
Meshes(verts=verts, faces=sphere_list[i].faces_padded())
)
joined_sphere_mesh = join_meshes_as_scene(sphere_mesh_list)
joined_sphere_mesh.textures = TexturesVertex(
verts_features=torch.ones_like(joined_sphere_mesh.verts_padded())
)
# Init rasterizer settings
R, T = look_at_view_transform(2.7, 0.0, 0.0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0,
faces_per_pixel=1,
perspective_correct=False,
)
# Init shader settings
materials = Materials(device=device)
lights = PointLights(device=device)
lights.location = torch.tensor([0.0, 0.0, +2.0], device=device)[None]
blend_params = BlendParams(1e-4, 1e-4, (0, 0, 0))
# Init renderer
rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings)
shaders = {
"phong": HardPhongShader,
"gouraud": HardGouraudShader,
"flat": HardFlatShader,
}
for (name, shader_init) in shaders.items():
shader = shader_init(
lights=lights,
cameras=cameras,
materials=materials,
blend_params=blend_params,
)
renderer = MeshRenderer(rasterizer=rasterizer, shader=shader)
image = renderer(joined_sphere_mesh)
rgb = image[..., :3].squeeze().cpu()
if DEBUG:
file_name = "DEBUG_joined_spheres_%s.png" % name
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / file_name
)
image_ref = load_rgb_image("test_joined_spheres_%s.png" % name, DATA_DIR)
self.assertClose(rgb, image_ref, atol=0.05)
def test_join_atlas(self):
"""Meshes with TexturesAtlas joined into a scene"""
# Test the result of rendering two tori with separate textures.
# The expected result is consistent with rendering them each alone.
torch.manual_seed(1)
device = torch.device("cuda:0")
plain_torus = torus(r=1, R=4, sides=5, rings=6, device=device)
[verts] = plain_torus.verts_list()
verts_shifted1 = verts.clone()
verts_shifted1 *= 1.2
verts_shifted1[:, 0] += 4
verts_shifted1[:, 1] += 5
verts[:, 0] -= 4
verts[:, 1] -= 4
[faces] = plain_torus.faces_list()
map_size = 3
# Two random atlases.
# The averaging of the random numbers here is not consistent with the
# meaning of the atlases, but makes each face a bit smoother than
# if everything had a random color.
atlas1 = torch.rand(size=(faces.shape[0], map_size, map_size, 3), device=device)
atlas1[:, 1] = 0.5 * atlas1[:, 0] + 0.5 * atlas1[:, 2]
atlas1[:, :, 1] = 0.5 * atlas1[:, :, 0] + 0.5 * atlas1[:, :, 2]
atlas2 = torch.rand(size=(faces.shape[0], map_size, map_size, 3), device=device)
atlas2[:, 1] = 0.5 * atlas2[:, 0] + 0.5 * atlas2[:, 2]
atlas2[:, :, 1] = 0.5 * atlas2[:, :, 0] + 0.5 * atlas2[:, :, 2]
textures1 = TexturesAtlas(atlas=[atlas1])
textures2 = TexturesAtlas(atlas=[atlas2])
mesh1 = Meshes(verts=[verts], faces=[faces], textures=textures1)
mesh2 = Meshes(verts=[verts_shifted1], faces=[faces], textures=textures2)
mesh_joined = join_meshes_as_scene([mesh1, mesh2])
R, T = look_at_view_transform(18, 0, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0,
faces_per_pixel=1,
perspective_correct=False,
)
lights = PointLights(
device=device,
ambient_color=((1.0, 1.0, 1.0),),
diffuse_color=((0.0, 0.0, 0.0),),
specular_color=((0.0, 0.0, 0.0),),
)
blend_params = BlendParams(
sigma=1e-1,
gamma=1e-4,
background_color=torch.tensor([1.0, 1.0, 1.0], device=device),
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(
device=device, blend_params=blend_params, cameras=cameras, lights=lights
),
)
output = renderer(mesh_joined)
image_ref = load_rgb_image("test_joinatlas_final.png", DATA_DIR)
if DEBUG:
debugging_outputs = []
for mesh_ in [mesh1, mesh2]:
debugging_outputs.append(renderer(mesh_))
Image.fromarray(
(output[0, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_joinatlas_final_.png")
Image.fromarray(
(debugging_outputs[0][0, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_joinatlas_1.png")
Image.fromarray(
(debugging_outputs[1][0, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_joinatlas_2.png")
result = output[0, ..., :3].cpu()
self.assertClose(result, image_ref, atol=0.05)
def test_join_uvs(self):
"""Meshes with TexturesUV joined into a scene"""
# Test the result of rendering three tori with separate textures.
# The expected result is consistent with rendering them each alone.
# This tests TexturesUV.join_scene with rectangle flipping,
# and we check the form of the merged map as well.
torch.manual_seed(1)
device = torch.device("cuda:0")
R, T = look_at_view_transform(18, 0, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=256, blur_radius=0.0, faces_per_pixel=1
)
lights = PointLights(
device=device,
ambient_color=((1.0, 1.0, 1.0),),
diffuse_color=((0.0, 0.0, 0.0),),
specular_color=((0.0, 0.0, 0.0),),
)
blend_params = BlendParams(
sigma=1e-1,
gamma=1e-4,
background_color=torch.tensor([1.0, 1.0, 1.0], device=device),
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(
device=device, blend_params=blend_params, cameras=cameras, lights=lights
),
)
plain_torus = torus(r=1, R=4, sides=5, rings=6, device=device)
[verts] = plain_torus.verts_list()
verts_shifted1 = verts.clone()
verts_shifted1 *= 0.5
verts_shifted1[:, 1] += 7
verts_shifted2 = verts.clone()
verts_shifted2 *= 0.5
verts_shifted2[:, 1] -= 7
[faces] = plain_torus.faces_list()
nocolor = torch.zeros((100, 100), device=device)
color_gradient = torch.linspace(0, 1, steps=100, device=device)
color_gradient1 = color_gradient[None].expand_as(nocolor)
color_gradient2 = color_gradient[:, None].expand_as(nocolor)
colors1 = torch.stack([nocolor, color_gradient1, color_gradient2], dim=2)
colors2 = torch.stack([color_gradient1, color_gradient2, nocolor], dim=2)
verts_uvs1 = torch.rand(size=(verts.shape[0], 2), device=device)
verts_uvs2 = torch.rand(size=(verts.shape[0], 2), device=device)
for i, align_corners, padding_mode in [
(0, True, "border"),
(1, False, "border"),
(2, False, "zeros"),
]:
textures1 = TexturesUV(
maps=[colors1],
faces_uvs=[faces],
verts_uvs=[verts_uvs1],
align_corners=align_corners,
padding_mode=padding_mode,
)
# These downsamplings of colors2 are chosen to ensure a flip and a non flip
# when the maps are merged.
# We have maps of size (100, 100), (50, 99) and (99, 50).
textures2 = TexturesUV(
maps=[colors2[::2, :-1]],
faces_uvs=[faces],
verts_uvs=[verts_uvs2],
align_corners=align_corners,
padding_mode=padding_mode,
)
offset = torch.tensor([0, 0, 0.5], device=device)
textures3 = TexturesUV(
maps=[colors2[:-1, ::2] + offset],
faces_uvs=[faces],
verts_uvs=[verts_uvs2],
align_corners=align_corners,
padding_mode=padding_mode,
)
mesh1 = Meshes(verts=[verts], faces=[faces], textures=textures1)
mesh2 = Meshes(verts=[verts_shifted1], faces=[faces], textures=textures2)
mesh3 = Meshes(verts=[verts_shifted2], faces=[faces], textures=textures3)
mesh = join_meshes_as_scene([mesh1, mesh2, mesh3])
output = renderer(mesh)[0, ..., :3].cpu()
output1 = renderer(mesh1)[0, ..., :3].cpu()
output2 = renderer(mesh2)[0, ..., :3].cpu()
output3 = renderer(mesh3)[0, ..., :3].cpu()
# The background color is white and the objects do not overlap, so we can
# predict the merged image by taking the minimum over every channel
merged = torch.min(torch.min(output1, output2), output3)
image_ref = load_rgb_image(f"test_joinuvs{i}_final.png", DATA_DIR)
map_ref = load_rgb_image(f"test_joinuvs{i}_map.png", DATA_DIR)
if DEBUG:
Image.fromarray((output.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_final_.png"
)
Image.fromarray((output.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_merged.png"
)
Image.fromarray((output1.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_1.png"
)
Image.fromarray((output2.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_2.png"
)
Image.fromarray((output3.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / f"test_joinuvs{i}_3.png"
)
Image.fromarray(
(mesh.textures.maps_padded()[0].cpu().numpy() * 255).astype(
np.uint8
)
).save(DATA_DIR / f"test_joinuvs{i}_map_.png")
Image.fromarray(
(mesh2.textures.maps_padded()[0].cpu().numpy() * 255).astype(
np.uint8
)
).save(DATA_DIR / f"test_joinuvs{i}_map2.png")
Image.fromarray(
(mesh3.textures.maps_padded()[0].cpu().numpy() * 255).astype(
np.uint8
)
).save(DATA_DIR / f"test_joinuvs{i}_map3.png")
self.assertClose(output, merged, atol=0.015)
self.assertClose(output, image_ref, atol=0.05)
self.assertClose(mesh.textures.maps_padded()[0].cpu(), map_ref, atol=0.05)
def test_batch_uvs(self):
"""Test that two random tori with TexturesUV render the same as each individually."""
torch.manual_seed(1)
device = torch.device("cuda:0")
plain_torus = torus(r=1, R=4, sides=10, rings=10, device=device)
[verts] = plain_torus.verts_list()
[faces] = plain_torus.faces_list()
nocolor = torch.zeros((100, 100), device=device)
color_gradient = torch.linspace(0, 1, steps=100, device=device)
color_gradient1 = color_gradient[None].expand_as(nocolor)
color_gradient2 = color_gradient[:, None].expand_as(nocolor)
colors1 = torch.stack([nocolor, color_gradient1, color_gradient2], dim=2)
colors2 = torch.stack([color_gradient1, color_gradient2, nocolor], dim=2)
verts_uvs1 = torch.rand(size=(verts.shape[0], 2), device=device)
verts_uvs2 = torch.rand(size=(verts.shape[0], 2), device=device)
textures1 = TexturesUV(
maps=[colors1], faces_uvs=[faces], verts_uvs=[verts_uvs1]
)
textures2 = TexturesUV(
maps=[colors2], faces_uvs=[faces], verts_uvs=[verts_uvs2]
)
mesh1 = Meshes(verts=[verts], faces=[faces], textures=textures1)
mesh2 = Meshes(verts=[verts], faces=[faces], textures=textures2)
mesh_both = join_meshes_as_batch([mesh1, mesh2])
R, T = look_at_view_transform(10, 10, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=128, blur_radius=0.0, faces_per_pixel=1
)
# Init shader settings
lights = PointLights(device=device)
lights.location = torch.tensor([0.0, 0.0, 2.0], device=device)[None]
blend_params = BlendParams(
sigma=1e-1,
gamma=1e-4,
background_color=torch.tensor([1.0, 1.0, 1.0], device=device),
)
# Init renderer
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(
device=device, lights=lights, cameras=cameras, blend_params=blend_params
),
)
outputs = []
for meshes in [mesh_both, mesh1, mesh2]:
outputs.append(renderer(meshes))
if DEBUG:
Image.fromarray(
(outputs[0][0, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_batch_uvs0.png")
Image.fromarray(
(outputs[1][0, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_batch_uvs1.png")
Image.fromarray(
(outputs[0][1, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_batch_uvs2.png")
Image.fromarray(
(outputs[2][0, ..., :3].cpu().numpy() * 255).astype(np.uint8)
).save(DATA_DIR / "test_batch_uvs3.png")
diff = torch.abs(outputs[0][0, ..., :3] - outputs[1][0, ..., :3])
Image.fromarray(((diff > 1e-5).cpu().numpy().astype(np.uint8) * 255)).save(
DATA_DIR / "test_batch_uvs01.png"
)
diff = torch.abs(outputs[0][1, ..., :3] - outputs[2][0, ..., :3])
Image.fromarray(((diff > 1e-5).cpu().numpy().astype(np.uint8) * 255)).save(
DATA_DIR / "test_batch_uvs23.png"
)
self.assertClose(outputs[0][0, ..., :3], outputs[1][0, ..., :3], atol=1e-5)
self.assertClose(outputs[0][1, ..., :3], outputs[2][0, ..., :3], atol=1e-5)
def test_texture_map(self):
"""
Test a mesh with a texture map is loaded and rendered correctly.
The pupils in the eyes of the cow should always be looking to the left.
"""
device = torch.device("cuda:0")
obj_dir = Path(__file__).resolve().parent.parent / "docs/tutorials/data"
obj_filename = obj_dir / "cow_mesh/cow.obj"
# Load mesh + texture
verts, faces, aux = load_obj(
obj_filename, device=device, load_textures=True, texture_wrap=None
)
tex_map = list(aux.texture_images.values())[0]
tex_map = tex_map[None, ...].to(faces.textures_idx.device)
textures = TexturesUV(
maps=tex_map, faces_uvs=[faces.textures_idx], verts_uvs=[aux.verts_uvs]
)
mesh = Meshes(verts=[verts], faces=[faces.verts_idx], textures=textures)
# Init rasterizer settings
R, T = look_at_view_transform(2.7, 0, 0)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=512, blur_radius=0.0, faces_per_pixel=1
)
# Init shader settings
materials = Materials(device=device)
lights = PointLights(device=device)
# Place light behind the cow in world space. The front of
# the cow is facing the -z direction.
lights.location = torch.tensor([0.0, 0.0, 2.0], device=device)[None]
blend_params = BlendParams(
sigma=1e-1,
gamma=1e-4,
background_color=torch.tensor([1.0, 1.0, 1.0], device=device),
)
# Init renderer
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings),
shader=TexturedSoftPhongShader(
lights=lights,
cameras=cameras,
materials=materials,
blend_params=blend_params,
),
)
# Load reference image
image_ref = load_rgb_image("test_texture_map_back.png", DATA_DIR)
for bin_size in [0, None]:
# Check both naive and coarse to fine produce the same output.
renderer.rasterizer.raster_settings.bin_size = bin_size
images = renderer(mesh)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / "DEBUG_texture_map_back.png"
)
# NOTE some pixels can be flaky and will not lead to
# `cond1` being true. Add `cond2` and check `cond1 or cond2`
cond1 = torch.allclose(rgb, image_ref, atol=0.05)
cond2 = ((rgb - image_ref).abs() > 0.05).sum() < 5
self.assertTrue(cond1 or cond2)
# Check grad exists
[verts] = mesh.verts_list()
verts.requires_grad = True
mesh2 = Meshes(verts=[verts], faces=mesh.faces_list(), textures=mesh.textures)
images = renderer(mesh2)
images[0, ...].sum().backward()
self.assertIsNotNone(verts.grad)
##########################################
# Check rendering of the front of the cow
##########################################
R, T = look_at_view_transform(2.7, 0, 180)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
# Move light to the front of the cow in world space
lights.location = torch.tensor([0.0, 0.0, -2.0], device=device)[None]
# Load reference image
image_ref = load_rgb_image("test_texture_map_front.png", DATA_DIR)
for bin_size in [0, None]:
# Check both naive and coarse to fine produce the same output.
renderer.rasterizer.raster_settings.bin_size = bin_size
images = renderer(mesh, cameras=cameras, lights=lights)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / "DEBUG_texture_map_front.png"
)
# NOTE some pixels can be flaky and will not lead to
# `cond1` being true. Add `cond2` and check `cond1 or cond2`
cond1 = torch.allclose(rgb, image_ref, atol=0.05)
cond2 = ((rgb - image_ref).abs() > 0.05).sum() < 5
self.assertTrue(cond1 or cond2)
#################################
# Add blurring to rasterization
#################################
R, T = look_at_view_transform(2.7, 0, 180)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
blend_params = BlendParams(sigma=5e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=np.log(1.0 / 1e-4 - 1.0) * blend_params.sigma,
faces_per_pixel=100,
clip_barycentric_coords=True,
perspective_correct=False,
)
# Load reference image
image_ref = load_rgb_image("test_blurry_textured_rendering.png", DATA_DIR)
for bin_size in [0, None]:
# Check both naive and coarse to fine produce the same output.
renderer.rasterizer.raster_settings.bin_size = bin_size
images = renderer(
mesh.clone(),
cameras=cameras,
raster_settings=raster_settings,
blend_params=blend_params,
)
rgb = images[0, ..., :3].squeeze().cpu()
if DEBUG:
Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / "DEBUG_blurry_textured_rendering.png"
)
self.assertClose(rgb, image_ref, atol=0.05)
