def forward(self, meshes_world, **kwargs) -> torch.Tensor:
"""
Render a batch of images from a batch of meshes by rasterizing and then shading.
NOTE: If the blur radius for rasterizaiton is > 0.0, some pixels can have one or more barycentric coordinates lying outside the range [0, 1].
For a pixel with out of bounds barycentric coordinates with respect to a face f, clipping is required before interpolating the texture uv coordinates and z buffer so that the colors and depths are limited to the range for the corresponding face.
"""
fragments = self.rasterizer(meshes_world, **kwargs)
raster_setting = kwargs.get("raster_settings",
self.rasterizer.raster_settings)
if raster_setting.blur_radius > 0.0:
# TODO: potentially move barycenteric clipping to the rasterizer.
# If no downstream functions requiers upclipped values.
# This will avoid uncessary re-interpolation of the z buffer.
clipped_bary_coords = _clip_barycentric_coordinates(
fragments.bary_coords)
clipped_zbuf = _interpolate_zbuf(fragments.pix_to_face,
clipped_bary_coords, meshes_world)
fragments = Fragments(
bary_coords=clipped_bary_coords,
zbuf=clipped_zbuf,
dists=fragments.dists,
pix_to_face=fragments.pix_to_face,
)
images = self.shader(fragments, meshes_world, **kwargs)
depth = fragments.zbuf
return images, depth
def test_bary_clip(self):
N = 10
bary = torch.randn(size=(N, 3))
# randomly make some values negative
bary[bary < 0.3] *= -1.0
# randomly make some values be greater than 1
bary[bary > 0.8] *= 2.0
negative_mask = bary < 0.0
positive_mask = bary > 1.0
clipped = _clip_barycentric_coordinates(bary)
self.assertTrue(clipped[negative_mask].sum() == 0)
self.assertTrue(clipped[positive_mask].gt(1.0).sum() == 0)
self.assertTrue(torch.allclose(clipped.sum(dim=-1), torch.ones(N)))
def raster_fn():
fragments = rasterizer(sphere_meshes)
# Clip bary and reinterpolate
clipped_bary_coords = _clip_barycentric_coordinates(
fragments.bary_coords)
clipped_zbuf = _interpolate_zbuf(fragments.pix_to_face,
clipped_bary_coords, sphere_meshes)
fragments = Fragments(
bary_coords=clipped_bary_coords,
zbuf=clipped_zbuf,
dists=fragments.dists,
pix_to_face=fragments.pix_to_face,
)
torch.cuda.synchronize()
def rasterize(R, T, meshes, rasterizer, blur_radius=0):
# It will automatically update the camera settings -> R, T in rasterizer.camera
fragments = rasterizer(meshes, R=R, T=T)
# Copy from pytorch3D source code, try if it is necessary to do gradient decent
if blur_radius > 0.0:
clipped_bary_coords = utils._clip_barycentric_coordinates(
fragments.bary_coords)
clipped_zbuf = utils._interpolate_zbuf(fragments.pix_to_face,
clipped_bary_coords, meshes)
fragments = Fragments(
bary_coords=clipped_bary_coords,
zbuf=clipped_zbuf,
dists=fragments.dists,
pix_to_face=fragments.pix_to_face,
)
return fragments
