def test_mesh_renderer_to(self):
"""
Test moving all the tensors in the mesh renderer to a new device.
"""
device1 = torch.device("cpu")
R, T = look_at_view_transform(1500, 0.0, 0.0)
# Init shader settings
materials = Materials(device=device1)
lights = PointLights(device=device1)
lights.location = torch.tensor([0.0, 0.0, +1000.0], device=device1)[None]
raster_settings = RasterizationSettings(
image_size=256, blur_radius=0.0, faces_per_pixel=1
)
cameras = FoVPerspectiveCameras(
device=device1, R=R, T=T, aspect_ratio=1.0, fov=60.0, zfar=100
)
rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings)
blend_params = BlendParams(
1e-4,
1e-4,
background_color=torch.zeros(3, dtype=torch.float32, device=device1),
)
shader = SoftPhongShader(
lights=lights,
cameras=cameras,
materials=materials,
blend_params=blend_params,
)
renderer = MeshRenderer(rasterizer=rasterizer, shader=shader)
mesh = ico_sphere(2, device1)
verts_padded = mesh.verts_padded()
textures = TexturesVertex(
verts_features=torch.ones_like(verts_padded, device=device1)
)
mesh.textures = textures
self._check_mesh_renderer_props_on_device(renderer, device1)
# Test rendering on cpu
output_images = renderer(mesh)
self.assertEqual(output_images.device, device1)
# Move renderer and mesh to another device and re render
# This also tests that background_color is correctly moved to
# the new device
device2 = torch.device("cuda:0")
renderer = renderer.to(device2)
mesh = mesh.to(device2)
self._check_mesh_renderer_props_on_device(renderer, device2)
output_images = renderer(mesh)
self.assertEqual(output_images.device, device2)
def __init__(self, rasterizer, shader_rgb, shader_mask, cameras):
super(Renderer, self).__init__()
self.cameras = cameras
self.rasterizer = rasterizer
self.shader_rgb = shader_rgb
self.shader_mask = shader_mask
self.lights = DirectionalLights(
ambient_color=((1, 1, 1),),
diffuse_color=((0., 0., 0.),),
specular_color=((0., 0., 0.),),
direction=((0, 0, 1),),
device='cuda',
)
self.materials = Materials(device='cuda')
def __init__(self,
device="cpu",
cameras=None,
lights=None,
materials=None,
blend_params=None):
super().__init__()
self.lights = lights if lights is not None else PointLights(
device=device)
self.materials = (materials if materials is not None else Materials(
device=device))
self.cameras = cameras
self.blend_params = blend_params if blend_params is not None else BlendParams(
)
def _compare_with_meshes_renderer(self,
image_size,
batch_size=11,
sphere_diameter=0.6):
"""
Generate a spherical RGB volumetric function and its corresponding mesh
and check whether MeshesRenderer returns the same images as the
corresponding ImplicitRenderer.
"""
# generate NDC camera extrinsics and intrinsics
cameras = init_cameras(batch_size, image_size=image_size, ndc=True)
# get rand offset of the volume
sphere_centroid = torch.randn(batch_size, 3,
device=cameras.device) * 0.1
sphere_centroid.requires_grad = True
# init the grid raysampler with the ndc grid
raysampler = NDCMultinomialRaysampler(
image_width=image_size[1],
image_height=image_size[0],
n_pts_per_ray=256,
min_depth=0.1,
max_depth=2.0,
)
# get the EA raymarcher
raymarcher = EmissionAbsorptionRaymarcher()
# jitter the camera intrinsics a bit for each render
cameras_randomized = cameras.clone()
cameras_randomized.principal_point = (
torch.randn_like(cameras.principal_point) * 0.3)
cameras_randomized.focal_length = (
cameras.focal_length +
torch.randn_like(cameras.focal_length) * 0.2)
# the list of differentiable camera vars
cam_vars = ("R", "T", "focal_length", "principal_point")
# enable the gradient caching for the camera variables
for cam_var in cam_vars:
getattr(cameras_randomized, cam_var).requires_grad = True
# get the implicit renderer
images_opacities = ImplicitRenderer(
raysampler=raysampler, raymarcher=raymarcher)(
cameras=cameras_randomized,
volumetric_function=spherical_volumetric_function,
sphere_centroid=sphere_centroid,
sphere_diameter=sphere_diameter,
)[0]
# check that the renderer does not erase gradients
loss = images_opacities.sum()
loss.backward()
for check_var in (
*[
getattr(cameras_randomized, cam_var)
for cam_var in cam_vars
],
sphere_centroid,
):
self.assertIsNotNone(check_var.grad)
# instantiate the corresponding spherical mesh
ico = ico_sphere(level=4, device=cameras.device).extend(batch_size)
verts = (torch.nn.functional.normalize(ico.verts_padded(), dim=-1) *
sphere_diameter + sphere_centroid[:, None])
meshes = Meshes(
verts=verts,
faces=ico.faces_padded(),
textures=TexturesVertex(verts_features=(
torch.nn.functional.normalize(verts, dim=-1) * 0.5 + 0.5)),
)
# instantiate the corresponding mesh renderer
lights = PointLights(device=cameras.device, location=[[0.0, 0.0, 0.0]])
renderer_textured = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras_randomized,
raster_settings=RasterizationSettings(
image_size=image_size,
blur_radius=1e-3,
faces_per_pixel=10,
z_clip_value=None,
perspective_correct=False,
),
),
shader=SoftPhongShader(
device=cameras.device,
cameras=cameras_randomized,
lights=lights,
materials=Materials(
ambient_color=((2.0, 2.0, 2.0), ),
diffuse_color=((0.0, 0.0, 0.0), ),
specular_color=((0.0, 0.0, 0.0), ),
shininess=64,
device=cameras.device,
),
blend_params=BlendParams(sigma=1e-3,
gamma=1e-4,
background_color=(0.0, 0.0, 0.0)),
),
)
# get the mesh render
images_opacities_meshes = renderer_textured(meshes,
cameras=cameras_randomized,
lights=lights)
if DEBUG:
outdir = tempfile.gettempdir() + "/test_implicit_vs_mesh_renderer"
os.makedirs(outdir, exist_ok=True)
frames = []
for (image_opacity,
image_opacity_mesh) in zip(images_opacities,
images_opacities_meshes):
image, opacity = image_opacity.split([3, 1], dim=-1)
image_mesh, opacity_mesh = image_opacity_mesh.split([3, 1],
dim=-1)
diff_image = (((image - image_mesh) * 0.5 + 0.5).mean(
dim=2, keepdim=True).repeat(1, 1, 3))
image_pil = Image.fromarray((torch.cat(
(
image,
image_mesh,
diff_image,
opacity.repeat(1, 1, 3),
opacity_mesh.repeat(1, 1, 3),
),
dim=1,
).detach().cpu().numpy() * 255.0).astype(np.uint8))
frames.append(image_pil)
# export gif
outfile = os.path.join(outdir, "implicit_vs_mesh_render.gif")
frames[0].save(
outfile,
save_all=True,
append_images=frames[1:],
duration=batch_size // 15,
loop=0,
)
print(f"exported {outfile}")
# export concatenated frames
outfile_cat = os.path.join(outdir, "implicit_vs_mesh_render.png")
Image.fromarray(
np.concatenate([np.array(f) for f in frames],
axis=0)).save(outfile_cat)
print(f"exported {outfile_cat}")
# compare the renders
diff = (images_opacities - images_opacities_meshes).abs().mean(dim=-1)
mu_diff = diff.mean(dim=(1, 2))
std_diff = diff.std(dim=(1, 2))
self.assertClose(mu_diff, torch.zeros_like(mu_diff), atol=5e-2)
self.assertClose(std_diff, torch.zeros_like(std_diff), atol=6e-2)
image_size=args.window_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=
None, # this setting controls whether naive or coarse-to-fine rasterization is used
max_faces_per_bin=None # this setting is for coarse rasterization
))
# ライトの作成
lights = PointLights(
device=device,
location=[[args.light_pos_x, args.light_pos_y, args.light_pos_z]])
# マテリアルの作成
materials = Materials(device=device,
specular_color=[[0.2, 0.2, 0.2]],
shininess=10.0)
# シェーダーの作成
if (args.shader == "soft_silhouette_shader"):
shader = SoftSilhouetteShader()
elif (args.shader == "soft_phong_shader"):
shader = SoftPhongShader(device=device,
cameras=cameras,
lights=lights,
materials=materials)
elif (args.shader == "textured_soft_phong_shader"):
shader = TexturedSoftPhongShader(device=device,
cameras=cameras,
lights=lights,
materials=materials)
if num_views != 1:
camera = PerspectiveCameras(device=device,
focal_length=4500,
principal_point=((512, 512), ),
R=Rtotal[None, 1, ...],
T=Ttotal[None, 1, ...],
image_size=((1024, 1024), ))
else:
camera = PerspectiveCameras(device=device,
focal_length=4500,
principal_point=((512, 512), ),
R=Rtotal,
T=Ttotal,
image_size=((1024, 1024), ))
mymaterials = Materials(device=device, shininess=8)
raster_settings = RasterizationSettings(
image_size=1024,
blur_radius=0.0,
faces_per_pixel=1,
)
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=camera, raster_settings=raster_settings),
shader=SoftPhongShader(
device=device,
cameras=camera,
lights=lights,
materials=mymaterials,
))
meshes = mesh.extend(num_views)
image_size=args.window_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=
None, # this setting controls whether naive or coarse-to-fine rasterization is used
max_faces_per_bin=None # this setting is for coarse rasterization
))
# ライトの作成
lights = PointLights(
device=device,
location=[[args.light_pos_x, args.light_pos_y, args.light_pos_z]])
# マテリアルの作成
materials = Materials(device=device,
specular_color=[[0.0, 0.0, 0.0]],
shininess=10.0)
# シェーダーの作成
if (args.shader == "soft_silhouette_shader"):
shader = SoftSilhouetteShader()
elif (args.shader == "soft_phong_shader"):
shader = SoftPhongShader(device=device,
cameras=cameras,
lights=lights,
materials=materials)
elif (args.shader == "textured_soft_phong_shader"):
# The textured phong shader will interpolate the texture uv coordinates for each vertex, sample from a texture image and apply the Phong lighting model
shader = TexturedSoftPhongShader(device=device,
cameras=cameras,
lights=lights,
def init_differential_renderer(self):
distance = 0.3
R, T = look_at_view_transform(distance, 0, 0)
cameras = OpenGLPerspectiveCameras(device=self.device, R=R, T=T)
raster_settings = RasterizationSettings(image_size=self.opt.crop_size,
blur_radius=0.0,
faces_per_pixel=1,
perspective_correct=True,
cull_backfaces=True)
silhouette_raster_settings = RasterizationSettings(
image_size=self.opt.crop_size,
blur_radius=0.0,
faces_per_pixel=1,
perspective_correct=True,
)
# Change specular color to green and change material shininess
self.materials = Materials(
device=self.device,
ambient_color=[[1.0, 1.0, 1.0]],
specular_color=[[0.0, 0.0, 0.0]],
diffuse_color=[[1.0, 1.0, 1.0]],
)
bp = BlendParams(background_color=(0, 0, 0)) # black
# bp = BlendParams(background_color=(1, 1, 1)) # white is default
lights = PointLights(device=self.device, location=((0.0, 0.0, 2.0), ))
self.renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=TexturedSoftPhongShader(
# blend_params=bp,
device=self.device,
lights=lights,
cameras=cameras,
))
import cv2
# segmentation_texture_map = cv2.imread(str(Path('resources') / 'part_segmentation_map_2048_gray_n_h.png'))[...,
segmentation_texture_map = cv2.imread(
str(Path('resources') /
'Color_Map_Sag_symmetric.png'))[..., ::-1].astype(np.uint8)
import matplotlib.pyplot as plt
plt.imshow(segmentation_texture_map)
plt.show()
segmentation_texture_map = (torch.from_numpy(
np.array(segmentation_texture_map))).unsqueeze(0).float()
self.segmentation_3d_renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=UVsCorrespondenceShader(blend_params=bp,
device=self.device,
cameras=cameras,
colormap=segmentation_texture_map))
# Create a silhouette mesh renderer by composing a rasterizer and a shader.
self.silhouette_renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=silhouette_raster_settings),
shader=SoftSilhouetteShader(
blend_params=BlendParams(sigma=1e-10, gamma=1e-4)))
self.negative_silhouette_renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftSilhouetteShader(
blend_params=BlendParams(sigma=1e-10, gamma=1e-4)))
self.texture_data = np.load('smpl_model/texture_data.npy',
allow_pickle=True,
encoding='latin1').item()
self.verts_uvs1 = torch.tensor(self.texture_data['vt'],
dtype=torch.float32).unsqueeze(0).cuda(
self.device)
self.faces_uvs1 = torch.tensor(
self.texture_data['ft'].astype(np.int64),
dtype=torch.int64).unsqueeze(0).cuda(self.device)