def __init__(self, meshes: Meshes, image_size=256, device='cuda'):
"""
Initialization of MaskRenderer. Renderer is initialized with predefined rasterizer and shader.
A soft silhouette shader is used to compute the projection mask.
:param meshes: A batch of meshes. pytorch3d.structures.Meshes. Dimension meshes \in R^N.
View https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py
for additional information.
In our case it is usually only one batch which is the template for a certain category.
:param device: The device, on which the computation is done.
:param image_size: Image size for the rasterization. Default is 256.
"""
super(MaskRenderer, self).__init__()
self.device = device
self._meshes = meshes
cameras = OpenGLOrthographicCameras(device=device)
# parameter settings as of Pytorch3D Tutorial
# (https://pytorch3d.org/tutorials/camera_position_optimization_with_differentiable_rendering)
self._rasterizer = MeshRasterizer(
cameras=cameras,
raster_settings=RasterizationSettings(image_size=image_size))
self._shader = SoftSilhouetteShader(
blend_params=(BlendParams(sigma=1e-4, gamma=1e-4)))
def test_silhouette_with_grad(self):
"""
Test silhouette blending. Also check that gradient calculation works.
"""
device = torch.device("cuda:0")
sphere_mesh = ico_sphere(5, device)
verts, faces = sphere_mesh.get_mesh_verts_faces(0)
sphere_mesh = Meshes(verts=[verts], faces=[faces])
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=np.log(1.0 / 1e-4 - 1.0) * blend_params.sigma,
faces_per_pixel=80,
clip_barycentric_coords=True,
)
# Init rasterizer settings
R, T = look_at_view_transform(2.7, 0, 0)
for cam_type in (
FoVPerspectiveCameras,
FoVOrthographicCameras,
PerspectiveCameras,
OrthographicCameras,
):
cameras = cam_type(device=device, R=R, T=T)
# Init renderer
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings
),
shader=SoftSilhouetteShader(blend_params=blend_params),
)
images = renderer(sphere_mesh)
alpha = images[0, ..., 3].squeeze().cpu()
if DEBUG:
filename = os.path.join(
DATA_DIR, "DEBUG_%s_silhouette.png" % (cam_type.__name__)
)
Image.fromarray((alpha.detach().numpy() * 255).astype(np.uint8)).save(
filename
)
ref_filename = "test_%s_silhouette.png" % (cam_type.__name__)
image_ref_filename = DATA_DIR / ref_filename
with Image.open(image_ref_filename) as raw_image_ref:
image_ref = torch.from_numpy(np.array(raw_image_ref))
image_ref = image_ref.to(dtype=torch.float32) / 255.0
self.assertClose(alpha, image_ref, atol=0.055)
# Check grad exist
verts.requires_grad = True
sphere_mesh = Meshes(verts=[verts], faces=[faces])
images = renderer(sphere_mesh)
images[0, ...].sum().backward()
self.assertIsNotNone(verts.grad)
def test_silhouette_with_grad(self):
"""
Test silhouette blending. Also check that gradient calculation works.
"""
device = torch.device("cuda:0")
ref_filename = "test_silhouette.png"
image_ref_filename = DATA_DIR / ref_filename
sphere_mesh = ico_sphere(5, device)
verts, faces = sphere_mesh.get_mesh_verts_faces(0)
sphere_mesh = Meshes(verts=[verts], faces=[faces])
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=np.log(1.0 / 1e-4 - 1.0) * blend_params.sigma,
faces_per_pixel=80,
bin_size=0,
)
# Init rasterizer settings
R, T = look_at_view_transform(2.7, 10, 20)
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
# Init renderer
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params),
)
images = renderer(sphere_mesh)
alpha = images[0, ..., 3].squeeze().cpu()
if DEBUG:
Image.fromarray((alpha.numpy() * 255).astype(np.uint8)).save(
DATA_DIR / "DEBUG_silhouette_grad.png")
with Image.open(image_ref_filename) as raw_image_ref:
image_ref = torch.from_numpy(np.array(raw_image_ref))
image_ref = image_ref.to(dtype=torch.float32) / 255.0
self.assertTrue(torch.allclose(alpha, image_ref, atol=0.055))
# Check grad exist
verts.requires_grad = True
sphere_mesh = Meshes(verts=[verts], faces=[faces])
images = renderer(sphere_mesh)
images[0, ...].sum().backward()
self.assertIsNotNone(verts.grad)
def __init__(self, meshes: Meshes, image_size=256):
"""
Initialization of the Renderer Class. Instances of the mask and depth renderer are create on corresponding
device.
:param device: The device, on which the computation is done.
:param image_size: Image size for the rasterization. Default is 256.
"""
super().__init__()
self.meshes = meshes
device = meshes.device
# TODO: check how to implement weak perspective (scaled orthographic).
cameras = OpenGLOrthographicCameras(device=device)
self._rasterizer = MeshRasterizer(
cameras=cameras,
raster_settings=RasterizationSettings(image_size=image_size,
faces_per_pixel=100))
self._shader = SoftSilhouetteShader(
blend_params=(BlendParams(sigma=1e-4, gamma=1e-4)))
def __init__(self, meshes: Meshes, device: str, image_size: int = 256):
"""
Initialization of DepthRenderer. Initialization of the default mesh rasterizer and silhouette shader which is
used because of simplicity.
:param device: The device, on which the computation is done, e.g. cpu or cuda.
:param image_size: Image size for the rasterization. Default is 256.
:param meshes: A batch of meshes. pytorch3d.structures.Meshes. Dimension meshes in R^N.
View https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py
for additional information.
"""
super(DepthRenderer, self).__init__()
self._meshes = meshes
# TODO: check how to implement weak perspective (scaled orthographic).
cameras = OpenGLOrthographicCameras(device=device)
raster_settings = RasterizationSettings(image_size=image_size)
self._rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
self._shader = SoftSilhouetteShader(
blend_params=(BlendParams(sigma=1e-4, gamma=1e-4)))
max_faces_per_bin=None # this setting is for coarse rasterization
))
# ライトの作成
lights = PointLights(
device=device,
location=[[args.light_pos_x, args.light_pos_y, args.light_pos_z]])
# マテリアルの作成
materials = Materials(device=device,
specular_color=[[0.2, 0.2, 0.2]],
shininess=10.0)
# シェーダーの作成
if (args.shader == "soft_silhouette_shader"):
shader = SoftSilhouetteShader()
elif (args.shader == "soft_phong_shader"):
shader = SoftPhongShader(device=device,
cameras=cameras,
lights=lights,
materials=materials)
elif (args.shader == "textured_soft_phong_shader"):
shader = TexturedSoftPhongShader(device=device,
cameras=cameras,
lights=lights,
materials=materials)
else:
NotImplementedError()
# レンダラーの作成
renderer = MeshRenderer(rasterizer=rasterizer, shader=shader)