def __init__(self, image_size):
super(Renderer, self).__init__()
self.image_size = image_size
self.dog_obj = load_objs_as_meshes(['data/dog_B/dog_B/dog_B_tpose.obj'])
raster_settings = RasterizationSettings(
image_size=self.image_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=None
)
R, T = look_at_view_transform(2.7, 0, 0)
cameras = OpenGLPerspectiveCameras(device=R.device, R=R, T=T)
lights = PointLights(device=R.device, location=[[0.0, 1.0, 0.0]])
self.renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings
),
shader=SoftPhongShader(
device=R.device,
cameras=cameras,
lights=lights
)
)
def _get_renderer(self, device):
R, T = look_at_view_transform(10, 0, 0) # camera's position
cameras = FoVPerspectiveCameras(
device=device,
R=R,
T=T,
znear=0.01,
zfar=50,
fov=2 * np.arctan(self.img_size // 2 / self.focal) * 180. / np.pi)
lights = PointLights(device=device,
location=[[0.0, 0.0, 1e5]],
ambient_color=[[1, 1, 1]],
specular_color=[[0., 0., 0.]],
diffuse_color=[[0., 0., 0.]])
raster_settings = RasterizationSettings(
image_size=self.img_size,
blur_radius=0.0,
faces_per_pixel=1,
)
blend_params = blending.BlendParams(background_color=[0, 0, 0])
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftPhongShader(device=device,
cameras=cameras,
lights=lights,
blend_params=blend_params))
return renderer
def setup(self, device):
R, T = look_at_view_transform(self.viewpoint_distance,
self.viewpoint_elevation,
self.viewpoint_azimuth,
device=device)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=self.opt.fast_image_size,
blur_radius=self.opt.raster_blur_radius,
faces_per_pixel=self.opt.raster_faces_per_pixel,
)
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
lights = PointLights(device=device,
location=[self.opt.lights_location])
lights = DirectionalLights(device=device,
direction=[self.opt.lights_direction])
shader = SoftPhongShader(
device=device,
cameras=cameras,
lights=lights,
blend_params=BlendParams(
self.opt.blend_params_sigma,
self.opt.blend_params_gamma,
self.opt.blend_params_background_color,
),
)
self.renderer = MeshRenderer(
rasterizer=rasterizer,
shader=shader,
)
def define_render(num):
shapenet_cam_params_file = '../data/metadata/rendering_metadata.json'
with open(shapenet_cam_params_file) as f:
shapenet_cam_params = json.load(f)
param_num = num
R, T = look_at_view_transform(
dist=shapenet_cam_params["distance"][param_num] * 5,
elev=shapenet_cam_params["elevation"][param_num],
azim=shapenet_cam_params["azimuth"][param_num])
cameras = FoVPerspectiveCameras(
device=device,
R=R,
T=T,
fov=shapenet_cam_params["field_of_view"][param_num])
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0,
faces_per_pixel=1,
)
lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]])
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=SoftPhongShader(device=device,
cameras=cameras,
lights=lights))
return renderer
def __get_renderer(self, render_size, lights):
cameras = FoVOrthographicCameras(
device=self.device,
znear=0.1,
zfar=10.0,
max_y=1.0,
min_y=-1.0,
max_x=1.0,
min_x=-1.0,
scale_xyz=((1.0, 1.0, 1.0), ), # (1, 3)
)
raster_settings = RasterizationSettings(
image_size=render_size,
blur_radius=0,
faces_per_pixel=1,
)
blend_params = BlendParams(sigma=1e-4,
gamma=1e-4,
background_color=(0, 0, 0))
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftPhongShader(device=self.device,
cameras=cameras,
lights=lights,
blend_params=blend_params))
return renderer
def test_mesh_renderer_to(self):
"""
Test moving all the tensors in the mesh renderer to a new device.
"""
device1 = torch.device("cpu")
R, T = look_at_view_transform(1500, 0.0, 0.0)
# Init shader settings
materials = Materials(device=device1)
lights = PointLights(device=device1)
lights.location = torch.tensor([0.0, 0.0, +1000.0], device=device1)[None]
raster_settings = RasterizationSettings(
image_size=256, blur_radius=0.0, faces_per_pixel=1
)
cameras = FoVPerspectiveCameras(
device=device1, R=R, T=T, aspect_ratio=1.0, fov=60.0, zfar=100
)
rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings)
blend_params = BlendParams(
1e-4,
1e-4,
background_color=torch.zeros(3, dtype=torch.float32, device=device1),
)
shader = SoftPhongShader(
lights=lights,
cameras=cameras,
materials=materials,
blend_params=blend_params,
)
renderer = MeshRenderer(rasterizer=rasterizer, shader=shader)
mesh = ico_sphere(2, device1)
verts_padded = mesh.verts_padded()
textures = TexturesVertex(
verts_features=torch.ones_like(verts_padded, device=device1)
)
mesh.textures = textures
self._check_mesh_renderer_props_on_device(renderer, device1)
# Test rendering on cpu
output_images = renderer(mesh)
self.assertEqual(output_images.device, device1)
# Move renderer and mesh to another device and re render
# This also tests that background_color is correctly moved to
# the new device
device2 = torch.device("cuda:0")
renderer = renderer.to(device2)
mesh = mesh.to(device2)
self._check_mesh_renderer_props_on_device(renderer, device2)
output_images = renderer(mesh)
self.assertEqual(output_images.device, device2)
def on_change(value):
img_copy = image.copy()
x = (cv2.getTrackbarPos("x", "image") - 1000) / 1000
y = (cv2.getTrackbarPos("y", "image") - 1000) / 1000
z = cv2.getTrackbarPos("z", "image") / 1000
rx = cv2.getTrackbarPos("rx", "image")
ry = cv2.getTrackbarPos("ry", "image")
rz = cv2.getTrackbarPos("rz", "image")
T = torch.tensor([[x, y, z]],
dtype=torch.float32,
device=device)
R = Rotation.from_euler("zyx", [rz, ry, rx],
degrees=True).as_matrix()
renderR = torch.from_numpy(R.T.reshape((1, 3, 3))).to(device)
cameras = PerspectiveCameras(
R=renderR,
T=T,
focal_length=-self.f,
principal_point=self.p,
image_size=(self.img_size, ),
device=device,
)
raster_settings = RasterizationSettings(
image_size=(self.intrinsics.height, self.intrinsics.width),
blur_radius=0.0,
faces_per_pixel=1,
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftPhongShader(
device=device,
cameras=cameras,
),
)
overlay = renderer(mesh)[0, ..., :3].cpu().numpy()[:, :, ::-1]
render_img = overlay * 0.7 + img_copy / 255 * 0.3
cv2.imshow(windowName, render_img)
store_and_exit = cv2.getTrackbarPos(
"0 : Manual Match \n1 : Store and Exit", "image")
if store_and_exit:
cv2.destroyAllWindows()
pose[mesh_name] = {
"translation": T.cpu().numpy(),
"rotation": R
}
def __init__(self,
dist=2,
elev=0,
azimuth=180,
fov=40,
image_size=256,
R=None,
T=None,
cameras=None,
return_format="torch",
device='cuda'):
super().__init__()
# If you provide R and T, you don't need dist, elev, azimuth, fov
self.device = device
self.return_format = return_format
# Data structures and functions for rendering
if cameras is None:
if R is None and T is None:
R, T = look_at_view_transform(dist, elev, azimuth)
cameras = FoVPerspectiveCameras(R=R,
T=T,
znear=1,
zfar=10000,
fov=fov,
degrees=True,
device=device)
# cameras = PerspectiveCameras(R=R, T=T, focal_length=1.6319*10, device=device)
self.raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=0.0, # no blur
bin_size=0,
)
# Place lights at the same point as the camera
location = T
if location is None:
location = ((0, 0, 0), )
lights = PointLights(ambient_color=((0.3, 0.3, 0.3), ),
diffuse_color=((0.7, 0.7, 0.7), ),
device=device,
location=location)
self.mesh_rasterizer = MeshRasterizer(
cameras=cameras, raster_settings=self.raster_settings)
self._renderer = MeshRenderer(rasterizer=self.mesh_rasterizer,
shader=SoftPhongShader(device=device,
cameras=cameras,
lights=lights))
self.cameras = self.mesh_rasterizer.cameras
def save_p3d_mesh(verts, faces, filling_factors):
features = [(int(i * 255), 0, 0) for i in filling_factors]
features = torch.unsqueeze(torch.Tensor(features), 0)
if torch.cuda.is_available():
device = torch.device("cuda:0")
torch.cuda.set_device(device)
else:
device = torch.device("cpu")
texture = TexturesVertex(features)
mesh = Meshes(torch.unsqueeze(torch.Tensor(verts), 0),
torch.unsqueeze(torch.Tensor(faces), 0), texture).cuda()
# Initialize a camera.
# Rotate the object by increasing the elevation and azimuth angles
R, T = look_at_view_transform(dist=2.0, elev=-50, azim=-90)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
# Define the settings for rasterization and shading. Here we set the output image to be of size
# 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1
# and blur_radius=0.0. We also set bin_size and max_faces_per_bin to None which ensure that
# the faster coarse-to-fine rasterization method is used. Refer to rasterize_meshes.py for
# explanations of these parameters. Refer to docs/notes/renderer.md for an explanation of
# the difference between naive and coarse-to-fine rasterization.
raster_settings = RasterizationSettings(
image_size=1024,
blur_radius=0.0,
faces_per_pixel=1,
)
# Place a point light in front of the object. As mentioned above, the front of the cow is facing the
# -z direction.
lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]])
# Create a phong renderer by composing a rasterizer and a shader. The textured phong shader will
# interpolate the texture uv coordinates for each vertex, sample from a texture image and
# apply the Phong lighting model
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=SoftPhongShader(device=device,
cameras=cameras,
lights=lights))
img = renderer(mesh)
plt.figure(figsize=(10, 10))
plt.imshow(img[0].cpu().numpy())
plt.show()
def _setup_render(self):
# Unpack options ...
opts = self.opts
# Initialize a camera.
# TODO(ycho): Alternatively, specify the intrinsic matrix `K` instead.
cameras = FoVPerspectiveCameras(znear=opts.znear,
zfar=opts.zfar,
aspect_ratio=opts.aspect,
fov=opts.fov,
degrees=True,
device=self.device)
# Define the settings for rasterization and shading.
# As we are rendering images for visualization purposes only we will set faces_per_pixel=1
# and blur_radius=0.0. Refer to raster_points.py for explanations of
# these parameters.
# points_per_pixel (Optional): We will keep track of this many points per
# pixel, returning the nearest points_per_pixel points along the z-axis
# Create a points renderer by compositing points using an alpha compositor (nearer points
# are weighted more heavily). See [1] for an explanation.
if self.opts.use_mesh:
raster_settings = RasterizationSettings(
image_size=opts.image_size,
blur_radius=0.0, # hmm...
faces_per_pixel=1)
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
lights = PointLights(device=self.device,
location=[[0.0, 0.0, -3.0]])
renderer = MeshRenderer(rasterizer=rasterizer,
shader=SoftPhongShader(device=self.device,
cameras=cameras,
lights=lights))
else:
raster_settings = PointsRasterizationSettings(
image_size=opts.image_size, radius=0.1, points_per_pixel=8)
rasterizer = PointsRasterizer(cameras=cameras,
raster_settings=raster_settings)
renderer = PointsRenderer(rasterizer=rasterizer,
compositor=AlphaCompositor())
return renderer
def test_cameras(self):
"""
DVR cameras
"""
device = torch.device('cuda:0')
input_dir = '/home/ywang/Documents/points/neural_splatter/differentiable_volumetric_rendering_upstream/data/DTU/scan106/scan106'
out_dir = os.path.join('tests', 'outputs', 'test_dvr_data')
if not os.path.exists(out_dir):
os.makedirs(out_dir)
dvr_camera_file = os.path.join(input_dir, 'cameras.npz')
dvr_camera_dict = np.load(dvr_camera_file)
n_views = len(glob.glob(os.path.join(input_dir, 'image', '*.png')))
focal_lengths = dvr_camera_dict['camera_mat_0'][(0,1),(0,1)].reshape(1,2)
principal_point = dvr_camera_dict['camera_mat_0'][(0,1),(2,2)].reshape(1,2)
cameras = PerspectiveCameras(focal_length=focal_lengths, principal_point=principal_point).to(device)
# Define the settings for rasterization and shading.
# Refer to raster_points.py for explanations of these parameters.
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0,
faces_per_pixel=5,
# this setting controls whether naive or coarse-to-fine rasterization is used
bin_size=None,
max_faces_per_bin=None # this setting is for coarse rasterization
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=None, raster_settings=raster_settings),
shader=SoftPhongShader(device=device)
)
mesh = trimesh.load_mesh('/home/ywang/Documents/points/neural_splatter/differentiable_volumetric_rendering_upstream/out/multi_view_reconstruction/birds/ours_depth_mvs/vis/000_0000477500.ply')
textures = TexturesVertex(verts_features=torch.ones(
1, mesh.vertices.shape[0], 3)).to(device=device)
meshes = Meshes(verts=[torch.tensor(mesh.vertices).float()], faces=[torch.tensor(mesh.faces)],
textures=textures).to(device=device)
for i in range(n_views):
transform_mat = torch.from_numpy(dvr_camera_dict['scale_mat_%d' % i].T @ dvr_camera_dict['world_mat_%d' % i].T).to(device).unsqueeze(0).float()
cameras.R, cameras.T = decompose_to_R_and_t(transform_mat)
cameras._N = cameras.R.shape[0]
imgs = renderer(meshes, cameras=cameras, zfar=1e4, znear=1.0)
import pdb; pdb.set_trace()
imageio.imwrite(os.path.join(out_dir, '%06d.png' % i), (imgs[0].detach().cpu().numpy()*255).astype('uint8'))
def differentiable_face_render(vert, tri, colors, bg_img, h, w):
"""
vert: (N, nver, 3)
tri: (ntri, 3)
colors: (N, nver. 3)
bg_img: (N, 3, H, W)
"""
assert h == w
N, nver, _ = vert.shape
ntri = tri.shape[0]
tri = torch.from_numpy(tri).to(vert.device).unsqueeze(0).expand(N, ntri, 3)
# Transform to Pytorch3D world space
vert_t = vert + torch.tensor((0.5, 0.5, 0), dtype=torch.float, device=vert.device).view(1, 1, 3)
vert_t = vert_t * torch.tensor((-1, 1, -1), dtype=torch.float, device=vert.device).view(1, 1, 3)
mesh_torch = Meshes(verts=vert_t, faces=tri, textures=TexturesVertex(verts_features=colors))
# Render
R = look_at_rotation(camera_position=((0, 0, -300),)).to(vert.device).expand(N, 3, 3)
T = torch.tensor((0, 0, 300), dtype=torch.float, device=vert.device).view(1, 3).expand(N, 3)
focal = torch.tensor((2. / float(w), 2. / float(h)), dtype=torch.float, device=vert.device).view(1, 2).expand(N, 2)
cameras = OrthographicCameras(device=vert.device, R=R, T=T, focal_length=focal)
raster_settings = RasterizationSettings(image_size=h, blur_radius=0.0, faces_per_pixel=1)
lights = DirectionalLights(ambient_color=((1., 1., 1.),), diffuse_color=((0., 0., 0.),),
specular_color=((0., 0., 0.),), direction=((0, 0, 1),), device=vert.device)
blend_params = BlendParams(background_color=(0, 0, 0))
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings
),
shader=SoftPhongShader(
device=vert.device,
cameras=cameras,
lights=lights,
blend_params=blend_params
)
)
images = renderer(mesh_torch)[:, :, :, :3] # (N, H, W, 3)
# Add background
if bg_img is not None:
bg_img = bg_img.permute(0, 2, 3, 1) # (N, H, W, 3)
images = torch.where(torch.eq(images.sum(dim=3, keepdim=True).expand(N, h, w, 3), 0), bg_img, images)
return images
def render_mesh(mesh, R, T, device, img_size=512, silhouette=False):
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
if silhouette:
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=img_size,
blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma,
faces_per_pixel=100,
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings
),
shader=SoftSilhouetteShader(blend_params=blend_params)
)
else:
raster_settings = RasterizationSettings(
image_size=img_size,
blur_radius=0.0,
faces_per_pixel=1,
)
lights = PointLights(device=device, location=[[0.0, 5.0, -10.0]])
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings
),
shader=SoftPhongShader(
device=device,
cameras=cameras,
lights=lights
)
)
rendered_images = renderer(mesh, cameras=cameras)
return rendered_images
def setup_shader(self, cameras, lights):
if self.shader_type == 'soft_phong':
self.shader = SoftPhongShader(device=self.device,
cameras=cameras,
lights=lights)
def _create_image_renderer(self):
raster_settings = self._get_rasterization_settings()
renderer = MeshRenderer(rasterizer=MeshRasterizer(cameras=self.cameras, raster_settings=raster_settings),
shader=SoftPhongShader(device=self.device, cameras=self.cameras))
return renderer
def initRender(self, method, image_size):
cameras = OpenGLPerspectiveCameras(device=self.device, fov=15)
if (method == "soft-silhouette"):
blend_params = BlendParams(sigma=1e-7, gamma=1e-7)
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=np.log(1. / 1e-7 - 1.) * blend_params.sigma,
faces_per_pixel=self.faces_per_pixel)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params))
elif (method == "hard-silhouette"):
blend_params = BlendParams(sigma=1e-7, gamma=1e-7)
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=np.log(1. / 1e-7 - 1.) * blend_params.sigma,
faces_per_pixel=1)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params))
elif (method == "soft-depth"):
# Soft Rasterizer - from https://github.com/facebookresearch/pytorch3d/issues/95
#blend_params = BlendParams(sigma=1e-7, gamma=1e-7)
blend_params = BlendParams(sigma=1e-3, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=image_size,
#blur_radius= np.log(1. / 1e-7 - 1.) * blend_params.sigma,
blur_radius=np.log(1. / 1e-3 - 1.) * blend_params.sigma,
faces_per_pixel=self.faces_per_pixel)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftDepthShader(blend_params=blend_params))
elif (method == "hard-depth"):
raster_settings = RasterizationSettings(image_size=image_size,
blur_radius=0,
faces_per_pixel=20)
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardDepthShader())
elif (method == "blurry-depth"):
# Soft Rasterizer - from https://github.com/facebookresearch/pytorch3d/issues/95
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma,
faces_per_pixel=self.faces_per_pixel)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftDepthShader(blend_params=blend_params))
elif (method == "soft-phong"):
blend_params = BlendParams(sigma=1e-3, gamma=1e-3)
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=np.log(1. / 1e-3 - 1.) * blend_params.sigma,
faces_per_pixel=self.faces_per_pixel)
# lights = DirectionalLights(device=self.device,
# ambient_color=[[0.25, 0.25, 0.25]],
# diffuse_color=[[0.6, 0.6, 0.6]],
# specular_color=[[0.15, 0.15, 0.15]],
# direction=[[0.0, 1.0, 0.0]])
lights = DirectionalLights(device=self.device,
direction=[[0.0, 1.0, 0.0]])
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftPhongShader(device=self.device,
blend_params=blend_params,
lights=lights))
elif (method == "hard-phong"):
blend_params = BlendParams(sigma=1e-8, gamma=1e-8)
raster_settings = RasterizationSettings(image_size=image_size,
blur_radius=0.0,
faces_per_pixel=1)
lights = DirectionalLights(device=self.device,
ambient_color=[[0.25, 0.25, 0.25]],
diffuse_color=[[0.6, 0.6, 0.6]],
specular_color=[[0.15, 0.15, 0.15]],
direction=[[-1.0, -1.0, 1.0]])
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=self.device,
lights=lights))
else:
print("Unknown render method!")
return None
return renderer
def _compare_with_meshes_renderer(self,
image_size,
batch_size=11,
sphere_diameter=0.6):
"""
Generate a spherical RGB volumetric function and its corresponding mesh
and check whether MeshesRenderer returns the same images as the
corresponding ImplicitRenderer.
"""
# generate NDC camera extrinsics and intrinsics
cameras = init_cameras(batch_size, image_size=image_size, ndc=True)
# get rand offset of the volume
sphere_centroid = torch.randn(batch_size, 3,
device=cameras.device) * 0.1
sphere_centroid.requires_grad = True
# init the grid raysampler with the ndc grid
raysampler = NDCMultinomialRaysampler(
image_width=image_size[1],
image_height=image_size[0],
n_pts_per_ray=256,
min_depth=0.1,
max_depth=2.0,
)
# get the EA raymarcher
raymarcher = EmissionAbsorptionRaymarcher()
# jitter the camera intrinsics a bit for each render
cameras_randomized = cameras.clone()
cameras_randomized.principal_point = (
torch.randn_like(cameras.principal_point) * 0.3)
cameras_randomized.focal_length = (
cameras.focal_length +
torch.randn_like(cameras.focal_length) * 0.2)
# the list of differentiable camera vars
cam_vars = ("R", "T", "focal_length", "principal_point")
# enable the gradient caching for the camera variables
for cam_var in cam_vars:
getattr(cameras_randomized, cam_var).requires_grad = True
# get the implicit renderer
images_opacities = ImplicitRenderer(
raysampler=raysampler, raymarcher=raymarcher)(
cameras=cameras_randomized,
volumetric_function=spherical_volumetric_function,
sphere_centroid=sphere_centroid,
sphere_diameter=sphere_diameter,
)[0]
# check that the renderer does not erase gradients
loss = images_opacities.sum()
loss.backward()
for check_var in (
*[
getattr(cameras_randomized, cam_var)
for cam_var in cam_vars
],
sphere_centroid,
):
self.assertIsNotNone(check_var.grad)
# instantiate the corresponding spherical mesh
ico = ico_sphere(level=4, device=cameras.device).extend(batch_size)
verts = (torch.nn.functional.normalize(ico.verts_padded(), dim=-1) *
sphere_diameter + sphere_centroid[:, None])
meshes = Meshes(
verts=verts,
faces=ico.faces_padded(),
textures=TexturesVertex(verts_features=(
torch.nn.functional.normalize(verts, dim=-1) * 0.5 + 0.5)),
)
# instantiate the corresponding mesh renderer
lights = PointLights(device=cameras.device, location=[[0.0, 0.0, 0.0]])
renderer_textured = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras_randomized,
raster_settings=RasterizationSettings(
image_size=image_size,
blur_radius=1e-3,
faces_per_pixel=10,
z_clip_value=None,
perspective_correct=False,
),
),
shader=SoftPhongShader(
device=cameras.device,
cameras=cameras_randomized,
lights=lights,
materials=Materials(
ambient_color=((2.0, 2.0, 2.0), ),
diffuse_color=((0.0, 0.0, 0.0), ),
specular_color=((0.0, 0.0, 0.0), ),
shininess=64,
device=cameras.device,
),
blend_params=BlendParams(sigma=1e-3,
gamma=1e-4,
background_color=(0.0, 0.0, 0.0)),
),
)
# get the mesh render
images_opacities_meshes = renderer_textured(meshes,
cameras=cameras_randomized,
lights=lights)
if DEBUG:
outdir = tempfile.gettempdir() + "/test_implicit_vs_mesh_renderer"
os.makedirs(outdir, exist_ok=True)
frames = []
for (image_opacity,
image_opacity_mesh) in zip(images_opacities,
images_opacities_meshes):
image, opacity = image_opacity.split([3, 1], dim=-1)
image_mesh, opacity_mesh = image_opacity_mesh.split([3, 1],
dim=-1)
diff_image = (((image - image_mesh) * 0.5 + 0.5).mean(
dim=2, keepdim=True).repeat(1, 1, 3))
image_pil = Image.fromarray((torch.cat(
(
image,
image_mesh,
diff_image,
opacity.repeat(1, 1, 3),
opacity_mesh.repeat(1, 1, 3),
),
dim=1,
).detach().cpu().numpy() * 255.0).astype(np.uint8))
frames.append(image_pil)
# export gif
outfile = os.path.join(outdir, "implicit_vs_mesh_render.gif")
frames[0].save(
outfile,
save_all=True,
append_images=frames[1:],
duration=batch_size // 15,
loop=0,
)
print(f"exported {outfile}")
# export concatenated frames
outfile_cat = os.path.join(outdir, "implicit_vs_mesh_render.png")
Image.fromarray(
np.concatenate([np.array(f) for f in frames],
axis=0)).save(outfile_cat)
print(f"exported {outfile_cat}")
# compare the renders
diff = (images_opacities - images_opacities_meshes).abs().mean(dim=-1)
mu_diff = diff.mean(dim=(1, 2))
std_diff = diff.std(dim=(1, 2))
self.assertClose(mu_diff, torch.zeros_like(mu_diff), atol=5e-2)
self.assertClose(std_diff, torch.zeros_like(std_diff), atol=6e-2)
def batch_render(
verts,
faces,
faces_per_pixel=10,
K=None,
rot=None,
trans=None,
colors=None,
color=(0.53, 0.53, 0.8), # light_purple
ambient_col=0.5,
specular_col=0.2,
diffuse_col=0.3,
face_colors=None,
# color = (0.74117647, 0.85882353, 0.65098039), # light_blue
image_sizes=None,
out_res=512,
bin_size=0,
shading="soft",
mode="rgb",
blend_gamma=1e-4,
min_depth=None,
):
device = torch.device("cuda:0")
K = K.to(device)
width, height = image_sizes[0]
out_size = int(max(image_sizes[0]))
raster_settings = RasterizationSettings(
image_size=out_size,
blur_radius=0.0,
faces_per_pixel=faces_per_pixel,
bin_size=bin_size,
)
fx = K[:, 0, 0]
fy = K[:, 1, 1]
focals = torch.stack([fx, fy], 1)
px = K[:, 0, 2]
py = K[:, 1, 2]
principal_point = torch.stack([width - px, height - py], 1)
if rot is None:
rot = torch.eye(3).unsqueeze(0).to(device)
if trans is None:
trans = torch.zeros(3).unsqueeze(0).to(device)
cameras = PerspectiveCameras(
device=device,
focal_length=focals,
principal_point=principal_point,
image_size=[(out_size, out_size) for _ in range(len(verts))],
R=rot,
T=trans,
)
if mode == "rgb":
lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]])
lights = DirectionalLights(
device=device,
direction=((0.6, -0.6, -0.6), ),
ambient_color=((ambient_col, ambient_col, ambient_col), ),
diffuse_color=((diffuse_col, diffuse_col, diffuse_col), ),
specular_color=((specular_col, specular_col, specular_col), ),
)
if shading == "soft":
shader = SoftPhongShader(device=device,
cameras=cameras,
lights=lights)
elif shading == "hard":
shader = HardPhongShader(device=device,
cameras=cameras,
lights=lights)
else:
raise ValueError(
f"Shading {shading} for mode rgb not in [sort|hard]")
elif mode == "silh":
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
shader = SoftSilhouetteShader(blend_params=blend_params)
elif shading == "faceidx":
shader = FaceIdxShader()
elif (mode == "facecolor") and (shading == "hard"):
shader = FaceColorShader(face_colors=face_colors)
elif (mode == "facecolor") and (shading == "soft"):
shader = SoftFaceColorShader(face_colors=face_colors,
blend_gamma=blend_gamma)
else:
raise ValueError(
f"Unhandled mode {mode} and shading {shading} combination")
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=shader,
)
if min_depth is not None:
verts = torch.cat([verts[:, :, :2], verts[:, :, 2:].clamp(min_depth)],
2)
if mode == "rgb":
if colors is None:
colors = get_colors(verts, color)
tex = textures.TexturesVertex(verts_features=colors)
meshes = Meshes(verts=verts, faces=faces, textures=tex)
elif mode in ["silh", "facecolor"]:
meshes = Meshes(verts=verts, faces=faces)
else:
raise ValueError(f"Render mode {mode} not in [rgb|silh]")
square_images = renderer(meshes, cameras=cameras)
square_images = torch.flip(square_images, (1, 2))
height_off = abs(int(width - height))
if width > height:
images = square_images[:, height_off:, :]
else:
images = square_images[:, :, height_off:]
return images
def render_img(face_shape,
face_color,
facemodel,
image_size=224,
fx=1015.0,
fy=1015.0,
px=112.0,
py=112.0,
device='cuda:0'):
'''
ref: https://github.com/facebookresearch/pytorch3d/issues/184
The rendering function (just for test)
Input:
face_shape: Tensor[1, 35709, 3]
face_color: Tensor[1, 35709, 3] in [0, 1]
facemodel: contains `tri` (triangles[70789, 3], index start from 1)
'''
from pytorch3d.structures import Meshes
from pytorch3d.renderer.mesh.textures import TexturesVertex
from pytorch3d.renderer import (PerspectiveCameras, PointLights,
RasterizationSettings, MeshRenderer,
MeshRasterizer, SoftPhongShader,
BlendParams)
face_color = TexturesVertex(verts_features=face_color.to(device))
face_buf = torch.from_numpy(facemodel.tri - 1) # index start from 1
face_idx = face_buf.unsqueeze(0)
mesh = Meshes(face_shape.to(device), face_idx.to(device), face_color)
R = torch.eye(3).view(1, 3, 3).to(device)
R[0, 0, 0] *= -1.0
T = torch.zeros([1, 3]).to(device)
half_size = (image_size - 1.0) / 2
focal_length = torch.tensor([fx / half_size, fy / half_size],
dtype=torch.float32).reshape(1, 2).to(device)
principal_point = torch.tensor([(half_size - px) / half_size,
(py - half_size) / half_size],
dtype=torch.float32).reshape(1,
2).to(device)
cameras = PerspectiveCameras(device=device,
R=R,
T=T,
focal_length=focal_length,
principal_point=principal_point)
raster_settings = RasterizationSettings(image_size=image_size,
blur_radius=0.0,
faces_per_pixel=1)
lights = PointLights(device=device,
ambient_color=((1.0, 1.0, 1.0), ),
diffuse_color=((0.0, 0.0, 0.0), ),
specular_color=((0.0, 0.0, 0.0), ),
location=((0.0, 0.0, 1e5), ))
blend_params = BlendParams(background_color=(0.0, 0.0, 0.0))
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=SoftPhongShader(device=device,
cameras=cameras,
lights=lights,
blend_params=blend_params))
images = renderer(mesh)
images = torch.clamp(images, 0.0, 1.0)
return images
trans = cameras.unproject_points(trans.unsqueeze(0),
world_coordinates=False,
scaled_depth_input=True)[0]
rot = random_rotations(1)[0].to(device)
transform = Transform3d() \
.scale(scale) \
.compose(Rotate(rot)) \
.translate(*trans)
# TODO: transform mesh
# Create a phong renderer by composing a rasterizer and a shader. The textured phong shader will
# interpolate the texture uv coordinates for each vertex, sample from a texture image and
# apply the Phong lighting model
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=SoftPhongShader(
device=device,
cameras=cameras,
lights=lights,
))
images = renderer(mesh.scale_verts(scale),
R=rot.unsqueeze(0),
T=trans.unsqueeze(0))
plt.figure(figsize=(10, 10))
plt.imshow(images[0, ..., :3].cpu().numpy())
plt.grid("off")
plt.axis("off")
plt.show()
plt.close('all')
principal_point=((512, 512), ),
R=Rtotal,
T=Ttotal,
image_size=((1024, 1024), ))
mymaterials = Materials(device=device, shininess=8)
raster_settings = RasterizationSettings(
image_size=1024,
blur_radius=0.0,
faces_per_pixel=1,
)
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=camera, raster_settings=raster_settings),
shader=SoftPhongShader(
device=device,
cameras=camera,
lights=lights,
materials=mymaterials,
))
meshes = mesh.extend(num_views)
target_images = renderer(meshes, cameras=cameras, lights=lights)
target_rgb = [target_images[i, ..., :3] for i in range(num_views)]
target_cameras = [
PerspectiveCameras(device=device,
focal_length=4500,
principal_point=((512, 512), ),
R=Rtotal[None, i, ...],
T=Ttotal[None, i, ...],
image_size=((1024, 1024), ))
for i in range(num_views)
]
def generate_cow_renders(num_views: int = 40,
data_dir: str = DATA_DIR,
azimuth_range: float = 180):
"""
This function generates `num_views` renders of a cow mesh.
The renders are generated from viewpoints sampled at uniformly distributed
azimuth intervals. The elevation is kept constant so that the camera's
vertical position coincides with the equator.
For a more detailed explanation of this code, please refer to the
docs/tutorials/fit_textured_mesh.ipynb notebook.
Args:
num_views: The number of generated renders.
data_dir: The folder that contains the cow mesh files. If the cow mesh
files do not exist in the folder, this function will automatically
download them.
Returns:
cameras: A batch of `num_views` `FoVPerspectiveCameras` from which the
images are rendered.
images: A tensor of shape `(num_views, height, width, 3)` containing
the rendered images.
silhouettes: A tensor of shape `(num_views, height, width)` containing
the rendered silhouettes.
"""
# set the paths
# download the cow mesh if not done before
cow_mesh_files = [
os.path.join(data_dir, fl)
for fl in ("cow.obj", "cow.mtl", "cow_texture.png")
]
if any(not os.path.isfile(f) for f in cow_mesh_files):
os.makedirs(data_dir, exis_ok=True)
os.system(
f"wget -P {data_dir} " +
"https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.obj")
os.system(
f"wget -P {data_dir} " +
"https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.mtl")
os.system(
f"wget -P {data_dir} " +
"https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png"
)
# Setup
if torch.cuda.is_available():
device = torch.device("cuda:0")
torch.cuda.set_device(device)
else:
device = torch.device("cpu")
# Load obj file
obj_filename = os.path.join(data_dir, "cow.obj")
mesh = load_objs_as_meshes([obj_filename], device=device)
# We scale normalize and center the target mesh to fit in a sphere of radius 1
# centered at (0,0,0). (scale, center) will be used to bring the predicted mesh
# to its original center and scale. Note that normalizing the target mesh,
# speeds up the optimization but is not necessary!
verts = mesh.verts_packed()
N = verts.shape[0]
center = verts.mean(0)
scale = max((verts - center).abs().max(0)[0])
mesh.offset_verts_(-(center.expand(N, 3)))
mesh.scale_verts_((1.0 / float(scale)))
# Get a batch of viewing angles.
elev = torch.linspace(0, 0, num_views) # keep constant
azim = torch.linspace(-azimuth_range, azimuth_range, num_views) + 180.0
# Place a point light in front of the object. As mentioned above, the front of
# the cow is facing the -z direction.
lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]])
# Initialize an OpenGL perspective camera that represents a batch of different
# viewing angles. All the cameras helper methods support mixed type inputs and
# broadcasting. So we can view the camera from the a distance of dist=2.7, and
# then specify elevation and azimuth angles for each viewpoint as tensors.
R, T = look_at_view_transform(dist=2.7, elev=elev, azim=azim)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
# Define the settings for rasterization and shading. Here we set the output
# image to be of size 128X128. As we are rendering images for visualization
# purposes only we will set faces_per_pixel=1 and blur_radius=0.0. Refer to
# rasterize_meshes.py for explanations of these parameters. We also leave
# bin_size and max_faces_per_bin to their default values of None, which sets
# their values using huristics and ensures that the faster coarse-to-fine
# rasterization method is used. Refer to docs/notes/renderer.md for an
# explanation of the difference between naive and coarse-to-fine rasterization.
raster_settings = RasterizationSettings(image_size=128,
blur_radius=0.0,
faces_per_pixel=1)
# Create a phong renderer by composing a rasterizer and a shader. The textured
# phong shader will interpolate the texture uv coordinates for each vertex,
# sample from a texture image and apply the Phong lighting model
blend_params = BlendParams(sigma=1e-4,
gamma=1e-4,
background_color=(0.0, 0.0, 0.0))
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftPhongShader(device=device,
cameras=cameras,
lights=lights,
blend_params=blend_params),
)
# Create a batch of meshes by repeating the cow mesh and associated textures.
# Meshes has a useful `extend` method which allows us do this very easily.
# This also extends the textures.
meshes = mesh.extend(num_views)
# Render the cow mesh from each viewing angle
target_images = renderer(meshes, cameras=cameras, lights=lights)
# Rasterization settings for silhouette rendering
sigma = 1e-4
raster_settings_silhouette = RasterizationSettings(
image_size=128,
blur_radius=np.log(1.0 / 1e-4 - 1.0) * sigma,
faces_per_pixel=50)
# Silhouette renderer
renderer_silhouette = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings_silhouette),
shader=SoftSilhouetteShader(),
)
# Render silhouette images. The 3rd channel of the rendering output is
# the alpha/silhouette channel
silhouette_images = renderer_silhouette(meshes,
cameras=cameras,
lights=lights)
# binary silhouettes
silhouette_binary = (silhouette_images[..., 3] > 1e-4).float()
return cameras, target_images[..., :3], silhouette_binary
# Define the settings for rasterization and shading. Here we set the output image to be of size
# 256x256. To form the blended image we use 100 faces for each pixel. Refer to rasterize_meshes.py
# for an explanation of this parameter.
raster_settings = RasterizationSettings(image_size=256,
blur_radius=np.log(1. / 0.001 - 1.) *
blend_params.sigma,
faces_per_pixel=80,
bin_size=0)
# Create a silhouette mesh renderer by composing a rasterizer and a shader.
lights = PointLights(device=device, location=((2.0, 2.0, -2.0), ))
silhouette_renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftPhongShader(blend_params=blend_params,
device=device,
lights=lights))
# We will also create a phong renderer. This is simpler and only needs to render one face per pixel.
raster_settings = RasterizationSettings(image_size=256,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
# We can add a point light in front of the object.
lights = PointLights(device=device, location=((2.0, 2.0, -2.0), ))
phong_renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=device,
lights=lights))
# Select the viewpoint using spherical angles
