def test_points_renderer_to(self):
"""
Test moving all the tensors in the points renderer to a new device.
"""
device1 = torch.device("cpu")
R, T = look_at_view_transform(1500, 0.0, 0.0)
raster_settings = PointsRasterizationSettings(image_size=256,
radius=0.001,
points_per_pixel=1)
cameras = FoVPerspectiveCameras(device=device1,
R=R,
T=T,
aspect_ratio=1.0,
fov=60.0,
zfar=100)
rasterizer = PointsRasterizer(cameras=cameras,
raster_settings=raster_settings)
renderer = PointsRenderer(rasterizer=rasterizer,
compositor=AlphaCompositor())
mesh = ico_sphere(2, device1)
verts_padded = mesh.verts_padded()
pointclouds = Pointclouds(points=verts_padded,
features=torch.randn_like(verts_padded))
self._check_points_renderer_props_on_device(renderer, device1)
# Test rendering on cpu
output_images = renderer(pointclouds)
self.assertEqual(output_images.device, device1)
# Move renderer and pointclouds to another device and re render
device2 = torch.device("cuda:0")
renderer = renderer.to(device2)
pointclouds = pointclouds.to(device2)
self._check_points_renderer_props_on_device(renderer, device2)
output_images = renderer(pointclouds)
self.assertEqual(output_images.device, device2)
def _setup_render(self):
# Unpack options ...
opts = self.opts
# Initialize a camera.
# TODO(ycho): Alternatively, specify the intrinsic matrix `K` instead.
cameras = FoVPerspectiveCameras(znear=opts.znear,
zfar=opts.zfar,
aspect_ratio=opts.aspect,
fov=opts.fov,
degrees=True,
device=self.device)
# Define the settings for rasterization and shading.
# As we are rendering images for visualization purposes only we will set faces_per_pixel=1
# and blur_radius=0.0. Refer to raster_points.py for explanations of
# these parameters.
# points_per_pixel (Optional): We will keep track of this many points per
# pixel, returning the nearest points_per_pixel points along the z-axis
# Create a points renderer by compositing points using an alpha compositor (nearer points
# are weighted more heavily). See [1] for an explanation.
if self.opts.use_mesh:
raster_settings = RasterizationSettings(
image_size=opts.image_size,
blur_radius=0.0, # hmm...
faces_per_pixel=1)
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
lights = PointLights(device=self.device,
location=[[0.0, 0.0, -3.0]])
renderer = MeshRenderer(rasterizer=rasterizer,
shader=SoftPhongShader(device=self.device,
cameras=cameras,
lights=lights))
else:
raster_settings = PointsRasterizationSettings(
image_size=opts.image_size, radius=0.1, points_per_pixel=8)
rasterizer = PointsRasterizer(cameras=cameras,
raster_settings=raster_settings)
renderer = PointsRenderer(rasterizer=rasterizer,
compositor=AlphaCompositor())
return renderer
def __init__(self, cfgs):
super(Renderer, self).__init__()
self.device = cfgs.get('device', 'cuda:0')
self.image_size = cfgs.get('image_size', 64)
self.min_depth = cfgs.get('min_depth', 0.9)
self.max_depth = cfgs.get('max_depth', 1.1)
self.rot_center_depth = cfgs.get('rot_center_depth',
(self.min_depth + self.max_depth) / 2)
# todo: FoV (Field of View) was set to be an fixed value of 10 degree (according to the paper).
self.fov = cfgs.get('fov', 10)
self.tex_cube_size = cfgs.get('tex_cube_size', 2)
self.renderer_min_depth = cfgs.get('renderer_min_depth', 0.1)
self.renderer_max_depth = cfgs.get('renderer_max_depth', 10.)
#### camera intrinsics
# (u) (x)
# d * K^-1 (v) = (y)
# (1) (z)
## renderer for visualization
R = [[[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]]]
R = torch.FloatTensor(R).to(self.device)
t = torch.zeros(1, 3, dtype=torch.float32).to(self.device)
## todo: K is the camera intrinsic matrix.
fx = (self.image_size - 1) / 2 / (math.tan(
self.fov / 2 * math.pi / 180))
fy = (self.image_size - 1) / 2 / (math.tan(
self.fov / 2 * math.pi / 180))
cx = (self.image_size - 1) / 2
cy = (self.image_size - 1) / 2
K = [[fx, 0., cx], [0., fy, cy], [0., 0., 1.]]
K = torch.FloatTensor(K).to(self.device)
self.inv_K = torch.inverse(K).unsqueeze(0)
self.K = K.unsqueeze(0)
## todo: define Renderer.
## use renderer from pytorch3d.
# fixme: znear and zfar is equivalent to the neural renderer default settings.
cameras = OpenGLOrthographicCameras(device=self.device,
R=R,
T=t,
znear=0.01,
zfar=100)
# cameras = OpenGLPerspectiveCameras(device=self.device, R=R, T=t,
# znear=self.renderer_min_depth,
# zfar=self.renderer_max_depth,
# fov=self.fov)
raster_settings = PointsRasterizationSettings(
image_size=self.image_size,
radius=0.003,
points_per_pixel=10,
bin_size=None,
max_points_per_bin=None)
self.renderer = PointsRenderer(
rasterizer=PointsRasterizer(cameras=cameras,
raster_settings=raster_settings),
compositor=AlphaCompositor(composite_params=None))
# assert len(cameras) == 1
# Define the settings for rasterization and shading. Here we set the output image to be of size
# 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1
# and blur_radius=0.0. Refer to raster_points.py for explanations of these parameters.
raster_settings = PointsRasterizationSettings(
image_size=scene.image_size,
# radius=0.003,
radius=0.005,
points_per_pixel=10)
# Create a points renderer by compositing points using an alpha compositor (nearer points
# are weighted more heavily). See [1] for an explanation.
rasterizer = PointsRasterizer(cameras=cameras,
raster_settings=raster_settings)
renderer = PointsRenderer(rasterizer=rasterizer,
compositor=AlphaCompositor())
images = renderer(scene.point_cloud)
PrintTensorInfo(images)
save_image(images[0].permute((2, 0, 1)), "debug/img0.jpg")
gt = scene.GetGroundTruth(1)
print("gt")
PrintTensorInfo(gt)
save_image(gt, "debug/img0_gt.jpg")
plt.figure(figsize=(10, 10))
plt.imshow(images[0, ..., :3].cpu().numpy())
plt.axis("off")
plt.waitforbuttonpress()
def test_compare_with_pointclouds_renderer(self,
batch_size=11,
volume_size=(30, 30, 30),
image_size=(200, 250)):
"""
Generate a volume and its corresponding point cloud and check whether
PointsRenderer returns the same images as the corresponding VolumeRenderer.
"""
# generate NDC camera extrinsics and intrinsics
cameras = init_cameras(batch_size, image_size=image_size, ndc=True)
# init the boundary volume
for shape in ("sphere", "cube"):
if not DEBUG and shape == "cube":
# do not run numeric checks for the cube as the
# differences in rendering equations make the renders incomparable
continue
# get rand offset of the volume
volume_translation = torch.randn(batch_size, 3) * 0.1
# volume_translation[2] = 0.1
volumes = init_boundary_volume(
volume_size=volume_size,
batch_size=batch_size,
shape=shape,
volume_translation=volume_translation,
)[0]
# convert the volumes to a pointcloud
points = []
points_features = []
for densities_one, features_one, grid_one in zip(
volumes.densities(),
volumes.features(),
volumes.get_coord_grid(world_coordinates=True),
):
opaque = densities_one.view(-1) > 1e-4
points.append(grid_one.view(-1, 3)[opaque])
points_features.append(features_one.reshape(3, -1).t()[opaque])
pointclouds = Pointclouds(points, features=points_features)
# init the grid raysampler with the ndc grid
coord_range = 1.0
half_pix_size = coord_range / max(*image_size)
raysampler = NDCMultinomialRaysampler(
image_width=image_size[1],
image_height=image_size[0],
n_pts_per_ray=256,
min_depth=0.1,
max_depth=2.0,
)
# get the EA raymarcher
raymarcher = EmissionAbsorptionRaymarcher()
# jitter the camera intrinsics a bit for each render
cameras_randomized = cameras.clone()
cameras_randomized.principal_point = (
torch.randn_like(cameras.principal_point) * 0.3)
cameras_randomized.focal_length = (
cameras.focal_length +
torch.randn_like(cameras.focal_length) * 0.2)
# get the volumetric render
images = VolumeRenderer(raysampler=raysampler,
raymarcher=raymarcher,
sample_mode="bilinear")(
cameras=cameras_randomized,
volumes=volumes)[0][..., :3]
# instantiate the points renderer
point_radius = 6 * half_pix_size
points_renderer = PointsRenderer(
rasterizer=PointsRasterizer(
cameras=cameras_randomized,
raster_settings=PointsRasterizationSettings(
image_size=image_size,
radius=point_radius,
points_per_pixel=10),
),
compositor=AlphaCompositor(),
)
# get the point render
images_pts = points_renderer(pointclouds)
if shape == "sphere":
diff = (images - images_pts).abs().mean(dim=-1)
mu_diff = diff.mean(dim=(1, 2))
std_diff = diff.std(dim=(1, 2))
self.assertClose(mu_diff, torch.zeros_like(mu_diff), atol=3e-2)
self.assertClose(std_diff,
torch.zeros_like(std_diff),
atol=6e-2)
if DEBUG:
outdir = tempfile.gettempdir() + "/test_volume_vs_pts_renderer"
os.makedirs(outdir, exist_ok=True)
frames = []
for (image, image_pts) in zip(images, images_pts):
diff_image = (((image - image_pts) * 0.5 + 0.5).mean(
dim=2, keepdim=True).repeat(1, 1, 3))
image_pil = Image.fromarray(
(torch.cat((image, image_pts, diff_image),
dim=1).detach().cpu().numpy() *
255.0).astype(np.uint8))
frames.append(image_pil)
# export gif
outfile = os.path.join(outdir,
f"volume_vs_pts_render_{shape}.gif")
frames[0].save(
outfile,
save_all=True,
append_images=frames[1:],
duration=batch_size // 15,
loop=0,
)
print(f"exported {outfile}")
# export concatenated frames
outfile_cat = os.path.join(
outdir, f"volume_vs_pts_render_{shape}.png")
Image.fromarray(
np.concatenate([np.array(f) for f in frames],
axis=0)).save(outfile_cat)
print(f"exported {outfile_cat}")
# pespective projection: x=fX/Z assuming px=py=0, normalization of Z
verts[:, :,
1] = verts[:, :, 1].clone() * proj_cam[:, :1] / verts[:, :,
2].clone()
verts[:, :,
0] = verts[:, :, 0].clone() * proj_cam[:, :1] / verts[:, :,
2].clone()
verts[:, :,
2] = ((verts[:, :, 2] - verts[:, :, 2].min()) /
(verts[:, :, 2].max() - verts[:, :, 2].min()) - 0.5).detach()
verts[:, :, 2] += 10
features = torch.ones_like(verts)
point_cloud = Pointclouds(points=verts[:, :, :3],
features=torch.Tensor(
mesh.visual.vertex_colors[None]).cuda())
cameras = OrthographicCameras(device=device)
raster_settings = PointsRasterizationSettings(image_size=img_size,
radius=0.005,
points_per_pixel=10)
renderer = PointsRenderer(
rasterizer=PointsRasterizer(cameras=cameras,
raster_settings=raster_settings),
compositor=AlphaCompositor(background_color=(33, 33, 33)))
img_pred = renderer(point_cloud)
frames.append(img_pred[0, :, :, :3].cpu())
#cv2.imwrite('%s/points%04d.png'%(args.outdir,i), np.asarray(img_pred[0,:,:,:3].cpu())[:,:,::-1])
imageio.mimsave('./output-depth.gif', frames, duration=5. / len(frames))