def test_simple_sphere(self): device = torch.device("cuda:0") sphere_mesh = ico_sphere(1, device) verts_padded = sphere_mesh.verts_padded() # Shift vertices to check coordinate frames are correct. verts_padded[..., 1] += 0.2 verts_padded[..., 0] += 0.2 pointclouds = Pointclouds( points=verts_padded, features=torch.ones_like(verts_padded) ) R, T = look_at_view_transform(2.7, 0.0, 0.0) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) raster_settings = PointsRasterizationSettings( image_size=256, radius=5e-2, points_per_pixel=1 ) rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings) compositor = NormWeightedCompositor() renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor) # Load reference image filename = "simple_pointcloud_sphere.png" image_ref = load_rgb_image("test_%s" % filename, DATA_DIR) for bin_size in [0, None]: # Check both naive and coarse to fine produce the same output. renderer.rasterizer.raster_settings.bin_size = bin_size images = renderer(pointclouds) rgb = images[0, ..., :3].squeeze().cpu() if DEBUG: filename = "DEBUG_%s" % filename Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save( DATA_DIR / filename ) self.assertClose(rgb, image_ref)
def test_render_pointcloud(self): """ Test a textured point cloud is rendered correctly in a non square image. """ device = torch.device("cuda:0") pointclouds = Pointclouds( points=[torus_points * 2.0], features=torch.ones_like(torus_points[None, ...]), ).to(device) R, T = look_at_view_transform(2.7, 0.0, 0.0) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) raster_settings = PointsRasterizationSettings( image_size=(512, 1024), radius=5e-2, points_per_pixel=1 ) rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings) compositor = AlphaCompositor() renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor) # Load reference image image_ref = load_rgb_image("test_pointcloud_rectangle_image.png", DATA_DIR) for bin_size in [0, None]: # Check both naive and coarse to fine produce the same output. renderer.rasterizer.raster_settings.bin_size = bin_size images = renderer(pointclouds) rgb = images[0, ..., :3].squeeze().cpu() if DEBUG: Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save( DATA_DIR / "DEBUG_pointcloud_rectangle_image.png" ) # NOTE some pixels can be flaky cond1 = torch.allclose(rgb, image_ref, atol=0.05) self.assertTrue(cond1)
def test_simple_sphere_batched(self): device = torch.device("cuda:0") sphere_mesh = ico_sphere(1, device) verts_padded = sphere_mesh.verts_padded() verts_padded[..., 1] += 0.2 verts_padded[..., 0] += 0.2 pointclouds = Pointclouds( points=verts_padded, features=torch.ones_like(verts_padded) ) batch_size = 20 pointclouds = pointclouds.extend(batch_size) R, T = look_at_view_transform(2.7, 0.0, 0.0) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) raster_settings = PointsRasterizationSettings( image_size=256, radius=5e-2, points_per_pixel=1 ) rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings) compositor = NormWeightedCompositor() renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor) # Load reference image filename = "simple_pointcloud_sphere.png" image_ref = load_rgb_image("test_%s" % filename, DATA_DIR) images = renderer(pointclouds) for i in range(batch_size): rgb = images[i, ..., :3].squeeze().cpu() if i == 0 and DEBUG: filename = "DEBUG_%s" % filename Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save( DATA_DIR / filename ) self.assertClose(rgb, image_ref)
def test_texture_sampling_cow(self): # test texture sampling for the cow example by converting # the cow mesh and its texture uv to a pointcloud with texture device = torch.device("cuda:0") obj_dir = get_pytorch3d_dir() / "docs/tutorials/data" obj_filename = obj_dir / "cow_mesh/cow.obj" for text_type in ("uv", "atlas"): # Load mesh + texture if text_type == "uv": mesh = load_objs_as_meshes( [obj_filename], device=device, load_textures=True, texture_wrap=None ) elif text_type == "atlas": mesh = load_objs_as_meshes( [obj_filename], device=device, load_textures=True, create_texture_atlas=True, texture_atlas_size=8, texture_wrap=None, ) points, normals, textures = sample_points_from_meshes( mesh, num_samples=50000, return_normals=True, return_textures=True ) pointclouds = Pointclouds(points, normals=normals, features=textures) for pos in ("front", "back"): # Init rasterizer settings if pos == "back": azim = 0.0 elif pos == "front": azim = 180 R, T = look_at_view_transform(2.7, 0, azim) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) raster_settings = PointsRasterizationSettings( image_size=512, radius=1e-2, points_per_pixel=1 ) rasterizer = PointsRasterizer( cameras=cameras, raster_settings=raster_settings ) compositor = NormWeightedCompositor() renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor) images = renderer(pointclouds) rgb = images[0, ..., :3].squeeze().cpu() if DEBUG: filename = "DEBUG_cow_mesh_to_pointcloud_%s_%s.png" % ( text_type, pos, ) Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save( DATA_DIR / filename )
def __init__(self): super(SceneModel, self).__init__() self.gamma = 1.0 # Points. torch.manual_seed(1) vert_pos = torch.rand(N_POINTS, 3, dtype=torch.float32, device=DEVICE) * 10.0 vert_pos[:, 2] += 25.0 vert_pos[:, :2] -= 5.0 self.register_parameter("vert_pos", nn.Parameter(vert_pos, requires_grad=True)) self.register_parameter( "vert_col", nn.Parameter( torch.ones(N_POINTS, 3, dtype=torch.float32, device=DEVICE) * 0.5, requires_grad=True, ), ) self.register_parameter( "vert_rad", nn.Parameter( torch.ones(N_POINTS, dtype=torch.float32) * 0.3, requires_grad=True ), ) self.register_buffer( "cam_params", torch.tensor( [0.0, 0.0, 0.0, 0.0, math.pi, 0.0, 5.0, 2.0], dtype=torch.float32 ), ) self.cameras = PerspectiveCameras( # The focal length must be double the size for PyTorch3D because of the NDC # coordinates spanning a range of two - and they must be normalized by the # sensor width (see the pulsar example). This means we need here # 5.0 * 2.0 / 2.0 to get the equivalent results as in pulsar. focal_length=5.0, R=torch.eye(3, dtype=torch.float32, device=DEVICE)[None, ...], T=torch.zeros((1, 3), dtype=torch.float32, device=DEVICE), image_size=((HEIGHT, WIDTH),), device=DEVICE, ) raster_settings = PointsRasterizationSettings( image_size=(HEIGHT, WIDTH), radius=self.vert_rad, ) rasterizer = PointsRasterizer( cameras=self.cameras, raster_settings=raster_settings ) self.renderer = PulsarPointsRenderer(rasterizer=rasterizer, n_track=32)
def test_simple_sphere_pulsar(self): for device in [torch.device("cpu"), torch.device("cuda")]: sphere_mesh = ico_sphere(1, device) verts_padded = sphere_mesh.verts_padded() # Shift vertices to check coordinate frames are correct. verts_padded[..., 1] += 0.2 verts_padded[..., 0] += 0.2 pointclouds = Pointclouds( points=verts_padded, features=torch.ones_like(verts_padded) ) for azimuth in [0.0, 90.0]: R, T = look_at_view_transform(2.7, 0.0, azimuth) for camera_name, cameras in [ ("fovperspective", FoVPerspectiveCameras(device=device, R=R, T=T)), ( "fovorthographic", FoVOrthographicCameras(device=device, R=R, T=T), ), ("perspective", PerspectiveCameras(device=device, R=R, T=T)), ("orthographic", OrthographicCameras(device=device, R=R, T=T)), ]: raster_settings = PointsRasterizationSettings( image_size=256, radius=5e-2, points_per_pixel=1 ) rasterizer = PointsRasterizer( cameras=cameras, raster_settings=raster_settings ) renderer = PulsarPointsRenderer(rasterizer=rasterizer).to(device) # Load reference image filename = ( "pulsar_simple_pointcloud_sphere_" f"azimuth{azimuth}_{camera_name}.png" ) image_ref = load_rgb_image("test_%s" % filename, DATA_DIR) images = renderer( pointclouds, gamma=(1e-3,), znear=(1.0,), zfar=(100.0,) ) rgb = images[0, ..., :3].squeeze().cpu() if DEBUG: filename = "DEBUG_%s" % filename Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save( DATA_DIR / filename ) self.assertClose(rgb, image_ref, rtol=7e-3, atol=5e-3)
def test_pointcloud_with_features(self): device = torch.device("cuda:0") file_dir = Path(__file__).resolve().parent.parent / "docs/tutorials/data" pointcloud_filename = file_dir / "PittsburghBridge/pointcloud.npz" # Note, this file is too large to check in to the repo. # Download the file to run the test locally. if not path.exists(pointcloud_filename): url = "https://dl.fbaipublicfiles.com/pytorch3d/data/PittsburghBridge/pointcloud.npz" msg = ( "pointcloud.npz not found, download from %s, save it at the path %s, and rerun" % (url, pointcloud_filename) ) warnings.warn(msg) return True # Load point cloud pointcloud = np.load(pointcloud_filename) verts = torch.Tensor(pointcloud["verts"]).to(device) rgb_feats = torch.Tensor(pointcloud["rgb"]).to(device) verts.requires_grad = True rgb_feats.requires_grad = True point_cloud = Pointclouds(points=[verts], features=[rgb_feats]) R, T = look_at_view_transform(20, 10, 0) cameras = FoVOrthographicCameras(device=device, R=R, T=T, znear=0.01) raster_settings = PointsRasterizationSettings( # Set image_size so it is not a multiple of 16 (min bin_size) # in order to confirm that there are no errors in coarse rasterization. image_size=500, radius=0.003, points_per_pixel=10, ) renderer = PointsRenderer( rasterizer=PointsRasterizer( cameras=cameras, raster_settings=raster_settings ), compositor=AlphaCompositor(), ) images = renderer(point_cloud) # Load reference image filename = "bridge_pointcloud.png" image_ref = load_rgb_image("test_%s" % filename, DATA_DIR) for bin_size in [0, None]: # Check both naive and coarse to fine produce the same output. renderer.rasterizer.raster_settings.bin_size = bin_size images = renderer(point_cloud) rgb = images[0, ..., :3].squeeze().cpu() if DEBUG: filename = "DEBUG_%s" % filename Image.fromarray((rgb.detach().numpy() * 255).astype(np.uint8)).save( DATA_DIR / filename ) self.assertClose(rgb, image_ref, atol=0.015) # Check grad exists. grad_images = torch.randn_like(images) images.backward(grad_images) self.assertIsNotNone(verts.grad) self.assertIsNotNone(rgb_feats.grad)
def test_unified_inputs_pulsar(self): # Test data on different devices. for device in [torch.device("cpu"), torch.device("cuda")]: sphere_mesh = ico_sphere(1, device) verts_padded = sphere_mesh.verts_padded() pointclouds = Pointclouds( points=verts_padded, features=torch.ones_like(verts_padded) ) R, T = look_at_view_transform(2.7, 0.0, 0.0) # Test the different camera types. for _, cameras in [ ("fovperspective", FoVPerspectiveCameras(device=device, R=R, T=T)), ( "fovorthographic", FoVOrthographicCameras(device=device, R=R, T=T), ), ("perspective", PerspectiveCameras(device=device, R=R, T=T)), ("orthographic", OrthographicCameras(device=device, R=R, T=T)), ]: # Test different ways for image size specification. for image_size in (256, (256, 256)): raster_settings = PointsRasterizationSettings( image_size=image_size, radius=5e-2, points_per_pixel=1 ) rasterizer = PointsRasterizer( cameras=cameras, raster_settings=raster_settings ) # Test that the compositor can be provided. It's value is ignored # so use a dummy. _ = PulsarPointsRenderer(rasterizer=rasterizer, compositor=1).to( device ) # Constructor without compositor. _ = PulsarPointsRenderer(rasterizer=rasterizer).to(device) # Constructor with n_channels. _ = PulsarPointsRenderer(rasterizer=rasterizer, n_channels=3).to( device ) # Constructor with max_num_spheres. renderer = PulsarPointsRenderer( rasterizer=rasterizer, max_num_spheres=1000 ).to(device) # Test the forward function. if isinstance(cameras, (PerspectiveCameras, OrthographicCameras)): # znear and zfar is required in this case. self.assertRaises( ValueError, lambda: renderer.forward( point_clouds=pointclouds, gamma=(1e-4,) ), ) renderer.forward( point_clouds=pointclouds, gamma=(1e-4,), znear=(1.0,), zfar=(2.0,), ) # znear and zfar must be batched. self.assertRaises( TypeError, lambda: renderer.forward( point_clouds=pointclouds, gamma=(1e-4,), znear=1.0, zfar=(2.0,), ), ) self.assertRaises( TypeError, lambda: renderer.forward( point_clouds=pointclouds, gamma=(1e-4,), znear=(1.0,), zfar=2.0, ), ) else: # gamma must be batched. self.assertRaises( TypeError, lambda: renderer.forward( point_clouds=pointclouds, gamma=1e-4 ), ) renderer.forward(point_clouds=pointclouds, gamma=(1e-4,)) # rasterizer width and height change. renderer.rasterizer.raster_settings.image_size = 0 self.assertRaises( ValueError, lambda: renderer.forward( point_clouds=pointclouds, gamma=(1e-4,) ), )
def __init__(self): super(SceneModel, self).__init__() self.gamma = 0.1 # Points. torch.manual_seed(1) vert_pos = torch.rand(N_POINTS, 3, dtype=torch.float32) * 10.0 vert_pos[:, 2] += 25.0 vert_pos[:, :2] -= 5.0 self.register_parameter("vert_pos", nn.Parameter(vert_pos, requires_grad=False)) self.register_parameter( "vert_col", nn.Parameter( torch.rand(N_POINTS, 3, dtype=torch.float32), requires_grad=False, ), ) self.register_parameter( "vert_rad", nn.Parameter( torch.rand(N_POINTS, dtype=torch.float32), requires_grad=False, ), ) self.register_parameter( "cam_pos", nn.Parameter( torch.tensor([0.1, 0.1, 0.0], dtype=torch.float32), requires_grad=True, ), ) self.register_parameter( "cam_rot", # We're using the 6D rot. representation for better gradients. nn.Parameter( axis_angle_to_matrix( torch.tensor( [ [0.02, 0.02, 0.01], ], dtype=torch.float32, ))[0], requires_grad=True, ), ) self.register_parameter( "focal_length", nn.Parameter( torch.tensor( [ 4.8 * 2.0 / 2.0, ], dtype=torch.float32, ), requires_grad=True, ), ) self.cameras = PerspectiveCameras( # The focal length must be double the size for PyTorch3D because of the NDC # coordinates spanning a range of two - and they must be normalized by the # sensor width (see the pulsar example). This means we need here # 5.0 * 2.0 / 2.0 to get the equivalent results as in pulsar. # # R, T and f are provided here, but will be provided again # at every call to the forward method. The reason are problems # with PyTorch which makes device placement for gradients problematic # for tensors which are themselves on a 'gradient path' but not # leafs in the calculation tree. This will be addressed by an architectural # change in PyTorch3D in the future. Until then, this workaround is # recommended. focal_length=self.focal_length, R=self.cam_rot[None, ...], T=self.cam_pos[None, ...], image_size=((HEIGHT, WIDTH), ), device=DEVICE, ) raster_settings = PointsRasterizationSettings( image_size=(HEIGHT, WIDTH), radius=self.vert_rad, ) rasterizer = PointsRasterizer(cameras=self.cameras, raster_settings=raster_settings) self.renderer = PulsarPointsRenderer(rasterizer=rasterizer)