def test_simple_sphere(self):
        device = torch.device("cuda:0")
        sphere_mesh = ico_sphere(1, device)
        verts_padded = sphere_mesh.verts_padded()
        # Shift vertices to check coordinate frames are correct.
        verts_padded[..., 1] += 0.2
        verts_padded[..., 0] += 0.2
        pointclouds = Pointclouds(
            points=verts_padded, features=torch.ones_like(verts_padded)
        )
        R, T = look_at_view_transform(2.7, 0.0, 0.0)
        cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
        raster_settings = PointsRasterizationSettings(
            image_size=256, radius=5e-2, points_per_pixel=1
        )
        rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings)
        compositor = NormWeightedCompositor()
        renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor)

        # Load reference image
        filename = "simple_pointcloud_sphere.png"
        image_ref = load_rgb_image("test_%s" % filename, DATA_DIR)

        for bin_size in [0, None]:
            # Check both naive and coarse to fine produce the same output.
            renderer.rasterizer.raster_settings.bin_size = bin_size
            images = renderer(pointclouds)
            rgb = images[0, ..., :3].squeeze().cpu()
            if DEBUG:
                filename = "DEBUG_%s" % filename
                Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
                    DATA_DIR / filename
                )
            self.assertClose(rgb, image_ref)
Exemple #2
0
    def test_render_pointcloud(self):
        """
        Test a textured point cloud is rendered correctly in a non square image.
        """
        device = torch.device("cuda:0")
        pointclouds = Pointclouds(
            points=[torus_points * 2.0],
            features=torch.ones_like(torus_points[None, ...]),
        ).to(device)
        R, T = look_at_view_transform(2.7, 0.0, 0.0)
        cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
        raster_settings = PointsRasterizationSettings(
            image_size=(512, 1024), radius=5e-2, points_per_pixel=1
        )
        rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings)
        compositor = AlphaCompositor()
        renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor)

        # Load reference image
        image_ref = load_rgb_image("test_pointcloud_rectangle_image.png", DATA_DIR)

        for bin_size in [0, None]:
            # Check both naive and coarse to fine produce the same output.
            renderer.rasterizer.raster_settings.bin_size = bin_size
            images = renderer(pointclouds)
            rgb = images[0, ..., :3].squeeze().cpu()

            if DEBUG:
                Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
                    DATA_DIR / "DEBUG_pointcloud_rectangle_image.png"
                )

            # NOTE some pixels can be flaky
            cond1 = torch.allclose(rgb, image_ref, atol=0.05)
            self.assertTrue(cond1)
    def test_simple_sphere_batched(self):
        device = torch.device("cuda:0")
        sphere_mesh = ico_sphere(1, device)
        verts_padded = sphere_mesh.verts_padded()
        verts_padded[..., 1] += 0.2
        verts_padded[..., 0] += 0.2
        pointclouds = Pointclouds(
            points=verts_padded, features=torch.ones_like(verts_padded)
        )
        batch_size = 20
        pointclouds = pointclouds.extend(batch_size)
        R, T = look_at_view_transform(2.7, 0.0, 0.0)
        cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
        raster_settings = PointsRasterizationSettings(
            image_size=256, radius=5e-2, points_per_pixel=1
        )
        rasterizer = PointsRasterizer(cameras=cameras, raster_settings=raster_settings)
        compositor = NormWeightedCompositor()
        renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor)

        # Load reference image
        filename = "simple_pointcloud_sphere.png"
        image_ref = load_rgb_image("test_%s" % filename, DATA_DIR)

        images = renderer(pointclouds)
        for i in range(batch_size):
            rgb = images[i, ..., :3].squeeze().cpu()
            if i == 0 and DEBUG:
                filename = "DEBUG_%s" % filename
                Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
                    DATA_DIR / filename
                )
            self.assertClose(rgb, image_ref)
    def test_texture_sampling_cow(self):
        # test texture sampling for the cow example by converting
        # the cow mesh and its texture uv to a pointcloud with texture

        device = torch.device("cuda:0")
        obj_dir = get_pytorch3d_dir() / "docs/tutorials/data"
        obj_filename = obj_dir / "cow_mesh/cow.obj"

        for text_type in ("uv", "atlas"):
            # Load mesh + texture
            if text_type == "uv":
                mesh = load_objs_as_meshes(
                    [obj_filename], device=device, load_textures=True, texture_wrap=None
                )
            elif text_type == "atlas":
                mesh = load_objs_as_meshes(
                    [obj_filename],
                    device=device,
                    load_textures=True,
                    create_texture_atlas=True,
                    texture_atlas_size=8,
                    texture_wrap=None,
                )

            points, normals, textures = sample_points_from_meshes(
                mesh, num_samples=50000, return_normals=True, return_textures=True
            )
            pointclouds = Pointclouds(points, normals=normals, features=textures)

            for pos in ("front", "back"):
                # Init rasterizer settings
                if pos == "back":
                    azim = 0.0
                elif pos == "front":
                    azim = 180
                R, T = look_at_view_transform(2.7, 0, azim)
                cameras = FoVPerspectiveCameras(device=device, R=R, T=T)

                raster_settings = PointsRasterizationSettings(
                    image_size=512, radius=1e-2, points_per_pixel=1
                )

                rasterizer = PointsRasterizer(
                    cameras=cameras, raster_settings=raster_settings
                )
                compositor = NormWeightedCompositor()
                renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor)
                images = renderer(pointclouds)

                rgb = images[0, ..., :3].squeeze().cpu()
                if DEBUG:
                    filename = "DEBUG_cow_mesh_to_pointcloud_%s_%s.png" % (
                        text_type,
                        pos,
                    )
                    Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
                        DATA_DIR / filename
                    )
Exemple #5
0
 def __init__(self):
     super(SceneModel, self).__init__()
     self.gamma = 1.0
     # Points.
     torch.manual_seed(1)
     vert_pos = torch.rand(N_POINTS, 3, dtype=torch.float32, device=DEVICE) * 10.0
     vert_pos[:, 2] += 25.0
     vert_pos[:, :2] -= 5.0
     self.register_parameter("vert_pos", nn.Parameter(vert_pos, requires_grad=True))
     self.register_parameter(
         "vert_col",
         nn.Parameter(
             torch.ones(N_POINTS, 3, dtype=torch.float32, device=DEVICE) * 0.5,
             requires_grad=True,
         ),
     )
     self.register_parameter(
         "vert_rad",
         nn.Parameter(
             torch.ones(N_POINTS, dtype=torch.float32) * 0.3, requires_grad=True
         ),
     )
     self.register_buffer(
         "cam_params",
         torch.tensor(
             [0.0, 0.0, 0.0, 0.0, math.pi, 0.0, 5.0, 2.0], dtype=torch.float32
         ),
     )
     self.cameras = PerspectiveCameras(
         # The focal length must be double the size for PyTorch3D because of the NDC
         # coordinates spanning a range of two - and they must be normalized by the
         # sensor width (see the pulsar example). This means we need here
         # 5.0 * 2.0 / 2.0 to get the equivalent results as in pulsar.
         focal_length=5.0,
         R=torch.eye(3, dtype=torch.float32, device=DEVICE)[None, ...],
         T=torch.zeros((1, 3), dtype=torch.float32, device=DEVICE),
         image_size=((HEIGHT, WIDTH),),
         device=DEVICE,
     )
     raster_settings = PointsRasterizationSettings(
         image_size=(HEIGHT, WIDTH),
         radius=self.vert_rad,
     )
     rasterizer = PointsRasterizer(
         cameras=self.cameras, raster_settings=raster_settings
     )
     self.renderer = PulsarPointsRenderer(rasterizer=rasterizer, n_track=32)
 def test_simple_sphere_pulsar(self):
     for device in [torch.device("cpu"), torch.device("cuda")]:
         sphere_mesh = ico_sphere(1, device)
         verts_padded = sphere_mesh.verts_padded()
         # Shift vertices to check coordinate frames are correct.
         verts_padded[..., 1] += 0.2
         verts_padded[..., 0] += 0.2
         pointclouds = Pointclouds(
             points=verts_padded, features=torch.ones_like(verts_padded)
         )
         for azimuth in [0.0, 90.0]:
             R, T = look_at_view_transform(2.7, 0.0, azimuth)
             for camera_name, cameras in [
                 ("fovperspective", FoVPerspectiveCameras(device=device, R=R, T=T)),
                 (
                     "fovorthographic",
                     FoVOrthographicCameras(device=device, R=R, T=T),
                 ),
                 ("perspective", PerspectiveCameras(device=device, R=R, T=T)),
                 ("orthographic", OrthographicCameras(device=device, R=R, T=T)),
             ]:
                 raster_settings = PointsRasterizationSettings(
                     image_size=256, radius=5e-2, points_per_pixel=1
                 )
                 rasterizer = PointsRasterizer(
                     cameras=cameras, raster_settings=raster_settings
                 )
                 renderer = PulsarPointsRenderer(rasterizer=rasterizer).to(device)
                 # Load reference image
                 filename = (
                     "pulsar_simple_pointcloud_sphere_"
                     f"azimuth{azimuth}_{camera_name}.png"
                 )
                 image_ref = load_rgb_image("test_%s" % filename, DATA_DIR)
                 images = renderer(
                     pointclouds, gamma=(1e-3,), znear=(1.0,), zfar=(100.0,)
                 )
                 rgb = images[0, ..., :3].squeeze().cpu()
                 if DEBUG:
                     filename = "DEBUG_%s" % filename
                     Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
                         DATA_DIR / filename
                     )
                 self.assertClose(rgb, image_ref, rtol=7e-3, atol=5e-3)
    def test_pointcloud_with_features(self):
        device = torch.device("cuda:0")
        file_dir = Path(__file__).resolve().parent.parent / "docs/tutorials/data"
        pointcloud_filename = file_dir / "PittsburghBridge/pointcloud.npz"

        # Note, this file is too large to check in to the repo.
        # Download the file to run the test locally.
        if not path.exists(pointcloud_filename):
            url = "https://dl.fbaipublicfiles.com/pytorch3d/data/PittsburghBridge/pointcloud.npz"
            msg = (
                "pointcloud.npz not found, download from %s, save it at the path %s, and rerun"
                % (url, pointcloud_filename)
            )
            warnings.warn(msg)
            return True

        # Load point cloud
        pointcloud = np.load(pointcloud_filename)
        verts = torch.Tensor(pointcloud["verts"]).to(device)
        rgb_feats = torch.Tensor(pointcloud["rgb"]).to(device)

        verts.requires_grad = True
        rgb_feats.requires_grad = True
        point_cloud = Pointclouds(points=[verts], features=[rgb_feats])

        R, T = look_at_view_transform(20, 10, 0)
        cameras = FoVOrthographicCameras(device=device, R=R, T=T, znear=0.01)

        raster_settings = PointsRasterizationSettings(
            # Set image_size so it is not a multiple of 16 (min bin_size)
            # in order to confirm that there are no errors in coarse rasterization.
            image_size=500,
            radius=0.003,
            points_per_pixel=10,
        )

        renderer = PointsRenderer(
            rasterizer=PointsRasterizer(
                cameras=cameras, raster_settings=raster_settings
            ),
            compositor=AlphaCompositor(),
        )

        images = renderer(point_cloud)

        # Load reference image
        filename = "bridge_pointcloud.png"
        image_ref = load_rgb_image("test_%s" % filename, DATA_DIR)

        for bin_size in [0, None]:
            # Check both naive and coarse to fine produce the same output.
            renderer.rasterizer.raster_settings.bin_size = bin_size
            images = renderer(point_cloud)
            rgb = images[0, ..., :3].squeeze().cpu()
            if DEBUG:
                filename = "DEBUG_%s" % filename
                Image.fromarray((rgb.detach().numpy() * 255).astype(np.uint8)).save(
                    DATA_DIR / filename
                )
            self.assertClose(rgb, image_ref, atol=0.015)

        # Check grad exists.
        grad_images = torch.randn_like(images)
        images.backward(grad_images)
        self.assertIsNotNone(verts.grad)
        self.assertIsNotNone(rgb_feats.grad)
 def test_unified_inputs_pulsar(self):
     # Test data on different devices.
     for device in [torch.device("cpu"), torch.device("cuda")]:
         sphere_mesh = ico_sphere(1, device)
         verts_padded = sphere_mesh.verts_padded()
         pointclouds = Pointclouds(
             points=verts_padded, features=torch.ones_like(verts_padded)
         )
         R, T = look_at_view_transform(2.7, 0.0, 0.0)
         # Test the different camera types.
         for _, cameras in [
             ("fovperspective", FoVPerspectiveCameras(device=device, R=R, T=T)),
             (
                 "fovorthographic",
                 FoVOrthographicCameras(device=device, R=R, T=T),
             ),
             ("perspective", PerspectiveCameras(device=device, R=R, T=T)),
             ("orthographic", OrthographicCameras(device=device, R=R, T=T)),
         ]:
             # Test different ways for image size specification.
             for image_size in (256, (256, 256)):
                 raster_settings = PointsRasterizationSettings(
                     image_size=image_size, radius=5e-2, points_per_pixel=1
                 )
                 rasterizer = PointsRasterizer(
                     cameras=cameras, raster_settings=raster_settings
                 )
                 # Test that the compositor can be provided. It's value is ignored
                 # so use a dummy.
                 _ = PulsarPointsRenderer(rasterizer=rasterizer, compositor=1).to(
                     device
                 )
                 # Constructor without compositor.
                 _ = PulsarPointsRenderer(rasterizer=rasterizer).to(device)
                 # Constructor with n_channels.
                 _ = PulsarPointsRenderer(rasterizer=rasterizer, n_channels=3).to(
                     device
                 )
                 # Constructor with max_num_spheres.
                 renderer = PulsarPointsRenderer(
                     rasterizer=rasterizer, max_num_spheres=1000
                 ).to(device)
                 # Test the forward function.
                 if isinstance(cameras, (PerspectiveCameras, OrthographicCameras)):
                     # znear and zfar is required in this case.
                     self.assertRaises(
                         ValueError,
                         lambda: renderer.forward(
                             point_clouds=pointclouds, gamma=(1e-4,)
                         ),
                     )
                     renderer.forward(
                         point_clouds=pointclouds,
                         gamma=(1e-4,),
                         znear=(1.0,),
                         zfar=(2.0,),
                     )
                     # znear and zfar must be batched.
                     self.assertRaises(
                         TypeError,
                         lambda: renderer.forward(
                             point_clouds=pointclouds,
                             gamma=(1e-4,),
                             znear=1.0,
                             zfar=(2.0,),
                         ),
                     )
                     self.assertRaises(
                         TypeError,
                         lambda: renderer.forward(
                             point_clouds=pointclouds,
                             gamma=(1e-4,),
                             znear=(1.0,),
                             zfar=2.0,
                         ),
                     )
                 else:
                     # gamma must be batched.
                     self.assertRaises(
                         TypeError,
                         lambda: renderer.forward(
                             point_clouds=pointclouds, gamma=1e-4
                         ),
                     )
                     renderer.forward(point_clouds=pointclouds, gamma=(1e-4,))
                     # rasterizer width and height change.
                     renderer.rasterizer.raster_settings.image_size = 0
                     self.assertRaises(
                         ValueError,
                         lambda: renderer.forward(
                             point_clouds=pointclouds, gamma=(1e-4,)
                         ),
                     )
 def __init__(self):
     super(SceneModel, self).__init__()
     self.gamma = 0.1
     # Points.
     torch.manual_seed(1)
     vert_pos = torch.rand(N_POINTS, 3, dtype=torch.float32) * 10.0
     vert_pos[:, 2] += 25.0
     vert_pos[:, :2] -= 5.0
     self.register_parameter("vert_pos",
                             nn.Parameter(vert_pos, requires_grad=False))
     self.register_parameter(
         "vert_col",
         nn.Parameter(
             torch.rand(N_POINTS, 3, dtype=torch.float32),
             requires_grad=False,
         ),
     )
     self.register_parameter(
         "vert_rad",
         nn.Parameter(
             torch.rand(N_POINTS, dtype=torch.float32),
             requires_grad=False,
         ),
     )
     self.register_parameter(
         "cam_pos",
         nn.Parameter(
             torch.tensor([0.1, 0.1, 0.0], dtype=torch.float32),
             requires_grad=True,
         ),
     )
     self.register_parameter(
         "cam_rot",
         # We're using the 6D rot. representation for better gradients.
         nn.Parameter(
             axis_angle_to_matrix(
                 torch.tensor(
                     [
                         [0.02, 0.02, 0.01],
                     ],
                     dtype=torch.float32,
                 ))[0],
             requires_grad=True,
         ),
     )
     self.register_parameter(
         "focal_length",
         nn.Parameter(
             torch.tensor(
                 [
                     4.8 * 2.0 / 2.0,
                 ],
                 dtype=torch.float32,
             ),
             requires_grad=True,
         ),
     )
     self.cameras = PerspectiveCameras(
         # The focal length must be double the size for PyTorch3D because of the NDC
         # coordinates spanning a range of two - and they must be normalized by the
         # sensor width (see the pulsar example). This means we need here
         # 5.0 * 2.0 / 2.0 to get the equivalent results as in pulsar.
         #
         # R, T and f are provided here, but will be provided again
         # at every call to the forward method. The reason are problems
         # with PyTorch which makes device placement for gradients problematic
         # for tensors which are themselves on a 'gradient path' but not
         # leafs in the calculation tree. This will be addressed by an architectural
         # change in PyTorch3D in the future. Until then, this workaround is
         # recommended.
         focal_length=self.focal_length,
         R=self.cam_rot[None, ...],
         T=self.cam_pos[None, ...],
         image_size=((HEIGHT, WIDTH), ),
         device=DEVICE,
     )
     raster_settings = PointsRasterizationSettings(
         image_size=(HEIGHT, WIDTH),
         radius=self.vert_rad,
     )
     rasterizer = PointsRasterizer(cameras=self.cameras,
                                   raster_settings=raster_settings)
     self.renderer = PulsarPointsRenderer(rasterizer=rasterizer)