def create_scene(self, hws, alphas, N): batch = [] for i in range(N): scene = [] for mesh_name in self.models: hw = hws[mesh_name] alpha = alphas[mesh_name] N, K, _ = hw.shape for k in range(K): c = self.colors[mesh_name].clone() c[..., 3] = alpha[i, k] textures = TexturesVertex(verts_features=[c]) m = Meshes(verts=[self.verts[mesh_name].clone()], faces=[self.faces[mesh_name].clone()], textures=textures) t = Translate(y=hw[i, k, 0], x=hw[i, k, 1], z=torch.zeros(1, device=self.device), device=str(self.device)) m = m.update_padded(t.transform_points(m.verts_padded())) scene += [m] batch += [join_meshes_as_scene(scene)] batch = join_meshes_as_batch(batch) return batch
def normalize_to_sphere_(self): """ Center and scale the point clouds to a unit sphere Returns: normalizing_trans (Transform3D) """ # (B,3,2) boxMinMax = self.get_bounding_boxes() boxCenter = boxMinMax.sum(dim=-1) / 2 # (B,) boxRange, _ = (boxMinMax[:, :, 1] - boxMinMax[:, :, 0]).max(dim=-1) if boxRange == 0: boxRange = 1 # center and scale the point clouds, likely faster than calling obj2world_trans directly? pointOffsets = torch.repeat_interleave(-boxCenter, self.num_points_per_cloud(), dim=0) self.offset_(pointOffsets) # (P) norms = torch.norm(self.points_packed(), dim=-1) # List[(Pi)] norms = torch.split(norms, self.num_points_per_cloud()) # (N) scale = torch.stack([x.max() for x in norms], dim=0) self.scale_(1 / eps_denom(scale)) normalizing_trans = Translate(-boxCenter).compose( Scale(1 / eps_denom(scale))).to(device=self.device) self.obj2world_trans = normalizing_trans.inverse().compose( self.obj2world_trans) return normalizing_trans
def normalize_to_box_(self): """ center and scale the point clouds to a unit cube, Returns: normalizing_trans (Transform3D): Transform3D used to normalize the pointclouds """ # (B,3,2) boxMinMax = self.get_bounding_boxes() boxCenter = boxMinMax.sum(dim=-1) / 2 # (B,) boxRange, _ = (boxMinMax[:, :, 1] - boxMinMax[:, :, 0]).max(dim=-1) if boxRange == 0: boxRange = 1 # center and scale the point clouds, likely faster than calling obj2world_trans directly? pointOffsets = torch.repeat_interleave(-boxCenter, self.num_points_per_cloud(), dim=0) self.offset_(pointOffsets) self.scale_(1 / boxRange) # update obj2world_trans normalizing_trans = Translate(-boxCenter).compose(Scale( 1 / boxRange)).to(device=self.device) self.obj2world_trans = normalizing_trans.inverse().compose( self.obj2world_trans) return normalizing_trans
def test_scene(): world = engine.World() world.add_mesh('red_box', verts, faces, red) world.add_mesh('green_box', verts, faces, green) world.add_mesh('blue_box', verts, faces, blue) scene_spec = [ {'red_box_0': 'red_box', 'green_box_0': 'green_box'}, {'blue_box_0': 'blue_box', 'blue_box_1': 'blue_box'} ] world.create_scenes(scene_spec) poses = [ [Translate(0, -30, 0), Translate(-10, -10, 0)], [Translate(40, 0, 0), Translate(-10, -10, 0)] ] world.update_scenes(poses) batch = world.batch() labels = world.labels() distance = 30 elevation = 0.0 azimuth = 0 R, T = look_at_view_transform(distance, elevation, azimuth) cameras = FoVOrthographicCameras(max_x=64.0, max_y=64.0, min_x=-64.0, min_y=-64.0, scale_xyz=((1, 1, 1),), R=R, T=T) raster_settings = RasterizationSettings( image_size=128, blur_radius=0, faces_per_pixel=6, ) renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings, ), shader=IdentityShader() ) boxes = world.bounding_boxes(cameras, (128, 128)) image = renderer(batch) fig, ax = plt.subplots(nrows=1, ncols=1) ax.imshow(image[0, :, :, 0, :]) for box in boxes[0]: ax.add_patch(box.get_patch()) plt.show()
def get_world_to_view_transform(self, **kwargs) -> Transform3d: """ Return the world-to-view transform. Args: **kwargs: parameters for the camera extrinsics can be passed in as keyword arguments to override the default values set in __init__. Setting R and T here will update the values set in init as these values may be needed later on in the rendering pipeline e.g. for lighting calculations. Returns: T: a Transform3d object which represents a batch of transforms of shape (N, 3, 3) """ R = self.R = kwargs.get("R", self.R) # pyre-ignore[16] T = self.T = kwargs.get("T", self.T) # pyre-ignore[16] if T.shape[0] != R.shape[0]: msg = "Expected R, T to have the same batch dimension; got %r, %r" raise ValueError(msg % (R.shape[0], T.shape[0])) if T.dim() != 2 or T.shape[1:] != (3,): msg = "Expected T to have shape (N, 3); got %r" raise ValueError(msg % repr(T.shape)) if R.dim() != 3 or R.shape[1:] != (3, 3): msg = "Expected R to have shape (N, 3, 3); got %r" raise ValueError(msg % R.shape) # Create a Transform3d object T = Translate(T, device=T.device) R = Rotate(R, device=R.device) world_to_view_transform = R.compose(T) return world_to_view_transform
def normalize_to_sphere_(self): """ Center and scale the point clouds to a unit sphere Returns: normalizing_trans (Transform3D) """ # packed offset center = torch.stack([x.mean(dim=0) for x in self.points_list()], dim=0) center_packed = torch.repeat_interleave(-center, self.num_points_per_cloud(), dim=0) self.offset_(center_packed) # (P) norms = torch.norm(self.points_packed(), dim=-1) # List[(Pi)] norms = torch.split(norms, self.num_points_per_cloud()) # (N) scale = torch.stack([x.max() for x in norms], dim=0) self.scale_(1 / eps_denom(scale)) normalizing_trans = Translate(-center).compose( Scale(1 / eps_denom(scale))).to(device=self.device) self.obj2world_trans = normalizing_trans.inverse().compose( self.obj2world_trans) return normalizing_trans
def get_world_to_view_transform(R=r, T=t) -> Transform3d: """ This function returns a Transform3d representing the transformation matrix to go from world space to view space by applying a rotation and a translation. Pytorch3d uses the same convention as Hartley & Zisserman. I.e., for camera extrinsic parameters R (rotation) and T (translation), we map a 3D point `X_world` in world coordinates to a point `X_cam` in camera coordinates with: `X_cam = X_world R + T` Args: R: (N, 3, 3) matrix representing the rotation. T: (N, 3) matrix representing the translation. Returns: a Transform3d object which represents the composed RT transformation. """ # TODO: also support the case where RT is specified as one matrix # of shape (N, 4, 4). if T.shape[0] != R.shape[0]: msg = "Expected R, T to have the same batch dimension; got %r, %r" raise ValueError(msg % (R.shape[0], T.shape[0])) if T.dim() != 2 or T.shape[1:] != (3, ): msg = "Expected T to have shape (N, 3); got %r" raise ValueError(msg % repr(T.shape)) if R.dim() != 3 or R.shape[1:] != (3, 3): msg = "Expected R to have shape (N, 3, 3); got %r" raise ValueError(msg % repr(R.shape)) # Create a Transform3d object T = Translate(T, device=T.device) R = Rotate(R, device=R.device) return R.compose(T)
def marching_cubes_naive( volume_data_batch: torch.Tensor, isolevel: Optional[float] = None, spacing: int = 1, return_local_coords: bool = True, ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: """ Runs the classic marching cubes algorithm, iterating over the coordinates of the volume_data and using a given isolevel for determining intersected edges of cubes of size `spacing`. Returns vertices and faces of the obtained mesh. This operation is non-differentiable. This is a naive implementation, and is not optimized for efficiency. Args: volume_data_batch: a Tensor of size (N, D, H, W) corresponding to a batch of 3D scalar fields isolevel: the isosurface value to use as the threshold to determine whether points are within a volume. If None, then the average of the maximum and minimum value of the scalar field will be used. spacing: an integer specifying the cube size to use return_local_coords: bool. If True the output vertices will be in local coordinates in the range [-1, 1] x [-1, 1] x [-1, 1]. If False they will be in the range [0, W-1] x [0, H-1] x [0, D-1] Returns: verts: [(V_0, 3), (V_1, 3), ...] List of N FloatTensors of vertices. faces: [(F_0, 3), (F_1, 3), ...] List of N LongTensors of faces. """ volume_data_batch = volume_data_batch.detach().cpu() batched_verts, batched_faces = [], [] D, H, W = volume_data_batch.shape[1:] # pyre-ignore [16] volume_size_xyz = volume_data_batch.new_tensor([W, H, D])[None] if return_local_coords: # Convert from local coordinates in the range [-1, 1] range to # world coordinates in the range [0, D-1], [0, H-1], [0, W-1] local_to_world_transform = Translate( x=+1.0, y=+1.0, z=+1.0, device=volume_data_batch.device).scale( (volume_size_xyz - 1) * spacing * 0.5) # Perform the inverse to go from world to local world_to_local_transform = local_to_world_transform.inverse() for i in range(len(volume_data_batch)): volume_data = volume_data_batch[i] curr_isolevel = (((volume_data.max() + volume_data.min()) / 2).item() if isolevel is None else isolevel) edge_vertices_to_index = {} vertex_coords_to_index = {} verts, faces = [], [] # Use length - spacing for the bounds since we are using # cubes of size spacing, with the lowest x,y,z values # (bottom front left) for x in range(0, W - spacing, spacing): for y in range(0, H - spacing, spacing): for z in range(0, D - spacing, spacing): cube = Cube((x, y, z), spacing) new_verts, new_faces = polygonise( cube, curr_isolevel, volume_data, edge_vertices_to_index, vertex_coords_to_index, ) verts.extend(new_verts) faces.extend(new_faces) if len(faces) > 0 and len(verts) > 0: verts = torch.tensor(verts, dtype=torch.float32) # Convert vertices from world to local coords if return_local_coords: verts = world_to_local_transform.transform_points(verts[None, ...]) verts = verts.squeeze() batched_verts.append(verts) batched_faces.append(torch.tensor(faces, dtype=torch.int64)) return batched_verts, batched_faces
return i // ncols, i % ncols plt.ion() fig = plt.figure(figsize=(10, 5)) spec = plt.GridSpec(ncols=N // 2, nrows=2, figure=fig) ax = [fig.add_subplot(spec[gridnum(i, ncols=N // 2)]) for i in range(N)] plt.tight_layout() for b in range(50): transforms = [] for _ in range(N): model_matrices = [] for _ in objects: xy = torch.rand(2) * 128 - 64 model_matrices += [Translate(x=xy[0], y=xy[1], z=0)] transforms += [model_matrices] distance = 30 elevation = 0.0 azimuth = 0 R, T = look_at_view_transform(distance, elevation, azimuth, device=device) cameras = FoVOrthographicCameras(device=device, max_x=64.0, max_y=64.0, min_x=-64.0, min_y=-64.0,
for i in range(1): for mesh_name in models: hw = hws[mesh_name] alpha = alphas[mesh_name] N, K, _ = hw.shape for k in range(K): c = colors[mesh_name].clone() c[..., 3] = c[..., 3] * alpha[i, k] textures = TexturesVertex(verts_features=[c]) m = Meshes(verts=[verts[mesh_name].clone()], faces=[faces[mesh_name].clone()], textures=textures) #m = meshes[mesh_name].clone().detach().to(device) t = Translate(y=hw[i, k, 0], x=hw[i, k, 1], z=torch.zeros(1, device=device), device=str(device)) m = m.update_padded(t.transform_points(m.verts_padded())) scene += [m] teapot_mesh = join_meshes_as_scene(scene) #for i, (name, particles) in particles_per_mesh: #textures = TexturesVertex(verts_features=colors) # meshes = Meshes( # verts=verts, # faces=faces, # textures=textures, # )