def forward(self, depth_logits, depth_maps, gt_boxes2d): """ Gets DDN loss Args: depth_logits: (B, D+1, H, W), Predicted depth logits depth_maps: (B, H, W), Depth map [m] gt_boxes2d: torch.Tensor (B, N, 4), 2D box labels for foreground/background balancing Returns: loss: (1), Depth distribution network loss tb_dict: dict[float], All losses to log in tensorboard """ tb_dict = {} # Bin depth map to create target depth_target = transform_utils.bin_depths(depth_maps, **self.disc_cfg, target=True) # Compute loss loss = self.loss_func(depth_logits, depth_target) # Compute foreground/background balancing loss, tb_dict = self.balancer(loss=loss, gt_boxes2d=gt_boxes2d) # Final loss loss *= self.weight tb_dict.update({"ddn_loss": loss.item()}) return loss, tb_dict
def transform_grid(self, voxel_grid, grid_to_lidar, lidar_to_cam, cam_to_img): """ Transforms voxel sampling grid into frustum sampling grid Args: grid: (B, X, Y, Z, 3), Voxel sampling grid grid_to_lidar: (4, 4), Voxel grid to LiDAR unprojection matrix lidar_to_cam: (B, 4, 4), LiDAR to camera frame transformation cam_to_img: (B, 3, 4), Camera projection matrix Returns: frustum_grid: (B, X, Y, Z, 3), Frustum sampling grid """ B = lidar_to_cam.shape[0] # Create transformation matricies V_G = grid_to_lidar # Voxel Grid -> LiDAR (4, 4) C_V = lidar_to_cam # LiDAR -> Camera (B, 4, 4) I_C = cam_to_img # Camera -> Image (B, 3, 4) trans = C_V @ V_G # Reshape to match dimensions trans = trans.reshape(B, 1, 1, 4, 4) voxel_grid = voxel_grid.repeat_interleave(repeats=B, dim=0) # Transform to camera frame camera_grid = kornia.transform_points(trans_01=trans, points_1=voxel_grid) # Project to image I_C = I_C.reshape(B, 1, 1, 3, 4) image_grid, image_depths = transform_utils.project_to_image( project=I_C, points=camera_grid) # Convert depths to depth bins image_depths = transform_utils.bin_depths(depth_map=image_depths, **self.disc_cfg) # Stack to form frustum grid image_depths = image_depths.unsqueeze(-1) frustum_grid = torch.cat((image_grid, image_depths), dim=-1) return frustum_grid