def test_box3diou(self): box1 = box_corners_from_param( torch.tensor([2, 2, 3]).float(), 0, torch.tensor([1, 1, 0])).numpy() box2 = box_corners_from_param( torch.tensor([1, 1, 1]).float(), 0, torch.tensor([0.5, 0.5, 0.5])).numpy() self.assertAlmostEqual(box3d_iou(box1, box2), 1.0 / (2 * 3 * 2), places=5)
def test_intersection_area(self): box1 = box_corners_from_param( torch.tensor([1, 1, 3]).float(), 0, torch.tensor([0, 0, 0])).numpy() box2 = box_corners_from_param( torch.tensor([1, 1, 3]).float(), np.pi / 2, torch.tensor([0, 0, 0])).numpy() rect1 = [(box1[i, 0], box1[i, 1]) for i in range(4)] rect2 = [(box2[i, 0], box2[i, 1]) for i in range(4)] self.assertAlmostEqual(intersection_area(rect1, rect2), 1, places=5) box1 = box_corners_from_param( torch.tensor([2, 2, 3]).float(), 0, torch.tensor([1, 1, 0])).numpy() box2 = box_corners_from_param( torch.tensor([2, 2, 3]).float(), 0, torch.tensor([0, 0, 0])).numpy() rect1 = [(box1[i, 0], box1[i, 1]) for i in range(4)] rect2 = [(box2[i, 0], box2[i, 1]) for i in range(4)] self.assertAlmostEqual(intersection_area(rect1, rect2), 1, places=5) rect1 = [(0, 0), (1, 0), (1, 1), (0, 1)] rect2 = [(0, 0), (1, 1), (0, 2), (-1, 1)] self.assertAlmostEqual(intersection_area(rect1, rect2), 0.5, places=5)
def test_nms(self): res = VoteNetResults(center=torch.zeros((2, 4, 3))) box = box_corners_from_param( torch.tensor([1, 1, 1]).float(), 0, torch.tensor([0.5, 0.5, 0.5])) boxes = box.unsqueeze(0).unsqueeze(0) boxes.repeat((res.batch_size, res.num_proposal, 1, 1)) objectness = torch.tensor([[0, 1, 0.5, 0], [1, 0.8, 0, 0]]) classes = torch.tensor([[0, 0, 0, 0], [2, 1, 1, 1]]).long() mask = res._nms_mask(boxes, objectness, classes) np.testing.assert_equal( mask, np.asarray([[False, True, False, False], [True, True, False, False]]))
def test_cornerfromparams(self): box = box_corners_from_param( torch.tensor([1, 2, 3]).float(), np.pi / 2, torch.tensor([1, 1, 1])) torch.testing.assert_allclose( box, 1 + torch.tensor([ [1.0, -0.5, -1.5], [1.0, 0.5, -1.5], [-1.0, 0.5, -1.5], [-1.0, -0.5, -1.5], [1.0, -0.5, 1.5], [1.0, 0.5, 1.5], [-1.0, 0.5, 1.5], [-1.0, -0.5, 1.5], ]), )
def test_evaldetection(self): box = box_corners_from_param( torch.tensor([1, 1, 1]).float(), 0, torch.tensor([0.5, 0.5, 0.5])) # Image1 -> 1 class1 and 1 class2 # Image2 -> 1 class1 gt = { "0": [BoxData("class1", box), BoxData("class2", box)], "1": [BoxData("class1", box)], } pred = { "0": [ BoxData("class1", box, score=0.5), BoxData("class2", box, score=0.5) ], "1": [BoxData("class2", box, score=1)], } rec, prec, ap = eval_detection(pred, gt) np.testing.assert_allclose(rec["class2"], np.asarray([0, 1])) np.testing.assert_allclose(prec["class2"], np.asarray([0, 0.5])) self.assertAlmostEqual(ap["class2"], 0.5)
def _set_extra_labels(self, data): """ Adds extra labels for the instance and object segmentation tasks instance_box_corners: (MAX_NUM_OBJ, 8, 3) corners of the bounding boxes in this room center_label: (MAX_NUM_OBJ,3) for GT box center XYZ sem_cls_label: (MAX_NUM_OBJ,) semantic class index angle_residual_label: (MAX_NUM_OBJ,) size_residual_label: (MAX_NUM_OBJ,3) box_label_mask: (MAX_NUM_OBJ) as 0/1 with 1 indicating a unique box vote_label: (N,3) with votes XYZ vote_label_mask: (N,) with 0/1 with 1 indicating the point is in one of the object's OBB. """ # Initaliase variables num_points = data.pos.shape[0] semantic_labels = data.y instance_labels = data.instance_labels center_label = torch.zeros((self.MAX_NUM_OBJ, 3)) target_bboxes_mask = torch.zeros((self.MAX_NUM_OBJ), dtype=torch.bool) angle_residuals = torch.zeros((self.MAX_NUM_OBJ, )) size_classes = torch.zeros((self.MAX_NUM_OBJ, )) size_residuals = torch.zeros((self.MAX_NUM_OBJ, 3)) # compute votes *AFTER* augmentation # generate votes # Note: since there's no map between bbox instance labels and # pc instance_labels (it had been filtered # in the data preparation step) we'll compute the instance bbox # from the points sharing the same instance label. point_votes = torch.zeros([num_points, 3]) point_votes_mask = torch.zeros(num_points, dtype=torch.bool) instance_box_corners = [] box_sizes = [] centers = [] instance_classes = [] for i_instance in np.unique(instance_labels): # find all points belong to that instance ind = np.where(instance_labels == i_instance)[0] # find the semantic label instance_class = semantic_labels[ind[0]].item() if instance_class in self.NYU40IDS: pos = data.pos[ind, :3] max_pox = pos.max(0)[0] min_pos = pos.min(0)[0] center = 0.5 * (min_pos + max_pox) point_votes[ind, :] = center - pos point_votes_mask[ind] = True box_size = max_pox - min_pos instance_box_corners.append( box_corners_from_param(box_size, 0, center)) box_sizes.append(box_size) centers.append(center) instance_classes.append(self.NYU40ID2CLASS[instance_class]) point_votes = point_votes.repeat((1, 3)) # make 3 votes identical instance_classes = torch.tensor(instance_classes) # Keep only boxes with valid ids num_instances = len(instance_classes) target_bboxes_mask[0:num_instances] = True # Set box semantic label target_bboxes_semcls = np.zeros((self.MAX_NUM_OBJ)) target_bboxes_semcls[0:num_instances] = instance_classes # Set size residual and box centres size_classes[0:num_instances] = instance_classes if num_instances > 0: box_sizes = torch.stack(box_sizes) centers = torch.stack(centers) size_residuals[0:num_instances, :] = box_sizes - torch.from_numpy( self.MEAN_SIZE_ARR[instance_classes, :]) center_label[0:num_instances, :] = centers data.center_label = center_label data.heading_class_label = torch.zeros((self.MAX_NUM_OBJ, )) data.heading_residual_label = angle_residuals.float() data.size_class_label = size_classes data.size_residual_label = size_residuals.float() data.sem_cls_label = torch.from_numpy(target_bboxes_semcls).int() data.box_label_mask = target_bboxes_mask data.vote_label = point_votes.float() data.vote_label_mask = point_votes_mask data.instance_box_corners = torch.zeros((self.MAX_NUM_OBJ, 8, 3)) if len(instance_box_corners): data.instance_box_corners[:len(instance_box_corners ), :, :] = torch.stack( instance_box_corners) delattr(data, "instance_bboxes") delattr(data, "instance_labels") delattr(data, "y") return data
def get_boxes( self, dataset, apply_nms=False, objectness_threshold=0.05, duplicate_boxes=False ) -> List[List[BoxData]]: """ Generates boxes from predictions Parameters ---------- dataset : Must provide a class2size method and a class2angle method that return the angle and size for a given object class and residual value apply_nms: bool If True then we apply non max suppression before returning the boxes duplicate_boxes: bool If True then we duplicate predicted boxes accross all classes. Else we assign the box to the most likely class Returns ------- List[List[BoxData]] contains the list of predicted boxes for each batch """ # Size and Heading prediciton pred_heading_class = torch.argmax(self.heading_scores, -1) # B,num_proposal pred_heading_residual = torch.gather( self.heading_residuals, 2, pred_heading_class.unsqueeze(-1) ) # B,num_proposal,1 pred_size_class = torch.argmax(self.size_scores, -1) # B,num_proposal pred_size_residual = torch.gather( self.size_residuals, 2, pred_size_class.unsqueeze(-1).unsqueeze(-1).repeat(1, 1, 1, 3) ).squeeze( 2 ) # B,num_proposal,3 # Generate box corners pred_corners_3d = torch.zeros((self.batch_size, self.num_proposal, 8, 3)) for i in range(self.batch_size): for j in range(self.num_proposal): heading_angle = dataset.class2angle(pred_heading_class[i, j], pred_heading_residual[i, j]) box_size = dataset.class2size(pred_size_class[i, j], pred_size_residual[i, j]) corners_3d = box_corners_from_param(box_size, heading_angle, self.center[i, j, :].cpu()) pred_corners_3d[i, j] = corners_3d # Objectness and class pred_obj = torch.nn.functional.softmax(self.objectness_scores, -1)[:, :, 1] # B,num_proposal pred_sem_cls = torch.argmax(self.sem_cls_scores, -1) # B,num_proposal sem_cls_proba = torch.softmax(self.sem_cls_scores, -1) # Apply nms if required if apply_nms: mask = self._nms_mask(pred_corners_3d, pred_obj, pred_sem_cls) else: mask = np.ones((self.batch_size, self.num_proposal), dtype=np.bool) detected_boxes = [] for i in range(self.batch_size): corners = pred_corners_3d[i, mask[i]] objectness = pred_obj[i, mask[i]] sem_cls_scores = sem_cls_proba[i, mask[i]] clsname = pred_sem_cls[i, mask[i]] # Build box data for each detected object and add it to the list batch_detection = [] for j in range(len(corners)): if objectness[j] > objectness_threshold: if duplicate_boxes and self.has_class: for classname in range(sem_cls_proba.shape[-1]): batch_detection.append( BoxData(classname, corners[j], score=objectness[j] * sem_cls_scores[j, classname]) ) else: batch_detection.append(BoxData(clsname[j], corners[j], score=objectness[j])) detected_boxes.append(batch_detection) return detected_boxes
def test_box3dvol(self): box = box_corners_from_param( torch.tensor([1, 2, 3]).float(), np.pi / 2, torch.tensor([0, 0, 0])) self.assertEqual(box3d_vol(box), 6)