def test_box3diou(self):
box1 = box_corners_from_param(
torch.tensor([2, 2, 3]).float(), 0, torch.tensor([1, 1,
0])).numpy()
box2 = box_corners_from_param(
torch.tensor([1, 1, 1]).float(), 0, torch.tensor([0.5, 0.5,
0.5])).numpy()
self.assertAlmostEqual(box3d_iou(box1, box2),
1.0 / (2 * 3 * 2),
places=5)
def test_intersection_area(self):
box1 = box_corners_from_param(
torch.tensor([1, 1, 3]).float(), 0, torch.tensor([0, 0,
0])).numpy()
box2 = box_corners_from_param(
torch.tensor([1, 1, 3]).float(), np.pi / 2,
torch.tensor([0, 0, 0])).numpy()
rect1 = [(box1[i, 0], box1[i, 1]) for i in range(4)]
rect2 = [(box2[i, 0], box2[i, 1]) for i in range(4)]
self.assertAlmostEqual(intersection_area(rect1, rect2), 1, places=5)
box1 = box_corners_from_param(
torch.tensor([2, 2, 3]).float(), 0, torch.tensor([1, 1,
0])).numpy()
box2 = box_corners_from_param(
torch.tensor([2, 2, 3]).float(), 0, torch.tensor([0, 0,
0])).numpy()
rect1 = [(box1[i, 0], box1[i, 1]) for i in range(4)]
rect2 = [(box2[i, 0], box2[i, 1]) for i in range(4)]
self.assertAlmostEqual(intersection_area(rect1, rect2), 1, places=5)
rect1 = [(0, 0), (1, 0), (1, 1), (0, 1)]
rect2 = [(0, 0), (1, 1), (0, 2), (-1, 1)]
self.assertAlmostEqual(intersection_area(rect1, rect2), 0.5, places=5)
def test_nms(self):
res = VoteNetResults(center=torch.zeros((2, 4, 3)))
box = box_corners_from_param(
torch.tensor([1, 1, 1]).float(), 0, torch.tensor([0.5, 0.5, 0.5]))
boxes = box.unsqueeze(0).unsqueeze(0)
boxes.repeat((res.batch_size, res.num_proposal, 1, 1))
objectness = torch.tensor([[0, 1, 0.5, 0], [1, 0.8, 0, 0]])
classes = torch.tensor([[0, 0, 0, 0], [2, 1, 1, 1]]).long()
mask = res._nms_mask(boxes, objectness, classes)
np.testing.assert_equal(
mask,
np.asarray([[False, True, False, False],
[True, True, False, False]]))
def test_cornerfromparams(self):
box = box_corners_from_param(
torch.tensor([1, 2, 3]).float(), np.pi / 2, torch.tensor([1, 1,
1]))
torch.testing.assert_allclose(
box,
1 + torch.tensor([
[1.0, -0.5, -1.5],
[1.0, 0.5, -1.5],
[-1.0, 0.5, -1.5],
[-1.0, -0.5, -1.5],
[1.0, -0.5, 1.5],
[1.0, 0.5, 1.5],
[-1.0, 0.5, 1.5],
[-1.0, -0.5, 1.5],
]),
)
def test_evaldetection(self):
box = box_corners_from_param(
torch.tensor([1, 1, 1]).float(), 0, torch.tensor([0.5, 0.5, 0.5]))
# Image1 -> 1 class1 and 1 class2
# Image2 -> 1 class1
gt = {
"0": [BoxData("class1", box),
BoxData("class2", box)],
"1": [BoxData("class1", box)],
}
pred = {
"0": [
BoxData("class1", box, score=0.5),
BoxData("class2", box, score=0.5)
],
"1": [BoxData("class2", box, score=1)],
}
rec, prec, ap = eval_detection(pred, gt)
np.testing.assert_allclose(rec["class2"], np.asarray([0, 1]))
np.testing.assert_allclose(prec["class2"], np.asarray([0, 0.5]))
self.assertAlmostEqual(ap["class2"], 0.5)
def _set_extra_labels(self, data):
""" Adds extra labels for the instance and object segmentation tasks
instance_box_corners: (MAX_NUM_OBJ, 8, 3) corners of the bounding boxes in this room
center_label: (MAX_NUM_OBJ,3) for GT box center XYZ
sem_cls_label: (MAX_NUM_OBJ,) semantic class index
angle_residual_label: (MAX_NUM_OBJ,)
size_residual_label: (MAX_NUM_OBJ,3)
box_label_mask: (MAX_NUM_OBJ) as 0/1 with 1 indicating a unique box
vote_label: (N,3) with votes XYZ
vote_label_mask: (N,) with 0/1 with 1 indicating the point is in one of the object's OBB.
"""
# Initaliase variables
num_points = data.pos.shape[0]
semantic_labels = data.y
instance_labels = data.instance_labels
center_label = torch.zeros((self.MAX_NUM_OBJ, 3))
target_bboxes_mask = torch.zeros((self.MAX_NUM_OBJ), dtype=torch.bool)
angle_residuals = torch.zeros((self.MAX_NUM_OBJ, ))
size_classes = torch.zeros((self.MAX_NUM_OBJ, ))
size_residuals = torch.zeros((self.MAX_NUM_OBJ, 3))
# compute votes *AFTER* augmentation
# generate votes
# Note: since there's no map between bbox instance labels and
# pc instance_labels (it had been filtered
# in the data preparation step) we'll compute the instance bbox
# from the points sharing the same instance label.
point_votes = torch.zeros([num_points, 3])
point_votes_mask = torch.zeros(num_points, dtype=torch.bool)
instance_box_corners = []
box_sizes = []
centers = []
instance_classes = []
for i_instance in np.unique(instance_labels):
# find all points belong to that instance
ind = np.where(instance_labels == i_instance)[0]
# find the semantic label
instance_class = semantic_labels[ind[0]].item()
if instance_class in self.NYU40IDS:
pos = data.pos[ind, :3]
max_pox = pos.max(0)[0]
min_pos = pos.min(0)[0]
center = 0.5 * (min_pos + max_pox)
point_votes[ind, :] = center - pos
point_votes_mask[ind] = True
box_size = max_pox - min_pos
instance_box_corners.append(
box_corners_from_param(box_size, 0, center))
box_sizes.append(box_size)
centers.append(center)
instance_classes.append(self.NYU40ID2CLASS[instance_class])
point_votes = point_votes.repeat((1, 3)) # make 3 votes identical
instance_classes = torch.tensor(instance_classes)
# Keep only boxes with valid ids
num_instances = len(instance_classes)
target_bboxes_mask[0:num_instances] = True
# Set box semantic label
target_bboxes_semcls = np.zeros((self.MAX_NUM_OBJ))
target_bboxes_semcls[0:num_instances] = instance_classes
# Set size residual and box centres
size_classes[0:num_instances] = instance_classes
if num_instances > 0:
box_sizes = torch.stack(box_sizes)
centers = torch.stack(centers)
size_residuals[0:num_instances, :] = box_sizes - torch.from_numpy(
self.MEAN_SIZE_ARR[instance_classes, :])
center_label[0:num_instances, :] = centers
data.center_label = center_label
data.heading_class_label = torch.zeros((self.MAX_NUM_OBJ, ))
data.heading_residual_label = angle_residuals.float()
data.size_class_label = size_classes
data.size_residual_label = size_residuals.float()
data.sem_cls_label = torch.from_numpy(target_bboxes_semcls).int()
data.box_label_mask = target_bboxes_mask
data.vote_label = point_votes.float()
data.vote_label_mask = point_votes_mask
data.instance_box_corners = torch.zeros((self.MAX_NUM_OBJ, 8, 3))
if len(instance_box_corners):
data.instance_box_corners[:len(instance_box_corners
), :, :] = torch.stack(
instance_box_corners)
delattr(data, "instance_bboxes")
delattr(data, "instance_labels")
delattr(data, "y")
return data
def get_boxes(
self, dataset, apply_nms=False, objectness_threshold=0.05, duplicate_boxes=False
) -> List[List[BoxData]]:
""" Generates boxes from predictions
Parameters
----------
dataset :
Must provide a class2size method and a class2angle method that return the angle and size
for a given object class and residual value
apply_nms: bool
If True then we apply non max suppression before returning the boxes
duplicate_boxes: bool
If True then we duplicate predicted boxes accross all classes. Else we assign the box to the
most likely class
Returns
-------
List[List[BoxData]] contains the list of predicted boxes for each batch
"""
# Size and Heading prediciton
pred_heading_class = torch.argmax(self.heading_scores, -1) # B,num_proposal
pred_heading_residual = torch.gather(
self.heading_residuals, 2, pred_heading_class.unsqueeze(-1)
) # B,num_proposal,1
pred_size_class = torch.argmax(self.size_scores, -1) # B,num_proposal
pred_size_residual = torch.gather(
self.size_residuals, 2, pred_size_class.unsqueeze(-1).unsqueeze(-1).repeat(1, 1, 1, 3)
).squeeze(
2
) # B,num_proposal,3
# Generate box corners
pred_corners_3d = torch.zeros((self.batch_size, self.num_proposal, 8, 3))
for i in range(self.batch_size):
for j in range(self.num_proposal):
heading_angle = dataset.class2angle(pred_heading_class[i, j], pred_heading_residual[i, j])
box_size = dataset.class2size(pred_size_class[i, j], pred_size_residual[i, j])
corners_3d = box_corners_from_param(box_size, heading_angle, self.center[i, j, :].cpu())
pred_corners_3d[i, j] = corners_3d
# Objectness and class
pred_obj = torch.nn.functional.softmax(self.objectness_scores, -1)[:, :, 1] # B,num_proposal
pred_sem_cls = torch.argmax(self.sem_cls_scores, -1) # B,num_proposal
sem_cls_proba = torch.softmax(self.sem_cls_scores, -1)
# Apply nms if required
if apply_nms:
mask = self._nms_mask(pred_corners_3d, pred_obj, pred_sem_cls)
else:
mask = np.ones((self.batch_size, self.num_proposal), dtype=np.bool)
detected_boxes = []
for i in range(self.batch_size):
corners = pred_corners_3d[i, mask[i]]
objectness = pred_obj[i, mask[i]]
sem_cls_scores = sem_cls_proba[i, mask[i]]
clsname = pred_sem_cls[i, mask[i]]
# Build box data for each detected object and add it to the list
batch_detection = []
for j in range(len(corners)):
if objectness[j] > objectness_threshold:
if duplicate_boxes and self.has_class:
for classname in range(sem_cls_proba.shape[-1]):
batch_detection.append(
BoxData(classname, corners[j], score=objectness[j] * sem_cls_scores[j, classname])
)
else:
batch_detection.append(BoxData(clsname[j], corners[j], score=objectness[j]))
detected_boxes.append(batch_detection)
return detected_boxes
def test_box3dvol(self):
box = box_corners_from_param(
torch.tensor([1, 2, 3]).float(), np.pi / 2, torch.tensor([0, 0,
0]))
self.assertEqual(box3d_vol(box), 6)
