def test_chamfer_disrance():
from mmdet3d.models.losses import ChamferDistance, chamfer_distance
with pytest.raises(AssertionError):
# test invalid mode
ChamferDistance(mode='smoothl1')
# test invalid type of reduction
ChamferDistance(mode='l2', reduction=None)
self = ChamferDistance(mode='l2',
reduction='sum',
loss_src_weight=1.0,
loss_dst_weight=1.0)
source = torch.tensor([[[-0.9888, 0.9683, -0.8494],
[-6.4536, 4.5146,
1.6861], [2.0482, 5.6936, -1.4701],
[-0.5173, 5.6472, 2.1748],
[-2.8010, 5.4423, -1.2158],
[2.4018, 2.4389, -0.2403],
[-2.8811, 3.8486, 1.4750],
[-0.2031, 3.8969,
-1.5245], [1.3827, 4.9295, 1.1537],
[-2.6961, 2.2621, -1.0976]],
[[0.3692, 1.8409,
-1.4983], [1.9995, 6.3602, 0.1798],
[-2.1317, 4.6011,
-0.7028], [2.4158, 3.1482, 0.3169],
[-0.5836, 3.6250, -1.2650],
[-1.9862, 1.6182, -1.4901],
[2.5992, 1.2847, -0.8471],
[-0.3467, 5.3681, -1.4755],
[-0.8576, 3.3400, -1.7399],
[2.7447, 4.6349, 0.1994]]])
target = torch.tensor([[[-0.4758, 1.0094, -0.8645],
[-0.3130, 0.8564, -0.9061],
[-0.1560, 2.0394, -0.8936],
[-0.3685, 1.6467, -0.8271],
[-0.2740, 2.2212, -0.7980]],
[[1.4856, 2.5299,
-1.0047], [2.3262, 3.3065, -0.9475],
[2.4593, 2.5870,
-0.9423], [0.0000, 0.0000, 0.0000],
[0.0000, 0.0000, 0.0000]]])
loss_source, loss_target, indices1, indices2 = self(source,
target,
return_indices=True)
assert torch.allclose(loss_source, torch.tensor(219.5936))
assert torch.allclose(loss_target, torch.tensor(22.3705))
assert (indices1 == indices1.new_tensor([[0, 4, 4, 4, 4, 2, 4, 4, 4, 3],
[0, 1, 0, 1, 0, 4, 2, 0, 0,
1]])).all()
assert (indices2 == indices2.new_tensor([[0, 0, 0, 0, 0], [0, 3, 6, 0,
0]])).all()
loss_source, loss_target, indices1, indices2 = chamfer_distance(
source, target, reduction='sum')
assert torch.allclose(loss_source, torch.tensor(219.5936))
assert torch.allclose(loss_target, torch.tensor(22.3705))
assert (indices1 == indices1.new_tensor([[0, 4, 4, 4, 4, 2, 4, 4, 4, 3],
[0, 1, 0, 1, 0, 4, 2, 0, 0,
1]])).all()
assert (indices2 == indices2.new_tensor([[0, 0, 0, 0, 0], [0, 3, 6, 0,
0]])).all()
def get_targets_single(self,
points,
gt_bboxes_3d,
gt_labels_3d,
pts_semantic_mask=None,
pts_instance_mask=None,
aggregated_points=None):
"""Generate targets of vote head for single batch.
Args:
points (torch.Tensor): Points of each batch.
gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth \
boxes of each batch.
gt_labels_3d (torch.Tensor): Labels of each batch.
pts_semantic_mask (None | torch.Tensor): Point-wise semantic
label of each batch.
pts_instance_mask (None | torch.Tensor): Point-wise instance
label of each batch.
aggregated_points (torch.Tensor): Aggregated points from
vote aggregation layer.
Returns:
tuple[torch.Tensor]: Targets of vote head.
"""
assert self.bbox_coder.with_rot or pts_semantic_mask is not None
gt_bboxes_3d = gt_bboxes_3d.to(points.device)
# generate votes target
num_points = points.shape[0]
if self.bbox_coder.with_rot:
vote_targets = points.new_zeros([num_points, 3 * self.gt_per_seed])
vote_target_masks = points.new_zeros([num_points],
dtype=torch.long)
vote_target_idx = points.new_zeros([num_points], dtype=torch.long)
box_indices_all = gt_bboxes_3d.points_in_boxes(points)
for i in range(gt_labels_3d.shape[0]):
box_indices = box_indices_all[:, i]
indices = torch.nonzero(
box_indices, as_tuple=False).squeeze(-1)
selected_points = points[indices]
vote_target_masks[indices] = 1
vote_targets_tmp = vote_targets[indices]
votes = gt_bboxes_3d.gravity_center[i].unsqueeze(
0) - selected_points[:, :3]
for j in range(self.gt_per_seed):
column_indices = torch.nonzero(
vote_target_idx[indices] == j,
as_tuple=False).squeeze(-1)
vote_targets_tmp[column_indices,
int(j * 3):int(j * 3 +
3)] = votes[column_indices]
if j == 0:
vote_targets_tmp[column_indices] = votes[
column_indices].repeat(1, self.gt_per_seed)
vote_targets[indices] = vote_targets_tmp
vote_target_idx[indices] = torch.clamp(
vote_target_idx[indices] + 1, max=2)
elif pts_semantic_mask is not None:
vote_targets = points.new_zeros([num_points, 3])
vote_target_masks = points.new_zeros([num_points],
dtype=torch.long)
for i in torch.unique(pts_instance_mask):
indices = torch.nonzero(
pts_instance_mask == i, as_tuple=False).squeeze(-1)
if pts_semantic_mask[indices[0]] < self.num_classes:
selected_points = points[indices, :3]
center = 0.5 * (
selected_points.min(0)[0] + selected_points.max(0)[0])
vote_targets[indices, :] = center - selected_points
vote_target_masks[indices] = 1
vote_targets = vote_targets.repeat((1, self.gt_per_seed))
else:
raise NotImplementedError
(center_targets, size_class_targets, size_res_targets,
dir_class_targets,
dir_res_targets) = self.bbox_coder.encode(gt_bboxes_3d, gt_labels_3d)
proposal_num = aggregated_points.shape[0]
distance1, _, assignment, _ = chamfer_distance(
aggregated_points.unsqueeze(0),
center_targets.unsqueeze(0),
reduction='none')
assignment = assignment.squeeze(0)
euclidean_distance1 = torch.sqrt(distance1.squeeze(0) + 1e-6)
objectness_targets = points.new_zeros((proposal_num), dtype=torch.long)
objectness_targets[
euclidean_distance1 < self.train_cfg['pos_distance_thr']] = 1
objectness_masks = points.new_zeros((proposal_num))
objectness_masks[
euclidean_distance1 < self.train_cfg['pos_distance_thr']] = 1.0
objectness_masks[
euclidean_distance1 > self.train_cfg['neg_distance_thr']] = 1.0
dir_class_targets = dir_class_targets[assignment]
dir_res_targets = dir_res_targets[assignment]
dir_res_targets /= (np.pi / self.num_dir_bins)
size_class_targets = size_class_targets[assignment]
size_res_targets = size_res_targets[assignment]
one_hot_size_targets = gt_bboxes_3d.tensor.new_zeros(
(proposal_num, self.num_sizes))
one_hot_size_targets.scatter_(1, size_class_targets.unsqueeze(-1), 1)
one_hot_size_targets = one_hot_size_targets.unsqueeze(-1).repeat(
1, 1, 3)
mean_sizes = size_res_targets.new_tensor(
self.bbox_coder.mean_sizes).unsqueeze(0)
pos_mean_sizes = torch.sum(one_hot_size_targets * mean_sizes, 1)
size_res_targets /= pos_mean_sizes
mask_targets = gt_labels_3d[assignment]
assigned_center_targets = center_targets[assignment]
return (vote_targets, vote_target_masks, size_class_targets,
size_res_targets, dir_class_targets,
dir_res_targets, center_targets, assigned_center_targets,
mask_targets.long(), objectness_targets, objectness_masks)
def get_targets_single(self,
points,
gt_bboxes_3d,
gt_labels_3d,
pts_semantic_mask=None,
pts_instance_mask=None,
aggregated_points=None,
pred_surface_center=None,
pred_line_center=None,
pred_obj_surface_center=None,
pred_obj_line_center=None,
pred_surface_sem=None,
pred_line_sem=None):
"""Generate targets for primitive cues for single batch.
Args:
points (torch.Tensor): Points of each batch.
gt_bboxes_3d (:obj:`BaseInstance3DBoxes`): Ground truth \
boxes of each batch.
gt_labels_3d (torch.Tensor): Labels of each batch.
pts_semantic_mask (None | torch.Tensor): Point-wise semantic
label of each batch.
pts_instance_mask (None | torch.Tensor): Point-wise instance
label of each batch.
aggregated_points (torch.Tensor): Aggregated points from
vote aggregation layer.
pred_surface_center (torch.Tensor): Prediction of surface center.
pred_line_center (torch.Tensor): Prediction of line center.
pred_obj_surface_center (torch.Tensor): Objectness prediction \
of surface center.
pred_obj_line_center (torch.Tensor): Objectness prediction of \
line center.
pred_surface_sem (torch.Tensor): Semantic prediction of \
surface center.
pred_line_sem (torch.Tensor): Semantic prediction of line center.
Returns:
tuple[torch.Tensor]: Targets for primitive cues.
"""
device = points.device
gt_bboxes_3d = gt_bboxes_3d.to(device)
num_proposals = aggregated_points.shape[0]
gt_center = gt_bboxes_3d.gravity_center
dist1, dist2, ind1, _ = chamfer_distance(
aggregated_points.unsqueeze(0),
gt_center.unsqueeze(0),
reduction='none')
# Set assignment
object_assignment = ind1.squeeze(0)
# Generate objectness label and mask
# objectness_label: 1 if pred object center is within
# self.train_cfg['near_threshold'] of any GT object
# objectness_mask: 0 if pred object center is in gray
# zone (DONOTCARE), 1 otherwise
euclidean_dist1 = torch.sqrt(dist1.squeeze(0) + 1e-6)
proposal_objectness_label = euclidean_dist1.new_zeros(num_proposals,
dtype=torch.long)
proposal_objectness_mask = euclidean_dist1.new_zeros(num_proposals)
gt_sem = gt_labels_3d[object_assignment]
obj_surface_center, obj_line_center = \
gt_bboxes_3d.get_surface_line_center()
obj_surface_center = obj_surface_center.reshape(-1, 6,
3).transpose(0, 1)
obj_line_center = obj_line_center.reshape(-1, 12, 3).transpose(0, 1)
obj_surface_center = obj_surface_center[:, object_assignment].reshape(
1, -1, 3)
obj_line_center = obj_line_center[:,
object_assignment].reshape(1, -1, 3)
surface_sem = torch.argmax(pred_surface_sem, dim=1).float()
line_sem = torch.argmax(pred_line_sem, dim=1).float()
dist_surface, _, surface_ind, _ = chamfer_distance(
obj_surface_center,
pred_surface_center.unsqueeze(0),
reduction='none')
dist_line, _, line_ind, _ = chamfer_distance(
obj_line_center, pred_line_center.unsqueeze(0), reduction='none')
surface_sel = pred_surface_center[surface_ind.squeeze(0)]
line_sel = pred_line_center[line_ind.squeeze(0)]
surface_sel_sem = surface_sem[surface_ind.squeeze(0)]
line_sel_sem = line_sem[line_ind.squeeze(0)]
surface_sel_sem_gt = gt_sem.repeat(6).float()
line_sel_sem_gt = gt_sem.repeat(12).float()
euclidean_dist_surface = torch.sqrt(dist_surface.squeeze(0) + 1e-6)
euclidean_dist_line = torch.sqrt(dist_line.squeeze(0) + 1e-6)
objectness_label_surface = euclidean_dist_line.new_zeros(
num_proposals * 6, dtype=torch.long)
objectness_mask_surface = euclidean_dist_line.new_zeros(num_proposals *
6)
objectness_label_line = euclidean_dist_line.new_zeros(num_proposals *
12,
dtype=torch.long)
objectness_mask_line = euclidean_dist_line.new_zeros(num_proposals *
12)
objectness_label_surface_sem = euclidean_dist_line.new_zeros(
num_proposals * 6, dtype=torch.long)
objectness_label_line_sem = euclidean_dist_line.new_zeros(
num_proposals * 12, dtype=torch.long)
euclidean_dist_obj_surface = torch.sqrt((
(pred_obj_surface_center - surface_sel)**2).sum(dim=-1) + 1e-6)
euclidean_dist_obj_line = torch.sqrt(
torch.sum((pred_obj_line_center - line_sel)**2, dim=-1) + 1e-6)
# Objectness score just with centers
proposal_objectness_label[
euclidean_dist1 < self.train_cfg['near_threshold']] = 1
proposal_objectness_mask[
euclidean_dist1 < self.train_cfg['near_threshold']] = 1
proposal_objectness_mask[
euclidean_dist1 > self.train_cfg['far_threshold']] = 1
objectness_label_surface[
(euclidean_dist_obj_surface <
self.train_cfg['label_surface_threshold']) *
(euclidean_dist_surface <
self.train_cfg['mask_surface_threshold'])] = 1
objectness_label_surface_sem[
(euclidean_dist_obj_surface <
self.train_cfg['label_surface_threshold']) *
(euclidean_dist_surface < self.train_cfg['mask_surface_threshold'])
* (surface_sel_sem == surface_sel_sem_gt)] = 1
objectness_label_line[
(euclidean_dist_obj_line < self.train_cfg['label_line_threshold'])
*
(euclidean_dist_line < self.train_cfg['mask_line_threshold'])] = 1
objectness_label_line_sem[
(euclidean_dist_obj_line < self.train_cfg['label_line_threshold'])
* (euclidean_dist_line < self.train_cfg['mask_line_threshold']) *
(line_sel_sem == line_sel_sem_gt)] = 1
objectness_label_surface_obj = proposal_objectness_label.repeat(6)
objectness_mask_surface_obj = proposal_objectness_mask.repeat(6)
objectness_label_line_obj = proposal_objectness_label.repeat(12)
objectness_mask_line_obj = proposal_objectness_mask.repeat(12)
objectness_mask_surface = objectness_mask_surface_obj
objectness_mask_line = objectness_mask_line_obj
cues_objectness_label = torch.cat(
(objectness_label_surface, objectness_label_line), 0)
cues_sem_label = torch.cat(
(objectness_label_surface_sem, objectness_label_line_sem), 0)
cues_mask = torch.cat((objectness_mask_surface, objectness_mask_line),
0)
objectness_label_surface *= objectness_label_surface_obj
objectness_label_line *= objectness_label_line_obj
cues_matching_label = torch.cat(
(objectness_label_surface, objectness_label_line), 0)
objectness_label_surface_sem *= objectness_label_surface_obj
objectness_label_line_sem *= objectness_label_line_obj
cues_match_mask = (torch.sum(cues_objectness_label.view(
18, num_proposals),
dim=0) >= 1).float()
obj_surface_line_center = torch.cat(
(obj_surface_center, obj_line_center), 1).squeeze(0)
return (cues_objectness_label, cues_sem_label,
proposal_objectness_label, cues_mask, cues_match_mask,
proposal_objectness_mask, cues_matching_label,
obj_surface_line_center)
def get_targets_single(self,
points,
gt_bboxes_3d,
gt_labels_3d,
pts_semantic_mask=None,
pts_instance_mask=None,
aggregated_points=None):
gt_bboxes_3d = gt_bboxes_3d.to(points.device)
# generate votes target
num_points = points.shape[0]
if self.bbox_coder.with_rot:
vote_targets = points.new_zeros([num_points, 3 * self.gt_per_seed])
vote_target_masks = points.new_zeros([num_points],
dtype=torch.long)
vote_target_idx = points.new_zeros([num_points], dtype=torch.long)
box_indices_all = gt_bboxes_3d.points_in_boxes(points)
for i in range(gt_labels_3d.shape[0]):
box_indices = box_indices_all[:, i]
indices = torch.nonzero(box_indices,
as_tuple=False).squeeze(-1)
selected_points = points[indices]
vote_target_masks[indices] = 1
vote_targets_tmp = vote_targets[indices]
votes = gt_bboxes_3d.gravity_center[i].unsqueeze(
0) - selected_points[:, :3]
for j in range(self.gt_per_seed):
column_indices = torch.nonzero(
vote_target_idx[indices] == j,
as_tuple=False).squeeze(-1)
vote_targets_tmp[column_indices,
int(j * 3):int(j * 3 +
3)] = votes[column_indices]
if j == 0:
vote_targets_tmp[column_indices] = votes[
column_indices].repeat(1, self.gt_per_seed)
vote_targets[indices] = vote_targets_tmp
vote_target_idx[indices] = torch.clamp(
vote_target_idx[indices] + 1, max=2)
elif pts_semantic_mask is not None:
vote_targets = points.new_zeros([num_points, 3])
vote_target_masks = points.new_zeros([num_points],
dtype=torch.long)
for i in torch.unique(pts_instance_mask):
indices = torch.nonzero(pts_instance_mask == i,
as_tuple=False).squeeze(-1)
if pts_semantic_mask[indices[0]] < self.num_classes:
selected_points = points[indices, :3]
center = 0.5 * (selected_points.min(0)[0] +
selected_points.max(0)[0])
vote_targets[indices, :] = center - selected_points
vote_target_masks[indices] = 1
vote_targets = vote_targets.repeat((1, self.gt_per_seed))
else:
raise NotImplementedError
(center_targets, size_targets, dir_class_targets, dir_res_targets,
dir_targets) = self.bbox_coder.encode(gt_bboxes_3d,
gt_labels_3d,
ret_dir_target=True)
proposal_num = aggregated_points.shape[0]
distance1, _, assignment, _ = chamfer_distance(
aggregated_points.unsqueeze(0),
center_targets.unsqueeze(0),
reduction='none')
assignment = assignment.squeeze(0)
euclidean_distance1 = torch.sqrt(distance1.squeeze(0) + 1e-6)
objectness_masks = points.new_zeros((proposal_num))
objectness_masks[
euclidean_distance1 < self.train_cfg['pos_distance_thr']] = 1.0
objectness_masks[
euclidean_distance1 > self.train_cfg['neg_distance_thr']] = 1.0
center_targets = center_targets[assignment]
dir_class_targets = dir_class_targets[assignment]
dir_res_targets = dir_res_targets[assignment]
dir_res_targets /= (np.pi / self.num_dir_bins)
size_res_targets = size_targets[assignment]
dir_targets = dir_targets[assignment]
mask_targets = gt_labels_3d[assignment]
# Centerness loss targets
canonical_xyz = aggregated_points - center_targets
# print(canonical_xyz.shape)
# print(gt_bboxes_3d.yaw[assignment].shape)
if self.bbox_coder.with_rot:
canonical_xyz = rotation_3d_in_axis(
canonical_xyz.unsqueeze(0).transpose(0, 1),
-gt_bboxes_3d.yaw[assignment], 2).squeeze(1)
distance_front = size_res_targets[:, 0] - canonical_xyz[:, 0]
distance_left = size_res_targets[:, 1] - canonical_xyz[:, 1]
distance_top = size_res_targets[:, 2] - canonical_xyz[:, 2]
distance_back = size_res_targets[:, 0] + canonical_xyz[:, 0]
distance_right = size_res_targets[:, 1] + canonical_xyz[:, 1]
distance_bottom = size_res_targets[:, 2] + canonical_xyz[:, 2]
distance_targets = torch.cat(
(distance_front.unsqueeze(-1), distance_left.unsqueeze(-1),
distance_top.unsqueeze(-1), distance_back.unsqueeze(-1),
distance_right.unsqueeze(-1), distance_bottom.unsqueeze(-1)),
dim=-1)
inside_mask = (distance_targets >= 0.).all(dim=-1)
objectness_targets = points.new_zeros((proposal_num), dtype=torch.long)
pos_mask = (euclidean_distance1 <
self.train_cfg['pos_distance_thr']) & inside_mask
objectness_targets[pos_mask] = 1
distance_targets.clamp_(min=0)
deltas = torch.cat(
(distance_targets[:, 0:3, None], distance_targets[:, 3:6, None]),
dim=-1)
nominators = deltas.min(dim=-1).values.prod(dim=-1)
denominators = deltas.max(dim=-1).values.prod(dim=-1) + 1e-6
centerness_targets = (nominators / denominators + 1e-6)**(1 / 3)
centerness_targets = torch.clamp(centerness_targets, min=0, max=1)
return (vote_targets, vote_target_masks, size_res_targets,
dir_class_targets, dir_res_targets, centerness_targets,
mask_targets.long(), objectness_targets, objectness_masks,
distance_targets, centerness_targets, dir_targets)
