def generate_recall_record(self,index, box_preds,batch_dict,recall_dict, thresh_list=(0.5, 0.7)):
rois = batch_dict["rois"][index]
if batch_dict["gt_boxes"] is not None:
gt_boxes = batch_dict["gt_boxes"][index]
cur_gt = gt_boxes
k = gt_boxes.__len__() - 1
while k > 0 and cur_gt[k].sum() == 0:
k -= 1
cur_gt = cur_gt[:k + 1]
cur_rois = rois
if cur_gt.shape[0] > 0:
iou3d_rcnn = iou3d_nms_utils.boxes_iou3d_gpu(box_preds, cur_gt[:, :7])
iou3d_rois = iou3d_nms_utils.boxes_iou3d_gpu(cur_rois, cur_gt[:, :7])
if recall_dict.__len__() == 0:
recall_dict["gt"] = 0
for thresh in thresh_list:
recall_dict["roi_%s" % str(thresh)] = 0
recall_dict["rcnn_%s" % str(thresh)] = 0
recall_dict["gt"] += cur_gt.shape[0]
for thresh in thresh_list:
rois_iou_max = iou3d_rois.max(dim=-1)[0]
rcnn_iou_max = iou3d_rcnn.max(dim=-1)[0]
recall_dict["roi_%s" % str(thresh)] += (rois_iou_max > thresh).sum().item()
recall_dict["rcnn_%s" % str(thresh)] += (rcnn_iou_max > thresh).sum().item()
else:
return {}
return recall_dict
def get_maxiou3d_with_same_class(rois, roi_labels, gt_boxes, gt_labels):
"""
三类物体的rois与真实的gt boxes进行匹配，得到每个roi最匹配的gt boxe的indx
:param rois: (N, 7)
:param roi_labels: (N)
:param gt_boxes: (N, 8)
:return:
"""
max_overlaps = rois.new_zeros(rois.shape[0])
gt_assignment = roi_labels.new_zeros(roi_labels.shape[0])
for k in range(gt_labels.min().item(), gt_labels.max().item() + 1):
roi_mask = (roi_labels == k)
gt_mask = (gt_labels == k)
if roi_mask.sum() > 0 and gt_mask.sum() > 0:
cur_roi = rois[roi_mask]
cur_gt = gt_boxes[gt_mask]
original_gt_assignment = gt_mask.nonzero().view(-1)
iou3d = iou3d_nms_utils.boxes_iou3d_gpu(cur_roi, cur_gt) # (M, N)
cur_max_overlaps, cur_gt_assignment = torch.max(iou3d, dim=1)
max_overlaps[roi_mask] = cur_max_overlaps
gt_assignment[roi_mask] = original_gt_assignment[cur_gt_assignment]
return max_overlaps, gt_assignment
def get_iou_with_same_class(rois, roi_labels, gt_boxes):
"""
gt_boxes:(num of box per image,8)
rois:(512,7)
roi—labels
"""
gt_labels = gt_boxes[:, 7]
gt_boxes = gt_boxes[:, :7]
max_overlap = torch.zeros(rois.shape[0],
dtype=torch.float32,
device=gt_boxes.device)
gt_assignment = torch.zeros(rois.shape[0],
dtype=torch.long,
device=gt_boxes.device)
for idx in np.arange(gt_labels.min().item(),
gt_labels.max().item() + 1, 1):
rois_mask_class = roi_labels == idx
# rois_mask_class_for_see = rois_mask_class.nonzero().view(-1)
gt_mask_class = gt_labels == idx
gt_origin_assign = torch.nonzero(gt_mask_class).view(-1)
if gt_mask_class.sum() > 0 and rois_mask_class.sum() > 0:
cur_gt = gt_boxes[gt_mask_class]
cur_rois = rois[rois_mask_class]
iou3d = iou3d_nms_utils.boxes_iou3d_gpu(cur_rois, cur_gt)
cur_max_overlap, cur_gt_assign = torch.max(iou3d, dim=1)
# cur_max_overlaps_see = cur_max_overlap.cpu().numpy()
max_overlap[rois_mask_class] = cur_max_overlap
gt_assignment[rois_mask_class] = gt_origin_assign[
cur_gt_assign]
return max_overlap, gt_assignment
def sample_rois_for_rcnn(roi_boxes3d, gt_boxes3d, roi_raw_scores, roi_labels,
roi_sampler_cfg):
"""
:param roi_boxes3d: (B,512,7)
:param gt_boxes3d: (B,M,7+1)这个1是box的种类（“car”,1,"Pedestrian",2,"Cyclist",3）
:param roi_raw_scores:(B,512)
:param roi_labels:(B,512)
:param roi_samlpler_cfg:
:return:
"""
batch_size = roi_boxes3d.size(0)
fg_rois_per_image = int(
np.round(roi_sampler_cfg.FG_RATIO * roi_sampler_cfg.ROI_PER_IMAGE))
code_size = roi_boxes3d.shape[-1]
batch_rois = gt_boxes3d.new(batch_size, roi_sampler_cfg.ROI_PER_IMAGE,
code_size).zero_()
batch_gt_of_rois = gt_boxes3d.new(batch_size,
roi_sampler_cfg.ROI_PER_IMAGE,
code_size + 1).zero_()
batch_roi_iou = gt_boxes3d.new(batch_size,
roi_sampler_cfg.ROI_PER_IMAGE).zero_()
batch_roi_raw_scores = gt_boxes3d.new(
batch_size, roi_sampler_cfg.ROI_PER_IMAGE).zero_()
batch_roi_labels = gt_boxes3d.new(
batch_size, roi_sampler_cfg.ROI_PER_IMAGE).zero_().long()
for idx in range(batch_size):
cur_roi, cur_gt, cur_roi_raw_scores, cur_roi_labels = \
roi_boxes3d[idx], gt_boxes3d[idx], roi_raw_scores[idx], roi_labels[idx]
k = cur_gt.__len__() - 1
while k > 0 and cur_gt[k].sum() == 0:
k -= 1
cur_gt = cur_gt[:k + 1]
if len(cfg.CLASS_NAMES) == 1:
iou3d = iou3d_nms_utils.boxes_iou3d_gpu(cur_roi,
cur_gt[:, 0:7]) # (M, N)
max_overlaps, gt_assignment = torch.max(iou3d, dim=1)
else:
cur_gt_labels = cur_gt[:, -1].long()
max_overlaps, gt_assignment = get_maxiou3d_with_same_class(
cur_roi, cur_roi_labels, cur_gt[:, 0:7], cur_gt_labels)
fg_thresh = min(roi_sampler_cfg.REG_FG_THRESH,
roi_sampler_cfg.CLS_FG_THRESH) #0.55
fg_inds = torch.nonzero((max_overlaps >= fg_thresh)).view(-1)
#torch.nonzero这个函数是返回非0的元素的编号，这里返回的是roi的编号
easy_bg_inds = torch.nonzero(
(max_overlaps < roi_sampler_cfg.CLS_BG_THRESH_LO)).view(
-1) #小于0.1的是easy bg
hard_bg_inds = torch.nonzero(
(max_overlaps < roi_sampler_cfg.REG_FG_THRESH)
& (max_overlaps >= roi_sampler_cfg.CLS_BG_THRESH_LO)).view(
-1) #在0.1到0.55之间的是hard
fg_num_rois = fg_inds.numel()
bg_num_rois = hard_bg_inds.numel() + easy_bg_inds.numel()
if fg_num_rois > 0 and bg_num_rois > 0:
# sampling fg
fg_rois_per_this_image = min(fg_rois_per_image, fg_num_rois)
rand_num = torch.from_numpy(
np.random.permutation(fg_num_rois)).type_as(gt_boxes3d).long()
fg_inds = fg_inds[rand_num[:fg_rois_per_this_image]]
# sampling bg
bg_rois_per_this_image = roi_sampler_cfg.ROI_PER_IMAGE - fg_rois_per_this_image
bg_inds = sample_bg_inds(hard_bg_inds, easy_bg_inds,
bg_rois_per_this_image, roi_sampler_cfg)
elif fg_num_rois > 0 and bg_num_rois == 0:
# sampling fg
rand_num = np.floor(
np.random.rand(roi_sampler_cfg.ROI_PER_IMAGE) * fg_num_rois)
rand_num = torch.from_numpy(rand_num).type_as(gt_boxes3d).long()
fg_inds = fg_inds[rand_num]
fg_rois_per_this_image = roi_sampler_cfg.ROI_PER_IMAGE
bg_rois_per_this_image = 0
elif bg_num_rois > 0 and fg_num_rois == 0:
# sampling bg
bg_rois_per_this_image = roi_sampler_cfg.ROI_PER_IMAGE
bg_inds = sample_bg_inds(hard_bg_inds, easy_bg_inds,
bg_rois_per_this_image, roi_sampler_cfg)
fg_rois_per_this_image = 0
else:
print('maxoverlaps:(min=%f, max=%f)' %
(max_overlaps.min().item(), max_overlaps.max().item()))
print('ERROR: FG=%d, BG=%d' % (fg_num_rois, bg_num_rois))
raise NotImplementedError
roi_list, roi_iou_list, roi_gt_list, roi_score_list, roi_labels_list = [], [], [], [], []
if fg_rois_per_this_image > 0:
fg_rois = cur_roi[fg_inds]
gt_of_fg_rois = cur_gt[gt_assignment[fg_inds]]
fg_iou3d = max_overlaps[fg_inds]
roi_list.append(fg_rois)
roi_iou_list.append(fg_iou3d)
roi_gt_list.append(gt_of_fg_rois)
roi_score_list.append(cur_roi_raw_scores[fg_inds])
roi_labels_list.append(cur_roi_labels[fg_inds])
if bg_rois_per_this_image > 0:
bg_rois = cur_roi[bg_inds]
gt_of_bg_rois = cur_gt[gt_assignment[bg_inds]]
bg_iou3d = max_overlaps[bg_inds]
roi_list.append(bg_rois)
roi_iou_list.append(bg_iou3d)
roi_gt_list.append(gt_of_bg_rois)
roi_score_list.append(cur_roi_raw_scores[bg_inds])
roi_labels_list.append(cur_roi_labels[bg_inds])
rois = torch.cat(roi_list, dim=0)
iou_of_rois = torch.cat(roi_iou_list, dim=0)
gt_of_rois = torch.cat(roi_gt_list, dim=0)
cur_roi_raw_scores = torch.cat(roi_score_list, dim=0)
cur_roi_labels = torch.cat(roi_labels_list, dim=0)
batch_rois[idx] = rois
batch_gt_of_rois[idx] = gt_of_rois
batch_roi_iou[idx] = iou_of_rois
batch_roi_raw_scores[idx] = cur_roi_raw_scores
batch_roi_labels[idx] = cur_roi_labels
return batch_rois, batch_gt_of_rois, batch_roi_iou, batch_roi_raw_scores, batch_roi_labels
def sample_rois_for_rcnn(self, batch_dict):
"""
Args:
batch_dict:
batch_size:
rois: (B, num_rois, 7 + C)
roi_scores: (B, num_rois)
gt_boxes: (B, N, 7 + C + 1)
roi_labels: (B, num_rois)
Returns:
"""
batch_size = batch_dict['batch_size']
rois = batch_dict['rois']
roi_scores = batch_dict['roi_scores']
roi_labels = batch_dict['roi_labels']
gt_boxes = batch_dict['gt_boxes']
code_size = rois.shape[-1]
batch_rois = rois.new_zeros(batch_size,
self.roi_sampler_cfg.ROI_PER_IMAGE,
code_size)
batch_gt_of_rois = rois.new_zeros(batch_size,
self.roi_sampler_cfg.ROI_PER_IMAGE,
code_size + 1)
batch_roi_ious = rois.new_zeros(batch_size,
self.roi_sampler_cfg.ROI_PER_IMAGE)
batch_roi_scores = rois.new_zeros(batch_size,
self.roi_sampler_cfg.ROI_PER_IMAGE)
batch_roi_labels = rois.new_zeros(
(batch_size, self.roi_sampler_cfg.ROI_PER_IMAGE), dtype=torch.long)
for index in range(batch_size):
cur_roi, cur_gt, cur_roi_labels, cur_roi_scores = \
rois[index], gt_boxes[index], roi_labels[index], roi_scores[index]
k = cur_gt.__len__() - 1
while k > 0 and cur_gt[k].sum() == 0:
k -= 1
cur_gt = cur_gt[:k + 1]
cur_gt = cur_gt.new_zeros(
(1, cur_gt.shape[1])) if len(cur_gt) == 0 else cur_gt
if self.roi_sampler_cfg.get('SAMPLE_ROI_BY_EACH_CLASS', False):
max_overlaps, gt_assignment = self.get_max_iou_with_same_class(
rois=cur_roi,
roi_labels=cur_roi_labels,
gt_boxes=cur_gt[:, 0:7],
gt_labels=cur_gt[:, -1].long())
else:
iou3d = iou3d_nms_utils.boxes_iou3d_gpu(
cur_roi, cur_gt[:, 0:7]) # (M, N)
max_overlaps, gt_assignment = torch.max(iou3d, dim=1)
sampled_inds = self.subsample_rois(max_overlaps=max_overlaps)
batch_rois[index] = cur_roi[sampled_inds]
batch_roi_labels[index] = cur_roi_labels[sampled_inds]
batch_roi_ious[index] = max_overlaps[sampled_inds]
batch_roi_scores[index] = cur_roi_scores[sampled_inds]
batch_gt_of_rois[index] = cur_gt[gt_assignment[sampled_inds]]
return batch_rois, batch_gt_of_rois, batch_roi_ious, batch_roi_scores, batch_roi_labels
def assign_targets_single(self, anchors,
gt_boxes,
gt_classes,
matched_threshold=0.6,
unmatched_threshold=0.45
):
num_anchors = anchors.shape[0]
num_gt = gt_boxes.shape[0]
box_ndim = anchors.shape[1]
labels = torch.ones((num_anchors,), dtype=torch.int32, device=anchors.device) * -1
gt_ids = torch.ones((num_anchors,), dtype=torch.int32, device=anchors.device) * -1
if len(gt_boxes) > 0 and anchors.shape[0] > 0:
anchor_by_gt_overlap = iou3d_nms_utils.boxes_iou3d_gpu(anchors,
gt_boxes) if self.match_height else box_utils.boxes3d_nearest_bev_iou(
anchors, gt_boxes)
anchor_to_gt_argmax = anchor_by_gt_overlap.argmax(dim=1)
anchor_to_gt_max = anchor_by_gt_overlap[torch.arange(num_anchors),
anchor_to_gt_argmax]
gt_to_anchor_argmax = anchor_by_gt_overlap.argmax(dim=0)
gt_to_anchor_max = anchor_by_gt_overlap[
gt_to_anchor_argmax,
torch.arange(num_gt)]
empty_gt_mask = gt_to_anchor_max == 0
gt_to_anchor_max[empty_gt_mask] = -1
anchors_with_max_overlap = torch.nonzero(
anchor_by_gt_overlap == gt_to_anchor_max)[:, 0]
gt_inds_force = anchor_to_gt_argmax[anchors_with_max_overlap]
labels[anchors_with_max_overlap] = gt_classes[gt_inds_force]
gt_ids[anchors_with_max_overlap] = gt_inds_force.int()
pos_inds = anchor_to_gt_max >= matched_threshold
gt_inds_over_thresh = anchor_to_gt_argmax[pos_inds]
labels[pos_inds] = gt_classes[gt_inds_over_thresh]
gt_ids[pos_inds] = gt_inds_over_thresh.int()
bg_inds = torch.nonzero(anchor_to_gt_max < unmatched_threshold)[:, 0]
else:
bg_inds = torch.arange(num_anchors)
fg_inds = torch.nonzero(labels > 0)[:, 0]
if self.pos_fraction is not None:
num_fg = int(self.pos_fraction * self.sample_size)
if len(fg_inds) > num_fg:
num_disabled = len(fg_inds) - num_fg
disable_inds = torch.randperm(len(fg_inds))[:num_disabled]
labels[disable_inds] = -1
fg_inds = torch.nonzero(labels > 0)[:, 0]
num_bg = self.sample_size - (labels > 0).sum()
if len(bg_inds) > num_bg:
enable_inds = bg_inds[torch.randint(0, len(bg_inds), size=(num_bg,))]
labels[enable_inds] = 0
bg_inds = torch.nonzero(labels == 0)[:, 0]
else:
if len(gt_boxes) == 0 or anchors.shape[0] == 0:
labels[:] = 0
else:
labels[bg_inds] = 0
labels[anchors_with_max_overlap] = gt_classes[gt_inds_force]
bbox_targets = anchors.new_zeros((num_anchors, self.box_coder.code_size))
if len(gt_boxes) > 0 and anchors.shape[0] > 0:
fg_gt_boxes = gt_boxes[anchor_to_gt_argmax[fg_inds], :]
fg_anchors = anchors[fg_inds, :]
bbox_targets[fg_inds, :] = self.box_coder.encode_torch(fg_gt_boxes, fg_anchors)
reg_weights = anchors.new_zeros((num_anchors,))
if self.norm_by_num_examples:
num_examples = (labels >= 0).sum()
num_examples = num_examples if num_examples > 1.0 else 1.0
reg_weights[labels > 0] = 1.0 / num_examples
else:
reg_weights[labels > 0] = 1.0
ret_dict = {
'box_cls_labels': labels,
'box_reg_targets': bbox_targets,
'reg_weights': reg_weights,
}
return ret_dict
def forward(self, x_in, gt_boxes):
# x_in = batch_dict["spatial_features"]
x = x_in
ups = []
ret_dict = {}
for i in range(len(self.blocks)):
x = self.blocks[i](x)
stride = int(x_in.shape[2] / x.shape[2])
ret_dict['spatial_features_%dx' % stride] = x
ups.append(self.deblocks[i](x))
if self._concat_input:
ups.append(x_in)
if len(ups) > 1:
x = torch.cat(ups, dim=1)
else:
x = ups[0]
if len(self.deblocks) > len(self.blocks):
x = self.deblocks[-1](x)
ret_dict['spatial_features_last'] = x
box_preds = self.conv_box(x)
cls_preds = self.conv_cls(x)
# [N, C, y(H), x(W)]
box_preds = box_preds.permute(0, 2, 3, 1).contiguous()
cls_preds = cls_preds.permute(0, 2, 3, 1).contiguous()
ret_dict.update({
'box_preds': box_preds,
'cls_preds': cls_preds,
})
if self.config["use_direction_classifier"]:
dir_pred = self.conv_dir_cls(x)
ret_dict.update({"dir_preds": dir_pred.contiguous()})
self.forward_ret_dict = ret_dict
if self.training:
targets_dict = self.get_assigner_target(gt_boxes=gt_boxes, )
# 验证target
batch_size = targets_dict["box_reg_targets"].shape[0]
anchors = torch.cat([anchor for anchor in self.anchors], dim=-2)
anchors = anchors.reshape(1, -1, 7).repeat(batch_size, 1, 1)
box_reg_targets = targets_dict["box_reg_targets"]
# box_reg_targets_np = box_reg_targets.cpu().numpy()
# box_reg_targets_np = box_reg_targets_np[box_reg_targets_np!=0].reshape(batch_size,-1,7)
box_gt_recover = self.box_coder.decode_torch(
box_reg_targets, anchors)
iou3d = iou3d_nms_utils.boxes_iou3d_gpu(box_gt_recover[0],
gt_boxes[0][..., :7])
idx = (iou3d > 0.98).int()
idx = torch.nonzero(idx)
iou3d = iou3d[idx[:, 0], idx[:, 1]]
box_gt_recover = box_gt_recover[0][idx[:, 0], :]
ret_dict.update(targets_dict)
self.forward_ret_dict.update(ret_dict)
# loss,tb_dict = self.get_loss()
rpn_cls_preds, rpn_box_preds = self.predict_box()
ret_dict.update({
"rpn_box_preds": rpn_box_preds,
"rpn_cls_preds": rpn_cls_preds
})
self.forward_ret_dict.update(ret_dict)
return ret_dict