def generate_recall_record(self,index, box_preds,batch_dict,recall_dict, thresh_list=(0.5, 0.7)): rois = batch_dict["rois"][index] if batch_dict["gt_boxes"] is not None: gt_boxes = batch_dict["gt_boxes"][index] cur_gt = gt_boxes k = gt_boxes.__len__() - 1 while k > 0 and cur_gt[k].sum() == 0: k -= 1 cur_gt = cur_gt[:k + 1] cur_rois = rois if cur_gt.shape[0] > 0: iou3d_rcnn = iou3d_nms_utils.boxes_iou3d_gpu(box_preds, cur_gt[:, :7]) iou3d_rois = iou3d_nms_utils.boxes_iou3d_gpu(cur_rois, cur_gt[:, :7]) if recall_dict.__len__() == 0: recall_dict["gt"] = 0 for thresh in thresh_list: recall_dict["roi_%s" % str(thresh)] = 0 recall_dict["rcnn_%s" % str(thresh)] = 0 recall_dict["gt"] += cur_gt.shape[0] for thresh in thresh_list: rois_iou_max = iou3d_rois.max(dim=-1)[0] rcnn_iou_max = iou3d_rcnn.max(dim=-1)[0] recall_dict["roi_%s" % str(thresh)] += (rois_iou_max > thresh).sum().item() recall_dict["rcnn_%s" % str(thresh)] += (rcnn_iou_max > thresh).sum().item() else: return {} return recall_dict
def get_maxiou3d_with_same_class(rois, roi_labels, gt_boxes, gt_labels): """ 三类物体的rois与真实的gt boxes进行匹配,得到每个roi最匹配的gt boxe的indx :param rois: (N, 7) :param roi_labels: (N) :param gt_boxes: (N, 8) :return: """ max_overlaps = rois.new_zeros(rois.shape[0]) gt_assignment = roi_labels.new_zeros(roi_labels.shape[0]) for k in range(gt_labels.min().item(), gt_labels.max().item() + 1): roi_mask = (roi_labels == k) gt_mask = (gt_labels == k) if roi_mask.sum() > 0 and gt_mask.sum() > 0: cur_roi = rois[roi_mask] cur_gt = gt_boxes[gt_mask] original_gt_assignment = gt_mask.nonzero().view(-1) iou3d = iou3d_nms_utils.boxes_iou3d_gpu(cur_roi, cur_gt) # (M, N) cur_max_overlaps, cur_gt_assignment = torch.max(iou3d, dim=1) max_overlaps[roi_mask] = cur_max_overlaps gt_assignment[roi_mask] = original_gt_assignment[cur_gt_assignment] return max_overlaps, gt_assignment
def get_iou_with_same_class(rois, roi_labels, gt_boxes): """ gt_boxes:(num of box per image,8) rois:(512,7) roi—labels """ gt_labels = gt_boxes[:, 7] gt_boxes = gt_boxes[:, :7] max_overlap = torch.zeros(rois.shape[0], dtype=torch.float32, device=gt_boxes.device) gt_assignment = torch.zeros(rois.shape[0], dtype=torch.long, device=gt_boxes.device) for idx in np.arange(gt_labels.min().item(), gt_labels.max().item() + 1, 1): rois_mask_class = roi_labels == idx # rois_mask_class_for_see = rois_mask_class.nonzero().view(-1) gt_mask_class = gt_labels == idx gt_origin_assign = torch.nonzero(gt_mask_class).view(-1) if gt_mask_class.sum() > 0 and rois_mask_class.sum() > 0: cur_gt = gt_boxes[gt_mask_class] cur_rois = rois[rois_mask_class] iou3d = iou3d_nms_utils.boxes_iou3d_gpu(cur_rois, cur_gt) cur_max_overlap, cur_gt_assign = torch.max(iou3d, dim=1) # cur_max_overlaps_see = cur_max_overlap.cpu().numpy() max_overlap[rois_mask_class] = cur_max_overlap gt_assignment[rois_mask_class] = gt_origin_assign[ cur_gt_assign] return max_overlap, gt_assignment
def sample_rois_for_rcnn(roi_boxes3d, gt_boxes3d, roi_raw_scores, roi_labels, roi_sampler_cfg): """ :param roi_boxes3d: (B,512,7) :param gt_boxes3d: (B,M,7+1)这个1是box的种类(“car”,1,"Pedestrian",2,"Cyclist",3) :param roi_raw_scores:(B,512) :param roi_labels:(B,512) :param roi_samlpler_cfg: :return: """ batch_size = roi_boxes3d.size(0) fg_rois_per_image = int( np.round(roi_sampler_cfg.FG_RATIO * roi_sampler_cfg.ROI_PER_IMAGE)) code_size = roi_boxes3d.shape[-1] batch_rois = gt_boxes3d.new(batch_size, roi_sampler_cfg.ROI_PER_IMAGE, code_size).zero_() batch_gt_of_rois = gt_boxes3d.new(batch_size, roi_sampler_cfg.ROI_PER_IMAGE, code_size + 1).zero_() batch_roi_iou = gt_boxes3d.new(batch_size, roi_sampler_cfg.ROI_PER_IMAGE).zero_() batch_roi_raw_scores = gt_boxes3d.new( batch_size, roi_sampler_cfg.ROI_PER_IMAGE).zero_() batch_roi_labels = gt_boxes3d.new( batch_size, roi_sampler_cfg.ROI_PER_IMAGE).zero_().long() for idx in range(batch_size): cur_roi, cur_gt, cur_roi_raw_scores, cur_roi_labels = \ roi_boxes3d[idx], gt_boxes3d[idx], roi_raw_scores[idx], roi_labels[idx] k = cur_gt.__len__() - 1 while k > 0 and cur_gt[k].sum() == 0: k -= 1 cur_gt = cur_gt[:k + 1] if len(cfg.CLASS_NAMES) == 1: iou3d = iou3d_nms_utils.boxes_iou3d_gpu(cur_roi, cur_gt[:, 0:7]) # (M, N) max_overlaps, gt_assignment = torch.max(iou3d, dim=1) else: cur_gt_labels = cur_gt[:, -1].long() max_overlaps, gt_assignment = get_maxiou3d_with_same_class( cur_roi, cur_roi_labels, cur_gt[:, 0:7], cur_gt_labels) fg_thresh = min(roi_sampler_cfg.REG_FG_THRESH, roi_sampler_cfg.CLS_FG_THRESH) #0.55 fg_inds = torch.nonzero((max_overlaps >= fg_thresh)).view(-1) #torch.nonzero这个函数是返回非0的元素的编号,这里返回的是roi的编号 easy_bg_inds = torch.nonzero( (max_overlaps < roi_sampler_cfg.CLS_BG_THRESH_LO)).view( -1) #小于0.1的是easy bg hard_bg_inds = torch.nonzero( (max_overlaps < roi_sampler_cfg.REG_FG_THRESH) & (max_overlaps >= roi_sampler_cfg.CLS_BG_THRESH_LO)).view( -1) #在0.1到0.55之间的是hard fg_num_rois = fg_inds.numel() bg_num_rois = hard_bg_inds.numel() + easy_bg_inds.numel() if fg_num_rois > 0 and bg_num_rois > 0: # sampling fg fg_rois_per_this_image = min(fg_rois_per_image, fg_num_rois) rand_num = torch.from_numpy( np.random.permutation(fg_num_rois)).type_as(gt_boxes3d).long() fg_inds = fg_inds[rand_num[:fg_rois_per_this_image]] # sampling bg bg_rois_per_this_image = roi_sampler_cfg.ROI_PER_IMAGE - fg_rois_per_this_image bg_inds = sample_bg_inds(hard_bg_inds, easy_bg_inds, bg_rois_per_this_image, roi_sampler_cfg) elif fg_num_rois > 0 and bg_num_rois == 0: # sampling fg rand_num = np.floor( np.random.rand(roi_sampler_cfg.ROI_PER_IMAGE) * fg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes3d).long() fg_inds = fg_inds[rand_num] fg_rois_per_this_image = roi_sampler_cfg.ROI_PER_IMAGE bg_rois_per_this_image = 0 elif bg_num_rois > 0 and fg_num_rois == 0: # sampling bg bg_rois_per_this_image = roi_sampler_cfg.ROI_PER_IMAGE bg_inds = sample_bg_inds(hard_bg_inds, easy_bg_inds, bg_rois_per_this_image, roi_sampler_cfg) fg_rois_per_this_image = 0 else: print('maxoverlaps:(min=%f, max=%f)' % (max_overlaps.min().item(), max_overlaps.max().item())) print('ERROR: FG=%d, BG=%d' % (fg_num_rois, bg_num_rois)) raise NotImplementedError roi_list, roi_iou_list, roi_gt_list, roi_score_list, roi_labels_list = [], [], [], [], [] if fg_rois_per_this_image > 0: fg_rois = cur_roi[fg_inds] gt_of_fg_rois = cur_gt[gt_assignment[fg_inds]] fg_iou3d = max_overlaps[fg_inds] roi_list.append(fg_rois) roi_iou_list.append(fg_iou3d) roi_gt_list.append(gt_of_fg_rois) roi_score_list.append(cur_roi_raw_scores[fg_inds]) roi_labels_list.append(cur_roi_labels[fg_inds]) if bg_rois_per_this_image > 0: bg_rois = cur_roi[bg_inds] gt_of_bg_rois = cur_gt[gt_assignment[bg_inds]] bg_iou3d = max_overlaps[bg_inds] roi_list.append(bg_rois) roi_iou_list.append(bg_iou3d) roi_gt_list.append(gt_of_bg_rois) roi_score_list.append(cur_roi_raw_scores[bg_inds]) roi_labels_list.append(cur_roi_labels[bg_inds]) rois = torch.cat(roi_list, dim=0) iou_of_rois = torch.cat(roi_iou_list, dim=0) gt_of_rois = torch.cat(roi_gt_list, dim=0) cur_roi_raw_scores = torch.cat(roi_score_list, dim=0) cur_roi_labels = torch.cat(roi_labels_list, dim=0) batch_rois[idx] = rois batch_gt_of_rois[idx] = gt_of_rois batch_roi_iou[idx] = iou_of_rois batch_roi_raw_scores[idx] = cur_roi_raw_scores batch_roi_labels[idx] = cur_roi_labels return batch_rois, batch_gt_of_rois, batch_roi_iou, batch_roi_raw_scores, batch_roi_labels
def sample_rois_for_rcnn(self, batch_dict): """ Args: batch_dict: batch_size: rois: (B, num_rois, 7 + C) roi_scores: (B, num_rois) gt_boxes: (B, N, 7 + C + 1) roi_labels: (B, num_rois) Returns: """ batch_size = batch_dict['batch_size'] rois = batch_dict['rois'] roi_scores = batch_dict['roi_scores'] roi_labels = batch_dict['roi_labels'] gt_boxes = batch_dict['gt_boxes'] code_size = rois.shape[-1] batch_rois = rois.new_zeros(batch_size, self.roi_sampler_cfg.ROI_PER_IMAGE, code_size) batch_gt_of_rois = rois.new_zeros(batch_size, self.roi_sampler_cfg.ROI_PER_IMAGE, code_size + 1) batch_roi_ious = rois.new_zeros(batch_size, self.roi_sampler_cfg.ROI_PER_IMAGE) batch_roi_scores = rois.new_zeros(batch_size, self.roi_sampler_cfg.ROI_PER_IMAGE) batch_roi_labels = rois.new_zeros( (batch_size, self.roi_sampler_cfg.ROI_PER_IMAGE), dtype=torch.long) for index in range(batch_size): cur_roi, cur_gt, cur_roi_labels, cur_roi_scores = \ rois[index], gt_boxes[index], roi_labels[index], roi_scores[index] k = cur_gt.__len__() - 1 while k > 0 and cur_gt[k].sum() == 0: k -= 1 cur_gt = cur_gt[:k + 1] cur_gt = cur_gt.new_zeros( (1, cur_gt.shape[1])) if len(cur_gt) == 0 else cur_gt if self.roi_sampler_cfg.get('SAMPLE_ROI_BY_EACH_CLASS', False): max_overlaps, gt_assignment = self.get_max_iou_with_same_class( rois=cur_roi, roi_labels=cur_roi_labels, gt_boxes=cur_gt[:, 0:7], gt_labels=cur_gt[:, -1].long()) else: iou3d = iou3d_nms_utils.boxes_iou3d_gpu( cur_roi, cur_gt[:, 0:7]) # (M, N) max_overlaps, gt_assignment = torch.max(iou3d, dim=1) sampled_inds = self.subsample_rois(max_overlaps=max_overlaps) batch_rois[index] = cur_roi[sampled_inds] batch_roi_labels[index] = cur_roi_labels[sampled_inds] batch_roi_ious[index] = max_overlaps[sampled_inds] batch_roi_scores[index] = cur_roi_scores[sampled_inds] batch_gt_of_rois[index] = cur_gt[gt_assignment[sampled_inds]] return batch_rois, batch_gt_of_rois, batch_roi_ious, batch_roi_scores, batch_roi_labels
def assign_targets_single(self, anchors, gt_boxes, gt_classes, matched_threshold=0.6, unmatched_threshold=0.45 ): num_anchors = anchors.shape[0] num_gt = gt_boxes.shape[0] box_ndim = anchors.shape[1] labels = torch.ones((num_anchors,), dtype=torch.int32, device=anchors.device) * -1 gt_ids = torch.ones((num_anchors,), dtype=torch.int32, device=anchors.device) * -1 if len(gt_boxes) > 0 and anchors.shape[0] > 0: anchor_by_gt_overlap = iou3d_nms_utils.boxes_iou3d_gpu(anchors, gt_boxes) if self.match_height else box_utils.boxes3d_nearest_bev_iou( anchors, gt_boxes) anchor_to_gt_argmax = anchor_by_gt_overlap.argmax(dim=1) anchor_to_gt_max = anchor_by_gt_overlap[torch.arange(num_anchors), anchor_to_gt_argmax] gt_to_anchor_argmax = anchor_by_gt_overlap.argmax(dim=0) gt_to_anchor_max = anchor_by_gt_overlap[ gt_to_anchor_argmax, torch.arange(num_gt)] empty_gt_mask = gt_to_anchor_max == 0 gt_to_anchor_max[empty_gt_mask] = -1 anchors_with_max_overlap = torch.nonzero( anchor_by_gt_overlap == gt_to_anchor_max)[:, 0] gt_inds_force = anchor_to_gt_argmax[anchors_with_max_overlap] labels[anchors_with_max_overlap] = gt_classes[gt_inds_force] gt_ids[anchors_with_max_overlap] = gt_inds_force.int() pos_inds = anchor_to_gt_max >= matched_threshold gt_inds_over_thresh = anchor_to_gt_argmax[pos_inds] labels[pos_inds] = gt_classes[gt_inds_over_thresh] gt_ids[pos_inds] = gt_inds_over_thresh.int() bg_inds = torch.nonzero(anchor_to_gt_max < unmatched_threshold)[:, 0] else: bg_inds = torch.arange(num_anchors) fg_inds = torch.nonzero(labels > 0)[:, 0] if self.pos_fraction is not None: num_fg = int(self.pos_fraction * self.sample_size) if len(fg_inds) > num_fg: num_disabled = len(fg_inds) - num_fg disable_inds = torch.randperm(len(fg_inds))[:num_disabled] labels[disable_inds] = -1 fg_inds = torch.nonzero(labels > 0)[:, 0] num_bg = self.sample_size - (labels > 0).sum() if len(bg_inds) > num_bg: enable_inds = bg_inds[torch.randint(0, len(bg_inds), size=(num_bg,))] labels[enable_inds] = 0 bg_inds = torch.nonzero(labels == 0)[:, 0] else: if len(gt_boxes) == 0 or anchors.shape[0] == 0: labels[:] = 0 else: labels[bg_inds] = 0 labels[anchors_with_max_overlap] = gt_classes[gt_inds_force] bbox_targets = anchors.new_zeros((num_anchors, self.box_coder.code_size)) if len(gt_boxes) > 0 and anchors.shape[0] > 0: fg_gt_boxes = gt_boxes[anchor_to_gt_argmax[fg_inds], :] fg_anchors = anchors[fg_inds, :] bbox_targets[fg_inds, :] = self.box_coder.encode_torch(fg_gt_boxes, fg_anchors) reg_weights = anchors.new_zeros((num_anchors,)) if self.norm_by_num_examples: num_examples = (labels >= 0).sum() num_examples = num_examples if num_examples > 1.0 else 1.0 reg_weights[labels > 0] = 1.0 / num_examples else: reg_weights[labels > 0] = 1.0 ret_dict = { 'box_cls_labels': labels, 'box_reg_targets': bbox_targets, 'reg_weights': reg_weights, } return ret_dict
def forward(self, x_in, gt_boxes): # x_in = batch_dict["spatial_features"] x = x_in ups = [] ret_dict = {} for i in range(len(self.blocks)): x = self.blocks[i](x) stride = int(x_in.shape[2] / x.shape[2]) ret_dict['spatial_features_%dx' % stride] = x ups.append(self.deblocks[i](x)) if self._concat_input: ups.append(x_in) if len(ups) > 1: x = torch.cat(ups, dim=1) else: x = ups[0] if len(self.deblocks) > len(self.blocks): x = self.deblocks[-1](x) ret_dict['spatial_features_last'] = x box_preds = self.conv_box(x) cls_preds = self.conv_cls(x) # [N, C, y(H), x(W)] box_preds = box_preds.permute(0, 2, 3, 1).contiguous() cls_preds = cls_preds.permute(0, 2, 3, 1).contiguous() ret_dict.update({ 'box_preds': box_preds, 'cls_preds': cls_preds, }) if self.config["use_direction_classifier"]: dir_pred = self.conv_dir_cls(x) ret_dict.update({"dir_preds": dir_pred.contiguous()}) self.forward_ret_dict = ret_dict if self.training: targets_dict = self.get_assigner_target(gt_boxes=gt_boxes, ) # 验证target batch_size = targets_dict["box_reg_targets"].shape[0] anchors = torch.cat([anchor for anchor in self.anchors], dim=-2) anchors = anchors.reshape(1, -1, 7).repeat(batch_size, 1, 1) box_reg_targets = targets_dict["box_reg_targets"] # box_reg_targets_np = box_reg_targets.cpu().numpy() # box_reg_targets_np = box_reg_targets_np[box_reg_targets_np!=0].reshape(batch_size,-1,7) box_gt_recover = self.box_coder.decode_torch( box_reg_targets, anchors) iou3d = iou3d_nms_utils.boxes_iou3d_gpu(box_gt_recover[0], gt_boxes[0][..., :7]) idx = (iou3d > 0.98).int() idx = torch.nonzero(idx) iou3d = iou3d[idx[:, 0], idx[:, 1]] box_gt_recover = box_gt_recover[0][idx[:, 0], :] ret_dict.update(targets_dict) self.forward_ret_dict.update(ret_dict) # loss,tb_dict = self.get_loss() rpn_cls_preds, rpn_box_preds = self.predict_box() ret_dict.update({ "rpn_box_preds": rpn_box_preds, "rpn_cls_preds": rpn_cls_preds }) self.forward_ret_dict.update(ret_dict) return ret_dict