def inference(self, predictions, proposals): """ Returns: list[Instances]: same as `fast_rcnn_inference`. list[Tensor]: same as `fast_rcnn_inference`. """ # TODO 2. image_shapes = [x.image_size for x in proposals] if self.std_cls_loss_type == 'softmax': pred_class_logits, pred_standard_cls_softmax_logits, pred_proposal_deltas = predictions cls_scores = self.predict_probs(pred_class_logits, proposals) std_cls_softmax_score = self.predict_probs( pred_standard_cls_softmax_logits, proposals) boxes = self.predict_boxes(pred_proposal_deltas, proposals) # instances: (list[Instances]) # kept_indices: (list[Tensor]) instances, kept_indices = fast_rcnn_inference( boxes, cls_scores, image_shapes, self.test_score_thresh, self.test_nms_thresh, self.test_topk_per_image) for i, indices in enumerate(kept_indices): instances[i].pred_standards = torch.argmax( std_cls_softmax_score[i][indices], dim=-1) return instances, kept_indices elif self.std_cls_loss_type == 'arc': pass elif self.std_cls_loss_type == 'arc+softmax': pass else: raise NotImplementedError( '标准分类只支持三种损失模式{arc, softmax, arc+softmax}')
def _forward_box(self, features, proposals, targets=None, extra_info=None): """ CascadeROIHeads._forward_box modified to include per-class thresholds. Args: features, targets: the same as in Same as in :meth:`ROIHeads.forward`. proposals (list[Instances]): the per-image object proposals with their matching ground truth. Each has fields "proposal_boxes", and "objectness_logits", "gt_classes", "gt_boxes". """ features = [features[f] for f in self.box_in_features] head_outputs = [] # (predictor, predictions, proposals) prev_pred_boxes = None image_sizes = [x.image_size for x in proposals] for k in range(self.num_cascade_stages): if k > 0: # The output boxes of the previous stage are used to create the input # proposals of the next stage. proposals = self._create_proposals_from_boxes( prev_pred_boxes, image_sizes ) if self.training: proposals = self._match_and_label_boxes(proposals, k, targets) predictions = self._run_stage(features, proposals, k) prev_pred_boxes = self.box_predictor[k].predict_boxes( predictions, proposals ) head_outputs.append((self.box_predictor[k], predictions, proposals)) if self.training: # <modified> raise NotImplementedError # </modified> else: # Each is a list[Tensor] of length #image. Each tensor is Ri x (K+1) scores_per_stage = [h[0].predict_probs(h[1], h[2]) for h in head_outputs] # Average the scores across heads scores = [ sum(list(scores_per_image)) * (1.0 / self.num_cascade_stages) for scores_per_image in zip(*scores_per_stage) ] # Use the boxes of the last head predictor, predictions, proposals = head_outputs[-1] boxes = predictor.predict_boxes(predictions, proposals) # <modified> for image_scores in scores: image_scores[image_scores < score_thresholds] = -1 pred_instances, _ = fast_rcnn_inference( boxes, scores, image_sizes, score_thresh=0, nms_thresh=predictor.test_nms_thresh, topk_per_image=-1, ) return pred_instances
def roi_pooling(self, feature_lists, proposals, is_cascade=True): if is_cascade: head_outputs = [] # (predictor, predictions, proposals) prev_pred_boxes = None image_sizes = [x.image_size for x in proposals] for k in range(self.roi_heads_module.num_cascade_stages): if k > 0: # The output boxes of the previous stage are used to create the input # proposals of the next stage. proposals = self.roi_heads_module._create_proposals_from_boxes( prev_pred_boxes, image_sizes) predictions = self.roi_heads_module._run_stage( feature_lists, proposals, k) prev_pred_boxes = self.roi_heads_module.box_predictor[ k].predict_boxes(predictions, proposals) head_outputs.append((self.roi_heads_module.box_predictor[k], predictions, proposals)) scores_per_stage = [ h[0].predict_probs(h[1], h[2]) for h in head_outputs ] # Average the scores across heads scores = [ sum(list(scores_per_image)) * (1.0 / self.roi_heads_module.num_cascade_stages) for scores_per_image in zip(*scores_per_stage) ] predictor, predictions, proposals = head_outputs[-1] boxes = predictor.predict_boxes(predictions, proposals) pred_instances, _ = fast_rcnn_inference( boxes, scores, image_sizes, predictor.test_score_thresh, predictor.test_nms_thresh, predictor.test_topk_per_image, ) else: box_features = self.roi_heads_module.box_pooler( feature_lists, [x.proposal_boxes for x in proposals]) box_features = self.roi_heads_module.box_head(box_features) predictions = self.roi_heads_module.box_predictor(box_features) pred_instances, _ = self.roi_heads_module.box_predictor.inference( predictions, proposals) proposals_per_batch = [len(ins) for ins in pred_instances] return proposals_per_batch, [x.pred_boxes for x in pred_instances], pred_instances
def inference(self, predictions, proposals, tta=False): scores = self.predict_probs(predictions, proposals) if tta: return [torch.cat(scores, 0), predictions[1]], None boxes = self.predict_boxes(predictions, proposals) image_shapes = [x.image_size for x in proposals] return fast_rcnn_inference( boxes, scores, image_shapes, self.test_score_thresh, self.test_nms_thresh, self.test_topk_per_image, )
def inference(self, score_thresh, nms_thresh, topk_per_image): """ Args: score_thresh (float): same as fast_rcnn_inference. nms_thresh (float): same as fast_rcnn_inference. topk_per_image (int): same as fast_rcnn_inference. Returns: list[Instances]: same as fast_rcnn_inference. list[Tensor]: same as fast_rcnn_inference. """ boxes = self.predict_boxes() scores = self.predict_probs() image_shapes = self.image_shapes return fast_rcnn_inference(boxes, scores, image_shapes, score_thresh, nms_thresh, topk_per_image)
def box_inference(self, score_thresh, nms_thresh, topk_per_image): """ Args: score_thresh (float): same as fast_rcnn_inference. nms_thresh (float): same as fast_rcnn_inference. topk_per_image (int): same as fast_rcnn_inference. Returns: list[Instances]: same as fast_rcnn_inference. list[Tensor]: same as fast_rcnn_inference. """ boxes = self.predict_boxes() scores = self.pred_class_logits.split(self.num_preds_per_image, dim=0) image_shapes = self.image_shapes return fast_rcnn_inference(boxes, scores, image_shapes, score_thresh, nms_thresh, topk_per_image)
def inference(self, predictions, proposals): """ Returns: list[Instances]: same as `fast_rcnn_inference`. list[Tensor]: same as `fast_rcnn_inference`. """ boxes = self.predict_boxes(predictions, proposals) scores = self.predict_probs(predictions, proposals) image_shapes = [x.image_size for x in proposals] return fast_rcnn_inference( boxes, scores, image_shapes, self.test_score_thresh, self.test_nms_thresh, self.test_topk_per_image, )
def inference(self, predictions, proposals): """ enable use proposal boxes """ boxes = self.predict_boxes(predictions, proposals) scores = self.predict_probs(predictions, proposals) if self.cfg.MODEL.ROI_BOX_HEAD.MULT_PROPOSAL_SCORE: proposal_scores = [p.get('objectness_logits') for p in proposals] scores = [(s * ps[:, None]) ** 0.5 \ for s, ps in zip(scores, proposal_scores)] image_shapes = [x.image_size for x in proposals] return fast_rcnn_inference( boxes, scores, image_shapes, self.test_score_thresh, self.test_nms_thresh, self.test_topk_per_image, )
def inference(self, predictions, proposals, tta=False): scores = self.predict_probs(predictions, proposals) if tta: if self.oicr_regression_branch: regression_output = torch.mean(torch.stack(predictions[1], 0), 0) return [torch.cat(scores, 0), regression_output], None else: return [torch.cat(scores, 0), predictions[1]], None boxes = self.predict_boxes(predictions, proposals) image_shapes = [x.image_size for x in proposals] return fast_rcnn_inference( boxes, scores, image_shapes, self.test_score_thresh, self.test_nms_thresh, self.test_topk_per_image, )
def _inference(self, predictions, proposals): """ Returns: list[Instances]: same as `fast_rcnn_inference`. list[Tensor]: same as `fast_rcnn_inference`. """ boxes = self.predict_boxes(predictions, proposals) scores = self.predict_probs(predictions, proposals) # <modified> for image_scores in scores: image_scores[image_scores < score_thresholds] = -1 image_shapes = [x.image_size for x in proposals] return fast_rcnn_inference( boxes, scores, image_shapes, score_thresh=0, nms_thresh=self.test_nms_thresh, topk_per_image=-1, )
def inference(self, predictions, proposals): """ Returns: list[Instances]: same as `fast_rcnn_inference`. list[Tensor]: same as `fast_rcnn_inference`. """ image_shapes = [x.image_size for x in proposals] category_logits, proposal_deltas, standard_logits = predictions category_scores = self.predict_probs(category_logits, proposals) standard_scores = self.predict_probs(standard_logits, proposals) boxes = self.predict_boxes(proposal_deltas, proposals) # instances: (list[Instances]) # kept_indices: (list[Tensor]) instances, kept_indices = fast_rcnn_inference(boxes, category_scores, image_shapes, self.test_score_thresh, self.test_nms_thresh, self.test_topk_per_image) for i, indices in enumerate(kept_indices): instances[i].pred_standards = torch.argmax( standard_scores[i][indices], dim=-1) return instances, kept_indices
def _forward_box(self, features, proposals, targets=None, dataset_source=-1): features = [features[f] for f in self.box_in_features] head_outputs = [] # (predictor, predictions, proposals) prev_pred_boxes = None image_sizes = [x.image_size for x in proposals] for k in range(self.num_cascade_stages): if k > 0: # The output boxes of the previous stage are the input proposals of the next stage proposals = self._create_proposals_from_boxes( prev_pred_boxes, image_sizes) if self.training: proposals = self._match_and_label_boxes( proposals, k, targets) predictions = self._run_stage(features, proposals, k, dataset_source) prev_pred_boxes = self.box_predictor[k].predict_boxes( predictions, proposals) head_outputs.append( (self.box_predictor[k], predictions, proposals)) if self.training: losses = {} storage = get_event_storage() for stage, (predictor, predictions, proposals) in enumerate(head_outputs): with storage.name_scope("{}_stage{}".format( self.dataset_names[dataset_source], stage)): stage_losses = predictor.losses( predictions, proposals, use_advanced_loss=( dataset_source == self.openimage_index)) losses.update({ "{}_{}_stage{}".format(self.dataset_names[dataset_source], k, stage): v for k, v in stage_losses.items() }) return losses else: # Each is a list[Tensor] of length #image. Each tensor is Ri x (K+1) scores_per_stage = [ h[0].predict_probs(h[1], h[2]) for h in head_outputs ] scores = [ sum(list(scores_per_image)) * (1.0 / self.num_cascade_stages) for scores_per_image in zip(*scores_per_stage) ] predictor, predictions, proposals = head_outputs[-1] boxes = predictor.predict_boxes(predictions, proposals) pred_instances, _ = fast_rcnn_inference( boxes, scores, image_sizes, predictor.test_score_thresh, predictor.test_nms_thresh, predictor.test_topk_per_image, ) return pred_instances
def _forward_box(self, features, proposals, targets=None): """ Add mult proposal scores at testing """ if (not self.training) and self.mult_proposal_score: if len(proposals) > 0 and proposals[0].has('scores'): proposal_scores = [p.get('scores') for p in proposals] else: proposal_scores = [ p.get('objectness_logits') for p in proposals ] features = [features[f] for f in self.box_in_features] head_outputs = [] # (predictor, predictions, proposals) prev_pred_boxes = None image_sizes = [x.image_size for x in proposals] for k in range(self.num_cascade_stages): if k > 0: proposals = self._create_proposals_from_boxes( prev_pred_boxes, image_sizes) if self.training: proposals = self._match_and_label_boxes( proposals, k, targets) predictions = self._run_stage(features, proposals, k) prev_pred_boxes = self.box_predictor[k].predict_boxes( predictions, proposals) head_outputs.append( (self.box_predictor[k], predictions, proposals)) if self.training: losses = {} storage = get_event_storage() for stage, (predictor, predictions, proposals) in enumerate(head_outputs): with storage.name_scope("stage{}".format(stage)): stage_losses = predictor.losses(predictions, proposals) losses.update({ k + "_stage{}".format(stage): v for k, v in stage_losses.items() }) return losses else: # Each is a list[Tensor] of length #image. Each tensor is Ri x (K+1) scores_per_stage = [ h[0].predict_probs(h[1], h[2]) for h in head_outputs ] scores = [ sum(list(scores_per_image)) * (1.0 / self.num_cascade_stages) for scores_per_image in zip(*scores_per_stage) ] if self.mult_proposal_score: scores = [(s * ps[:, None]) ** 0.5 \ for s, ps in zip(scores, proposal_scores)] predictor, predictions, proposals = head_outputs[-1] boxes = predictor.predict_boxes(predictions, proposals) pred_instances, _ = fast_rcnn_inference( boxes, scores, image_sizes, predictor.test_score_thresh, predictor.test_nms_thresh, predictor.test_topk_per_image, ) return pred_instances
def test_caffe2_pytorch_eq(self): ims_per_batch = 8 post_nms_topk = 100 detections_per_im = 10 num_class = 80 score_thresh = 0.05 nms_thresh = 0.5 image_shapes = [torch.Size([800, 800])] * ims_per_batch batch_splits = [post_nms_topk] * ims_per_batch # NOTE: There're still some unsure minor implementation differences # (eg. ordering when equal score across classes) causing some seeds # don't pass the test. # Thus set a fixed seed to make sure this test passes consistantly. rng = torch.Generator() rng.manual_seed(42) boxes = [] for n in batch_splits: box = 1000.0 * 0.5 * torch.rand(n, num_class, 4, generator=rng) + 0.001 box[:, :, -2:] += box[:, :, :2] box = box.view(n, num_class * 4) boxes.append(box) scores = [ torch.rand(n, num_class + 1, generator=rng) for n in batch_splits ] ref_results, ref_kept_indices = fast_rcnn_inference( boxes, scores, image_shapes, score_thresh=score_thresh, nms_thresh=nms_thresh, topk_per_image=detections_per_im) for result, kept_index, score in zip(ref_results, ref_kept_indices, scores): torch.testing.assert_allclose( score[kept_index, result.pred_classes], result.scores, ) # clip is done in BBoxTransformOp c2_boxes = [] for box, image_shape in zip(boxes, image_shapes): num_bbox_reg_classes = box.shape[1] // 4 clipped_box = Boxes(box.reshape(-1, 4)) clipped_box.clip(image_shape) clipped_box = clipped_box.tensor.view(-1, num_bbox_reg_classes * 4) c2_boxes.append(clipped_box) c2_boxes = cat(c2_boxes) c2_scores = cat(scores) c2_batch_splits = torch.Tensor(batch_splits) nms_outputs = torch.ops._caffe2.BoxWithNMSLimit( c2_scores, c2_boxes, c2_batch_splits, score_thresh=float(score_thresh), nms=float(nms_thresh), detections_per_im=int(detections_per_im), soft_nms_enabled=False, soft_nms_method="linear", soft_nms_sigma=0.5, soft_nms_min_score_thres=0.001, rotated=False, cls_agnostic_bbox_reg=False, input_boxes_include_bg_cls=False, output_classes_include_bg_cls=False, legacy_plus_one=False, ) roi_score_nms, roi_bbox_nms, roi_class_nms, roi_batch_splits_nms, roi_keeps_nms, roi_keeps_size_nms = nms_outputs # noqa roi_score_nms = roi_score_nms.split( roi_batch_splits_nms.int().tolist()) roi_bbox_nms = roi_bbox_nms.split(roi_batch_splits_nms.int().tolist()) roi_class_nms = roi_class_nms.split( roi_batch_splits_nms.int().tolist()) roi_keeps_nms = roi_keeps_nms.split( roi_batch_splits_nms.int().tolist()) for _score_nms, _class_nms, _keeps_nms, _score in zip( roi_score_nms, roi_class_nms, roi_keeps_nms, scores): torch.testing.assert_allclose( _score[_keeps_nms.to(torch.int64), _class_nms.to(torch.int64)], _score_nms, ) for ref, s, b, c in zip(ref_results, roi_score_nms, roi_bbox_nms, roi_class_nms): s1, i1 = s.sort() s2, i2 = ref.scores.sort() torch.testing.assert_allclose(s1, s2) torch.testing.assert_allclose(b[i1], ref.pred_boxes.tensor[i2]) torch.testing.assert_allclose( c.to(torch.int64)[i1], ref.pred_classes[i2]) for ref, k in zip(ref_kept_indices, roi_keeps_nms): # NOTE: order might be different due to implementation ref_set = set(ref.tolist()) k_set = set(k.tolist()) self.assertEqual(ref_set, k_set)