def inference(self, predictions, proposals):
"""
Returns:
list[Instances]: same as `fast_rcnn_inference`.
list[Tensor]: same as `fast_rcnn_inference`.
"""
# TODO 2.
image_shapes = [x.image_size for x in proposals]
if self.std_cls_loss_type == 'softmax':
pred_class_logits, pred_standard_cls_softmax_logits, pred_proposal_deltas = predictions
cls_scores = self.predict_probs(pred_class_logits, proposals)
std_cls_softmax_score = self.predict_probs(
pred_standard_cls_softmax_logits, proposals)
boxes = self.predict_boxes(pred_proposal_deltas, proposals)
# instances: (list[Instances])
# kept_indices: (list[Tensor])
instances, kept_indices = fast_rcnn_inference(
boxes, cls_scores, image_shapes, self.test_score_thresh,
self.test_nms_thresh, self.test_topk_per_image)
for i, indices in enumerate(kept_indices):
instances[i].pred_standards = torch.argmax(
std_cls_softmax_score[i][indices], dim=-1)
return instances, kept_indices
elif self.std_cls_loss_type == 'arc':
pass
elif self.std_cls_loss_type == 'arc+softmax':
pass
else:
raise NotImplementedError(
'标准分类只支持三种损失模式{arc, softmax, arc+softmax}')
def _forward_box(self, features, proposals, targets=None, extra_info=None):
"""
CascadeROIHeads._forward_box modified to include per-class thresholds.
Args:
features, targets: the same as in
Same as in :meth:`ROIHeads.forward`.
proposals (list[Instances]): the per-image object proposals with
their matching ground truth.
Each has fields "proposal_boxes", and "objectness_logits",
"gt_classes", "gt_boxes".
"""
features = [features[f] for f in self.box_in_features]
head_outputs = [] # (predictor, predictions, proposals)
prev_pred_boxes = None
image_sizes = [x.image_size for x in proposals]
for k in range(self.num_cascade_stages):
if k > 0:
# The output boxes of the previous stage are used to create the input
# proposals of the next stage.
proposals = self._create_proposals_from_boxes(
prev_pred_boxes, image_sizes
)
if self.training:
proposals = self._match_and_label_boxes(proposals, k, targets)
predictions = self._run_stage(features, proposals, k)
prev_pred_boxes = self.box_predictor[k].predict_boxes(
predictions, proposals
)
head_outputs.append((self.box_predictor[k], predictions, proposals))
if self.training:
# <modified>
raise NotImplementedError
# </modified>
else:
# Each is a list[Tensor] of length #image. Each tensor is Ri x (K+1)
scores_per_stage = [h[0].predict_probs(h[1], h[2]) for h in head_outputs]
# Average the scores across heads
scores = [
sum(list(scores_per_image)) * (1.0 / self.num_cascade_stages)
for scores_per_image in zip(*scores_per_stage)
]
# Use the boxes of the last head
predictor, predictions, proposals = head_outputs[-1]
boxes = predictor.predict_boxes(predictions, proposals)
# <modified>
for image_scores in scores:
image_scores[image_scores < score_thresholds] = -1
pred_instances, _ = fast_rcnn_inference(
boxes,
scores,
image_sizes,
score_thresh=0,
nms_thresh=predictor.test_nms_thresh,
topk_per_image=-1,
)
return pred_instances
def roi_pooling(self, feature_lists, proposals, is_cascade=True):
if is_cascade:
head_outputs = [] # (predictor, predictions, proposals)
prev_pred_boxes = None
image_sizes = [x.image_size for x in proposals]
for k in range(self.roi_heads_module.num_cascade_stages):
if k > 0:
# The output boxes of the previous stage are used to create the input
# proposals of the next stage.
proposals = self.roi_heads_module._create_proposals_from_boxes(
prev_pred_boxes, image_sizes)
predictions = self.roi_heads_module._run_stage(
feature_lists, proposals, k)
prev_pred_boxes = self.roi_heads_module.box_predictor[
k].predict_boxes(predictions, proposals)
head_outputs.append((self.roi_heads_module.box_predictor[k],
predictions, proposals))
scores_per_stage = [
h[0].predict_probs(h[1], h[2]) for h in head_outputs
]
# Average the scores across heads
scores = [
sum(list(scores_per_image)) *
(1.0 / self.roi_heads_module.num_cascade_stages)
for scores_per_image in zip(*scores_per_stage)
]
predictor, predictions, proposals = head_outputs[-1]
boxes = predictor.predict_boxes(predictions, proposals)
pred_instances, _ = fast_rcnn_inference(
boxes,
scores,
image_sizes,
predictor.test_score_thresh,
predictor.test_nms_thresh,
predictor.test_topk_per_image,
)
else:
box_features = self.roi_heads_module.box_pooler(
feature_lists, [x.proposal_boxes for x in proposals])
box_features = self.roi_heads_module.box_head(box_features)
predictions = self.roi_heads_module.box_predictor(box_features)
pred_instances, _ = self.roi_heads_module.box_predictor.inference(
predictions, proposals)
proposals_per_batch = [len(ins) for ins in pred_instances]
return proposals_per_batch, [x.pred_boxes
for x in pred_instances], pred_instances
def inference(self, predictions, proposals, tta=False):
scores = self.predict_probs(predictions, proposals)
if tta:
return [torch.cat(scores, 0), predictions[1]], None
boxes = self.predict_boxes(predictions, proposals)
image_shapes = [x.image_size for x in proposals]
return fast_rcnn_inference(
boxes,
scores,
image_shapes,
self.test_score_thresh,
self.test_nms_thresh,
self.test_topk_per_image,
)
def inference(self, score_thresh, nms_thresh, topk_per_image):
"""
Args:
score_thresh (float): same as fast_rcnn_inference.
nms_thresh (float): same as fast_rcnn_inference.
topk_per_image (int): same as fast_rcnn_inference.
Returns:
list[Instances]: same as fast_rcnn_inference.
list[Tensor]: same as fast_rcnn_inference.
"""
boxes = self.predict_boxes()
scores = self.predict_probs()
image_shapes = self.image_shapes
return fast_rcnn_inference(boxes, scores, image_shapes, score_thresh,
nms_thresh, topk_per_image)
def box_inference(self, score_thresh, nms_thresh, topk_per_image):
"""
Args:
score_thresh (float): same as fast_rcnn_inference.
nms_thresh (float): same as fast_rcnn_inference.
topk_per_image (int): same as fast_rcnn_inference.
Returns:
list[Instances]: same as fast_rcnn_inference.
list[Tensor]: same as fast_rcnn_inference.
"""
boxes = self.predict_boxes()
scores = self.pred_class_logits.split(self.num_preds_per_image, dim=0)
image_shapes = self.image_shapes
return fast_rcnn_inference(boxes, scores, image_shapes, score_thresh,
nms_thresh, topk_per_image)
def inference(self, predictions, proposals):
"""
Returns:
list[Instances]: same as `fast_rcnn_inference`.
list[Tensor]: same as `fast_rcnn_inference`.
"""
boxes = self.predict_boxes(predictions, proposals)
scores = self.predict_probs(predictions, proposals)
image_shapes = [x.image_size for x in proposals]
return fast_rcnn_inference(
boxes,
scores,
image_shapes,
self.test_score_thresh,
self.test_nms_thresh,
self.test_topk_per_image,
)
def inference(self, predictions, proposals):
"""
enable use proposal boxes
"""
boxes = self.predict_boxes(predictions, proposals)
scores = self.predict_probs(predictions, proposals)
if self.cfg.MODEL.ROI_BOX_HEAD.MULT_PROPOSAL_SCORE:
proposal_scores = [p.get('objectness_logits') for p in proposals]
scores = [(s * ps[:, None]) ** 0.5 \
for s, ps in zip(scores, proposal_scores)]
image_shapes = [x.image_size for x in proposals]
return fast_rcnn_inference(
boxes,
scores,
image_shapes,
self.test_score_thresh,
self.test_nms_thresh,
self.test_topk_per_image,
)
def inference(self, predictions, proposals, tta=False):
scores = self.predict_probs(predictions, proposals)
if tta:
if self.oicr_regression_branch:
regression_output = torch.mean(torch.stack(predictions[1], 0),
0)
return [torch.cat(scores, 0), regression_output], None
else:
return [torch.cat(scores, 0), predictions[1]], None
boxes = self.predict_boxes(predictions, proposals)
image_shapes = [x.image_size for x in proposals]
return fast_rcnn_inference(
boxes,
scores,
image_shapes,
self.test_score_thresh,
self.test_nms_thresh,
self.test_topk_per_image,
)
def _inference(self, predictions, proposals):
"""
Returns:
list[Instances]: same as `fast_rcnn_inference`.
list[Tensor]: same as `fast_rcnn_inference`.
"""
boxes = self.predict_boxes(predictions, proposals)
scores = self.predict_probs(predictions, proposals)
# <modified>
for image_scores in scores:
image_scores[image_scores < score_thresholds] = -1
image_shapes = [x.image_size for x in proposals]
return fast_rcnn_inference(
boxes,
scores,
image_shapes,
score_thresh=0,
nms_thresh=self.test_nms_thresh,
topk_per_image=-1,
)
def inference(self, predictions, proposals):
"""
Returns:
list[Instances]: same as `fast_rcnn_inference`.
list[Tensor]: same as `fast_rcnn_inference`.
"""
image_shapes = [x.image_size for x in proposals]
category_logits, proposal_deltas, standard_logits = predictions
category_scores = self.predict_probs(category_logits, proposals)
standard_scores = self.predict_probs(standard_logits, proposals)
boxes = self.predict_boxes(proposal_deltas, proposals)
# instances: (list[Instances])
# kept_indices: (list[Tensor])
instances, kept_indices = fast_rcnn_inference(boxes, category_scores,
image_shapes,
self.test_score_thresh,
self.test_nms_thresh,
self.test_topk_per_image)
for i, indices in enumerate(kept_indices):
instances[i].pred_standards = torch.argmax(
standard_scores[i][indices], dim=-1)
return instances, kept_indices
def _forward_box(self,
features,
proposals,
targets=None,
dataset_source=-1):
features = [features[f] for f in self.box_in_features]
head_outputs = [] # (predictor, predictions, proposals)
prev_pred_boxes = None
image_sizes = [x.image_size for x in proposals]
for k in range(self.num_cascade_stages):
if k > 0:
# The output boxes of the previous stage are the input proposals of the next stage
proposals = self._create_proposals_from_boxes(
prev_pred_boxes, image_sizes)
if self.training:
proposals = self._match_and_label_boxes(
proposals, k, targets)
predictions = self._run_stage(features, proposals, k,
dataset_source)
prev_pred_boxes = self.box_predictor[k].predict_boxes(
predictions, proposals)
head_outputs.append(
(self.box_predictor[k], predictions, proposals))
if self.training:
losses = {}
storage = get_event_storage()
for stage, (predictor, predictions,
proposals) in enumerate(head_outputs):
with storage.name_scope("{}_stage{}".format(
self.dataset_names[dataset_source], stage)):
stage_losses = predictor.losses(
predictions,
proposals,
use_advanced_loss=(
dataset_source == self.openimage_index))
losses.update({
"{}_{}_stage{}".format(self.dataset_names[dataset_source],
k, stage): v
for k, v in stage_losses.items()
})
return losses
else:
# Each is a list[Tensor] of length #image. Each tensor is Ri x (K+1)
scores_per_stage = [
h[0].predict_probs(h[1], h[2]) for h in head_outputs
]
scores = [
sum(list(scores_per_image)) * (1.0 / self.num_cascade_stages)
for scores_per_image in zip(*scores_per_stage)
]
predictor, predictions, proposals = head_outputs[-1]
boxes = predictor.predict_boxes(predictions, proposals)
pred_instances, _ = fast_rcnn_inference(
boxes,
scores,
image_sizes,
predictor.test_score_thresh,
predictor.test_nms_thresh,
predictor.test_topk_per_image,
)
return pred_instances
def _forward_box(self, features, proposals, targets=None):
"""
Add mult proposal scores at testing
"""
if (not self.training) and self.mult_proposal_score:
if len(proposals) > 0 and proposals[0].has('scores'):
proposal_scores = [p.get('scores') for p in proposals]
else:
proposal_scores = [
p.get('objectness_logits') for p in proposals
]
features = [features[f] for f in self.box_in_features]
head_outputs = [] # (predictor, predictions, proposals)
prev_pred_boxes = None
image_sizes = [x.image_size for x in proposals]
for k in range(self.num_cascade_stages):
if k > 0:
proposals = self._create_proposals_from_boxes(
prev_pred_boxes, image_sizes)
if self.training:
proposals = self._match_and_label_boxes(
proposals, k, targets)
predictions = self._run_stage(features, proposals, k)
prev_pred_boxes = self.box_predictor[k].predict_boxes(
predictions, proposals)
head_outputs.append(
(self.box_predictor[k], predictions, proposals))
if self.training:
losses = {}
storage = get_event_storage()
for stage, (predictor, predictions,
proposals) in enumerate(head_outputs):
with storage.name_scope("stage{}".format(stage)):
stage_losses = predictor.losses(predictions, proposals)
losses.update({
k + "_stage{}".format(stage): v
for k, v in stage_losses.items()
})
return losses
else:
# Each is a list[Tensor] of length #image. Each tensor is Ri x (K+1)
scores_per_stage = [
h[0].predict_probs(h[1], h[2]) for h in head_outputs
]
scores = [
sum(list(scores_per_image)) * (1.0 / self.num_cascade_stages)
for scores_per_image in zip(*scores_per_stage)
]
if self.mult_proposal_score:
scores = [(s * ps[:, None]) ** 0.5 \
for s, ps in zip(scores, proposal_scores)]
predictor, predictions, proposals = head_outputs[-1]
boxes = predictor.predict_boxes(predictions, proposals)
pred_instances, _ = fast_rcnn_inference(
boxes,
scores,
image_sizes,
predictor.test_score_thresh,
predictor.test_nms_thresh,
predictor.test_topk_per_image,
)
return pred_instances
def test_caffe2_pytorch_eq(self):
ims_per_batch = 8
post_nms_topk = 100
detections_per_im = 10
num_class = 80
score_thresh = 0.05
nms_thresh = 0.5
image_shapes = [torch.Size([800, 800])] * ims_per_batch
batch_splits = [post_nms_topk] * ims_per_batch
# NOTE: There're still some unsure minor implementation differences
# (eg. ordering when equal score across classes) causing some seeds
# don't pass the test.
# Thus set a fixed seed to make sure this test passes consistantly.
rng = torch.Generator()
rng.manual_seed(42)
boxes = []
for n in batch_splits:
box = 1000.0 * 0.5 * torch.rand(n, num_class, 4,
generator=rng) + 0.001
box[:, :, -2:] += box[:, :, :2]
box = box.view(n, num_class * 4)
boxes.append(box)
scores = [
torch.rand(n, num_class + 1, generator=rng) for n in batch_splits
]
ref_results, ref_kept_indices = fast_rcnn_inference(
boxes,
scores,
image_shapes,
score_thresh=score_thresh,
nms_thresh=nms_thresh,
topk_per_image=detections_per_im)
for result, kept_index, score in zip(ref_results, ref_kept_indices,
scores):
torch.testing.assert_allclose(
score[kept_index, result.pred_classes],
result.scores,
)
# clip is done in BBoxTransformOp
c2_boxes = []
for box, image_shape in zip(boxes, image_shapes):
num_bbox_reg_classes = box.shape[1] // 4
clipped_box = Boxes(box.reshape(-1, 4))
clipped_box.clip(image_shape)
clipped_box = clipped_box.tensor.view(-1, num_bbox_reg_classes * 4)
c2_boxes.append(clipped_box)
c2_boxes = cat(c2_boxes)
c2_scores = cat(scores)
c2_batch_splits = torch.Tensor(batch_splits)
nms_outputs = torch.ops._caffe2.BoxWithNMSLimit(
c2_scores,
c2_boxes,
c2_batch_splits,
score_thresh=float(score_thresh),
nms=float(nms_thresh),
detections_per_im=int(detections_per_im),
soft_nms_enabled=False,
soft_nms_method="linear",
soft_nms_sigma=0.5,
soft_nms_min_score_thres=0.001,
rotated=False,
cls_agnostic_bbox_reg=False,
input_boxes_include_bg_cls=False,
output_classes_include_bg_cls=False,
legacy_plus_one=False,
)
roi_score_nms, roi_bbox_nms, roi_class_nms, roi_batch_splits_nms, roi_keeps_nms, roi_keeps_size_nms = nms_outputs # noqa
roi_score_nms = roi_score_nms.split(
roi_batch_splits_nms.int().tolist())
roi_bbox_nms = roi_bbox_nms.split(roi_batch_splits_nms.int().tolist())
roi_class_nms = roi_class_nms.split(
roi_batch_splits_nms.int().tolist())
roi_keeps_nms = roi_keeps_nms.split(
roi_batch_splits_nms.int().tolist())
for _score_nms, _class_nms, _keeps_nms, _score in zip(
roi_score_nms, roi_class_nms, roi_keeps_nms, scores):
torch.testing.assert_allclose(
_score[_keeps_nms.to(torch.int64),
_class_nms.to(torch.int64)],
_score_nms,
)
for ref, s, b, c in zip(ref_results, roi_score_nms, roi_bbox_nms,
roi_class_nms):
s1, i1 = s.sort()
s2, i2 = ref.scores.sort()
torch.testing.assert_allclose(s1, s2)
torch.testing.assert_allclose(b[i1], ref.pred_boxes.tensor[i2])
torch.testing.assert_allclose(
c.to(torch.int64)[i1], ref.pred_classes[i2])
for ref, k in zip(ref_kept_indices, roi_keeps_nms):
# NOTE: order might be different due to implementation
ref_set = set(ref.tolist())
k_set = set(k.tolist())
self.assertEqual(ref_set, k_set)
