def filter_results(self, boxlist, num_classes):
"""Returns bounding-box detection results by thresholding on scores and
applying non-maximum suppression (NMS).
"""
# unwrap the boxlist to avoid additional overhead.
# if we had multi-class NMS, we could perform this directly on the boxlist
boxes = boxlist.bbox.reshape(-1, num_classes * 4)
scores = boxlist.get_field("scores").reshape(-1, num_classes)
device = scores.device
result = []
# Apply threshold on detection probabilities and apply NMS
# Skip j = 0, because it's the background class
inds_all = scores > self.score_thresh
for j in range(1, num_classes):
inds = inds_all[:, j].nonzero().squeeze(1)
scores_j = scores[inds, j]
boxes_j = boxes[inds, j * 4:(j + 1) * 4]
boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy")
boxlist_for_class.add_field("scores", scores_j)
boxlist_for_class_old = boxlist_for_class
if cfg.TEST.SOFT_NMS.ENABLED:
boxlist_for_class = boxlist_soft_nms(
boxlist_for_class,
sigma=cfg.TEST.SOFT_NMS.SIGMA,
overlap_thresh=self.nms,
score_thresh=0.0001,
method=cfg.TEST.SOFT_NMS.METHOD)
else:
boxlist_for_class = boxlist_nms(boxlist_for_class, self.nms)
# Refine the post-NMS boxes using bounding-box voting
if cfg.TEST.BBOX_VOTE.ENABLED and boxes_j.shape[0] > 0:
boxlist_for_class = boxlist_box_voting(
boxlist_for_class,
boxlist_for_class_old,
cfg.TEST.BBOX_VOTE.VOTE_TH,
scoring_method=cfg.TEST.BBOX_VOTE.SCORING_METHOD)
num_labels = len(boxlist_for_class)
boxlist_for_class.add_field(
"labels",
torch.full((num_labels, ), j, dtype=torch.int64,
device=device))
result.append(boxlist_for_class)
result = cat_boxlist(result)
number_of_detections = len(result)
# Limit to max_per_image detections **over all classes**
if number_of_detections > self.detections_per_img > 0:
cls_scores = result.get_field("scores")
image_thresh, _ = torch.kthvalue(
cls_scores.cpu(),
number_of_detections - self.detections_per_img + 1)
keep = cls_scores >= image_thresh.item()
keep = torch.nonzero(keep).squeeze(1)
result = result[keep]
return result
def filter_results(boxlist, nms_thresh=0.5, detections_per_img=100):
num_classes = cfg.MODEL.NUM_CLASSES
if not cfg.TEST.SOFT_NMS.ENABLED and not cfg.TEST.BBOX_VOTE.ENABLED:
result = boxlist_ml_nms(boxlist, nms_thresh)
else:
boxes = boxlist.bbox
scores = boxlist.get_field("scores")
labels = boxlist.get_field("labels")
result = []
for j in range(1, num_classes): # skip the background
inds = (labels == j).nonzero().view(-1)
scores_j = scores[inds]
boxes_j = boxes[inds, :].view(-1, 4)
boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy")
boxlist_for_class.add_field("scores", scores_j)
boxlist_for_class_old = boxlist_for_class
if cfg.TEST.SOFT_NMS.ENABLED:
boxlist_for_class = boxlist_soft_nms(
boxlist_for_class,
sigma=cfg.TEST.SOFT_NMS.SIGMA,
overlap_thresh=nms_thresh,
score_thresh=0.0001,
method=cfg.TEST.SOFT_NMS.METHOD)
else:
boxlist_for_class = boxlist_nms(boxlist_for_class, nms_thresh)
# Refine the post-NMS boxes using bounding-box voting
if cfg.TEST.BBOX_VOTE.ENABLED and boxes_j.shape[0] > 0:
boxlist_for_class = boxlist_box_voting(
boxlist_for_class,
boxlist_for_class_old,
cfg.TEST.BBOX_VOTE.VOTE_TH,
scoring_method=cfg.TEST.BBOX_VOTE.SCORING_METHOD)
num_labels = len(boxlist_for_class)
boxlist_for_class.add_field(
"labels",
torch.full((num_labels, ),
j,
dtype=torch.int64,
device=scores.device))
result.append(boxlist_for_class)
result = cat_boxlist(result)
# Limit to max_per_image detections **over all classes**
number_of_detections = len(result)
if number_of_detections > detections_per_img > 0:
cls_scores = result.get_field("scores")
image_thresh, _ = torch.kthvalue(
cls_scores.cpu(), number_of_detections - detections_per_img + 1)
keep = cls_scores >= image_thresh.item()
keep = torch.nonzero(keep).squeeze(1)
result = result[keep]
return result
def forward_for_single_feature_map(self, anchors, objectness,
box_regression):
"""
Arguments:
anchors: list[BoxList]
objectness: tensor of size N, A, H, W
box_regression: tensor of size N, A * 4, H, W
"""
device = objectness.device
N, A, H, W = objectness.shape
# put in the same format as anchors
objectness = permute_and_flatten(objectness, N, A, 1, H, W).view(N, -1)
objectness = objectness.sigmoid()
box_regression = permute_and_flatten(box_regression, N, A, 4, H, W)
num_anchors = A * H * W
pre_nms_top_n = min(self.pre_nms_top_n, num_anchors)
objectness, topk_idx = objectness.topk(pre_nms_top_n,
dim=1,
sorted=True)
batch_idx = torch.arange(N, device=device)[:, None]
box_regression = box_regression[batch_idx, topk_idx]
image_shapes = [box.size for box in anchors]
concat_anchors = torch.cat([a.bbox for a in anchors], dim=0)
concat_anchors = concat_anchors.reshape(N, -1, 4)[batch_idx, topk_idx]
proposals = self.box_coder.decode(box_regression.view(-1, 4),
concat_anchors.view(-1, 4))
proposals = proposals.view(N, -1, 4)
result = []
for proposal, score, im_shape in zip(proposals, objectness,
image_shapes):
boxlist = BoxList(proposal, im_shape, mode="xyxy")
boxlist.add_field("objectness", score)
boxlist = boxlist.clip_to_image(remove_empty=False)
boxlist = remove_small_boxes(boxlist, self.min_size)
boxlist = boxlist_nms(
boxlist,
self.nms_thresh,
max_proposals=self.post_nms_top_n,
score_field="objectness",
)
result.append(boxlist)
return result
def filter_results(boxlist):
num_classes = cfg.MODEL.NUM_CLASSES
if not cfg.TEST.SOFT_NMS.ENABLED and not cfg.TEST.BBOX_VOTE.ENABLED:
# multiclass nms
scores = boxlist.get_field("scores")
device = scores.device
num_repeat = int(boxlist.bbox.shape[0] / num_classes)
labels = np.tile(np.arange(num_classes), num_repeat)
boxlist.add_field(
"labels",
torch.from_numpy(labels).to(dtype=torch.int64, device=device))
fg_labels = torch.from_numpy(
(np.arange(boxlist.bbox.shape[0]) % num_classes !=
0).astype(int)).to(dtype=torch.uint8, device=device)
_scores = scores > cfg.FAST_RCNN.SCORE_THRESH
inds_all = _scores & fg_labels
result = boxlist_ml_nms(boxlist[inds_all], cfg.FAST_RCNN.NMS)
else:
boxes = boxlist.bbox.reshape(-1, num_classes * 4)
scores = boxlist.get_field("scores").reshape(-1, num_classes)
device = scores.device
result = []
# Apply threshold on detection probabilities and apply NMS
# Skip j = 0, because it's the background class
inds_all = scores > cfg.FAST_RCNN.SCORE_THRESH
for j in range(1, num_classes):
inds = inds_all[:, j].nonzero().squeeze(1)
scores_j = scores[inds, j]
boxes_j = boxes[inds, j * 4:(j + 1) * 4]
boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy")
boxlist_for_class.add_field("scores", scores_j)
boxlist_for_class_old = boxlist_for_class
if cfg.TEST.SOFT_NMS.ENABLED:
boxlist_for_class = boxlist_soft_nms(
boxlist_for_class,
sigma=cfg.TEST.SOFT_NMS.SIGMA,
overlap_thresh=cfg.FAST_RCNN.NMS,
score_thresh=0.0001,
method=cfg.TEST.SOFT_NMS.METHOD)
else:
boxlist_for_class = boxlist_nms(boxlist_for_class,
cfg.FAST_RCNN.NMS)
# Refine the post-NMS boxes using bounding-box voting
if cfg.TEST.BBOX_VOTE.ENABLED and boxes_j.shape[0] > 0:
boxlist_for_class = boxlist_box_voting(
boxlist_for_class,
boxlist_for_class_old,
cfg.TEST.BBOX_VOTE.VOTE_TH,
scoring_method=cfg.TEST.BBOX_VOTE.SCORING_METHOD)
num_labels = len(boxlist_for_class)
boxlist_for_class.add_field(
"labels",
torch.full((num_labels, ), j, dtype=torch.int64,
device=device))
result.append(boxlist_for_class)
result = cat_boxlist(result)
number_of_detections = len(result)
# Limit to max_per_image detections **over all classes**
if number_of_detections > cfg.FAST_RCNN.DETECTIONS_PER_IMG > 0:
cls_scores = result.get_field("scores")
image_thresh, _ = torch.kthvalue(
cls_scores.cpu(),
number_of_detections - cfg.FAST_RCNN.DETECTIONS_PER_IMG + 1)
keep = cls_scores >= image_thresh.item()
keep = torch.nonzero(keep).squeeze(1)
result = result[keep]
return result
def select_over_all_levels(self, boxlists):
num_images = len(boxlists)
results = []
for i in range(num_images):
if not cfg.TEST.SOFT_NMS.ENABLED and not cfg.TEST.BBOX_VOTE.ENABLED:
# multiclass nms
result = boxlist_ml_nms(boxlists[i], self.nms_thresh)
else:
scores = boxlists[i].get_field("scores")
labels = boxlists[i].get_field("labels")
boxes = boxlists[i].bbox
boxlist = boxlists[i]
result = []
# skip the background
for j in range(2, self.num_classes + 1):
inds = (labels == j).nonzero().view(-1)
scores_j = scores[inds]
boxes_j = boxes[inds, :].view(-1, 4)
boxlist_for_class = BoxList(boxes_j,
boxlist.size,
mode="xyxy")
boxlist_for_class.add_field("scores", scores_j)
boxlist_for_class_old = boxlist_for_class
if cfg.TEST.SOFT_NMS.ENABLED:
boxlist_for_class = boxlist_soft_nms(
boxlist_for_class,
sigma=cfg.TEST.SOFT_NMS.SIGMA,
overlap_thresh=self.nms_thresh,
score_thresh=0.0001,
method=cfg.TEST.SOFT_NMS.METHOD)
else:
boxlist_for_class = boxlist_nms(boxlist_for_class,
self.nms_thresh,
score_field="scores")
# Refine the post-NMS boxes using bounding-box voting
if cfg.TEST.BBOX_VOTE.ENABLED and boxes_j.shape[0] > 0:
boxlist_for_class = boxlist_box_voting(
boxlist_for_class,
boxlist_for_class_old,
cfg.TEST.BBOX_VOTE.VOTE_TH,
scoring_method=cfg.TEST.BBOX_VOTE.SCORING_METHOD)
num_labels = len(boxlist_for_class)
boxlist_for_class.add_field(
"labels",
torch.full((num_labels, ),
j,
dtype=torch.int64,
device=scores.device))
result.append(boxlist_for_class)
result = cat_boxlist(result)
number_of_detections = len(result)
# Limit to max_per_image detections **over all classes**
if number_of_detections > self.fpn_post_nms_top_n > 0:
cls_scores = result.get_field("scores")
image_thresh, _ = torch.kthvalue(
cls_scores.cpu(),
number_of_detections - self.fpn_post_nms_top_n + 1)
keep = cls_scores >= image_thresh.item()
keep = torch.nonzero(keep).squeeze(1)
result = result[keep]
results.append(result)
return results
