def rpn_boxes(self, fmap, im_sizes, image_offset, gt_boxes=None, gt_classes=None, gt_rels=None,
train_anchor_inds=None, proposals=None):
"""
Gets boxes from the RPN
:param fmap:
:param im_sizes:
:param image_offset:
:param gt_boxes:
:param gt_classes:
:param gt_rels:
:param train_anchor_inds:
:return:
"""
rpn_feats = self.rpn_head(fmap)
rois = self.rpn_head.roi_proposals(
rpn_feats, im_sizes, nms_thresh=0.7,
pre_nms_topn=12000 if self.training and self.mode == 'rpntrain' else 6000,
post_nms_topn=2000 if self.training and self.mode == 'rpntrain' else 1000,
)
if self.training:
if gt_boxes is None or gt_classes is None or train_anchor_inds is None:
raise ValueError(
"Must supply GT boxes, GT classes, trainanchors when in train mode")
rpn_scores, rpn_box_deltas = self.rpn_head.anchor_preds(rpn_feats, train_anchor_inds,
image_offset)
# rpn_box_deltas, rpn_scores = None, None
# if gt_rels is not None and self.mode == 'rpntrain':
# raise ValueError("Training the object detector and the relationship model with detection"
# "at the same time isn't supported")
if self.mode == 'refinerels':
all_rois = Variable(rois)
# Potentially you could add in GT rois if none match
# is_match = (bbox_overlaps(rois[:,1:].contiguous(), gt_boxes.data) > 0.5).long()
# gt_not_matched = (is_match.sum(0) == 0).nonzero()
#
# if gt_not_matched.dim() > 0:
# gt_to_add = torch.cat((gt_classes[:,0,None][gt_not_matched.squeeze(1)].float(),
# gt_boxes[gt_not_matched.squeeze(1)]), 1)
#
# all_rois = torch.cat((all_rois, gt_to_add),0)
# num_gt = gt_to_add.size(0)
labels = None
bbox_targets = None
rel_labels = None
else:
all_rois, labels, bbox_targets = proposal_assignments_det(
rois, gt_boxes.data, gt_classes.data, image_offset, fg_thresh=0.5)
rel_labels = None
else:
all_rois = Variable(rois, volatile=True)
labels = None
bbox_targets = None
rel_labels = None
rpn_box_deltas = None
rpn_scores = None
return all_rois, labels, bbox_targets, rpn_scores, rpn_box_deltas, rel_labels
def proposal_boxes(self,
fmap,
im_sizes,
image_offset,
gt_boxes=None,
gt_classes=None,
gt_rels=None,
train_anchor_inds=None,
proposals=None):
"""
Gets boxes from the RPN
:param fmap:
:param im_sizes:
:param image_offset:
:param gt_boxes:
:param gt_classes:
:param gt_rels:
:param train_anchor_inds:
:return:
"""
assert proposals is not None
rois = filter_roi_proposals(
proposals[:, 2:].data.contiguous(),
proposals[:, 1].data.contiguous(),
np.array([2000] * len(im_sizes)),
nms_thresh=0.7,
pre_nms_topn=12000
if self.training and self.mode == 'rpntrain' else 6000,
post_nms_topn=2000
if self.training and self.mode == 'rpntrain' else 1000,
)
if self.training:
all_rois, labels, bbox_targets = proposal_assignments_det(
rois,
gt_boxes.data,
gt_classes.data,
image_offset,
fg_thresh=0.5)
# RETRAINING FOR DETECTION HERE.
all_rois = torch.cat((all_rois, Variable(rois)), 0)
else:
all_rois = Variable(rois)
labels = None
bbox_targets = None
rpn_scores = None
rpn_box_deltas = None
rel_labels = None
rels_to_gt = None
return all_rois, labels, bbox_targets, rpn_scores, rpn_box_deltas, rel_labels, rels_to_gt
def rpn_boxes(self,
fmap,
im_sizes,
image_offset,
gt_boxes=None,
gt_classes=None,
gt_rels=None,
train_anchor_inds=None,
proposals=None):
"""
Gets boxes from the RPN
:param fmap:
:param im_sizes:
:param image_offset:
:param gt_boxes:
:param gt_classes:
:param gt_rels:
:param train_anchor_inds:
:return:
"""
# rpn_feats:[6,37,37,20,6], scores+deltas of all anchors; RPN_Head.init & forward
rpn_feats = self.rpn_head(fmap)
# roi_proposals: pre_nms_topn=6000, post_nms_topn=1000, thres=0.7, filter boxes from 16w to 4275 (6 images)
# get the first max(6000, #boxes) boxes per img, then apply nms, get max(1000, #boxes) per img
# rois: [12000, 5] if rpntrain / [4000+, 5] if refinerels(sgdet)
rois = self.rpn_head.roi_proposals(
rpn_feats,
im_sizes,
nms_thresh=0.7,
pre_nms_topn=12000
if self.training and self.mode == 'rpntrain' else 6000,
post_nms_topn=2000
if self.training and self.mode == 'rpntrain' else 1000,
)
#ipdb.set_trace()
if self.training:
if gt_boxes is None or gt_classes is None or train_anchor_inds is None:
raise ValueError(
"Must supply GT boxes, GT classes, trainanchors when in train mode"
)
rpn_scores, rpn_box_deltas = self.rpn_head.anchor_preds(
rpn_feats, train_anchor_inds, image_offset)
if gt_rels is not None and self.mode == 'rpntrain':
raise ValueError(
"Training the object detector and the relationship model with detection"
"at the same time isn't supported")
# sgdet/refinerels
if self.mode == 'refinerels':
# NOTE: If we're doing this during training, we need to assign labels here.
#ipdb.set_trace()
pred_to_gtbox = bbox_overlaps(
rois[:, 1:], gt_boxes.data) # [4000+, #gtboxes]
im_inds = (rois[:, 0] + image_offset).long() # [4000+]
pred_to_gtbox[im_inds[:, None] != gt_classes.data[
None, :,
0]] = 0.0 # gt_classes, (im_inds, class); match the image index
max_overlaps, argmax_overlaps = pred_to_gtbox.max(1)
# gt labels assignment； gt_classes: [#gtbox,2]
labels = gt_classes[:, 1][argmax_overlaps]
labels[max_overlaps < 0.5] = 0 # bad boxes; invalid labels
# gt boxes assignment
bbox_targets = gt_boxes[argmax_overlaps, :]
tem = max_overlaps.view(max_overlaps.size()[0], 1)
bbox_targets.data[torch.cat(
(tem, tem, tem, tem), 1) < 0.5] = 1 # arbitrary value
# rois become not Tensor, unlike 'rpntrain' and 'gtbox'
all_rois = Variable(rois)
# Potentially you could add in GT rois if none match
# is_match = (bbox_overlaps(rois[:,1:].contiguous(), gt_boxes.data) > 0.5).long()
# gt_not_matched = (is_match.sum(0) == 0).nonzero()
#
# if gt_not_matched.dim() > 0:
# gt_to_add = torch.cat((gt_classes[:,0,None][gt_not_matched.squeeze(1)].float(),
# gt_boxes[gt_not_matched.squeeze(1)]), 1)
#
# all_rois = torch.cat((all_rois, gt_to_add),0)
# num_gt = gt_to_add.size(0)
#labels = None
#bbox_targets = None
rel_labels = None
# 'rpntrain' / 'gtbox'(sgcls)
else:
# all_rois:[1536,4], and labels, bbox_targets are all Tensor
all_rois, labels, bbox_targets = proposal_assignments_det(
rois,
gt_boxes.data,
gt_classes.data,
image_offset,
fg_thresh=0.5)
rel_labels = None
else:
all_rois = Variable(rois, volatile=True)
labels = None
bbox_targets = None
rel_labels = None
rpn_box_deltas = None
rpn_scores = None
return all_rois, labels, bbox_targets, rpn_scores, rpn_box_deltas, rel_labels
