def generate_proposal(self, rpn_cls_probs, anchors, rpn_bbox_preds,
im_info):
# TODO create a new Function
"""
Args:
rpn_cls_probs: FloatTensor,shape(N,2*num_anchors,H,W)
rpn_bbox_preds: FloatTensor,shape(N,num_anchors*4,H,W)
anchors: FloatTensor,shape(N,4,H,W)
Returns:
proposals_batch: FloatTensor, shape(N,post_nms_topN,4)
fg_probs_batch: FloatTensor, shape(N,post_nms_topN)
"""
# do not backward
rpn_cls_probs = rpn_cls_probs.detach()
rpn_bbox_preds = rpn_bbox_preds.detach()
batch_size = rpn_bbox_preds.shape[0]
coders = bbox_coders.build(
self.target_generators.target_generator_config['coder_config'])
proposals = coders.decode_batch(rpn_bbox_preds, anchors)
# filer and clip
proposals = box_ops.clip_boxes(proposals, im_info)
# fg prob
fg_probs = rpn_cls_probs[:, :, 1]
# sort fg
_, fg_probs_order = torch.sort(fg_probs, dim=1, descending=True)
proposals_order = torch.zeros(
batch_size, self.post_nms_topN).fill_(-1).type_as(fg_probs_order)
return proposals, proposals_order
def generate_proposal(self, rpn_cls_probs, anchors, rpn_bbox_preds,
im_info):
# TODO create a new Function
"""
Args:
rpn_cls_probs: FloatTensor,shape(N,2*num_anchors,H,W)
rpn_bbox_preds: FloatTensor,shape(N,num_anchors*4,H,W)
anchors: FloatTensor,shape(N,4,H,W)
Returns:
proposals_batch: FloatTensor, shape(N,post_nms_topN,4)
fg_probs_batch: FloatTensor, shape(N,post_nms_topN)
"""
# assert len(
# rpn_bbox_preds) == 1, 'just one feature maps is supported now'
# rpn_bbox_preds = rpn_bbox_preds[0]
anchors = anchors[0]
# do not backward
anchors = anchors
rpn_cls_probs = rpn_cls_probs.detach()
rpn_bbox_preds = rpn_bbox_preds.detach()
batch_size = rpn_bbox_preds.shape[0]
rpn_bbox_preds = rpn_bbox_preds.permute(0, 2, 3, 1).contiguous()
# shape(N,H*W*num_anchors,4)
rpn_bbox_preds = rpn_bbox_preds.view(batch_size, -1, 4)
# apply deltas to anchors to decode
# loop here due to many features maps
# proposals = []
# for rpn_bbox_preds_single_map, anchors_single_map in zip(
# rpn_bbox_preds, anchors):
# proposals.append(
# self.bbox_coder.decode(rpn_bbox_preds_single_map,
# anchors_single_map))
# proposals = torch.cat(proposals, dim=1)
proposals = self.bbox_coder.decode_batch(rpn_bbox_preds, anchors)
# filer and clip
proposals = box_ops.clip_boxes(proposals, im_info)
# fg prob
fg_probs = rpn_cls_probs[:, self.num_anchors:, :, :]
fg_probs = fg_probs.permute(0, 2, 3,
1).contiguous().view(batch_size, -1)
# sort fg
_, fg_probs_order = torch.sort(fg_probs, dim=1, descending=True)
# fg_probs_batch = torch.zeros(batch_size,
# self.post_nms_topN).type_as(rpn_cls_probs)
proposals_batch = torch.zeros(batch_size, self.post_nms_topN,
4).type_as(rpn_bbox_preds)
proposals_order = torch.zeros(
batch_size, self.post_nms_topN).fill_(-1).type_as(fg_probs_order)
for i in range(batch_size):
proposals_single = proposals[i]
fg_probs_single = fg_probs[i]
fg_order_single = fg_probs_order[i]
# pre nms
if self.pre_nms_topN > 0:
fg_order_single = fg_order_single[:self.pre_nms_topN]
proposals_single = proposals_single[fg_order_single]
fg_probs_single = fg_probs_single[fg_order_single]
# nms
keep_idx_i = nms(
torch.cat((proposals_single, fg_probs_single.unsqueeze(1)), 1),
self.nms_thresh)
keep_idx_i = keep_idx_i.long().view(-1)
# post nms
if self.post_nms_topN > 0:
keep_idx_i = keep_idx_i[:self.post_nms_topN]
proposals_single = proposals_single[keep_idx_i, :]
fg_probs_single = fg_probs_single[keep_idx_i]
fg_order_single = fg_order_single[keep_idx_i]
# padding 0 at the end.
num_proposal = keep_idx_i.numel()
proposals_batch[i, :num_proposal, :] = proposals_single
# fg_probs_batch[i, :num_proposal] = fg_probs_single
proposals_order[i, :num_proposal] = fg_order_single
return proposals_batch, proposals_order
def generate_proposal(self, rpn_cls_probs, anchors, rpn_bbox_preds,
im_info):
# TODO create a new Function
"""
Args:
rpn_cls_probs: FloatTensor,shape(N,2*num_anchors,H,W)
rpn_bbox_preds: FloatTensor,shape(N,num_anchors*4,H,W)
anchors: FloatTensor,shape(N,4,H,W)
Returns:
proposals_batch: FloatTensor, shape(N,post_nms_topN,4)
fg_probs_batch: FloatTensor, shape(N,post_nms_topN)
"""
# assert len(
# rpn_bbox_preds) == 1, 'just one feature maps is supported now'
# rpn_bbox_preds = rpn_bbox_preds[0]
# do not backward
rpn_cls_probs = rpn_cls_probs.detach()
rpn_bbox_preds = rpn_bbox_preds.detach()
batch_size = rpn_bbox_preds.shape[0]
coders = bbox_coders.build(
self.target_generators.target_generator_config['coder_config'])
proposals = coders.decode_batch(rpn_bbox_preds, anchors)
# filer and clip
proposals = box_ops.clip_boxes(proposals, im_info)
# fg prob
fg_probs = rpn_cls_probs[:, :, 1]
# sort fg
_, fg_probs_order = torch.sort(fg_probs, dim=1, descending=True)
# fg_probs_batch = torch.zeros(batch_size,
# self.post_nms_topN).type_as(rpn_cls_probs)
proposals_batch = torch.zeros(batch_size, self.post_nms_topN,
4).type_as(rpn_bbox_preds)
proposals_order = torch.zeros(
batch_size, self.post_nms_topN).fill_(-1).type_as(fg_probs_order)
for i in range(batch_size):
proposals_single = proposals[i]
fg_probs_single = fg_probs[i]
fg_order_single = fg_probs_order[i]
# pre nms
if self.pre_nms_topN > 0:
fg_order_single = fg_order_single[:self.pre_nms_topN]
proposals_single = proposals_single[fg_order_single]
fg_probs_single = fg_probs_single[fg_order_single]
# nms
keep_idx_i = nms(proposals_single, fg_probs_single,
self.nms_thresh)
keep_idx_i = keep_idx_i.long().view(-1)
# post nms
if self.post_nms_topN > 0:
keep_idx_i = keep_idx_i[:self.post_nms_topN]
proposals_single = proposals_single[keep_idx_i, :]
fg_probs_single = fg_probs_single[keep_idx_i]
fg_order_single = fg_order_single[keep_idx_i]
# padding 0 at the end.
num_proposal = keep_idx_i.numel()
proposals_batch[i, :num_proposal, :] = proposals_single
# fg_probs_batch[i, :num_proposal] = fg_probs_single
proposals_order[i, :num_proposal] = fg_order_single
return proposals_batch, proposals_order
def assign(self,
bboxes,
gt_boxes,
gt_labels=None,
cls_prob=None,
input_size=None):
"""
Assign each bboxes with label and bbox targets for training
Args:
bboxes: shape(N,K,4), encoded by xxyy
gt_boxes: shape(N,M,4), encoded likes as bboxes
"""
# import ipdb
# ipdb.set_trace()
# usually IoU overlaps is used as metric
bboxes = bboxes.detach()
if input_size is not None:
# clip bbox
# import ipdb
# ipdb.set_trace()
bboxes = box_ops.clip_boxes(bboxes, input_size)
match_quality_matrix = self.similarity_calc.compare_batch(
bboxes, gt_boxes)
iog_match_quality_matrix = self.iog_similarity_calc.compare_batch(
bboxes, gt_boxes)
iod_match_quality_matrix = self.iod_similarity_calc.compare_batch(
bboxes, gt_boxes)
self.matcher.iog_match_quality_matrix_batch = iog_match_quality_matrix
self.matcher.iod_match_quality_matrix_batch = iod_match_quality_matrix
# nonsuppress_match = self.matcher.match_batch(match_quality_matrix, 0)
# self.assigned_iog_batch = self._assign_iox(iog_match_quality_matrix,
# nonsuppress_match)
# self.assigned_iod_batch = self._assign_iox(iod_match_quality_matrix,
# nonsuppress_match)
# match
# shape(N,K)
# used for being matched for each one
match = self.matcher.match_batch(match_quality_matrix, self.fg_thresh)
self.analyzer.analyze(match, gt_boxes.shape[1])
# get assigned infomation
# shape (num_batch,num_boxes)
assigned_overlaps_batch = self.matcher.assigned_overlaps_batch
# assign regression targets
reg_targets = self._assign_regression_targets(match, bboxes, gt_boxes)
# assign classification targets
cls_targets = self._assign_classification_targets(match, gt_labels)
# cls_targets = assigned_overlaps_batch
# create regression weights
reg_weights = self._create_regression_weights(assigned_overlaps_batch)
# create classification weights
cls_weights = self._create_classification_weights(
assigned_overlaps_batch)
####################################
# postprocess
####################################
# match == -1 means unmatched
reg_targets[match == -1] = 0
# cls_targets[match == -1] = 0
reg_weights[match == -1] = 0
# as for cls weights, ignore according to bg_thresh
# if self.bg_thresh > 0:
# ignored_bg = (assigned_overlaps_batch > self.bg_thresh) & (
# match == -1)
# cls_weights[ignored_bg] = 0
return cls_targets, reg_targets, cls_weights, reg_weights
def forward(self, bottom_blobs):
base_feat = bottom_blobs['base_feat']
batch_size = base_feat.shape[0]
gt_boxes = bottom_blobs['gt_boxes']
im_info = bottom_blobs['im_info']
# rpn conv
rpn_conv = F.relu(self.rpn_conv(base_feat), inplace=True)
# rpn cls score
# shape(N,2*num_anchors,H,W)
rpn_cls_scores = self.rpn_cls_score(rpn_conv)
# rpn cls prob shape(N,2*num_anchors,H,W)
rpn_cls_score_reshape = rpn_cls_scores.view(batch_size, 2, -1)
rpn_cls_probs = F.softmax(rpn_cls_score_reshape, dim=1)
rpn_cls_probs = rpn_cls_probs.view_as(rpn_cls_scores)
# import ipdb
# ipdb.set_trace()
# rpn bbox pred
# shape(N,4*num_anchors,H,W)
if self.use_score:
# shape (N,2,num_anchoros*H*W)
rpn_cls_scores = rpn_cls_score_reshape.permute(0, 2, 1)
rpn_bbox_preds = []
for i in range(self.num_anchors):
rpn_bbox_feat = torch.cat(
[rpn_conv, rpn_cls_scores[:, ::self.num_anchors, :, :]],
dim=1)
rpn_bbox_preds.append(self.rpn_bbox_pred(rpn_bbox_feat))
rpn_bbox_preds = torch.cat(rpn_bbox_preds, dim=1)
else:
# get rpn offsets to the anchor boxes
rpn_bbox_preds = self.rpn_bbox_pred(rpn_conv)
# rpn_bbox_preds = [rpn_bbox_preds]
# generate anchors
feature_map_list = [base_feat.size()[-2:]]
im_shape = im_info[0][:-1]
anchors = self.anchor_generator.generate(feature_map_list, im_shape)
###############################
# Proposal
###############################
# note that proposals_order is used for track transform of propsoals
proposals_batch, proposals_order = self.generate_proposal(
rpn_cls_probs, anchors, rpn_bbox_preds, im_info)
batch_idx = torch.arange(batch_size).view(batch_size, 1).expand(
-1, proposals_batch.shape[1]).type_as(proposals_batch)
rois_batch = torch.cat((batch_idx.unsqueeze(-1), proposals_batch),
dim=2)
if self.training:
gt_boxes_cliped = box_ops.clip_boxes(gt_boxes, im_info)
rois_batch = self.append_gt(rois_batch, gt_boxes_cliped)
rpn_cls_scores = rpn_cls_scores.view(batch_size, 2, -1,
rpn_cls_scores.shape[2],
rpn_cls_scores.shape[3])
rpn_cls_scores = rpn_cls_scores.permute(0, 3, 4, 2,
1).contiguous().view(
batch_size, -1, 2)
# postprocess
rpn_cls_probs = rpn_cls_probs.view(batch_size, 2, -1,
rpn_cls_probs.shape[2],
rpn_cls_probs.shape[3])
rpn_cls_probs = rpn_cls_probs.permute(0, 3, 4, 2, 1).contiguous().view(
batch_size, -1, 2)
predict_dict = {
'proposals_batch': proposals_batch,
'rpn_cls_scores': rpn_cls_scores,
'rois_batch': rois_batch,
'anchors': anchors,
# used for loss
'rpn_bbox_preds': rpn_bbox_preds,
'rpn_cls_probs': rpn_cls_probs,
'proposals_order': proposals_order,
}
return predict_dict
def postprocess(self, instance, im_info):
# TODO create a new Function
"""
Args:
rpn_cls_probs: FloatTensor,shape(N,2*num_anchors,H,W)
rpn_bbox_preds: FloatTensor,shape(N,num_anchors*4,H,W)
anchors: FloatTensor,shape(N,4,H,W)
Returns:
proposals_batch: FloatTensor, shape(N,post_nms_topN,4)
fg_probs_batch: FloatTensor, shape(N,post_nms_topN)
"""
proposals = instance[constants.KEY_BOXES_2D]
rpn_cls_probs = instance[constants.KEY_OBJECTNESS]
batch_size = rpn_cls_probs.shape[0]
# filer and clip
proposals = box_ops.clip_boxes(proposals, im_info)
# fg prob
fg_probs = rpn_cls_probs[..., 1]
# sort fg
_, fg_probs_order = torch.sort(fg_probs, dim=1, descending=True)
proposals_batch = torch.zeros(batch_size, self.post_nms_topN,
4).type_as(proposals)
proposals_order = torch.zeros(
batch_size, self.post_nms_topN).fill_(-1).type_as(fg_probs_order)
for i in range(batch_size):
proposals_single = proposals[i]
fg_probs_single = fg_probs[i]
fg_order_single = fg_probs_order[i]
# pre nms
if self.pre_nms_topN > 0:
fg_order_single = fg_order_single[:self.pre_nms_topN]
proposals_single = proposals_single[fg_order_single]
fg_probs_single = fg_probs_single[fg_order_single]
# nms
keep_idx_i = nms(proposals_single, fg_probs_single,
self.nms_thresh)
keep_idx_i = keep_idx_i.long().view(-1)
# post nms
if self.post_nms_topN > 0:
keep_idx_i = keep_idx_i[:self.post_nms_topN]
proposals_single = proposals_single[keep_idx_i, :]
fg_probs_single = fg_probs_single[keep_idx_i]
fg_order_single = fg_order_single[keep_idx_i]
# padding 0 at the end.
num_proposal = keep_idx_i.numel()
proposals_batch[i, :num_proposal, :] = proposals_single
# fg_probs_batch[i, :num_proposal] = fg_probs_single
proposals_order[i, :num_proposal] = fg_order_single
instance[constants.KEY_BOXES_2D] = proposals_batch
# TODO(assign rpn_cls_probs)
return instance
