def assign(self, bboxes, gt_bboxes, gt_bboxes_ignore=None, gt_labels=None):
"""Assign gt to bboxes.
This method assign a gt bbox to every bbox (proposal/anchor), each bbox
will be assigned with -1, 0, or a positive number. -1 means don't care,
0 means negative sample, positive number is the index (1-based) of
assigned gt.
The assignment is done in following steps, the order matters.
1. assign every bbox to -1
2. assign proposals whose iou with all gts < neg_iou_thr to 0
3. for each bbox, if the iou with its nearest gt >= pos_iou_thr,
assign it to that bbox
4. for each gt bbox, assign its nearest proposals (may be more than
one) to itself
Args:
bboxes (Tensor): Bounding boxes to be assigned, shape(n, 4).
gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4).
gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are
labelled as `ignored`, e.g., crowd boxes in COCO.
gt_labels (Tensor, optional): Label of gt_bboxes, shape (k, ).
Returns:
:obj:`AssignResult`: The assign result.
"""
if bboxes.shape[0] == 0 or gt_bboxes.shape[0] == 0:
raise ValueError('No gt or bboxes')
bboxes = bboxes[:, :4]
## constructing effective gt areas
gt_eff = scale_boxes(
gt_bboxes,
self.pos_area_thr) # effective bboxes, i.e. center 0.2 part
bbox_centers = (bboxes[:, 2:4] + bboxes[:, 0:2] + 1) / 2
is_bbox_in_gt = is_located_in(
bbox_centers,
gt_bboxes) # the center points lie within the gt boxes
bbox_and_gt_eff_overlaps = bbox_overlaps(bboxes, gt_eff, mode='iof')
is_bbox_in_gt_eff = is_bbox_in_gt & (
bbox_and_gt_eff_overlaps > self.min_pos_iof) # shape (n, k)
# the center point of effective priors should be within the gt box
## constructing ignored gt areas
gt_ignore = scale_boxes(gt_bboxes, self.neg_area_thr)
is_bbox_in_gt_ignore = (bbox_overlaps(bboxes, gt_ignore, mode='iof') >
self.min_pos_iof)
is_bbox_in_gt_ignore &= (~is_bbox_in_gt_eff
) # rule out center effective pixels
gt_areas = bboxes_area(gt_bboxes)
_, sort_idx = gt_areas.sort(
descending=True) # smaller instances can overlay larger ones
assign_result = self.assign_wrt_areas(is_bbox_in_gt_eff,
is_bbox_in_gt_ignore,
gt_labels,
gt_priority=sort_idx)
return assign_result
def nms_resampling_linear(self, proposals, gt_bboxes, gt_labels, thresh):
assert any(gt_labels > 0)
iou = bbox_overlaps(proposals[:, :4], gt_bboxes)
max_iou, gt_assignment = iou.max(dim=1)
proposals_labels = gt_labels[gt_assignment]
# proposal is considered as background when its iou with gt < 0.3
proposals_labels[max_iou < 0.3] = 0
proposals_labels = proposals_labels.cpu().numpy()
t = thresh[proposals_labels]
keep = self.nms_py(proposals.cpu().numpy(), t)
keep = np.array(keep)
return proposals[keep, :]
def forward(self, img, gtbboxes, proposals, label):
backbonefeat, backbonefeat_ = self.backbone(img) # 14 *14
#print(backbonefeat.shape)
#print(backbonefeat_.shape)
#print(backbonefeat)
#print('backbonefeat shape {}'.format(backbonefeat.shape))
#print('out222 \n{}'.format(backbonefeat.shape))
#print('out333 \n{}'.format(backbonefeat_.shape))
globalfeat = F.adaptive_avg_pool2d(backbonefeat_, (1, 1))
globalfeat = globalfeat.view(globalfeat.shape[0], -1)
# proposal gtbboxes 都是 list 里面是tensor
#print('globalfeat \n {}'.format(globalfeat.shape))
# 筛选proposal
#print('before {}'.format(proposals[0].shape))
#print('gtbox {}'.format(gtbboxes[0].shape))
for ii in range(len(gtbboxes)):
tem_proposal = []
iofs = bbox_overlaps(proposals[ii][:, :4].double(),
gtbboxes[ii].double(),
mode='iof',
is_aligned=False) #
iofs = iofs.sum(1)
for jj, iof in enumerate(iofs):
if iof >= 0.2:
tem_proposal.append(proposals[ii][jj])
if tem_proposal:
proposals[ii] = torch.stack(tem_proposal, 0)
#print(proposals[ii].shape)
assert proposals[ii].dim() == 2
else:
proposals[ii] = torch.cat(
(gtbboxes[ii], torch.zeros(
(gtbboxes[ii].size(0), 1)).cuda()),
1) # 如果没有符合要求的proposal，就让person box作为 proposal
proposals[ii] = proposals[ii].double()
#print('after{}'.format(proposals[0].shape))
#list 中的proposal可以是5列或者4列 都可以转换一样的roi
proposals = bbox2roi(
proposals
) #(list[Tensor]) --》Tensor: shape (n, 5), [batch_ind, x1, y1, x2, y2]
proposals = proposals.cuda().float()
idxs = proposals[:, 0].data.cpu().numpy()
#print('proposal shape {}'.format(proposals.shape))
bbox_feat = self.roiextract([backbonefeat.float()],
proposals) #只有一个尺度无FPN，所以list里面只有一个尺度的feat
#print('bbox feat shape {}'.format(bbox_feat.shape))
#print(torch.sum(bbox_feat[0]))
split_bbox_feat = []
start = 0
for ii, index in enumerate(idxs):
if ii != len(idxs) - 1 and index == idxs[ii + 1]:
continue
else:
end = ii + 1
mean_proposal = bbox_feat[start:end].sum(0)
split_bbox_feat.append(mean_proposal)
start = ii + 1
#print(index)
#print((split_bbox_feat))
bbox_feat = torch.stack(split_bbox_feat, 0) # B , c ,7 ,7
bbox_feat = self.conv1(bbox_feat)
bbox_feat = self.conv2(bbox_feat)
#print(bbox_feat.shape)
bbox_feat = F.adaptive_avg_pool2d(bbox_feat, (1, 1)) #B , c ,1 ,1
#print('bboxfeat \n{}'.format(bbox_feat.shape))
#print(bbox_feat.shape)
assert bbox_feat.shape[1] == 2048
bbox_feat = bbox_feat.view(bbox_feat.size(0), -1)
end_feat = torch.cat((globalfeat, bbox_feat), 1)
output = self.fc(end_feat)
return output
def nms_resampling_discrete(self, proposals, gt_bboxes, gt_labels, a_r,
a_c, a_f):
assert any(gt_labels > 0)
# proposal is considered as background when its iou with gt < 0.3
select_thresh = 0.3
out = []
rare, common, frequent = self.get_category_frequency(gt_labels.device)
rare_gtbox = torch.zeros((2000, 4), device=gt_labels.device)
rare_gtbox_idx = 0
common_gtbox = torch.zeros((2000, 4), device=gt_labels.device)
common_gtbox_idx = 0
frequent_gtbox = torch.zeros((2000, 4), device=gt_labels.device)
frequent_gtbox_idx = 0
for gt_bbox, gt_label in zip(gt_bboxes, gt_labels):
if gt_label in rare:
rare_gtbox[rare_gtbox_idx, ...] = gt_bbox
rare_gtbox_idx += 1
elif gt_label in common:
common_gtbox[common_gtbox_idx, ...] = gt_bbox
common_gtbox_idx += 1
else:
frequent_gtbox[frequent_gtbox_idx, ...] = gt_bbox
frequent_gtbox_idx += 1
rare_gtbox = rare_gtbox[:rare_gtbox_idx, ...]
common_gtbox = common_gtbox[:common_gtbox_idx, ...]
frequent_proposals, _ = nms(proposals, a_f)
if len(rare_gtbox) > 0:
rare_proposals, _ = nms(proposals, a_r)
rare_overlaps = bbox_overlaps(rare_gtbox, rare_proposals[:, :4])
rare_max_overlaps, rare_argmax_overlaps = rare_overlaps.max(dim=0)
rare_pos_inds = rare_max_overlaps >= select_thresh
rare_proposals = rare_proposals[rare_pos_inds, :]
out.append(rare_proposals)
frequent_rare_overlaps = bbox_overlaps(rare_gtbox,
frequent_proposals[:, :4])
frequent_rare_max_overlaps, frequent_rare_argmax_overlaps = frequent_rare_overlaps.max(
dim=0)
valid_inds = frequent_rare_max_overlaps < select_thresh
frequent_proposals = frequent_proposals[valid_inds, :]
if len(common_gtbox) > 0:
common_proposals, _ = nms(proposals, a_c)
common_overlaps = bbox_overlaps(common_gtbox,
common_proposals[:, :4])
common_max_overlaps, common_argmax_overlaps = common_overlaps.max(
dim=0)
common_pos_inds = common_max_overlaps >= select_thresh
common_proposals = common_proposals[common_pos_inds, :]
out.append(common_proposals)
frequent_common_overlaps = bbox_overlaps(common_gtbox,
frequent_proposals[:, :4])
frequent_common_max_overlaps, frequent_common_argmax_overlaps = frequent_common_overlaps.max(
dim=0)
valid_inds = frequent_common_max_overlaps < select_thresh
frequent_proposals = frequent_proposals[valid_inds, :]
out.append(frequent_proposals)
if len(out) > 1:
out_proposals = torch.cat(out, 0)
else:
out_proposals = frequent_proposals
return out_proposals