def boxlist_nms_stereo_td(boxlist_left,
boxlist_right,
nms_thresh,
max_proposals=-1,
score_field="scores"):
if nms_thresh <= 0:
return boxlist_left, boxlist_right
mode = boxlist_left.mode
boxlist_left = boxlist_left.convert("xyxy")
boxlist_right = boxlist_right.convert("xyxy")
boxes_left = boxlist_left.bbox
boxes_right = boxlist_right.bbox
score = (boxlist_left.get_field(score_field) +
boxlist_right.get_field(score_field)) * 0.5
keep_left = _box_nms(boxes_left, score, nms_thresh)
keep_right = _box_nms(boxes_right, score, nms_thresh)
temp_keep = torch.zeros_like(score)
temp_keep[keep_left] = temp_keep[keep_left] + 1
temp_keep[keep_right] = temp_keep[keep_right] + 1
keep = (temp_keep == 2).nonzero().squeeze(1)
if max_proposals > 0:
keep = keep[:max_proposals]
return keep, mode
def test_nms():
boxes = get_random_boxes(1000)
scores = torch.randn(1000)
nms_thresh = .7
keep_cpu = _box_nms(boxes, scores, nms_thresh)
keep_gpu = _box_nms(boxes.cuda(), scores.cuda(), nms_thresh)
assert torch.allclose(keep_cpu, keep_gpu.cpu())
print("test_nms: OK")
return
def boxlist_nms(boxlist, nms_thresh, method='vanilla', sigma=0.5, min_score=0.001, max_proposals=-1, score_field="scores"):
"""
Performs non-maximum suppression on a boxlist, with scores specified
in a boxlist field via score_field.
Arguments:
boxlist(BoxList)
nms_thresh (float)
max_proposals (int): if > 0, then only the top max_proposals are kept
after non-maximum suppression
score_field (str)
"""
if nms_thresh <= 0:
return boxlist
mode = boxlist.mode
boxlist = boxlist.convert("xyxy")
boxes = boxlist.bbox
scores = boxlist.get_field(score_field)
method_codes = {'linear': 1, 'gaussian': 2}
if method in method_codes:
boxes = boxes.detach().cpu()
scores = scores.detach().cpu()
keep = _box_soft_nms(boxes, scores, nms_thresh, method_codes[method], sigma, min_score)
elif method == 'vanilla':
keep = _box_nms(boxes, scores, nms_thresh)
else:
raise ValueError("invalid nms method:", method)
if max_proposals > 0:
keep = keep[: max_proposals]
boxlist = boxlist[keep]
return boxlist.convert(mode)
def boxlist_nms(boxlist,
nms_thresh,
max_proposals=-1,
score_field="scores",
return_keep=False):
"""
Performs non-maximum suppression on a boxlist, with scores specified
in a boxlist field via score_field.
Arguments:
boxlist(BoxList)
nms_thresh (float)
max_proposals (int): if > 0, then only the top max_proposals are kept
after non-maximum suppression
score_field (str)
"""
if nms_thresh <= 0:
return boxlist
mode = boxlist.mode
boxlist = boxlist.convert("xyxy")
boxes = boxlist.bbox
score = boxlist.get_field(score_field)
keep = _box_nms(boxes, score, nms_thresh)
if max_proposals > 0:
keep = keep[:max_proposals]
boxlist = boxlist[keep]
if return_keep == False:
return boxlist.convert(mode)
else:
return boxlist.convert(mode), keep
def __call__(self, loc, score, anchor, img_size, train=True, scale=1.):
if train:
n_pre_nms = self.n_train_pre_nms
n_post_nms = self.n_train_post_nms
else:
n_pre_nms = self.n_test_pre_nms
n_post_nms = self.n_test_post_nms
# print('======================', n_pre_nms, n_post_nms, self.min_size, self.nms_thresh,
# '==============================')
h, w = img_size
roi = loc2bbox(loc, anchor)
roi[:, slice(0, 4, 2)] = torch.clamp(roi[:, slice(0, 4, 2)], 0, w)
roi[:, slice(1, 4, 2)] = torch.clamp(roi[:, slice(1, 4, 2)], 0, h)
hw = roi[:, 2:4] - roi[:, :2]
inds = hw >= self.min_size
inds = inds.all(dim=1)
roi = roi[inds]
score = score[inds]
score, inds = score.sort(descending=True)
score = score[:n_pre_nms]
inds = inds[:n_pre_nms]
roi = roi[inds]
inds = _box_nms(roi, score, self.nms_thresh)[:n_post_nms]
# inds = _box_nms(roi.cpu().cuda(), score.cpu().cuda(), self.nms_thresh)[:n_post_nms]
roi = roi[inds]
return roi
def boxlist_nms_inference(boxlist, nms_thresh, max_proposals=-1, score_field="scores"):
"""
COPIED FOR PDB PURPOSE, OTHERWISE IT WILL BE HELD UP BY RPN EVEY TIME
Performs non-maximum suppression on a boxlist, with scores specified
in a boxlist field via score_field.
Arguments:
boxlist(BoxList)
nms_thresh (float)
max_proposals (int): if > 0, then only the top max_proposals are kept
after non-maximum suppression
score_field (str)
"""
if nms_thresh <= 0:
return boxlist
mode = boxlist.mode
boxlist = boxlist.convert("xyxy")
boxes = boxlist.bbox
score = boxlist.get_field(score_field)
keep = _box_nms(boxes, score, nms_thresh) ## a _C module
'''Not only does nms removes scores<nms_thresh
It also do reginonal removal based on spatial information
'''
if max_proposals > 0:
keep = keep[: max_proposals]
boxlist = boxlist[keep]
return boxlist.convert(mode)
def boxlist_nms_no_convert_back(boxlist, nms_thresh, max_proposals=-1, score_field="scores"):
"""
Performs non-maximum suppression on a boxlist, with scores specified
in a boxlist field via score_field.
Arguments:
boxlist(BoxList)
nms_thresh (float)
max_proposals (int): if > 0, then only the top max_proposals are kept
after non-maximum suppression
score_field (str)
"""
if nms_thresh <= 0:
return boxlist
boxlist = boxlist.convert("xyxy")
boxes = boxlist.bbox
score = boxlist.get_field(score_field)
#if max_proposals > 0:
#keep = nms_max(boxes, score, nms_thresh, max_proposals)
#assert len(keep) <= max_proposals
#else:
keep = _box_nms(boxes, score, nms_thresh)
if max_proposals > 0:
keep = keep[: max_proposals]
boxlist = boxlist[keep]
return boxlist
def boxlist_nms(boxlist, nms_thresh, max_proposals=-1, score_field="score"):
"""
Performs non-maximum suppression on a boxlist, with scores specified
in a boxlist field via score_field.
Arguments:
boxlist(BoxList)
nms_thresh (float)
max_proposals (int): if > 0, then only the top max_proposals are kept
after non-maxium suppression
score_field (str)
"""
if nms_thresh <= 0:
return boxlist
mode = boxlist.mode
boxlist = boxlist.convert("xyxy")
boxes = boxlist.bbox
score = boxlist.get_field(score_field)
# this nms can only run on cuda:0
boxes = boxes.to('cuda:0')
score = score.to('cuda:0')
keep = _box_nms(boxes, score, nms_thresh)
keep = keep.to(boxlist.bbox.device)
if max_proposals > 0:
keep = keep[:max_proposals]
boxlist = boxlist[keep]
return boxlist.convert(mode)
def boxlist_nms(boxlist, nms_thresh, max_proposals=-1, score_field="scores"):
"""
Performs non-maximum suppression on a boxlist, with scores specified
in a boxlist field via score_field.
Arguments:
boxlist(BoxList)
nms_thresh (float)
max_proposals (int): if > 0, then only the top max_proposals are kept
after non-maximum suppression
score_field (str)
"""
if nms_thresh <= 0:
return boxlist
mode = boxlist.mode
boxlist = boxlist.convert("xyxy")
boxes = boxlist.bbox
score = boxlist.get_field(score_field)
# print('box', boxes.size())
# assert boxes.size(0) == score.size(0), [boxes.size(), score.size()]
keep = _box_nms(boxes, score, nms_thresh)
if max_proposals > 0:
keep = keep[:max_proposals]
boxlist = boxlist[keep]
return boxlist.convert(mode)
def get_R_net_res(self, x, tanchors):
h, w = x.shape[2:]
tanchors = bbox2square(tanchors[..., :4])
if tanchors.shape[0] > 0:
t, _ = tanchors.max(dim=0)
t = t + 1
t = t.long()
n_w, n_h = t[2], t[3]
x = F.pad(x, (0, max(n_w - w, 0), 0, max(n_h - h, 0)),
mode='constant',
value=127.5)
roi = tanchors
roi_inds = cuda(torch.zeros((roi.size()[0], 1)))
roi = torch.cat([roi_inds, roi], dim=1)
xx = roialign_24(x, roi)
xx = (xx - 127.5) / 128.0
R_net_logits, R_net_loc, R_net_landmarks = self.R_net(xx)
# R_net_logits = R_net_logits.permute(0, 2, 3, 1).view(-1, 2)
# R_net_loc = R_net_loc.permute(0, 2, 3, 1).view(-1, 4)
# R_net_landmarks = R_net_landmarks.permute(0, 2, 3, 1).view(-1, 10)
score = F.softmax(R_net_logits, dim=-1)[..., 1]
inds = score >= self.config.R_net_conf_thresh
if inds.sum() == 0:
return cuda(torch.zeros((0, 5)))
score = score[inds]
net_loc = R_net_loc[inds]
net_landmarks = R_net_landmarks[inds]
tanchors = tanchors[inds]
bboxes = loc2bbox(net_loc, tanchors)
bboxes[..., slice(0, 4, 2)] = torch.clamp(bboxes[...,
slice(0, 4, 2)], 0,
w - 1)
bboxes[..., slice(1, 4, 2)] = torch.clamp(bboxes[...,
slice(1, 4, 2)], 0,
h - 1)
hw = bboxes[:, 2:4] - bboxes[:, :2]
inds = hw >= self.config.roi_min_size[1]
inds = inds.all(dim=1)
bboxes = bboxes[inds]
score = score[inds]
if bboxes.shape[0] == 0:
return cuda(torch.zeros((0, 5)))
score, inds = score.sort(descending=True)
bboxes = bboxes[inds]
keep = _box_nms(bboxes, score, self.config.R_net_iou_thresh)
bboxes = bboxes[keep]
score = score[keep]
score = score.view(-1, 1)
return torch.cat([bboxes, score], dim=-1)
def __call__(self,
loc,
score,
anchor,
img_size,
map_HW,
train=True,
scale=1.):
self.min_size = [4, 8, 16, 32, 64]
if train:
n_pre_nms = self.n_train_pre_nms
n_post_nms = self.n_train_post_nms
else:
n_pre_nms = self.n_test_pre_nms
n_post_nms = self.n_test_post_nms
# print('======================', n_pre_nms, n_post_nms, self.min_size, self.nms_thresh,
# '==============================')
h, w = img_size
roi = loc2bbox(loc, anchor)
roi[:, slice(0, 4, 2)] = torch.clamp(roi[:, slice(0, 4, 2)], 0, w)
roi[:, slice(1, 4, 2)] = torch.clamp(roi[:, slice(1, 4, 2)], 0, h)
Roi = []
Score = []
C = 0
for i in range(5):
map_H, map_W = map_HW[i]
c = map_H * map_W * 3
tscore = score[C:C + c]
troi = roi[C:C + c]
C += c
hw = troi[:, 2:4] - troi[:, :2]
inds = hw >= self.min_size[i]
inds = inds.all(dim=1)
troi = troi[inds]
tscore = tscore[inds]
tscore, inds = tscore.sort(descending=True)
inds = inds[:n_post_nms]
troi = troi[inds]
tscore = tscore[:n_post_nms]
Roi.append(troi)
Score.append(tscore)
roi = torch.cat(Roi, dim=0)
score = torch.cat(Score, dim=0)
score, inds = score.sort(descending=True)
roi = roi[inds]
inds = _box_nms(roi, score, self.nms_thresh)[:n_post_nms]
roi = roi[inds]
return roi
def map_func(bbox, score, cls):
score, inds = score.sort(descending=True)
bbox = bbox[inds]
keep = _box_nms(bbox, score, iou_thresh)
bbox = bbox[keep]
score = score[keep]
cls = cuda(torch.zeros((bbox.shape[0], 1))) + cls
score = score.view(-1, 1)
return torch.cat([bbox, score, cls], dim=1)
def nms(self, merge_result, scores):
from maskrcnn_benchmark.layers import nms as _box_nms
import torch
if scores is None:
scores = torch.ones(size=(len(merge_result), ))
if not isinstance(scores, torch.Tensor):
scores = torch.Tensor(scores)
merge_result = torch.Tensor(merge_result)
keep = _box_nms(merge_result, scores, self.nms_th)
merge_result = merge_result[keep].detach().cpu().numpy()
return merge_result, keep.detach().cpu().numpy()
def boxlist_nms(boxlist_left,
boxlist_right,
nms_thresh,
max_proposals=-1,
score_field="scores"):
"""
Performs non-maximum suppression on a boxlist, with scores specified
in a boxlist field via score_field.
Arguments:
boxlist(BoxList)
nms_thresh (float)
max_proposals (int): if > 0, then only the top max_proposals are kept
after non-maximum suppression
score_field (str)
"""
if nms_thresh <= 0:
return boxlist_left, boxlist_right
mode = boxlist_left.mode
boxlist_left = boxlist_left.convert("xyxy")
boxlist_right = boxlist_right.convert("xyxy")
boxes_left = boxlist_left.bbox
boxes_right = boxlist_right.bbox
score_left = boxlist_left.get_field(score_field)
score_right = boxlist_right.get_field(score_field)
# new append
keep_left = _box_nms(boxes_left, score_left, nms_thresh)
keep_right = _box_nms(boxes_right, score_right, nms_thresh)
keep = torch.from_numpy(
np.intersect1d(keep_left.cpu().numpy(),
keep_right.cpu().numpy())).cuda()
#
if max_proposals > 0:
keep = keep[:max_proposals]
boxlist_left = boxlist_left[keep]
boxlist_right = boxlist_right[keep]
return boxlist_left.convert(mode), boxlist_right.convert(mode)
def filter_results(boxlists,
nms_thresh=0.8,
max_proposals=0,
score_field="scores"):
"""
do nms based on the tight box
"""
for b in range(len(boxlists)):
boxlist = boxlists[b]
# boxlist = shrink_box(proposals, mask)
score = boxlist.get_field(score_field)
boxes = boxlist.bbox
keep = _box_nms(boxes, score, nms_thresh)
if max_proposals > 0 and max_proposals < len(boxlist):
keep = keep[:max_proposals]
boxlists[b] = boxlist[keep]
return boxlists
def apply_nms(scores, boxes, im_inds=None, nms_thresh=0.7):
"""
Note - this function is non-differentiable so everything is assumed to be a tensor, not
a variable.
"""
just_inds = im_inds is None
if im_inds is None:
im_inds = scores.new(scores.size(0)).fill_(0).long()
keep = []
im_per = []
for i, s, e in enumerate_by_image(im_inds):
keep_im = _box_nms(boxes[s:e], scores[s:e], nms_thresh)
keep.append(keep_im + s)
im_per.append(keep_im.size(0))
inds = torch.cat(keep, 0)
if just_inds:
return inds
return inds, im_per
def boxlist_nms(boxlist, nms_thresh, max_proposals=-1, score_field="scores", nms_type='nms', vote_thresh=0.65):
if nms_thresh <= 0:
return boxlist
mode = boxlist.mode
boxlist = boxlist.convert("xyxy")
boxes = boxlist.bbox
score = boxlist.get_field(score_field)
if nms_type == 'nms':
keep = _box_nms(boxes, score, nms_thresh)
if max_proposals > 0:
keep = keep[: max_proposals]
boxlist = boxlist[keep]
else:
if nms_type == 'vote':
boxes_vote, scores_vote = bbox_vote(boxes, score, vote_thresh)
else:
boxes_vote, scores_vote = soft_bbox_vote(boxes, score, vote_thresh)
if len(boxes_vote) > 0:
boxlist.bbox = boxes_vote
boxlist.extra_fields['scores'] = scores_vote
return boxlist.convert(mode)
def boxlist_nms(boxlist, nms_thresh, max_proposals=-1, score_field="scores"):
"""根据 score_field 对 boxlist 中的所有 box 进行非极大值抑制
Arguments:
boxlist(BoxList)
nms_thresh (float)
max_proposals (int): if > 0, then only the top max_proposals are kept
after non-maximum suppression
score_field (str)
"""
if nms_thresh <= 0:
return boxlist
mode = boxlist.mode
boxlist = boxlist.convert("xyxy")
boxes = boxlist.bbox
score = boxlist.get_field(score_field)
# 调用 nms 函数
keep = _box_nms(boxes, score, nms_thresh)
if max_proposals > 0:
keep = keep[:max_proposals]
boxlist = boxlist[keep]
return boxlist.convert(mode)
def get_P_net_res(self, x):
h, w = x.shape[2:]
roi = cuda(torch.tensor([[0, 0, 0, w, h]]).float())
i = 0
all_bboxes = []
all_score = []
while True:
n_h, n_w = int(h * self.scale ** i), int(w * self.scale ** i)
if n_h < 12 or n_w < 12:
break
roialign = ROIAlign((n_h, n_w), 1 / 1., 2)
xx = roialign(x, roi)
# xx = F.interpolate(x, size=(n_h, n_w))
a = np.ceil(n_h / 12.) * 12
b = np.ceil(n_w / 12.) * 12
a = int(a)
b = int(b)
xx = F.pad(xx, (0, b - n_w, 0, a - n_h), mode='constant', value=127.5)
xx = (xx - 127.5) / 128.0
P_net_logits, P_net_loc, P_net_landmarks = self.P_net(xx)
map_H, map_W = P_net_logits.shape[2:]
P_net_logits = P_net_logits.permute(0, 2, 3, 1).contiguous().view(-1, 2)
P_net_loc = P_net_loc.permute(0, 2, 3, 1).contiguous().view(-1, 4)
P_net_landmarks = P_net_landmarks.permute(0, 2, 3, 1).contiguous().view(-1, 10)
anchors = self.anchors[:map_H, :map_W].contiguous().view(-1, 4) / self.scale ** i
i += 1
score = F.softmax(P_net_logits, dim=-1)[..., 1]
inds = score >= self.config.P_net_conf_thresh
if inds.sum() == 0:
continue
score = score[inds]
P_net_loc = P_net_loc[inds]
anchors = anchors[inds]
bboxes = loc2bbox(P_net_loc, anchors)
bboxes[..., slice(0, 4, 2)] = torch.clamp(bboxes[..., slice(0, 4, 2)], 0, w - 1)
bboxes[..., slice(1, 4, 2)] = torch.clamp(bboxes[..., slice(1, 4, 2)], 0, h - 1)
hw = bboxes[..., 2:4] - bboxes[..., :2]
inds = hw >= self.config.roi_min_size[0]
inds = inds.all(dim=-1)
if inds.sum() == 0:
continue
bboxes = bboxes[inds]
score = score[inds]
score, inds = score.sort(descending=True)
bboxes = bboxes[inds]
keep = _box_nms(bboxes, score, 0.5)
score = score[keep]
bboxes = bboxes[keep]
all_bboxes.append(bboxes)
all_score.append(score)
if len(all_bboxes) == 0:
return cuda(torch.zeros((0, 5)))
bboxes = torch.cat(all_bboxes, dim=0)
score = torch.cat(all_score, dim=0)
score, inds = score.sort(descending=True)
bboxes = bboxes[inds]
keep = _box_nms(bboxes, score, self.config.P_net_iou_thresh)
bboxes = bboxes[keep]
score = score[keep]
return torch.cat([bboxes, score.view(-1, 1)], dim=1)