def proposal_layer(rpn_cls_prob, rpn_bbox_pred, im_info, cfg_key, _feat_stride,
anchors, num_anchors, is_tfchannel=False):
"""A simplified version compared to fast/er RCNN
For details please see the technical report
"""
if type(cfg_key) == bytes:
cfg_key = cfg_key.decode('utf-8')
pre_nms_topN = cfg[cfg_key].RPN_PRE_NMS_TOP_N
post_nms_topN = cfg[cfg_key].RPN_POST_NMS_TOP_N
nms_thresh = cfg[cfg_key].RPN_NMS_THRESH
im_info = im_info[0]
# from IPython import embed; embed()
# Get the scores and bounding boxes
if is_tfchannel:
scores = rpn_cls_prob.reshape(-1, 2)
scores = scores[:, 1]
else:
scores = rpn_cls_prob[:, :, :, num_anchors:]
rpn_bbox_pred = rpn_bbox_pred.reshape((-1, 4))
#if cfg_key == 'TRAIN' and 'RPN_NORMALIZE_TARGETS' in cfg.TRAIN.keys() \
if 'RPN_NORMALIZE_TARGETS' in cfg.TRAIN.keys() \
and cfg.TRAIN.RPN_NORMALIZE_TARGETS:
rpn_bbox_pred *= cfg.TRAIN.RPN_NORMALIZE_STDS
rpn_bbox_pred += cfg.TRAIN.RPN_NORMALIZE_MEANS
scores = scores.reshape((-1, 1))
proposals = bbox_transform_inv(anchors, rpn_bbox_pred)
proposals = clip_boxes(proposals, im_info[:2])
#filter boxes
if 'RPN_MIN_SIZE' in cfg[cfg_key].keys():
min_size = cfg[cfg_key].RPN_MIN_SIZE
if min_size > 0:
keep = _filter_boxes(proposals, min_size * im_info[2])
proposals = proposals[keep, :]
scores = scores[keep]
# Pick the top region proposals
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
# Non-maximal suppression
keep = nms(np.hstack((proposals, scores)), nms_thresh)
if post_nms_topN > 0:
keep = keep[:post_nms_topN]
proposals = proposals[keep, :]
scores = scores[keep]
# Only support single image as input
batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False)))
return blob, scores.flatten()
def inference(val_func, inputs, data_dict):
image = data_dict['data']
ori_shape = image.shape
if config.eval_resize == False:
resized_img, scale = image, 1
else:
resized_img, scale = dataset.resize_img_by_short_and_max_size(
image, config.eval_image_short_size, config.eval_image_max_size)
height, width = resized_img.shape[0:2]
resized_img = resized_img.astype(np.float32) - config.image_mean
resized_img = np.ascontiguousarray(resized_img[:, :, [2, 1, 0]])
im_info = np.array([[height, width, scale, ori_shape[0], ori_shape[1], 0]],
dtype=np.float32)
feed_dict = {inputs[0]: resized_img[None, :, :, :], inputs[1]: im_info}
print('fd:\n', feed_dict)
#st = time.time()
_, scores, pred_boxes, rois = val_func(feed_dict=feed_dict)
#ed = time.time()
#print(ed -st)
boxes = rois[:, 1:5] / scale
if cfg.TEST.BBOX_REG:
pred_boxes = bbox_transform_inv(boxes, pred_boxes)
pred_boxes = clip_boxes(pred_boxes, ori_shape)
pred_boxes = pred_boxes.reshape(-1, config.num_classes, 4)
result_boxes = []
for j in range(1, config.num_classes):
inds = np.where(scores[:, j] > config.test_cls_threshold)[0]
cls_scores = scores[inds, j]
cls_bboxes = pred_boxes[inds, j, :]
cls_dets = np.hstack(
(cls_bboxes, cls_scores[:, np.newaxis])).astype(np.float32,
copy=False)
keep = nms(cls_dets, config.test_nms)
cls_dets = np.array(cls_dets[keep, :], dtype=np.float, copy=False)
for i in range(cls_dets.shape[0]):
db = cls_dets[i, :]
dbox = DetBox(db[0],
db[1],
db[2] - db[0],
db[3] - db[1],
tag=config.class_names[j],
score=db[-1])
result_boxes.append(dbox)
if len(result_boxes) > config.test_max_boxes_per_image:
result_boxes = sorted(
result_boxes, reverse=True, key=lambda t_res: t_res.score) \
[:config.test_max_boxes_per_image]
result_dict = data_dict.copy()
result_dict['result_boxes'] = result_boxes
return result_dict
def proposal_without_nms_layer(rpn_cls_prob,
rpn_bbox_pred,
im_info,
cfg_key,
feat_stride,
anchors,
num_anchors,
is_tfchannel=False):
if type(cfg_key) == bytes:
cfg_key = cfg_key.decode('utf-8')
if cfg_key == 'TRAIN':
pre_nms_topN = cfg.TRAIN.RPN_PRE_NMS_TOP_N
else:
pre_nms_topN = cfg.TEST.RPN_PRE_NMS_TOP_N
im_info = im_info[0]
# Get the scores and bounding boxes
if is_tfchannel:
scores = rpn_cls_prob.reshape(-1, 2)
scores = scores[:, 1]
else:
scores = rpn_cls_prob[:, :, :, num_anchors:]
rpn_bbox_pred = rpn_bbox_pred.reshape((-1, 4))
if 'RPN_NORMALIZE_TARGETS' in cfg.TRAIN.keys() \
and cfg.TRAIN.RPN_NORMALIZE_TARGETS:
rpn_bbox_pred *= cfg.TRAIN.RPN_NORMALIZE_STDS
rpn_bbox_pred += cfg.TRAIN.RPN_NORMALIZE_MEANS
scores = scores.reshape((-1, 1))
proposals = bbox_transform_inv(anchors, rpn_bbox_pred)
proposals = clip_boxes(proposals, im_info[:2])
# filter boxes
min_size = 0
if cfg_key == 'TRAIN':
if 'RPN_MIN_SIZE' in cfg.TRAIN.keys():
min_size = cfg.TRAIN.RPN_MIN_SIZE
elif cfg_key == 'TEST':
if 'RPN_MIN_SIZE' in cfg.TEST.keys():
min_size = cfg.TEST.RPN_MIN_SIZE
if min_size > 0:
keep = _filter_boxes(proposals, min_size * im_info[2])
proposals = proposals[keep, :]
scores = scores[keep]
# Pick the top region proposals
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order].flatten()
##why add one, because tf nms assume x2,y2 does not include border
proposals_addone = np.array(proposals)
proposals_addone[:, 2] += 1
proposals_addone[:, 3] += 1
return proposals, scores, proposals_addone
def inference(val_func, inputs, data_dict):
image = data_dict['data']
ori_shape = image.shape
if config.eval_resize == False:
resized_img, scale = image, 1
else:
resized_img, scale = dataset.resize_img_by_short_and_max_size(
image, config.eval_image_short_size, config.eval_image_max_size)
height, width = resized_img.shape[0:2]
resized_img = resized_img.astype(np.float32) - config.image_mean
resized_img = np.ascontiguousarray(resized_img[:, :, [2, 1, 0]])
im_info = np.array(
[[height, width, scale, ori_shape[0], ori_shape[1], 0]],
dtype=np.float32)
feed_dict = {inputs[0]: resized_img[None, :, :, :], inputs[1]: im_info}
#st = time.time()
_, scores, pred_boxes, rois = val_func(feed_dict=feed_dict)
#ed = time.time()
#print(ed -st)
boxes = rois[:, 1:5] / scale
if cfg.TEST.BBOX_REG:
pred_boxes = bbox_transform_inv(boxes, pred_boxes)
pred_boxes = clip_boxes(pred_boxes, ori_shape)
pred_boxes = pred_boxes.reshape(-1, config.num_classes, 4)
result_boxes = []
for j in range(1, config.num_classes):
inds = np.where(scores[:, j] > config.test_cls_threshold)[0]
cls_scores = scores[inds, j]
cls_bboxes = pred_boxes[inds, j, :]
cls_dets = np.hstack((cls_bboxes, cls_scores[:, np.newaxis])).astype(
np.float32, copy=False)
keep = nms(cls_dets, config.test_nms)
cls_dets = np.array(cls_dets[keep, :], dtype=np.float, copy=False)
for i in range(cls_dets.shape[0]):
db = cls_dets[i, :]
dbox = DetBox(
db[0], db[1], db[2] - db[0], db[3] - db[1],
tag=config.class_names[j], score=db[-1])
result_boxes.append(dbox)
if len(result_boxes) > config.test_max_boxes_per_image:
result_boxes = sorted(
result_boxes, reverse=True, key=lambda t_res: t_res.score) \
[:config.test_max_boxes_per_image]
result_dict = data_dict.copy()
result_dict['result_boxes'] = result_boxes
return result_dict
def proposal_without_nms_layer(rpn_cls_prob, rpn_bbox_pred, im_info, cfg_key,
feat_stride, anchors, num_anchors,
is_tfchannel=False):
if type(cfg_key) == bytes:
cfg_key = cfg_key.decode('utf-8')
if cfg_key == 'TRAIN':
pre_nms_topN = cfg.TRAIN.RPN_PRE_NMS_TOP_N
else:
pre_nms_topN = cfg.TEST.RPN_PRE_NMS_TOP_N
im_info = im_info[0]
# Get the scores and bounding boxes
if is_tfchannel:
scores = rpn_cls_prob.reshape(-1, 2)
scores = scores[:, 1]
else:
scores = rpn_cls_prob[:, :, :, num_anchors:]
rpn_bbox_pred = rpn_bbox_pred.reshape((-1, 4))
if 'RPN_NORMALIZE_TARGETS' in cfg.TRAIN.keys() \
and cfg.TRAIN.RPN_NORMALIZE_TARGETS:
rpn_bbox_pred *= cfg.TRAIN.RPN_NORMALIZE_STDS
rpn_bbox_pred += cfg.TRAIN.RPN_NORMALIZE_MEANS
scores = scores.reshape((-1, 1))
proposals = bbox_transform_inv(anchors, rpn_bbox_pred)
proposals = clip_boxes(proposals, im_info[:2])
# filter boxes
min_size = 0
if cfg_key == 'TRAIN':
if 'RPN_MIN_SIZE' in cfg.TRAIN.keys():
min_size = cfg.TRAIN.RPN_MIN_SIZE
elif cfg_key == 'TEST':
if 'RPN_MIN_SIZE' in cfg.TEST.keys():
min_size = cfg.TEST.RPN_MIN_SIZE
if min_size > 0:
keep = _filter_boxes(proposals, min_size * im_info[2])
proposals = proposals[keep, :]
scores = scores[keep]
# Pick the top region proposals
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order].flatten()
##why add one, because tf nms assume x2,y2 does not include border
proposals_addone = np.array(proposals)
proposals_addone[:, 2] += 1
proposals_addone[:, 3] += 1
return proposals, scores, proposals_addone
def coco_results_one_category_kernel(data_pack):
cat_id = data_pack['cat_id']
ann_type = data_pack['ann_type']
binary_thresh = data_pack['binary_thresh']
all_im_info = data_pack['all_im_info']
boxes = data_pack['boxes']
if ann_type == 'bbox':
masks = []
elif ann_type == 'segm':
masks = data_pack['masks']
else:
print('unimplemented ann_type: ' + ann_type)
cat_results = []
for im_ind, im_info in enumerate(all_im_info):
index = im_info['index']
dets = boxes[im_ind].astype(np.float)
from IPython import embed
embed()
# if len(masks[im_ind]) != dets.shape[0]:
# masks[im_ind] = masks[im_ind][:dets.shape[0]]
print(dets.shape, len(masks[im_ind]))
if len(dets) == 0:
continue
scores = dets[:, -1]
if ann_type == 'bbox':
xs = dets[:, 0]
ys = dets[:, 1]
ws = dets[:, 2] - xs + 1
hs = dets[:, 3] - ys + 1
result = [{
'image_id': index,
'category_id': cat_id,
'bbox': [xs[k], ys[k], ws[k], hs[k]],
'score': scores[k]
} for k in range(dets.shape[0])]
elif ann_type == 'segm':
width = im_info['width']
height = im_info['height']
dets[:, :4] = clip_boxes(dets[:, :4], [height, width])
mask_encode = mask_voc2coco(masks[im_ind], dets[:, :4], height,
width, binary_thresh)
result = [{
'image_id': index,
'category_id': cat_id,
'segmentation': mask_encode[k],
'score': scores[k]
} for k in range(len(mask_encode))]
cat_results.extend(result)
return cat_results
def proposal_top_layer(rpn_cls_prob, rpn_bbox_pred, im_info,
_feat_stride, anchors, num_anchors):
"""A layer that just selects the top region proposals
without using non-maximal suppression,
For details please see the technical report
"""
rpn_top_n = cfg.TEST.RPN_TOP_N
im_info = im_info[0]
scores = rpn_cls_prob[:, :, :, num_anchors:]
rpn_bbox_pred = rpn_bbox_pred.reshape((-1, 4))
scores = scores.reshape((-1, 1))
length = scores.shape[0]
if length < rpn_top_n:
# Random selection, maybe unnecessary and loses good proposals
# But such case rarely happens
top_inds = npr.choice(length, size=rpn_top_n, replace=True)
else:
top_inds = scores.argsort(0)[::-1]
top_inds = top_inds[:rpn_top_n]
top_inds = top_inds.reshape(rpn_top_n, )
# Do the selection here
anchors = anchors[top_inds, :]
rpn_bbox_pred = rpn_bbox_pred[top_inds, :]
scores = scores[top_inds]
# Convert anchors into proposals via bbox transformations
proposals = bbox_transform_inv(anchors, rpn_bbox_pred)
# Clip predicted boxes to image
proposals = clip_boxes(proposals, im_info[:2])
# Output rois blob
# Our RPN implementation only supports a single input image, so all
# batch inds are 0
batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False)))
#from IPython import embed; embed()
return blob, scores
def proposal_without_nms_layer(rpn_cls_prob_fg, rpn_bbox_pred, im_info,
anchors):
pre_nms_topN = cfg.TRAIN.RPN_PRE_NMS_TOP_N
im_info = im_info[0]
scores = rpn_cls_prob_fg
scores = scores.reshape((-1, 1))
rpn_bbox_pred[:, 2:4] = np.minimum(20, rpn_bbox_pred[:, 2:4])
proposals = bbox_transform_inv(anchors, rpn_bbox_pred)
proposals = clip_boxes(proposals, im_info[:2])
# Pick the top region proposals
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order].flatten()
##why add one, because tf nms assume x2,y2 does not include border
proposals_addone = np.array(proposals)
proposals_addone[:, 2] += 1
proposals_addone[:, 3] += 1
return proposals, scores, proposals_addone, order
def proposal_layer(rpn_cls_prob, rpn_bbox_pred, im_info, cfg_key, _feat_stride,
anchors, num_anchors, is_tfchannel=False):
"""A simplified version compared to fast/er RCNN
For details please see the technical report
"""
if type(cfg_key) == bytes:
cfg_key = cfg_key.decode('utf-8')
if cfg_key == 'TRAIN':
pre_nms_topN = cfg.TRAIN.RPN_PRE_NMS_TOP_N
post_nms_topN = cfg.TRAIN.RPN_POST_NMS_TOP_N
nms_thresh = cfg.TRAIN.RPN_NMS_THRESH
else:
pre_nms_topN = cfg.TEST.RPN_PRE_NMS_TOP_N
post_nms_topN = cfg.TEST.RPN_POST_NMS_TOP_N
nms_thresh = cfg.TEST.RPN_NMS_THRESH
im_info = im_info[0]
# from IPython import embed; embed()
# Get the scores and bounding boxes
if is_tfchannel:
scores = rpn_cls_prob.reshape(-1, 2)
scores = scores[:, 1]
else:
scores = rpn_cls_prob[:, :, :, num_anchors:]
rpn_bbox_pred = rpn_bbox_pred.reshape((-1, 4))
# if cfg_key == 'TRAIN' and 'RPN_NORMALIZE_TARGETS' in cfg.TRAIN.keys() \
if 'RPN_NORMALIZE_TARGETS' in cfg.TRAIN.keys() \
and cfg.TRAIN.RPN_NORMALIZE_TARGETS:
rpn_bbox_pred *= cfg.TRAIN.RPN_NORMALIZE_STDS
rpn_bbox_pred += cfg.TRAIN.RPN_NORMALIZE_MEANS
scores = scores.reshape((-1, 1))
proposals = bbox_transform_inv(anchors, rpn_bbox_pred)
proposals = clip_boxes(proposals, im_info[:2])
# filter boxes
min_size = 0
if cfg_key == 'TRAIN':
if 'RPN_MIN_SIZE' in cfg.TRAIN.keys():
min_size = cfg.TRAIN.RPN_MIN_SIZE
elif cfg_key == 'TEST':
if 'RPN_MIN_SIZE' in cfg.TEST.keys():
min_size = cfg.TEST.RPN_MIN_SIZE
if min_size > 0:
keep = _filter_boxes(proposals, min_size * im_info[2])
proposals = proposals[keep, :]
scores = scores[keep]
# Pick the top region proposals
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
# Non-maximal suppression
keep = nms(np.hstack((proposals, scores)), nms_thresh)
if post_nms_topN > 0:
keep = keep[:post_nms_topN]
proposals = proposals[keep, :]
scores = scores[keep]
# Only support single image as input
batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False)))
return blob, scores.flatten()