def _proposal_layer_py(rpn_bbox_cls_prob, rpn_bbox_pred, im_dims, cfg_key, _feat_stride,anchor_scales):
'''
# Algorithm:
#
# for each (H, W) location i
# generate A anchor boxes centered on cell i
# apply predicted bbox deltas at cell i to each of the A anchors
# clip predicted boxes to image
# remove predicted boxes with either height or width < threshold
# sort all (proposal, score) pairs by score from highest to lowest
# take top pre_nms_topN proposals before NMS
# apply NMS with threshold 0.7 to remaining proposals
# take after_nms_topN proposals after NMS
# return the top proposals (-> RoIs top, scores top)
'''
_anchors = generate_anchors(scales=np.array(anchor_scales))
_num_anchors = _anchors.shape[0]
rpn_bbox_cls_prob = np.transpose(rpn_bbox_cls_prob,[0,3,1,2])
rpn_bbox_pred = np.transpose(rpn_bbox_pred,[0,3,1,2])
# Only minibatch of 1 supported
assert rpn_bbox_cls_prob.shape[0] == 1, \
'Only single item batches are supported'
pre_nms_topN = 1000 # cfg[cfg_key].RPN_PRE_NMS_TOP_N
post_nms_topN = 100 # cfg[cfg_key].RPN_POST_NMS_TOP_N
nms_thresh = 0.8 # cfg[cfg_key].RPN_NMS_THRESH
min_size = 12 # cfg[cfg_key].RPN_MIN_SIZE
# the first set of _num_anchors channels are bg probs
# the second set are the fg probs, which we want
scores = rpn_bbox_cls_prob[:, _num_anchors:, :, :]
bbox_deltas = rpn_bbox_pred
# 1. Generate proposals from bbox deltas and shifted anchors
height, width = scores.shape[-2:]
# Enumerate all shifts
shift_x = np.arange(0, width) * _feat_stride
shift_y = np.arange(0, height) * _feat_stride
shift_x, shift_y = np.meshgrid(shift_x, shift_y)
shifts = np.vstack((shift_x.ravel(), shift_y.ravel(),
shift_x.ravel(), shift_y.ravel())).transpose()
# Enumerate all shifted anchors:
#
# add A anchors (1, A, 4) to
# cell K shifts (K, 1, 4) to get
# shift anchors (K, A, 4)
# reshape to (K*A, 4) shifted anchors
A = _num_anchors
K = shifts.shape[0]
anchors = _anchors.reshape((1, A, 4)) + \
shifts.reshape((1, K, 4)).transpose((1, 0, 2))
anchors = anchors.reshape((K * A, 4))
# Transpose and reshape predicted bbox transformations to get them
# into the same order as the anchors:
#
# bbox deltas will be (1, 4 * A, H, W) format
# transpose to (1, H, W, 4 * A)
# reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a)
# in slowest to fastest order
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
# Same story for the scores:
#
# scores are (1, A, H, W) format
# transpose to (1, H, W, A)
# reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a)
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
# Convert anchors into proposals via bbox transformations
proposals = bbox_transform_inv(anchors, bbox_deltas)
# 2. clip predicted boxes to image
proposals = clip_boxes(proposals, im_dims)
# 3. remove predicted boxes with either height or width < threshold
keep = _filter_boxes(proposals, min_size)
proposals = proposals[keep, :]
scores = scores[keep]
# 4. sort all (proposal, score) pairs by score from highest to lowest
# 5. take top pre_nms_topN (e.g. 6000)
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
# 6. apply nms (e.g. threshold = 0.7)
# 7. take after_nms_topN (e.g. 300)
# 8. return the top proposals (-> RoIs top)
keep = nms(np.hstack((proposals, scores)), nms_thresh)
if post_nms_topN > 0:
keep = keep[:post_nms_topN]
proposals = proposals[keep, :]
scores = scores[keep]
# Output rois blob
# Our RPN implementation only supports a single input image, so all
# batch inds are 0
batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False)))
return blob
def main():
args = get_arguments()
cfg = Config(dataset=args.dataset,
is_training=False,
filter_scale=args.filter_scale)
model = model_config[args.model]
image_filenames = get_filename_list(image_dir)
image_batch = tf.placeholder(tf.float32, shape=[1, 1024, 2048, 3])
label_batch = tf.placeholder(tf.uint8, shape=[1, 1024, 2048, 1])
gt_boxes = tf.placeholder(tf.int32, shape=[None, 5])
images, fnames = cityscapeInputs(image_filenames, 1)
img = tf.squeeze(image_batch)
img = _extract_mean(img, IMG_MEAN, swap_channel=True)
img = tf.reshape(img, [1, 1024, 2048, 3])
net = model(img, cfg=cfg, mode='eval')
feature_maps = net.conv5_3
model = createrpn(feature_maps, gt_boxes, 'EVAL')
boxes = tf.round(model.roi_proposal_net['EVAL'].get_rois() * 255 / 1024)
mean = fs_pool(net.output, boxes, [255, 511])
pred = tf.image.resize_bilinear(net.logits,
size=[1024, 2048],
align_corners=True)
pred = tf.argmax(pred, axis=3)
pred = tf.expand_dims(pred, axis=3)
preds_summary = tf.py_func(Config.decode_labels, [pred, 1, 19], tf.uint8)
preds_summary = tf.squeeze(preds_summary)
# mIoU
pred_flatten = tf.reshape(net.output, [
-1,
])
label_flatten = tf.reshape(label_batch, [
-1,
])
mask = tf.not_equal(label_flatten, cfg.param['ignore_label'])
indices = tf.squeeze(tf.where(mask), 1)
gt = tf.cast(tf.gather(label_flatten, indices), tf.int32)
pred = tf.gather(pred_flatten, indices)
if cfg.dataset == 'ade20k':
pred = tf.add(pred, tf.constant(1, dtype=tf.int64))
mIoU, update_op = tf.metrics.mean_iou(
predictions=pred,
labels=gt,
num_classes=cfg.param['num_classes'] + 1)
elif cfg.dataset == 'cityscapes':
mIoU, update_op = tf.metrics.mean_iou(
predictions=pred, labels=gt, num_classes=cfg.param['num_classes'])
net.create_session()
#net.restore(cfg.model_paths[args.model])
net.restore(SNAPSHOT_DIR)
saver = tf.train.Saver(tf.global_variables())
for step in trange(cfg.param['eval_steps'], desc='evaluation', leave=True):
image_bat, fname = net.sess.run([images, fnames])
ima = image_bat.reshape([1024, 2048, 3])
tf_outputs = (boxes, net.output, preds_summary, mean)
feed_dict = {image_batch: image_bat}
rois, temp, pred, roi_mean = net.sess.run(tf_outputs, feed_dict)
pred_fin = pred // 2 + ima // 3
f2 = time.time()
temp = temp.reshape(255, 511)
rois = rois[:, 1:5]
keep = np.where(roi_mean > 0.45)[0]
rois = rois[keep, :]
roi_mean = np.expand_dims(roi_mean[keep], axis=1)
roisnms = np.hstack((rois, roi_mean))
keep = np.where(roisnms[:, 4] > 0)[0]
roisnms = roisnms[keep, :]
keep = roisnms[:, 4].ravel().argsort() # [::-1]
roisnms = roisnms[keep, :]
keep = nms(roisnms, 0.3)
roisnms = roisnms[keep, :]
roisnms = roisnms.astype(np.int)
sign = danpeo(roisnms, temp, pred_fin)
f3 = time.time()
roisnms = roisnms * 4
drawbbox(sign, pred_fin, roisnms)
pred_fin = pred_fin.astype('uint8')
ima = ima.astype('uint8')
plt.subplot(121)
plt.imshow(pred_fin)
plt.subplot(122)
plt.imshow(ima)
plt.show()
def _proposal_layer_py(rpn_bbox_cls_prob, rpn_bbox_pred, im_dims, cfg_key, _feat_stride, anchor_scales):
'''
# Algorithm:
#
# for each (H, W) location i
# generate A anchor boxes centered on cell i
# apply predicted bbox deltas at cell i to each of the A anchors
# clip predicted boxes to image
# remove predicted boxes with either height or width < threshold
# sort all (proposal, score) pairs by score from highest to lowest
# take top pre_nms_topN proposals before NMS
# apply NMS with threshold 0.7 to remaining proposals
# take after_nms_topN proposals after NMS
# return the top proposals (-> RoIs top, scores top)
'''
_anchors = generate_anchors(scales=np.array(anchor_scales))
_num_anchors = _anchors.shape[0]
rpn_bbox_cls_prob = np.transpose(rpn_bbox_cls_prob,[0,3,1,2])
rpn_bbox_pred = np.transpose(rpn_bbox_pred,[0,3,1,2])
# Only minibatch of 1 supported
assert rpn_bbox_cls_prob.shape[0] == 1, \
'Only single item batches are supported'
if cfg_key == 'TRAIN':
pre_nms_topN = cfg.TRAIN.RPN_PRE_NMS_TOP_N
post_nms_topN = cfg.TRAIN.RPN_POST_NMS_TOP_N
nms_thresh = cfg.TRAIN.RPN_NMS_THRESH
min_size = cfg.TRAIN.RPN_MIN_SIZE
else: # cfg_key == 'TEST':
pre_nms_topN = cfg.TEST.RPN_PRE_NMS_TOP_N
post_nms_topN = cfg.TEST.RPN_POST_NMS_TOP_N
nms_thresh = cfg.TEST.RPN_NMS_THRESH
min_size = cfg.TEST.RPN_MIN_SIZE
# the first set of _num_anchors channels are bg probs
# the second set are the fg probs, which we want
scores = rpn_bbox_cls_prob[:, _num_anchors:, :, :]
bbox_deltas = rpn_bbox_pred
# 1. Generate proposals from bbox deltas and shifted anchors
height, width = scores.shape[-2:]
# Enumerate all shifts
shift_x = np.arange(0, width) * _feat_stride
shift_y = np.arange(0, height) * _feat_stride
shift_x, shift_y = np.meshgrid(shift_x, shift_y)
shifts = np.vstack((shift_x.ravel(), shift_y.ravel(),
shift_x.ravel(), shift_y.ravel())).transpose()
# Enumerate all shifted anchors:
#
# add A anchors (1, A, 4) to
# cell K shifts (K, 1, 4) to get
# shift anchors (K, A, 4)
# reshape to (K*A, 4) shifted anchors
A = _num_anchors
K = shifts.shape[0]
anchors = _anchors.reshape((1, A, 4)) + \
shifts.reshape((1, K, 4)).transpose((1, 0, 2))
anchors = anchors.reshape((K * A, 4))
# Transpose and reshape predicted bbox transformations to get them
# into the same order as the anchors:
#
# bbox deltas will be (1, 4 * A, H, W) format
# transpose to (1, H, W, 4 * A)
# reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a)
# in slowest to fastest order
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
# Same story for the scores:
#
# scores are (1, A, H, W) format
# transpose to (1, H, W, A)
# reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a)
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
# Convert anchors into proposals via bbox transformations
proposals = bbox_transform_inv(anchors, bbox_deltas)
# 2. clip predicted boxes to image
proposals = clip_boxes(proposals, im_dims)
# 3. remove predicted boxes with either height or width < threshold
keep = _filter_boxes(proposals, min_size)
proposals = proposals[keep, :]
scores = scores[keep]
# 4. sort all (proposal, score) pairs by score from highest to lowest
# 5. take top pre_nms_topN (e.g. 6000)
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
# 6. apply nms (e.g. threshold = 0.7)
# 7. take after_nms_topN (e.g. 300)
# 8. return the top proposals (-> RoIs top)
keep = nms(np.hstack((proposals, scores)), nms_thresh)
if post_nms_topN > 0:
keep = keep[:post_nms_topN]
proposals = proposals[keep, :]
scores = scores[keep]
# Output rois blob
# Our RPN implementation only supports a single input image, so all
# batch inds are 0
batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False)))
return blob