def vis_detections(lidar_bv, image, calib, bbox_pred_cnr, rpn_data, rpn_rois,
rcnn_roi, scores, gt_boxes_3d):
import matplotlib.pyplot as plt
from utils.transform import lidar_3d_to_corners, corners_to_bv
from fast_rcnn.bbox_transform import bbox_transform_inv_cnr
from utils.draw import show_lidar_corners, show_image_boxes, scale_to_255
from utils.cython_nms import nms, nms_new
image = image.reshape((image.shape[1], image.shape[2], image.shape[3]))
image += cfg.PIXEL_MEANS
image = image.astype(np.uint8, copy=False)
lidar_bv = lidar_bv.reshape(
(lidar_bv.shape[1], lidar_bv.shape[2], lidar_bv.shape[3]))[:, :, 8]
# visualize anchor_target_layer output
rpn_anchors_3d = rpn_data[3][:, 1:7]
rpn_bv = rpn_data[2][:, 1:5]
# rpn_label = rpn_data[0]
# print rpn_label.shape
# print rpn_label[rpn_label==1]
rpn_boxes_cnr = lidar_3d_to_corners(rpn_anchors_3d)
img = show_lidar_corners(image, rpn_boxes_cnr, calib)
img_bv = show_image_boxes(scale_to_255(lidar_bv, min=0, max=2), rpn_bv)
print img.shape
# plt.ion()
plt.title('anchor target layer before regression')
plt.subplot(211)
plt.imshow(img_bv)
plt.subplot(212)
plt.imshow(img)
plt.show()
# visualize proposal_layer output
boxes_3d = rpn_rois[2][:, 1:7]
boxes_bv = rpn_rois[0][:, 0:5]
boxes_img = rpn_rois[1][:, 0:5]
# keep = nms(boxes_img, cfg.TEST.NMS)
# boxes_img = boxes_img[keep]
# boxes_3d = boxes_3d[keep]
# boxes_cnr = lidar_3d_to_corners(boxes_3d[:100])
print boxes_3d.shape
print boxes_bv.shape
# image_cnr = show_lidar_corners(image, boxes_cnr, calib)
image_bv = show_image_boxes(lidar_bv, boxes_bv[:, 1:5])
image_img = show_image_boxes(image, boxes_img[:, 1:5])
plt.title('proposal_layer ')
plt.subplot(211)
plt.imshow(image_bv)
plt.subplot(212)
plt.imshow(image_img)
plt.show()
def test_net(sess,
net,
imdb,
weights_filename,
max_per_image=300,
thresh=0.05,
vis=False):
"""Test a Fast R-CNN network on an image database."""
num_images = len(imdb.image_index)
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
# all_boxes_cnr[cls][image] = N x 25 array of detections in
# (x0-x7, y0-y7, z0-z7, score)
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
all_boxes_img = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
all_boxes_cnr = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
output_dir = get_output_dir(imdb, weights_filename)
# timers
_t = {'im_detect': Timer(), 'misc': Timer()}
# conv1_1 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv1_1")
# conv1_2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv1_2")
# conv2_1 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv2_1")
# conv2_2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv2_2")
# conv3_1 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv3_1")
# conv3_2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv3_2")
# conv3_3 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv3_3")
# conv4_1 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv4_1")
# conv4_2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv4_2")
# conv4_3 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv4_3")
# conv5_1 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv5_1")
# conv5_2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv5_2")
# conv5_3 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="conv5_3")
# rpn_w = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="rpn_conv/3x3")[0]
# rpn_b = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="rpn_conv/3x3")[1]
# rpn_w2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="rpn_cls_score")[0]
# rpn_b2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="rpn_cls_score")[1]
# rpn_w3 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="rpn_bbox_pred")[0]
# rpn_b3 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="rpn_bbox_pred")[1]
# weights = {
# 'conv1_1' : {"weights" : conv1_1[0].eval(session=sess), "biases": conv1_1[1].eval(session=sess)},
# 'conv1_2' : {"weights" : conv1_2[0].eval(session=sess), "biases": conv1_2[1].eval(session=sess)},
# 'conv2_1' : {"weights" : conv2_1[0].eval(session=sess), "biases": conv2_1[1].eval(session=sess)},
# 'conv2_2' : {"weights" : conv2_2[0].eval(session=sess), "biases": conv2_2[1].eval(session=sess)},
# 'conv3_1' : {"weights" : conv3_1[0].eval(session=sess), "biases": conv3_1[1].eval(session=sess)},
# 'conv3_2' : {"weights" : conv3_2[0].eval(session=sess), "biases": conv3_2[1].eval(session=sess)},
# 'conv3_3' : {"weights" : conv3_3[0].eval(session=sess), "biases": conv3_3[1].eval(session=sess)},
# 'conv4_1' : {"weights" : conv4_1[0].eval(session=sess), "biases": conv4_1[1].eval(session=sess)},
# 'conv4_2' : {"weights" : conv4_2[0].eval(session=sess), "biases": conv4_2[1].eval(session=sess)},
# 'conv4_3' : {"weights" : conv4_3[0].eval(session=sess), "biases": conv4_3[1].eval(session=sess)},
# 'conv5_1' : {"weights" : conv5_1[0].eval(session=sess), "biases": conv5_1[1].eval(session=sess)},
# 'conv5_2' : {"weights" : conv5_2[0].eval(session=sess), "biases": conv5_2[1].eval(session=sess)},
# 'conv5_3' : {"weights" : conv5_3[0].eval(session=sess), "biases": conv5_3[1].eval(session=sess)},
# 'rpn_conv/3x3' : {"weights" : rpn_w.eval(session=sess), "biases": rpn_b.eval(session=sess)},
# 'rpn_cls_score' : {"weights" : rpn_w2.eval(session=sess), "biases": rpn_b2.eval(session=sess)},
# 'rpn_bbox_pred' : {"weights" : rpn_w3.eval(session=sess), "biases": rpn_b3.eval(session=sess)},
# }
# # print rpn_w.eval(session=sess)
# np.save('rpn_data.npy', weights)
# deconv2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="deconv_4x_1")[0]
# shape_conv5_3 = conv5_3.get_shape().as_list()
# shape1 = deconv1.get_shape().as_list()
# shape2 = deconv2.get_shape().as_list()
# print 'conv5_3 shape', shape_conv5_3
# print 'deconv_2x_1 shape', shape1
# print 'deconv_4x_1 shape', shape2
for i in xrange(num_images):
# filter out any ground truth boxes
if cfg.TEST.HAS_RPN:
box_proposals = None
im = cv2.imread(imdb.image_path_at(i))
bv = np.load(imdb.lidar_path_at(i))
calib = imdb.calib_at(i)
print "Inference: ", imdb.lidar_path_at(i)
_t['im_detect'].tic()
scores, boxes_bv, boxes_cnr, boxes_cnr_r = box_detect(
sess, net, im, bv, calib, box_proposals)
_t['im_detect'].toc()
_t['misc'].tic()
if vis:
image = im[:, :, (2, 1, 0)]
plt.cla()
plt.imshow(image)
thresh = 0.05
# skip j = 0, because it's the background class
for j in xrange(1, imdb.num_classes):
inds = np.where(scores[:, j] > thresh)[0]
cls_scores = scores[inds, j]
cls_boxes = boxes_bv[inds, j * 4:(j + 1) * 4]
cls_boxes_cnr = boxes_cnr[inds, j * 24:(j + 1) * 24]
cls_boxes_cnr_r = boxes_cnr_r[inds, j * 24:(j + 1) * 24]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
cls_dets_cnr = np.hstack((cls_boxes_cnr, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
cls_dets_cnr_r = np.hstack((cls_boxes_cnr_r, cls_scores[:, np.newaxis])) \
.astype(np.float32, copy=False)
# print "scores: ", cls_scores
# print "cls_dets : ", cls_dets.shape
keep = nms(cls_dets, cfg.TEST.NMS)
cls_dets = cls_dets[keep, :]
cls_dets_cnr = cls_dets_cnr[keep, :]
cls_dets_cnr_r = cls_dets_cnr_r[keep, :]
cls_scores = cls_scores[keep]
# project to image
if np.any(cls_dets_cnr):
plt.rcParams['figure.figsize'] = (10, 10)
img_boxes = lidar_cnr_to_img(cls_dets_cnr[:, :24], calib[3],
calib[2], calib[0])
img = show_image_boxes(im, img_boxes)
plt.imshow(img)
#plt.show()
print cls_dets_cnr.shape
image_bv = show_image_boxes(
scale_to_255(bv[:, :, 8], min=0, max=2), cls_dets[:, :4])
image_cnr = show_lidar_corners(im, cls_dets_cnr[:, :24], calib)
if 1:
import mayavi.mlab as mlab
filename = os.path.join(
imdb.lidar_path_at(i)[:-19], 'velodyne',
str(i).zfill(6) + '.bin')
print filename
scan = np.fromfile(filename, dtype=np.float32)
scan = scan.reshape((-1, 4))
corners = cls_dets_cnr[:, :24].reshape(
(-1, 3, 8)).transpose((0, 2, 1))
corners_r = cls_dets_cnr_r[:, :24].reshape(
(-1, 3, 8)).transpose((0, 2, 1))
print corners_r
fig = mlab.figure(figure=None,
bgcolor=(0, 0, 0),
fgcolor=None,
engine=None,
size=(1000, 500))
draw_lidar(scan, fig=fig)
draw_gt_boxes3d(corners, fig=fig)
draw_gt_boxes3d(corners_r, color=(1, 0, 1), fig=fig)
mlab.show()
plt.subplot(211)
plt.title('bv proposal')
plt.imshow(image_bv, cmap='jet')
plt.subplot(212)
plt.imshow(image_cnr)
plt.show()
all_boxes[j][i] = cls_dets
# all_boxes_img[j][i] = cls_des_img
all_boxes_cnr[j][i] = cls_dets_cnr
if vis:
plt.show()
# Limit to max_per_image detections *over all classes*
if max_per_image > 0:
image_scores = np.hstack(
[all_boxes[j][i][:, -1] for j in xrange(1, imdb.num_classes)])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in xrange(1, imdb.num_classes):
keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0]
all_boxes[j][i] = all_boxes[j][i][keep, :]
# all_boxes_img[j][i] = all_boxes_img[j][i][keep, :]
all_boxes_cnr[j][i] = all_boxes_cnr[j][i][keep, :]
_t['misc'].toc()
print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \
.format(i + 1, num_images, _t['im_detect'].average_time,
_t['misc'].average_time)
det_file = os.path.join(output_dir, 'detections.pkl')
with open(det_file, 'wb') as f:
cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL)
det_cnr_file = os.path.join(output_dir, 'detections_cnr.pkl')
with open(det_cnr_file, 'wb') as f:
cPickle.dump(all_boxes_cnr, f, cPickle.HIGHEST_PROTOCOL)
print 'Evaluating detections'
imdb.evaluate_detections(all_boxes, all_boxes_cnr, output_dir)
def demo(sess, net, root_dir, image_name):
"""Test a Fast R-CNN network on an image database."""
# Load the demo image
im_file = os.path.join(root_dir, 'image_2', image_name + '.png')
velo_file = os.path.join(root_dir, 'velodyne', image_name + '.bin')
calib_file = os.path.join(root_dir, 'calib', '000000' + '.txt')
bv_file = os.path.join(root_dir, 'lidar_bv', image_name + '.npy')
im = cv2.imread(im_file)
velo = make_bird_view(velo_file)[0]
bv = np.load(bv_file)
calib = make_calib(calib_file)
plt.imshow(bv[:, :, 5])
plt.show
# Detect all object classes and regress object bounds
timer = Timer()
timer.tic()
scores, boxes_bv, boxes_cnr, boxes_cnr_r = box_detect(
sess, net, im, bv, calib)
timer.toc()
print('Detection took {:.3f}s for '
'{:d} object proposals').format(timer.total_time, boxes_bv.shape[0])
# Visualize detections for each class
# im = im[:, :, (2, 1, 0)]
# fig, ax = plt.subplots(figsize=(12, 12))
# ax.imshow(im, aspect='equal')
# plt.imshow(im)
# plt.show()
CONF_THRESH = 0.1
NMS_THRESH = 0.1
for cls_ind, cls in enumerate(CLASSES[1:]):
cls_ind += 1 # because we skipped background
inds = np.where(scores[:, cls_ind] > CONF_THRESH)[0]
cls_boxes = boxes_bv[inds, 4 * cls_ind:4 * (cls_ind + 1)]
cls_boxes_cnr = boxes_cnr[inds, cls_ind * 24:(cls_ind + 1) * 24]
cls_boxes_cnr_r = boxes_cnr_r[inds, cls_ind * 24:(cls_ind + 1) * 24]
cls_scores = scores[inds, cls_ind]
# cls_boxes = boxes_bv[:, 4*cls_ind:4*(cls_ind + 1)]
# cls_boxes_cnr = boxes_cnr[:, cls_ind*24:(cls_ind+1)*24]
# cls_boxes_cnr_r = boxes_cnr_r[:, cls_ind*24:(cls_ind+1)*24]
# cls_scores = scores[:, cls_ind]
cls_dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
cls_dets_cnr = np.hstack(
(cls_boxes_cnr, cls_scores[:, np.newaxis])).astype(np.float32,
copy=False)
cls_dets_cnr_r = np.hstack(
(cls_boxes_cnr_r, cls_scores[:, np.newaxis])).astype(np.float32,
copy=False)
keep = nms(cls_dets, NMS_THRESH)
cls_dets = cls_dets[keep, :]
cls_dets_cnr = cls_dets_cnr[keep, :]
cls_dets_cnr_r = cls_dets_cnr_r[keep, :]
cls_scores = cls_scores[keep]
print cls_dets[:, :4]
print cls_scores
# vis_detections(im, cls, dets, ax, thresh=CONF_THRESH)
# project to image
# if np.any(cls_dets_cnr):
# img_boxes = lidar_cnr_to_img(cls_dets_cnr_r[:,:24], calib[3], calib[2], calib[0])
# # print img_boxes
# img = show_image_boxes(im, img_boxes)
# plt.imshow(img)
# cv2.imwrite('examples/' + image_name+'.png', img)
print cls_dets_cnr_r.shape
image_bv = show_image_boxes(scale_to_255(bv[:, :, 8], min=0, max=2),
cls_dets[:, :4])
image_cnr = show_lidar_corners(im, cls_dets_cnr_r[:, :24], calib)
cv2.imwrite(image_name + '.png', image_cnr)
# plt.imshow()
if 1:
corners = cls_dets_cnr[:, :24].reshape((-1, 3, 8)).transpose(
(0, 2, 1))
corners_r = cls_dets_cnr_r[:, :24].reshape((-1, 3, 8)).transpose(
(0, 2, 1))
fig = mlab.figure(figure=None,
bgcolor=(0, 0, 0),
fgcolor=None,
engine=None,
size=(1000, 500))
draw_lidar(velo, fig=fig)
# draw_gt_boxes3d(corners, fig=fig)
draw_gt_boxes3d(corners_r, color=(1, 0, 1), fig=fig)
mlab.savefig('lidar' + image_name + '.png', figure=fig)
mlab.close()