while blobs['im_id'] != '000103':
print(blobs['im_id'])
blobs = data_layer.forward()
voxel_coord = blobs['voxel_data']['coordinate_buffer']
voxel_size = blobs['im_info'][0]
print(blobs['im_info'][0])
voxel_full = np.zeros((int(voxel_size[0]), int(voxel_size[1])))
#show ground truth
gt_boxes_3d = blobs['gt_boxes_3d']
gt_ry = blobs['gt_rys']
gt_corners = np.zeros((gt_ry.shape[0], 24))
for i in range(gt_ry.shape[0]):
gt_corners[i, :] = computeCorners3D(gt_boxes_3d[i, :],
gt_ry[i, :]).flatten()
gt_voxel_cnr = lidar_in_cam_to_voxel(gt_corners.reshape((-1, 3, 8)))
print(gt_voxel_cnr)
for i in range(gt_voxel_cnr.shape[0]):
voxel_full = drawBox3D(voxel_full, gt_voxel_cnr[i, :, :])
print(voxel_full.shape)
voxel_full[voxel_coord[:, 3], voxel_coord[:, 2]] = 200
plt.figure()
img_bv = np.flipud(voxel_full) #scale_to_255(voxel_full, min=0, max=2)
plt.imshow(img_bv)
# show data for fusion
print('========================data for fusion=======================')
bv_size = blobs['bv_size']
def _load_kitti_annotation(self, index):
"""
Load image and bounding boxes info from txt file in the KITTI
format.
"""
# filename = '$Faster-RCNN_TF/data/KITTI/object/training/label_2/000000.txt'
filename = os.path.join(self._data_path, 'training/label_2',
index + '.txt')
# print("Loading: ", filename)
# calib
calib = self._load_kitti_calib(index)
Tr = calib['Tr_velo2cam']
# print 'Loading: {}'.format(filename)
with open(filename, 'r') as f:
lines = f.readlines()
num_objs = len(lines)
translation = np.zeros((num_objs, 3), dtype=np.float32)
rys = np.zeros((num_objs), dtype=np.float32)
lwh = np.zeros((num_objs, 3), dtype=np.float32)
boxes = np.zeros((num_objs, 4), dtype=np.float32)
boxes_bv = np.zeros((num_objs, 4), dtype=np.float32)
boxes3D = np.zeros((num_objs, 6), dtype=np.float32)
boxes3D_lidar = np.zeros((num_objs, 6), dtype=np.float32)
boxes3D_cam_cnr = np.zeros((num_objs, 24), dtype=np.float32)
boxes3D_corners = np.zeros((num_objs, 24), dtype=np.float32)
alphas = np.zeros((num_objs), dtype=np.float32)
gt_classes = np.zeros((num_objs), dtype=np.int32)
overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)
# print(boxes3D.shape)
# Load object bounding boxes into a data frame.
ix = -1
for line in lines:
obj = line.strip().split(' ')
try:
cls = self._class_to_ind[obj[0].strip()]
# print cls
except:
continue
# ignore objects with undetermined difficult level
# level = self._get_obj_level(obj)
# if level > 3:
# continue
ix += 1
# 0-based coordinates
alpha = float(obj[3])
x1 = float(obj[4])
y1 = float(obj[5])
x2 = float(obj[6])
y2 = float(obj[7])
h = float(obj[8])
w = float(obj[9])
l = float(obj[10])
tx = float(obj[11])
ty = float(obj[12])
tz = float(obj[13])
ry = float(obj[14])
rys[ix] = ry
lwh[ix, :] = [l, w, h]
alphas[ix] = alpha
translation[ix, :] = [tx, ty, tz]
boxes[ix, :] = [x1, y1, x2, y2]
boxes3D[ix, :] = [tx, ty, tz, l, w, h]
# convert boxes3D cam to 8 corners(cam)
boxes3D_cam_cnr_single = computeCorners3D(boxes3D[ix, :], ry)
boxes3D_cam_cnr[ix, :] = boxes3D_cam_cnr_single.reshape(24)
# convert 8 corners(cam) to 8 corners(lidar)
boxes3D_corners[ix, :] = camera_to_lidar_cnr(
boxes3D_cam_cnr_single, Tr)
# convert 8 corners(cam) to lidar boxes3D
boxes3D_lidar[ix, :] = lidar_cnr_to_3d(boxes3D_corners[ix, :],
lwh[ix, :])
# convert 8 corners(lidar) to lidar bird view
boxes_bv[ix, :] = lidar_3d_to_bv(boxes3D_lidar[ix, :])
# boxes3D_corners[ix, :] = lidar_to_corners_single(boxes3D_lidar[ix, :])
gt_classes[ix] = cls
overlaps[ix, cls] = 1.0
rys.resize(ix + 1)
lwh.resize(ix + 1, 3)
translation.resize(ix + 1, 3)
alphas.resize(ix + 1)
boxes.resize(ix + 1, 4)
boxes_bv.resize(ix + 1, 4)
boxes3D.resize(ix + 1, 6)
boxes3D_lidar.resize(ix + 1, 6)
boxes3D_cam_cnr.resize(ix + 1, 24)
boxes3D_corners.resize(ix + 1, 24)
gt_classes.resize(ix + 1)
# print(self.num_classes)
overlaps.resize(ix + 1, self.num_classes)
# if index == '000142':
# print(index)
# print(overlaps)
overlaps = scipy.sparse.csr_matrix(overlaps)
# if index == '000142':
# print(overlaps)
return {
'ry': rys,
'lwh': lwh,
'boxes': boxes,
'boxes_bv': boxes_bv,
'boxes_3D_cam': boxes3D,
'boxes_3D': boxes3D_lidar,
'boxes3D_cam_corners': boxes3D_cam_cnr,
'boxes_corners': boxes3D_corners,
'gt_classes': gt_classes,
'gt_overlaps': overlaps,
'xyz': translation,
'alphas': alphas,
'flipped': False
}
gt_ry,
im_info,
_feat_stride=2,
use_reward=False,
DEBUG=True)
print(rpn_labels.shape, rpn_bbox_targets.shape, all_anchors_3d_bbox.shape,
reward_and_orig_labels.shape)
#show ground truth
gt_boxes_3d = blobs['gt_boxes_3d']
gt_ry = blobs['gt_rys']
gt_corners = np.zeros((gt_ry.shape[0], 24))
print(gt_ry)
for i in range(gt_ry.shape[0]):
gt_corners[i, :] = computeCorners3D(gt_boxes_3d[i, :],
gt_ry[i, :]).flatten()
gt_voxel_cnr = lidar_in_cam_to_voxel(gt_corners.reshape((-1, 3, 8)))
voxel_coord = blobs['voxel_data']['coordinate_buffer']
voxel_size = blobs['im_info'][0]
print('im_info: ', blobs['im_info'][0])
#show voxels in transposed version so it suits my prefered POV
voxel_full = np.zeros((int(voxel_size[0]), int(voxel_size[1])))
for i in range(gt_voxel_cnr.shape[0]):
voxel_full = drawBox3D(voxel_full, gt_voxel_cnr[i, :, :])
print('gt_boxes_3d: ', gt_boxes_3d)
print('gt_rys: ', gt_ry)
print('anchors_3d: ', all_anchors_3d_bbox[rpn_labels > 0, :])
print('targets: ', rpn_bbox_targets[rpn_labels > 0, :])
print(' voxel_full.shape: ', voxel_full.shape)
def _load_kitti_annotation(self, index):
"""
Load image and bounding boxes info from txt file in the KITTI
format.
WZN: The non-interested area (Dontcare) is just ignored (treated same as background)
"""
if self._image_set == 'test':
return {
'ry': np.array([]),
'lwh': np.array([]),
'boxes': np.array([]), #xy box in image
#'boxes_bv' : boxes_bv, #xy box in bird view
'boxes_3D_cam':
np.array([]), #[xyz_center, lwh] in 3D, cam frame
#'boxes_3D' : boxes3D_lidar, #[xyz_center, lwh] in 3D, absolute
'boxes3D_cam_corners':
np.array([]), #8 corners of box in 3D, cam frame
#'boxes_corners' : boxes3D_corners, #8 corners of box in 3D
#'boxes_bv_corners' : boxes_bv_corners, #4 corners of box in bird view
'gt_classes': np.array([]), #classes
'gt_overlaps': np.array([]), #default 1, changed later
'xyz': np.array([]),
'alphas': np.array([]),
'diff_level': np.array([]),
'flipped': False
}
else:
# filename = '$Faster-RCNN_TF/data/KITTI/object/training/label_2/000000.txt'
filename = os.path.join(self.gt_dir, index + '.txt')
# print("Loading: ", filename)
# calib
calib = self._load_kitti_calib(index)
Tr = np.dot(calib['R0'], calib['Tr_velo2cam'])
# print 'Loading: {}'.format(filename)
with open(filename, 'r') as f:
lines = f.readlines()
num_objs = len(lines)
translation = np.zeros((num_objs, 3), dtype=np.float32)
rys = np.zeros((num_objs), dtype=np.float32)
lwh = np.zeros((num_objs, 3), dtype=np.float32)
boxes = np.zeros((num_objs, 4), dtype=np.float32)
boxes_bv = np.zeros((num_objs, 4), dtype=np.float32)
boxes3D = np.zeros((num_objs, 6), dtype=np.float32)
boxes3D_lidar = np.zeros((num_objs, 6), dtype=np.float32)
boxes3D_cam_cnr = np.zeros((num_objs, 24), dtype=np.float32)
boxes3D_corners = np.zeros((num_objs, 24), dtype=np.float32)
alphas = np.zeros((num_objs), dtype=np.float32)
gt_classes = np.zeros((num_objs), dtype=np.int32)
overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)
# new difficulty level for in training evaluation
diff_level = np.zeros((num_objs), dtype=np.int32)
# print(boxes3D.shape)
# Load object bounding boxes into a data frame.
ix = -1
for line in lines:
obj = line.strip().split(' ')
try:
#WZN.strip() removes white spaces
cls = self._class_to_ind[obj[0].strip()]
# print cls
except:
continue
# ignore objects with undetermined difficult level
level = self._get_obj_level(obj)
if level > 3:
continue
ix += 1
# 0-based coordinates
alpha = float(obj[3])
x1 = float(obj[4])
y1 = float(obj[5])
x2 = float(obj[6])
y2 = float(obj[7])
h = float(obj[8])
w = float(obj[9])
l = float(obj[10])
tx = float(obj[11])
ty = float(obj[12])
tz = float(obj[13])
ry = float(obj[14])
diff_level[ix] = level
if obj[0].strip() == 'Person_sitting':
diff_level[ix] = -1
rys[ix] = ry
lwh[ix, :] = [l, w, h]
alphas[ix] = alpha
translation[ix, :] = [tx, ty, tz]
boxes[ix, :] = [x1, y1, x2, y2]
boxes3D[ix, :] = [tx, ty, tz, l, w, h]
# convert boxes3D cam to 8 corners(cam)
boxes3D_cam_cnr_single = computeCorners3D(boxes3D[ix, :], ry)
boxes3D_cam_cnr[ix, :] = boxes3D_cam_cnr_single.reshape(24)
# convert 8 corners(cam) to 8 corners(lidar)
boxes3D_corners[ix, :] = camera_to_lidar_cnr(
boxes3D_cam_cnr_single, Tr)
# convert 8 corners(cam) to lidar boxes3D, note this is not ivertible because we LOSE ry!
boxes3D_lidar[ix, :] = lidar_cnr_to_3d(boxes3D_corners[ix, :],
lwh[ix, :])
# convert 8 corners(lidar) to lidar bird view
boxes_bv[ix, :] = lidar_3d_to_bv(boxes3D_lidar[ix, :])
# boxes3D_corners[ix, :] = lidar_to_corners_single(boxes3D_lidar[ix, :])
gt_classes[ix] = cls
overlaps[ix, cls] = 1.0
rys.resize(ix + 1)
lwh.resize(ix + 1, 3)
translation.resize(ix + 1, 3)
alphas.resize(ix + 1)
boxes.resize(ix + 1, 4)
boxes_bv.resize(ix + 1, 4)
boxes3D.resize(ix + 1, 6)
boxes3D_lidar.resize(ix + 1, 6)
boxes3D_cam_cnr.resize(ix + 1, 24)
boxes3D_corners.resize(ix + 1, 24)
boxes_bv_corners = lidar_cnr_to_bv_cnr(boxes3D_corners)
gt_classes.resize(ix + 1)
# print(self.num_classes)
overlaps.resize(ix + 1, self.num_classes)
diff_level.resize(ix + 1)
# if index == '000142':
# print(index)
# print(overlaps)
overlaps = scipy.sparse.csr_matrix(overlaps)
# if index == '000142':
# print(overlaps)
#if ix>=0:
# print index
return {
'ry': rys,
'lwh': lwh,
'boxes': boxes, #xy box in image
#'boxes_bv' : boxes_bv, #xy box in bird view
'boxes_3D_cam': boxes3D, #[xyz_center, lwh] in 3D, cam frame
#'boxes_3D' : boxes3D_lidar, #[xyz_center, lwh] in 3D, absolute
'boxes3D_cam_corners':
boxes3D_cam_cnr, #8 corners of box in 3D, cam frame
#'boxes_corners' : boxes3D_corners, #8 corners of box in 3D
#'boxes_bv_corners' : boxes_bv_corners, #4 corners of box in bird view
'gt_classes': gt_classes, #classes
'gt_overlaps': overlaps, #default 1, changed later
'xyz': translation,
'alphas': alphas,
'diff_level': diff_level,
'flipped': False
}
