def keep(self, label: KittiLabel, pc: np.array,
calib: KittiCalib) -> (KittiLabel, np.array):
# copy pc & label
pc_ = pc.copy()
label_ = KittiLabel()
label_.data = []
for obj in label.data:
label_.data.append(KittiObj(str(obj)))
res_label = KittiLabel()
for obj in label_.data:
res_label.add_obj(obj)
res_label.current_frame = "Cam2"
return res_label, pc_
def validate(data_dir: str, idx: str, label_Fcam: KittiLabel, save_dir: str):
'''
To validate the function of this script
inputs:
data_dir: dir of Carla data
idx: "000000.txt"
label_Fcam: result after converting
save_dir: dir for saving visulizaition figures
Note: the following code is dangerous, and should not be used in other place.
'''
from det3.dataloader.carladata import CarlaData
from det3.visualizer.vis import FVImage, BEVImage
from det3.dataloader.kittidata import KittiObj, KittiLabel, KittiCalib
from PIL import Image
tag = idx.split(".")[0]
pc, label, calib = CarlaData(data_dir, tag).read_data()
bevimg = BEVImage(x_range=(-100, 100),
y_range=(-50, 50),
grid_size=(0.05, 0.05))
bevimg.from_lidar(np.vstack([pc['velo_top']]), scale=1)
fvimg = FVImage()
fvimg.from_lidar(calib, np.vstack([pc['velo_top']]))
for obj in label.read_label_file().data:
if obj.type == 'Car':
bevimg.draw_box(obj, calib, bool_gt=True)
fvimg.draw_box(obj, calib, bool_gt=True)
kittilabel = KittiLabel()
kittilabel.data = []
kitticalib = KittiCalib(None)
kitticalib.P2 = np.array([[450, 0., 600, 0.], [0., 450, 180, 0.],
[0., 0., 1, 0.]])
kitticalib.R0_rect = np.eye(4)
kitticalib.Tr_velo_to_cam = np.zeros((4, 4))
kitticalib.Tr_velo_to_cam[0, 1] = -1
kitticalib.Tr_velo_to_cam[1, 2] = -1
kitticalib.Tr_velo_to_cam[2, 0] = 1
kitticalib.Tr_velo_to_cam[3, 3] = 1
kitticalib.data = []
for obj in label_Fcam.data:
kittilabel.data.append(KittiObj(str(obj)))
for obj in kittilabel.data:
if obj.type == 'Car':
fvimg.draw_box(obj, kitticalib, bool_gt=False)
bevimg.draw_box(obj, kitticalib, bool_gt=False)
bevimg_img = Image.fromarray(bevimg.data)
bevimg_img.save(os.path.join(save_dir, "{}_bev.png".format(tag)))
fvimg_img = Image.fromarray(fvimg.data)
fvimg_img.save(os.path.join(save_dir, "{}_fv.png".format(tag)))
def flip_pc(self, label: KittiLabel, pc: np.array,
calib: KittiCalib) -> (KittiLabel, np.array):
'''
flip point cloud along the y axis of the Kitti Lidar frame
inputs:
label: ground truth
pc: point cloud
calib:
'''
assert istype(label, "KittiLabel")
calib_is_iter = isinstance(calib, list)
# copy pc & label
pc_ = pc.copy()
label_ = KittiLabel()
label_.data = []
for obj in label.data:
label_.data.append(KittiObj(str(obj)))
# flip point cloud
pc_[:, 1] *= -1
# modify gt
if not calib_is_iter:
for obj in label_.data:
bottom_Fcam = np.array([obj.x, obj.y, obj.z]).reshape(1, -1)
bottom_Flidar = calib.leftcam2lidar(bottom_Fcam)
bottom_Flidar[0, 1] *= -1
bottom_Fcam = calib.lidar2leftcam(bottom_Flidar)
obj.x, obj.y, obj.z = bottom_Fcam.flatten()
obj.ry *= -1
else:
for obj, calib_ in zip(label_.data, calib):
bottom_Fcam = np.array([obj.x, obj.y, obj.z]).reshape(1, -1)
bottom_Flidar = calib_.leftcam2lidar(bottom_Fcam)
bottom_Flidar[0, 1] *= -1
bottom_Fcam = calib_.lidar2leftcam(bottom_Flidar)
obj.x, obj.y, obj.z = bottom_Fcam.flatten()
obj.ry *= -1
res_label = KittiLabel()
for obj in label_.data:
res_label.add_obj(obj)
res_label.current_frame = "Cam2"
return res_label, pc_
def main(is_val, data_dir, vis_dir, result_path):
dt_data = pickle.load(open(result_path, 'rb'))
save_dir = vis_dir
if is_val:
for dt in tqdm(dt_data):
idx = dt['metadata']['image_idx']
idx = "{:06d}".format(idx)
calib, img, label, pc = KittiData(root_dir=data_dir,
idx=idx).read_data()
label_est = KittiLabel()
label_est.data = []
num_dt = len(dt['name'])
for i in range(num_dt):
obj = KittiObj()
obj.type = dt['name'][i]
obj.truncated = dt['truncated'][i]
obj.occluded = dt['occluded'][i]
obj.alpha = dt['alpha'][i]
obj.bbox_l, obj.bbox_t, obj.bbox_r, obj.bbox_b = dt['bbox'][i]
obj.l, obj.h, obj.w = dt['dimensions'][i]
obj.x, obj.y, obj.z = dt['location'][i]
obj.ry = dt['rotation_y'][i]
obj.score = dt['score'][i]
label_est.data.append(obj)
fvimg = FVImage()
fvimg.from_image(img)
for obj in label.data:
if (obj.x > obj.z) or (-obj.x > obj.z):
continue
if obj.type in ['Car', 'Van']:
fvimg.draw_3dbox(obj, calib, bool_gt=True, width=3)
for obj in label_est.data:
if (obj.x > obj.z) or (-obj.x > obj.z):
continue
fvimg.draw_3dbox(obj,
calib,
bool_gt=False,
width=2,
c=(255, 255, int(255 * obj.score), 255))
fvimg_img = Image.fromarray(fvimg.data)
fvimg_img.save(os.path.join(save_dir, "fv_{}.png".format(idx)))
else:
pc_dir = os.path.join(data_dir, "velodyne_points", "data")
img2_dir = os.path.join(data_dir, "image_02", "data")
calib_dir = os.path.join(data_dir, "calib")
calib = read_calib(calib_dir)
for dt in tqdm(dt_data):
idx = dt['metadata']['image_idx']
idx = "{:010d}".format(idx)
pc = read_pc_from_bin(os.path.join(pc_dir, idx + ".bin"))
img = read_image(os.path.join(img2_dir, idx + ".png"))
label_est = KittiLabel()
label_est.data = []
num_dt = len(dt['name'])
for i in range(num_dt):
obj = KittiObj()
obj.type = dt['name'][i]
obj.truncated = dt['truncated'][i]
obj.occluded = dt['occluded'][i]
obj.alpha = dt['alpha'][i]
obj.bbox_l, obj.bbox_t, obj.bbox_r, obj.bbox_b = dt['bbox'][i]
obj.l, obj.h, obj.w = dt['dimensions'][i]
obj.x, obj.y, obj.z = dt['location'][i]
obj.ry = dt['rotation_y'][i]
obj.score = dt['score'][i]
label_est.data.append(obj)
fvimg = FVImage()
fvimg.from_image(img)
# for obj in label.data:
# if obj.type in ['Car', 'Van']:
# fvimg.draw_3dbox(obj, calib, bool_gt=True, width=3)
for obj in label_est.data:
if (obj.x > obj.z) or (-obj.x > obj.z):
continue
fvimg.draw_3dbox(obj,
calib,
bool_gt=False,
width=2,
c=(255, 255, int(obj.score * 255), 255))
fvimg_img = Image.fromarray(fvimg.data)
fvimg_img.save(os.path.join(save_dir, "fv_{}.png".format(idx)))
def rotate_obj(self, label: KittiLabel, pc: np.array, calib: KittiCalib,
dry_range: List[float]) -> (KittiLabel, np.array):
'''
rotate object along the z axis in the LiDAR frame
inputs:
label: gt
pc: [#pts, >= 3]
calib:
dry_range: [dry_min, dry_max] in radius
returns:
label_rot
pc_rot
Note: If the attemp times is over max_attemp, it will return the original copy
'''
assert istype(label, "KittiLabel")
dry_min, dry_max = dry_range
max_attemp = 10
num_attemp = 0
calib_is_iter = isinstance(calib, list)
while True:
num_attemp += 1
# copy pc & label
pc_ = pc.copy()
label_ = KittiLabel()
label_.data = []
for obj in label.data:
label_.data.append(KittiObj(str(obj)))
if num_attemp > max_attemp:
print(
"KittiAugmentor.rotate_obj: Warning: the attemp times is over {} times,"
"will return the original copy".format(max_attemp))
break
if not calib_is_iter:
for obj in label_.data:
# generate random number
dry = np.random.rand() * (dry_max - dry_min) + dry_min
# modify pc_
idx = obj.get_pts_idx(pc_[:, :3], calib)
bottom_Fcam = np.array([obj.x, obj.y,
obj.z]).reshape(1, -1)
bottom_Flidar = calib.leftcam2lidar(bottom_Fcam)
pc_[idx, :3] = apply_tr(pc_[idx, :3], -bottom_Flidar)
# obj.ry += dry is correspond to rotz(-dry)
# since obj is in cam frame
# pc_ is in LiDAR frame
pc_[idx, :3] = apply_R(pc_[idx, :3], rotz(-dry))
pc_[idx, :3] = apply_tr(pc_[idx, :3], bottom_Flidar)
# modify obj
obj.ry += dry
else:
for obj, calib_ in zip(label_.data, calib):
# generate random number
dry = np.random.rand() * (dry_max - dry_min) + dry_min
# modify pc_
idx = obj.get_pts_idx(pc_[:, :3], calib_)
bottom_Fcam = np.array([obj.x, obj.y,
obj.z]).reshape(1, -1)
bottom_Flidar = calib_.leftcam2lidar(bottom_Fcam)
pc_[idx, :3] = apply_tr(pc_[idx, :3], -bottom_Flidar)
# obj.ry += dry is correspond to rotz(-dry)
# since obj is in cam frame
# pc_ is in LiDAR frame
pc_[idx, :3] = apply_R(pc_[idx, :3], rotz(-dry))
pc_[idx, :3] = apply_tr(pc_[idx, :3], bottom_Flidar)
# modify obj
obj.ry += dry
if self.check_overlap(label_):
break
res_label = KittiLabel()
for obj in label_.data:
res_label.add_obj(obj)
res_label.current_frame = "Cam2"
return res_label, pc_
def tr_obj(self, label: KittiLabel, pc: np.array, calib: KittiCalib,
dx_range: List[float], dy_range: List[float],
dz_range: List[float]) -> (KittiLabel, np.array):
'''
translate object in the LiDAR frame
inputs:
label: gt
pc: [#pts, >= 3]
calib:
dx_range: [dx_min, dx_max] in LiDAR frame
dy_range: [dy_min, dy_max] in LiDAR frame
dz_range: [dz_min, dz_max] in LiDAR frame
returns:
label_tr
pc_tr
Note: If the attemp times is over max_attemp, it will return the original copy
'''
assert istype(label, "KittiLabel")
dx_min, dx_max = dx_range
dy_min, dy_max = dy_range
dz_min, dz_max = dz_range
max_attemp = 10
num_attemp = 0
calib_is_iter = isinstance(calib, list)
while True:
num_attemp += 1
# copy pc & label
pc_ = pc.copy()
label_ = KittiLabel()
label_.data = []
for obj in label.data:
label_.data.append(KittiObj(str(obj)))
if num_attemp > max_attemp:
print(
"KittiAugmentor.tr_obj: Warning: the attemp times is over {} times,"
"will return the original copy".format(max_attemp))
break
if not calib_is_iter:
for obj in label_.data:
# gennerate ramdom number
dx = np.random.rand() * (dx_max - dx_min) + dx_min
dy = np.random.rand() * (dy_max - dy_min) + dy_min
dz = np.random.rand() * (dz_max - dz_min) + dz_min
# modify pc_
idx = obj.get_pts_idx(pc_[:, :3], calib)
dtr = np.array([dx, dy, dz]).reshape(1, -1)
pc_[idx, :3] = apply_tr(pc_[idx, :3], dtr)
# modify obj
bottom_Fcam = np.array([obj.x, obj.y,
obj.z]).reshape(1, -1)
bottom_Flidar = calib.leftcam2lidar(bottom_Fcam)
bottom_Flidar = apply_tr(bottom_Flidar, dtr)
bottom_Fcam = calib.lidar2leftcam(bottom_Flidar)
obj.x, obj.y, obj.z = bottom_Fcam.flatten()
else:
for obj, calib_ in zip(label_.data, calib):
# gennerate ramdom number
dx = np.random.rand() * (dx_max - dx_min) + dx_min
dy = np.random.rand() * (dy_max - dy_min) + dy_min
dz = np.random.rand() * (dz_max - dz_min) + dz_min
# modify pc_
idx = obj.get_pts_idx(pc_[:, :3], calib_)
dtr = np.array([dx, dy, dz]).reshape(1, -1)
pc_[idx, :3] = apply_tr(pc_[idx, :3], dtr)
# modify obj
bottom_Fcam = np.array([obj.x, obj.y,
obj.z]).reshape(1, -1)
bottom_Flidar = calib_.leftcam2lidar(bottom_Fcam)
bottom_Flidar = apply_tr(bottom_Flidar, dtr)
bottom_Fcam = calib_.lidar2leftcam(bottom_Flidar)
obj.x, obj.y, obj.z = bottom_Fcam.flatten()
if self.check_overlap(label_):
break
res_label = KittiLabel()
for obj in label_.data:
res_label.add_obj(obj)
res_label.current_frame = "Cam2"
return res_label, pc_
