def show_image_with_boxes(img: np.ndarray,
objects: Iterable[ObjectLabelRecord],
calib: Calibration) -> np.ndarray:
"""Show image with 2D bounding boxes."""
img1 = np.copy(img)
h, w = np.shape(img1)[0:2]
planes = generate_frustum_planes(calib.K, calib.camera)
assert planes is not None
for obj in objects:
if obj.occlusion == 100:
continue
box3d_pts_3d = obj.as_3d_bbox()
uv_cam = calib.project_ego_to_cam(box3d_pts_3d)
img1 = obj.render_clip_frustum_cv2(
img1,
uv_cam[:, :3],
planes.copy(),
copy.deepcopy(calib.camera_config),
linewidth=3,
)
return img1
def test_generate_frustum_planes_stereo() -> None:
"""Test generate_frustum_planes() for a stereo camera.
Skew is 0.0.
"""
near_clip_dist = 3.56 # arbitrary value
K = np.eye(3)
# Set "focal_length_x_px_"
K[0, 0] = 3666.534329132812
# Set "focal_length_y_px_"
K[1, 1] = 3673.5030423482513
# Set "focal_center_x_px_"
K[0, 2] = 1235.0158218941356
# Set "focal_center_y_px_"
K[1, 2] = 1008.4536901420888
camera_name = "stereo_front_left"
img_height = 2056
img_width = 2464
planes = generate_frustum_planes(K,
camera_name,
near_clip_dist=near_clip_dist)
if planes is None:
assert False
left_plane, right_plane, near_plane, low_plane, top_plane = planes
fx = K[0, 0]
left_plane_gt = np.array([fx, 0.0, img_width / 2.0, 0.0])
right_plane_gt = np.array([-fx, 0.0, img_width / 2.0, 0.0])
near_plane_gt = np.array([0.0, 0.0, 1.0, -near_clip_dist])
low_plane_gt = np.array([0.0, -fx, img_height / 2.0, 0.0])
top_plane_gt = np.array([0.0, fx, img_height / 2.0, 0.0])
assert np.allclose(left_plane,
left_plane_gt / np.linalg.norm(left_plane_gt))
assert np.allclose(right_plane,
right_plane_gt / np.linalg.norm(right_plane_gt))
assert np.allclose(low_plane, low_plane_gt / np.linalg.norm(low_plane_gt))
assert np.allclose(top_plane, top_plane_gt / np.linalg.norm(top_plane_gt))
assert np.allclose(near_plane, near_plane_gt)
def test_generate_frustum_planes_ring_cam() -> None:
"""Test generate_frustum_planes() for a ring camera.
Skew is 0.0.
"""
near_clip_dist = 6.89 # arbitrary value
K = np.eye(3)
# Set "focal_length_x_px_"
K[0, 0] = 1402.4993697398709
# Set "focal_length_y_px_"
K[1, 1] = 1405.1207294310225
# Set "focal_center_x_px_"
K[0, 2] = 957.8471720086527
# Set "focal_center_y_px_"
K[1, 2] = 600.442948946496
camera_name = "ring_front_right"
img_height = 1200
img_width = 1920
planes = generate_frustum_planes(K,
camera_name,
near_clip_dist=near_clip_dist)
if planes is None:
assert False
left_plane, right_plane, near_plane, low_plane, top_plane = planes
fx = K[0, 0]
left_plane_gt = np.array([fx, 0.0, img_width / 2.0, 0.0])
right_plane_gt = np.array([-fx, 0.0, img_width / 2.0, 0.0])
near_plane_gt = np.array([0.0, 0.0, 1.0, -near_clip_dist])
low_plane_gt = np.array([0.0, -fx, img_height / 2.0, 0.0])
top_plane_gt = np.array([0.0, fx, img_height / 2.0, 0.0])
assert np.allclose(left_plane,
left_plane_gt / np.linalg.norm(left_plane_gt))
assert np.allclose(right_plane,
right_plane_gt / np.linalg.norm(right_plane_gt))
assert np.allclose(low_plane, low_plane_gt / np.linalg.norm(low_plane_gt))
assert np.allclose(top_plane, top_plane_gt / np.linalg.norm(top_plane_gt))
assert np.allclose(near_plane, near_plane_gt)
plot_frustum_planes_and_normals,
populate_frustum_voxels,
)
except ImportError:
MAYAVI_MISSING = True
else:
MAYAVI_MISSING = False
_TEST_DIR = pathlib.Path(__file__).parent.parent / "tests"
if not MAYAVI_MISSING:
calib = load_calib(
os.fspath(_TEST_DIR /
"test_data/tracking/1/vehicle_calibration_info.json")
)["ring_front_center"]
planes = generate_frustum_planes(calib.K, calib.camera)
assert planes is not None
skip_if_mayavi_missing = pytest.mark.skipif(
MAYAVI_MISSING,
reason=
"Could not test functionality that depends on mayavi because mayavi is missing.",
)
@skip_if_mayavi_missing # type: ignore
def test_plot_frustum_planes_and_normals() -> None:
""""""
assert planes is not None
plot_frustum_planes_and_normals(planes,
def dump_clipped_3d_cuboids_to_images(
log_ids: Sequence[str],
max_num_images_to_render: int,
data_dir: str,
experiment_prefix: str,
motion_compensate: bool = True,
) -> List[str]:
"""
We bring the 3D points into each camera coordinate system, and do the clipping there in 3D.
Args:
log_ids: A list of log IDs
max_num_images_to_render: maximum numbers of images to render.
data_dir: path to dataset with the latest data
experiment_prefix: Output directory
motion_compensate: Whether to motion compensate when projecting
Returns:
saved_img_fpaths
"""
saved_img_fpaths = []
dl = SimpleArgoverseTrackingDataLoader(data_dir=data_dir,
labels_dir=data_dir)
avm = ArgoverseMap()
for log_id in log_ids:
save_dir = f"{experiment_prefix}_{log_id}"
if not Path(save_dir).exists():
os.makedirs(save_dir)
city_name = dl.get_city_name(log_id)
log_calib_data = dl.get_log_calibration_data(log_id)
flag_done = False
for cam_idx, camera_name in enumerate(RING_CAMERA_LIST +
STEREO_CAMERA_LIST):
cam_im_fpaths = dl.get_ordered_log_cam_fpaths(log_id, camera_name)
for i, im_fpath in enumerate(cam_im_fpaths):
if i % 50 == 0:
logging.info("\tOn file %s of camera %s of %s", i,
camera_name, log_id)
cam_timestamp = Path(im_fpath).stem.split("_")[-1]
cam_timestamp = int(cam_timestamp)
# load PLY file path, e.g. 'PC_315978406032859416.ply'
ply_fpath = dl.get_closest_lidar_fpath(log_id, cam_timestamp)
if ply_fpath is None:
continue
lidar_pts = load_ply(ply_fpath)
save_img_fpath = f"{save_dir}/{camera_name}_{cam_timestamp}.jpg"
if Path(save_img_fpath).exists():
saved_img_fpaths += [save_img_fpath]
if max_num_images_to_render != -1 and len(
saved_img_fpaths) > max_num_images_to_render:
flag_done = True
break
continue
city_to_egovehicle_se3 = dl.get_city_to_egovehicle_se3(
log_id, cam_timestamp)
if city_to_egovehicle_se3 is None:
continue
lidar_timestamp = Path(ply_fpath).stem.split("_")[-1]
lidar_timestamp = int(lidar_timestamp)
labels = dl.get_labels_at_lidar_timestamp(
log_id, lidar_timestamp)
if labels is None:
logging.info("\tLabels missing at t=%s", lidar_timestamp)
continue
# Swap channel order as OpenCV expects it -- BGR not RGB
# must make a copy to make memory contiguous
img = imageio.imread(im_fpath)[:, :, ::-1].copy()
camera_config = get_calibration_config(log_calib_data,
camera_name)
planes = generate_frustum_planes(
camera_config.intrinsic.copy(), camera_name)
img = plot_lane_centerlines_in_img(
lidar_pts,
city_to_egovehicle_se3,
img,
city_name,
avm,
camera_config,
planes,
)
for label_idx, label in enumerate(labels):
obj_rec = json_label_dict_to_obj_record(label)
if obj_rec.occlusion == 100:
continue
cuboid_vertices = obj_rec.as_3d_bbox()
points_h = point_cloud_to_homogeneous(cuboid_vertices).T
if motion_compensate:
(
uv,
uv_cam,
valid_pts_bool,
K,
) = project_lidar_to_img_motion_compensated(
points_h, # these are recorded at lidar_time
copy.deepcopy(log_calib_data),
camera_name,
cam_timestamp,
lidar_timestamp,
data_dir,
log_id,
return_K=True,
)
else:
# project_lidar_to_img
(
uv,
uv_cam,
valid_pts_bool,
camera_config,
) = project_lidar_to_undistorted_img(
points_h, copy.deepcopy(log_calib_data),
camera_name)
if valid_pts_bool.sum() == 0:
continue
img = obj_rec.render_clip_frustum_cv2(
img,
uv_cam.T[:, :3],
planes.copy(),
copy.deepcopy(camera_config),
)
cv2.imwrite(save_img_fpath, img)
saved_img_fpaths += [save_img_fpath]
if max_num_images_to_render != -1 and len(
saved_img_fpaths) > max_num_images_to_render:
flag_done = True
break
if flag_done:
break
category_subdir = "amodal_labels"
if not Path(f"{experiment_prefix}_{category_subdir}").exists():
os.makedirs(f"{experiment_prefix}_{category_subdir}")
for cam_idx, camera_name in enumerate(RING_CAMERA_LIST +
STEREO_CAMERA_LIST):
# Write the cuboid video -- could also write w/ fps=20,30,40
if "stereo" in camera_name:
fps = 5
else:
fps = 30
img_wildcard = f"{save_dir}/{camera_name}_%*.jpg"
output_fpath = f"{experiment_prefix}_{category_subdir}/{log_id}_{camera_name}_{fps}fps.mp4"
write_nonsequential_idx_video(img_wildcard, output_fpath, fps)
return saved_img_fpaths
argoverse_loader = ArgoverseTrackingLoader(tracking_dataset_dir)
log_id = argoverse_loader.log_list[log_index]
argoverse_data = argoverse_loader[log_index]
city_name = argoverse_data.city_name
use_existing_files = True
domv = DatasetOnMapVisualizer(dataset_dir,
experiment_prefix,
use_existing_files=use_existing_files,
log_id=argoverse_data.current_log)
# %%
camera = 'ring_front_center'
# camera = 'ring_front_left'
calib = argoverse_data.get_calibration(camera)
planes = generate_frustum_planes(calib.camera_config.intrinsic.copy(), camera)
domv = DatasetOnMapVisualizer(tracking_dataset_dir,
experiment_prefix,
use_existing_files=True,
log_id=argoverse_data.current_log)
# %%
def get_plot(map_range, axis_off=True, for_network=False):
my_dpi = 96.0 # screen constant, check here https://www.infobyip.com/detectmonitordpi.php
if for_network:
fig = plt.figure(figsize=(72 / my_dpi, 72 / my_dpi), dpi=my_dpi)
else:
fig = plt.figure(figsize=(496 / my_dpi, 496 / my_dpi), dpi=my_dpi)
