def load_camera_config(calib_filepath, camera_list):
"""
In the calib file `calib_cam_to_cam.txt` the intrinsics of a camera can be extracted from
its projective matrix (3x4 matrix)
Parameters
----------
calib_filepath: str
The path to the calibration .json file
camera_list: list
list of camera for which to load the parameters. Must be in Argoverse's CAMERA_LIST
Returns
-------
dict
configs[camera_name] contains the a argoverse-api CameraConfig with the attributes:
extrinsic: extrinsic matrix
intrinsic: intrinsic matrix
img_width: image width
img_height: image height
distortion_coeffs: distortion coefficients
"""
with open(calib_filepath, "r") as f:
calib = json.load(f)
configs = {}
for camera in camera_list:
cam_config = get_calibration_config(calib, camera)
configs[camera] = cam_config
return configs
def load_camera_calibration(self, log_id: str, camera_name: str) -> None:
"""Load extrinsics and intrinsics from disk."""
calib_data = self._dl.get_log_calibration_data(log_id)
cam_config = get_calibration_config(calib_data, camera_name)
self._K = cam_config.intrinsic[:3, :3]
# square pixels, so fx and fy should be (nearly) identical
assert np.isclose(self._K[0, 0], self._K[1, 1], atol=0.1)
self._camera_SE3_egovehicle = SE3(
rotation=cam_config.extrinsic[:3, :3],
translation=cam_config.extrinsic[:3, 3])
self._egovehicle_SE3_camera = self._camera_SE3_egovehicle.inverse()
def dump_clipped_3d_cuboids_to_images(
log_ids: Sequence[str],
max_num_images_to_render: int,
data_dir: str,
experiment_prefix: str,
motion_compensate: bool = True,
) -> List[str]:
"""
We bring the 3D points into each camera coordinate system, and do the clipping there in 3D.
Args:
log_ids: A list of log IDs
max_num_images_to_render: maximum numbers of images to render.
data_dir: path to dataset with the latest data
experiment_prefix: Output directory
motion_compensate: Whether to motion compensate when projecting
Returns:
saved_img_fpaths
"""
saved_img_fpaths = []
dl = SimpleArgoverseTrackingDataLoader(data_dir=data_dir,
labels_dir=data_dir)
avm = ArgoverseMap()
for log_id in log_ids:
save_dir = f"{experiment_prefix}_{log_id}"
if not Path(save_dir).exists():
os.makedirs(save_dir)
city_name = dl.get_city_name(log_id)
log_calib_data = dl.get_log_calibration_data(log_id)
flag_done = False
for cam_idx, camera_name in enumerate(RING_CAMERA_LIST +
STEREO_CAMERA_LIST):
cam_im_fpaths = dl.get_ordered_log_cam_fpaths(log_id, camera_name)
for i, im_fpath in enumerate(cam_im_fpaths):
if i % 50 == 0:
logging.info("\tOn file %s of camera %s of %s", i,
camera_name, log_id)
cam_timestamp = Path(im_fpath).stem.split("_")[-1]
cam_timestamp = int(cam_timestamp)
# load PLY file path, e.g. 'PC_315978406032859416.ply'
ply_fpath = dl.get_closest_lidar_fpath(log_id, cam_timestamp)
if ply_fpath is None:
continue
lidar_pts = load_ply(ply_fpath)
save_img_fpath = f"{save_dir}/{camera_name}_{cam_timestamp}.jpg"
if Path(save_img_fpath).exists():
saved_img_fpaths += [save_img_fpath]
if max_num_images_to_render != -1 and len(
saved_img_fpaths) > max_num_images_to_render:
flag_done = True
break
continue
city_to_egovehicle_se3 = dl.get_city_to_egovehicle_se3(
log_id, cam_timestamp)
if city_to_egovehicle_se3 is None:
continue
lidar_timestamp = Path(ply_fpath).stem.split("_")[-1]
lidar_timestamp = int(lidar_timestamp)
labels = dl.get_labels_at_lidar_timestamp(
log_id, lidar_timestamp)
if labels is None:
logging.info("\tLabels missing at t=%s", lidar_timestamp)
continue
# Swap channel order as OpenCV expects it -- BGR not RGB
# must make a copy to make memory contiguous
img = imageio.imread(im_fpath)[:, :, ::-1].copy()
camera_config = get_calibration_config(log_calib_data,
camera_name)
planes = generate_frustum_planes(
camera_config.intrinsic.copy(), camera_name)
img = plot_lane_centerlines_in_img(
lidar_pts,
city_to_egovehicle_se3,
img,
city_name,
avm,
camera_config,
planes,
)
for label_idx, label in enumerate(labels):
obj_rec = json_label_dict_to_obj_record(label)
if obj_rec.occlusion == 100:
continue
cuboid_vertices = obj_rec.as_3d_bbox()
points_h = point_cloud_to_homogeneous(cuboid_vertices).T
if motion_compensate:
(
uv,
uv_cam,
valid_pts_bool,
K,
) = project_lidar_to_img_motion_compensated(
points_h, # these are recorded at lidar_time
copy.deepcopy(log_calib_data),
camera_name,
cam_timestamp,
lidar_timestamp,
data_dir,
log_id,
return_K=True,
)
else:
# project_lidar_to_img
(
uv,
uv_cam,
valid_pts_bool,
camera_config,
) = project_lidar_to_undistorted_img(
points_h, copy.deepcopy(log_calib_data),
camera_name)
if valid_pts_bool.sum() == 0:
continue
img = obj_rec.render_clip_frustum_cv2(
img,
uv_cam.T[:, :3],
planes.copy(),
copy.deepcopy(camera_config),
)
cv2.imwrite(save_img_fpath, img)
saved_img_fpaths += [save_img_fpath]
if max_num_images_to_render != -1 and len(
saved_img_fpaths) > max_num_images_to_render:
flag_done = True
break
if flag_done:
break
category_subdir = "amodal_labels"
if not Path(f"{experiment_prefix}_{category_subdir}").exists():
os.makedirs(f"{experiment_prefix}_{category_subdir}")
for cam_idx, camera_name in enumerate(RING_CAMERA_LIST +
STEREO_CAMERA_LIST):
# Write the cuboid video -- could also write w/ fps=20,30,40
if "stereo" in camera_name:
fps = 5
else:
fps = 30
img_wildcard = f"{save_dir}/{camera_name}_%*.jpg"
output_fpath = f"{experiment_prefix}_{category_subdir}/{log_id}_{camera_name}_{fps}fps.mp4"
write_nonsequential_idx_video(img_wildcard, output_fpath, fps)
return saved_img_fpaths
def get_log_camera_config(self, log_id: str,
camera_name: str) -> CameraConfig:
"""Return an object containing camera extrinsics, intrinsics, and image dimensions."""
log_calib_data = self.get_log_calibration_data(log_id)
camera_config = get_calibration_config(log_calib_data, camera_name)
return camera_config