def test_ffmpeg_nonseq_frame_vid_smokescreen(): """ """ img_wildcard = "imgs_%*.jpg" output_fpath = "out.mp4" fps = 10 write_nonsequential_idx_video(img_wildcard, output_fpath, fps)
def plot_log_one_at_a_time(self, log_id="", idx=-1, save_video=True, city=""): """ Playback a log in the static context of a map. In the far left frame, we show the car moving in the map. In the middle frame, we show the car's LiDAR returns (in the map frame). In the far right frame, we show the front camera's RGB image. """ avm = ArgoverseMap() for city_name, trajs in self.per_city_traj_dict.items(): if city != "": if city != city_name: continue if city_name not in ["PIT", "MIA"]: logger.info("Unknown city") continue log_ids = [] logger.info(f"{city_name} has {len(trajs)} tracks") if log_id == "": # first iterate over the instance axis for traj_idx, (traj, log_id) in enumerate(trajs): log_ids += [log_id] else: log_ids = [log_id] # eliminate the duplicates for log_id in set(log_ids): logger.info(f"Log: {log_id}") ply_fpaths = sorted( glob.glob(f"{self.dataset_dir}/{log_id}/lidar/PC_*.ply")) # then iterate over the time axis for i, ply_fpath in enumerate(ply_fpaths): if idx != -1: if i != idx: continue if i % 500 == 0: logger.info(f"\tOn file {i} of {log_id}") lidar_timestamp = ply_fpath.split("/")[-1].split( ".")[0].split("_")[-1] lidar_timestamp = int(lidar_timestamp) print("Lidar timestamp = ", lidar_timestamp) # added by Yike print("Log Egopose Dict = ", self.log_egopose_dict) if lidar_timestamp not in self.log_egopose_dict[log_id]: all_available_timestamps = sorted( self.log_egopose_dict[log_id].keys()) diff = (all_available_timestamps[0] - lidar_timestamp) / 1e9 logger.info( f"{diff:.2f} sec before first labeled sweep") continue logger.info(f"\tt={lidar_timestamp}") if self.plot_lidar_bev: fig = plt.figure(figsize=(15, 15)) plt.title( f"Log {log_id} @t={lidar_timestamp} in {city_name}" ) plt.axis("off") # ax_map = fig.add_subplot(131) ax_3d = fig.add_subplot(111) # ax_rgb = fig.add_subplot(133) plt.ion() # added by Yike # need the ego-track here pose_city_to_ego = self.log_egopose_dict[log_id][ lidar_timestamp] xcenter = pose_city_to_ego["translation"][0] ycenter = pose_city_to_ego["translation"][1] ego_center_xyz = np.array(pose_city_to_ego["translation"]) city_to_egovehicle_se3 = SE3( rotation=pose_city_to_ego["rotation"], translation=ego_center_xyz) if self.plot_lidar_bev: xmin = xcenter - 80 # 150 xmax = xcenter + 80 # 150 ymin = ycenter - 80 # 150 ymax = ycenter + 80 # 150 # ax_map.scatter(xcenter, ycenter, 200, color="g", marker=".", zorder=2) # ax_map.set_xlim([xmin, xmax]) # ax_map.set_ylim([ymin, ymax]) local_lane_polygons = avm.find_local_lane_polygons( [xmin, xmax, ymin, ymax], city_name) local_das = avm.find_local_driveable_areas( [xmin, xmax, ymin, ymax], city_name) lidar_pts = load_ply(ply_fpath) if self.plot_lidar_in_img: draw_ground_pts_in_image( self.sdb, copy.deepcopy(lidar_pts), city_to_egovehicle_se3, avm, log_id, lidar_timestamp, city_name, self.dataset_dir, self.experiment_prefix, plot_ground=True, ) if self.plot_lidar_bev: driveable_area_pts = copy.deepcopy(lidar_pts) driveable_area_pts = city_to_egovehicle_se3.transform_point_cloud( driveable_area_pts) # put into city coords driveable_area_pts = avm.remove_non_driveable_area_points( driveable_area_pts, city_name) driveable_area_pts = avm.remove_ground_surface( driveable_area_pts, city_name) driveable_area_pts = city_to_egovehicle_se3.inverse_transform_point_cloud( driveable_area_pts ) # put back into ego-vehicle coords self.render_bev_labels_mpl( city_name, ax_3d, "ego_axis", lidar_pts, copy.deepcopy(local_lane_polygons), copy.deepcopy(local_das), log_id, lidar_timestamp, city_to_egovehicle_se3, avm, ) fig.tight_layout() if not Path( f"{self.experiment_prefix}_per_log_viz/{log_id}" ).exists(): os.makedirs( f"{self.experiment_prefix}_per_log_viz/{log_id}" ) plt.savefig( f"{self.experiment_prefix}_per_log_viz/{log_id}/{city_name}_{log_id}_{lidar_timestamp}.png", dpi=400, ) # plt.show() # plt.close("all") # after all frames are processed, write video with saved images if save_video: if self.plot_lidar_bev: fps = 10 img_wildcard = f"{self.experiment_prefix}_per_log_viz/{log_id}/{city_name}_{log_id}_%*.png" output_fpath = f"{self.experiment_prefix}_per_log_viz/{log_id}_lidar_roi_nonground.mp4" write_nonsequential_idx_video(img_wildcard, output_fpath, fps) if self.plot_lidar_in_img: for camera_name in RING_CAMERA_LIST + STEREO_CAMERA_LIST: image_prefix = ( f"{self.experiment_prefix}_per_log_viz/{log_id}/{camera_name}/{camera_name}_%d.jpg" ) output_prefix = f"{self.experiment_prefix}_per_log_viz/{log_id}_{camera_name}" write_video(image_prefix, output_prefix)
def test_ffmpeg_nonseq_frame_vid_smokescreen() -> None: """Test writing video with non-consecutive, nanosecond, integer timestamps in `ffmpeg`.""" img_wildcard = "imgs_%*.jpg" output_fpath = "out.mp4" fps = 10 write_nonsequential_idx_video(img_wildcard, output_fpath, fps)
def dump_clipped_3d_cuboids_to_images( log_ids: Sequence[str], max_num_images_to_render: int, data_dir: str, experiment_prefix: str, motion_compensate: bool = True, ) -> List[str]: """ We bring the 3D points into each camera coordinate system, and do the clipping there in 3D. Args: log_ids: A list of log IDs max_num_images_to_render: maximum numbers of images to render. data_dir: path to dataset with the latest data experiment_prefix: Output directory motion_compensate: Whether to motion compensate when projecting Returns: saved_img_fpaths """ saved_img_fpaths = [] dl = SimpleArgoverseTrackingDataLoader(data_dir=data_dir, labels_dir=data_dir) avm = ArgoverseMap() for log_id in log_ids: save_dir = f"{experiment_prefix}_{log_id}" if not Path(save_dir).exists(): os.makedirs(save_dir) city_name = dl.get_city_name(log_id) log_calib_data = dl.get_log_calibration_data(log_id) flag_done = False for cam_idx, camera_name in enumerate(RING_CAMERA_LIST + STEREO_CAMERA_LIST): cam_im_fpaths = dl.get_ordered_log_cam_fpaths(log_id, camera_name) for i, im_fpath in enumerate(cam_im_fpaths): if i % 50 == 0: logging.info("\tOn file %s of camera %s of %s", i, camera_name, log_id) cam_timestamp = Path(im_fpath).stem.split("_")[-1] cam_timestamp = int(cam_timestamp) # load PLY file path, e.g. 'PC_315978406032859416.ply' ply_fpath = dl.get_closest_lidar_fpath(log_id, cam_timestamp) if ply_fpath is None: continue lidar_pts = load_ply(ply_fpath) save_img_fpath = f"{save_dir}/{camera_name}_{cam_timestamp}.jpg" if Path(save_img_fpath).exists(): saved_img_fpaths += [save_img_fpath] if max_num_images_to_render != -1 and len( saved_img_fpaths) > max_num_images_to_render: flag_done = True break continue city_to_egovehicle_se3 = dl.get_city_to_egovehicle_se3( log_id, cam_timestamp) if city_to_egovehicle_se3 is None: continue lidar_timestamp = Path(ply_fpath).stem.split("_")[-1] lidar_timestamp = int(lidar_timestamp) labels = dl.get_labels_at_lidar_timestamp( log_id, lidar_timestamp) if labels is None: logging.info("\tLabels missing at t=%s", lidar_timestamp) continue # Swap channel order as OpenCV expects it -- BGR not RGB # must make a copy to make memory contiguous img = imageio.imread(im_fpath)[:, :, ::-1].copy() camera_config = get_calibration_config(log_calib_data, camera_name) planes = generate_frustum_planes( camera_config.intrinsic.copy(), camera_name) img = plot_lane_centerlines_in_img( lidar_pts, city_to_egovehicle_se3, img, city_name, avm, camera_config, planes, ) for label_idx, label in enumerate(labels): obj_rec = json_label_dict_to_obj_record(label) if obj_rec.occlusion == 100: continue cuboid_vertices = obj_rec.as_3d_bbox() points_h = point_cloud_to_homogeneous(cuboid_vertices).T if motion_compensate: ( uv, uv_cam, valid_pts_bool, K, ) = project_lidar_to_img_motion_compensated( points_h, # these are recorded at lidar_time copy.deepcopy(log_calib_data), camera_name, cam_timestamp, lidar_timestamp, data_dir, log_id, return_K=True, ) else: # project_lidar_to_img ( uv, uv_cam, valid_pts_bool, camera_config, ) = project_lidar_to_undistorted_img( points_h, copy.deepcopy(log_calib_data), camera_name) if valid_pts_bool.sum() == 0: continue img = obj_rec.render_clip_frustum_cv2( img, uv_cam.T[:, :3], planes.copy(), copy.deepcopy(camera_config), ) cv2.imwrite(save_img_fpath, img) saved_img_fpaths += [save_img_fpath] if max_num_images_to_render != -1 and len( saved_img_fpaths) > max_num_images_to_render: flag_done = True break if flag_done: break category_subdir = "amodal_labels" if not Path(f"{experiment_prefix}_{category_subdir}").exists(): os.makedirs(f"{experiment_prefix}_{category_subdir}") for cam_idx, camera_name in enumerate(RING_CAMERA_LIST + STEREO_CAMERA_LIST): # Write the cuboid video -- could also write w/ fps=20,30,40 if "stereo" in camera_name: fps = 5 else: fps = 30 img_wildcard = f"{save_dir}/{camera_name}_%*.jpg" output_fpath = f"{experiment_prefix}_{category_subdir}/{log_id}_{camera_name}_{fps}fps.mp4" write_nonsequential_idx_video(img_wildcard, output_fpath, fps) return saved_img_fpaths