def convert_detection_to_kitti_annos(self, detection): class_names = self._class_names det_image_idxes = [k for k in detection.keys()] gt_image_idxes = [ str(info["image"]["image_idx"]) for info in self._kitti_infos ] # print(f"det_image_idxes: {det_image_idxes[:10]}") # print(f"gt_image_idxes: {gt_image_idxes[:10]}") annos = [] # for i in range(len(detection)): for det_idx in gt_image_idxes: det = detection[det_idx] info = self._kitti_infos[gt_image_idxes.index(det_idx)] # info = self._kitti_infos[i] calib = info["calib"] rect = calib["R0_rect"] Trv2c = calib["Tr_velo_to_cam"] P2 = calib["P2"] final_box_preds = det["box3d_lidar"].detach().cpu().numpy() label_preds = det["label_preds"].detach().cpu().numpy() scores = det["scores"].detach().cpu().numpy() anno = get_start_result_anno() num_example = 0 if final_box_preds.shape[0] != 0: final_box_preds[:, -1] = box_np_ops.limit_period( final_box_preds[:, -1], offset=0.5, period=np.pi * 2, ) final_box_preds[:, 2] -= final_box_preds[:, 5] / 2 # aim: x, y, z, w, l, h, r -> -y, -z, x, h, w, l, r # (x, y, z, w, l, h r) in lidar -> (x', y', z', l, h, w, r) in camera box3d_camera = box_np_ops.box_lidar_to_camera( final_box_preds, rect, Trv2c) camera_box_origin = [0.5, 1.0, 0.5] box_corners = box_np_ops.center_to_corner_box3d( box3d_camera[:, :3], box3d_camera[:, 3:6], box3d_camera[:, 6], camera_box_origin, axis=1, ) box_corners_in_image = box_np_ops.project_to_image( box_corners, P2) # box_corners_in_image: [N, 8, 2] minxy = np.min(box_corners_in_image, axis=1) maxxy = np.max(box_corners_in_image, axis=1) bbox = np.concatenate([minxy, maxxy], axis=1) for j in range(box3d_camera.shape[0]): image_shape = info["image"]["image_shape"] if bbox[j, 0] > image_shape[1] or bbox[j, 1] > image_shape[0]: continue if bbox[j, 2] < 0 or bbox[j, 3] < 0: continue bbox[j, 2:] = np.minimum(bbox[j, 2:], image_shape[::-1]) bbox[j, :2] = np.maximum(bbox[j, :2], [0, 0]) anno["bbox"].append(bbox[j]) anno["alpha"].append(-np.arctan2(-final_box_preds[j, 1], final_box_preds[j, 0]) + box3d_camera[j, 6]) # anno["dimensions"].append(box3d_camera[j, [4, 5, 3]]) anno["dimensions"].append(box3d_camera[j, 3:6]) anno["location"].append(box3d_camera[j, :3]) anno["rotation_y"].append(box3d_camera[j, 6]) anno["name"].append(class_names[int(label_preds[j])]) anno["truncated"].append(0.0) anno["occluded"].append(0) anno["score"].append(scores[j]) num_example += 1 if num_example != 0: anno = {n: np.stack(v) for n, v in anno.items()} annos.append(anno) else: annos.append(empty_result_anno()) num_example = annos[-1]["name"].shape[0] annos[-1]["metadata"] = det["metadata"] return annos
def convert_detection_to_kitti_annos(self, detection): class_names = self._class_names det_image_idxes = [k for k in detection.keys()] gt_image_idxes = [ str(info["image"]["image_idx"]) for info in self._kitti_infos ] annos = [] for det_idx in gt_image_idxes: det = detection[det_idx] dim = det['box3d_lidar'][:, 3:6] l, w, h = dim[:, 0:1], dim[:, 1:2], dim[:, 2:3] det['box3d_lidar'][:, 2] = (det['box3d_lidar'][:, 2].T + (h / 2).T).reshape(-1) det['box3d_lidar'][:, -1] = det['box3d_lidar'][:, -1] * -1 info = self._kitti_infos[gt_image_idxes.index(det_idx)] # info = self._kitti_infos[i] calib = info["calib"] rect = calib["R0_rect"] Trv2c = calib["Tr_velo_to_cam"] P2 = calib["P2"] # final_box_preds = det["box3d_lidar"].detach().cpu().numpy() # label_preds = det["label_preds"].detach().cpu().numpy() # scores = det["scores"].detach().cpu().numpy() final_box_preds = det["box3d_lidar"] label_preds = det["label_preds"] scores = det["scores"] anno = get_start_result_anno() num_example = 0 if final_box_preds.shape[0] != 0: final_box_preds[:, -1] = box_np_ops.limit_period( final_box_preds[:, -1], offset=0.5, period=np.pi * 2, ) box3d_camera = final_box_preds camera_box_origin = [0.5, 0.5, 0.5] box_corners = box_np_ops.center_to_corner_box3d( box3d_camera[:, :3], box3d_camera[:, 3:6], box3d_camera[:, 6], camera_box_origin, axis=2, ) box_corners_in_image = box_np_ops.project_to_image( box_corners, P2) # box_corners_in_image: [N, 8, 2] minxy = np.min(box_corners_in_image, axis=1) maxxy = np.max(box_corners_in_image, axis=1) bbox = np.concatenate([minxy, maxxy], axis=1) for j in range(box3d_camera.shape[0]): anno["bbox"].append([-1, -1, -1, -1]) anno["alpha"].append(0) # anno["dimensions"].append(box3d_camera[j, [4, 5, 3]]) anno["dimensions"].append(box3d_camera[j, 3:6]) anno["location"].append(box3d_camera[j, :3]) anno["rotation_y"].append(box3d_camera[j, 6]) anno["name"].append(class_names[int(label_preds[j] - 1)]) anno["truncated"].append(0.0) anno["occluded"].append(0) anno["score"].append(scores[j]) num_example += 1 if num_example != 0: anno = {n: np.stack(v) for n, v in anno.items()} annos.append(anno) else: annos.append(empty_result_anno()) num_example = annos[-1]["name"].shape[0] annos[-1]["metadata"] = det["metadata"] return annos
def prep_pointcloud_rpn( input_dict, root_path, task_class_names=[], prep_cfg=None, db_sampler=None, remove_outside_points=False, training=True, num_point_features=4, random_crop=False, reference_detections=None, out_dtype=np.float32, min_points_in_gt=-1, logger=None, ): """ convert point cloud to voxels, create targets if ground truths exists. input_dict format: dataset.get_sensor_data format """ assert prep_cfg is not None remove_environment = prep_cfg.REMOVE_UNKOWN_EXAMPLES if training: remove_unknown = prep_cfg.REMOVE_UNKOWN_EXAMPLES gt_rotation_noise = prep_cfg.GT_ROT_NOISE gt_loc_noise_std = prep_cfg.GT_LOC_NOISE global_rotation_noise = prep_cfg.GLOBAL_ROT_NOISE global_scaling_noise = prep_cfg.GLOBAL_SCALE_NOISE global_random_rot_range = prep_cfg.GLOBAL_ROT_PER_OBJ_RANGE global_translate_noise_std = prep_cfg.GLOBAL_TRANS_NOISE gt_points_drop = prep_cfg.GT_DROP_PERCENTAGE gt_drop_max_keep = prep_cfg.GT_DROP_MAX_KEEP_POINTS remove_points_after_sample = prep_cfg.REMOVE_POINTS_AFTER_SAMPLE class_names = list(itertools.chain(*task_class_names)) # points_only = input_dict["lidar"]["points"] # times = input_dict["lidar"]["times"] # points = np.hstack([points_only, times]) points = input_dict["lidar"]["points"] if training: anno_dict = input_dict["lidar"]["annotations"] gt_dict = { "gt_boxes": anno_dict["boxes"], "gt_names": np.array(anno_dict["names"]).reshape(-1), } if "difficulty" not in anno_dict: difficulty = np.zeros([anno_dict["boxes"].shape[0]], dtype=np.int32) gt_dict["difficulty"] = difficulty else: gt_dict["difficulty"] = anno_dict["difficulty"] # if use_group_id and "group_ids" in anno_dict: # group_ids = anno_dict["group_ids"] # gt_dict["group_ids"] = group_ids calib = None if "calib" in input_dict: calib = input_dict["calib"] if reference_detections is not None: assert calib is not None and "image" in input_dict C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2) frustums = box_np_ops.get_frustum_v2(reference_detections, C) frustums -= T frustums = np.einsum("ij, akj->aki", np.linalg.inv(R), frustums) frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c) surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums) masks = points_in_convex_polygon_3d_jit(points, surfaces) points = points[masks.any(-1)] if remove_outside_points: assert calib is not None image_shape = input_dict["image"]["image_shape"] points = box_np_ops.remove_outside_points(points, calib["rect"], calib["Trv2c"], calib["P2"], image_shape) if remove_environment is True and training: selected = kitti.keep_arrays_by_name(gt_names, target_assigner.classes) _dict_select(gt_dict, selected) masks = box_np_ops.points_in_rbbox(points, gt_dict["gt_boxes"]) points = points[masks.any(-1)] if training: selected = kitti.drop_arrays_by_name(gt_dict["gt_names"], ["DontCare", "ignore"]) _dict_select(gt_dict, selected) if remove_unknown: remove_mask = gt_dict["difficulty"] == -1 """ gt_boxes_remove = gt_boxes[remove_mask] gt_boxes_remove[:, 3:6] += 0.25 points = prep.remove_points_in_boxes(points, gt_boxes_remove) """ keep_mask = np.logical_not(remove_mask) _dict_select(gt_dict, keep_mask) gt_dict.pop("difficulty") gt_boxes_mask = np.array( [n in class_names for n in gt_dict["gt_names"]], dtype=np.bool_) # db_sampler = None if db_sampler is not None: group_ids = None # if "group_ids" in gt_dict: # group_ids = gt_dict["group_ids"] sampled_dict = db_sampler.sample_all( root_path, gt_dict["gt_boxes"], gt_dict["gt_names"], num_point_features, random_crop, gt_group_ids=group_ids, calib=calib, ) if sampled_dict is not None: sampled_gt_names = sampled_dict["gt_names"] sampled_gt_boxes = sampled_dict["gt_boxes"] sampled_points = sampled_dict["points"] sampled_gt_masks = sampled_dict["gt_masks"] gt_dict["gt_names"] = np.concatenate( [gt_dict["gt_names"], sampled_gt_names], axis=0) gt_dict["gt_boxes"] = np.concatenate( [gt_dict["gt_boxes"], sampled_gt_boxes]) gt_boxes_mask = np.concatenate( [gt_boxes_mask, sampled_gt_masks], axis=0) # if group_ids is not None: # sampled_group_ids = sampled_dict["group_ids"] # gt_dict["group_ids"] = np.concatenate( # [gt_dict["group_ids"], sampled_group_ids]) if remove_points_after_sample: masks = box_np_ops.points_in_rbbox(points, sampled_gt_boxes) points = points[np.logical_not(masks.any(-1))] points = np.concatenate([sampled_points, points], axis=0) # group_ids = None # if "group_ids" in gt_dict: # group_ids = gt_dict["group_ids"] prep.noise_per_object_v3_( gt_dict["gt_boxes"], points, gt_boxes_mask, rotation_perturb=gt_rotation_noise, center_noise_std=gt_loc_noise_std, global_random_rot_range=global_random_rot_range, group_ids=None, num_try=100, ) # should remove unrelated objects after noise per object # for k, v in gt_dict.items(): # print(k, v.shape) _dict_select(gt_dict, gt_boxes_mask) gt_classes = np.array( [class_names.index(n) + 1 for n in gt_dict["gt_names"]], dtype=np.int32) gt_dict["gt_classes"] = gt_classes gt_dict["gt_boxes"], points = prep.random_flip(gt_dict["gt_boxes"], points) gt_dict["gt_boxes"], points = prep.global_rotation( gt_dict["gt_boxes"], points, rotation=global_rotation_noise) gt_dict["gt_boxes"], points = prep.global_scaling_v2( gt_dict["gt_boxes"], points, *global_scaling_noise) prep.global_translate_(gt_dict["gt_boxes"], points, global_translate_noise_std) task_masks = [] flag = 0 for class_name in task_class_names: task_masks.append([ np.where(gt_dict["gt_classes"] == class_name.index(i) + 1 + flag) for i in class_name ]) flag += len(class_name) task_boxes = [] task_classes = [] task_names = [] flag2 = 0 for idx, mask in enumerate(task_masks): task_box = [] task_class = [] task_name = [] for m in mask: task_box.append(gt_dict["gt_boxes"][m]) task_class.append(gt_dict["gt_classes"][m] - flag2) task_name.append(gt_dict["gt_names"][m]) task_boxes.append(np.concatenate(task_box, axis=0)) task_classes.append(np.concatenate(task_class)) task_names.append(np.concatenate(task_name)) flag2 += len(mask) for task_box in task_boxes: # limit rad to [-pi, pi] task_box[:, -1] = box_np_ops.limit_period(task_box[:, -1], offset=0.5, period=2 * np.pi) # print(gt_dict.keys()) gt_dict["gt_classes"] = task_classes gt_dict["gt_names"] = task_names gt_dict["gt_boxes"] = task_boxes example = { "pts_input": points, "pts_rect": None, "pts_features": None, "gt_boxes3d": gt_dict["gt_boxes"], "rpn_cls_label": [], "rpn_reg_label": [], } if calib is not None: example["calib"] = calib return example
def convert_detection_to_lvx_annos(self, detection): class_names = self._class_names lvx_infos = [] for clips in self._start_idx: lvx_infos.extend(self._lvx_infos[clips[0] + 2:clips[1]]) gt_image_idxes = [str(info["token"]) for info in lvx_infos] annos = [] for det_idx in gt_image_idxes: det = detection[det_idx] final_box_preds = det["box3d_lidar"].detach().cpu().numpy() final_box_preds_1 = det["box3d_lidar_1"].detach().cpu().numpy() final_box_preds_2 = det["box3d_lidar_2"].detach().cpu().numpy() label_preds = det["label_preds"].detach().cpu().numpy() scores = det["scores"].detach().cpu().numpy() anno = get_start_result_anno() num_example = 0 if final_box_preds.shape[0] != 0: final_box_preds[:, -1] = box_np_ops.limit_period( final_box_preds[:, -1], offset=0.5, period=np.pi * 2, ) final_box_preds_1[:, -1] = box_np_ops.limit_period( final_box_preds_1[:, -1], offset=0.5, period=np.pi * 2, ) final_box_preds_2[:, -1] = box_np_ops.limit_period( final_box_preds_2[:, -1], offset=0.5, period=np.pi * 2, ) bbox = np.asarray([0, 0, 500, 500]) for j in range(final_box_preds.shape[0]): anno["bbox"].append(bbox) anno["alpha"].append(-10) anno["dimensions"].append(final_box_preds[j, 3:6]) anno["location"].append(final_box_preds[j, :3]) anno["rotation_y"].append(final_box_preds[j, 6]) anno["dimensions_1"].append(final_box_preds_1[j, 3:6]) anno["location_1"].append(final_box_preds_1[j, :3]) anno["rotation_y_1"].append(final_box_preds_1[j, 6]) anno["dimensions_2"].append(final_box_preds_2[j, 3:6]) anno["location_2"].append(final_box_preds_2[j, :3]) anno["rotation_y_2"].append(final_box_preds_2[j, 6]) anno["name"].append(class_names[int(label_preds[j])]) anno["truncated"].append(0.0) anno["occluded"].append(0) anno["score"].append(scores[j]) num_example += 1 if num_example != 0: anno = {n: np.stack(v) for n, v in anno.items()} annos.append(anno) else: annos.append(empty_result_anno()) num_example = annos[-1]["name"].shape[0] annos[-1]["metadata"] = det["metadata"] return annos
def prep_sequence_pointcloud( input_dict, root_path, voxel_generator, target_assigners, prep_cfg=None, db_sampler=None, remove_outside_points=False, training=True, create_targets=True, num_point_features=4, anchor_cache=None, random_crop=False, reference_detections=None, out_size_factor=2, out_dtype=np.float32, min_points_in_gt=-1, logger=None, ): """ convert point cloud to voxels, create targets if ground truths exists. input_dict format: dataset.get_sensor_data format """ assert prep_cfg is not None remove_environment = prep_cfg.REMOVE_ENVIRONMENT max_voxels = prep_cfg.MAX_VOXELS_NUM shuffle_points = prep_cfg.SHUFFLE anchor_area_threshold = prep_cfg.ANCHOR_AREA_THRES if training: remove_unknown = prep_cfg.REMOVE_UNKOWN_EXAMPLES gt_rotation_noise = prep_cfg.GT_ROT_NOISE gt_loc_noise_std = prep_cfg.GT_LOC_NOISE global_rotation_noise = prep_cfg.GLOBAL_ROT_NOISE global_scaling_noise = prep_cfg.GLOBAL_SCALE_NOISE global_random_rot_range = prep_cfg.GLOBAL_ROT_PER_OBJ_RANGE global_translate_noise_std = prep_cfg.GLOBAL_TRANS_NOISE gt_points_drop = prep_cfg.GT_DROP_PERCENTAGE gt_drop_max_keep = prep_cfg.GT_DROP_MAX_KEEP_POINTS remove_points_after_sample = prep_cfg.REMOVE_POINTS_AFTER_SAMPLE min_points_in_gt = prep_cfg.get("MIN_POINTS_IN_GT", -1) task_class_names = [ target_assigner.classes for target_assigner in target_assigners ] class_names = list(itertools.chain(*task_class_names)) # points_only = input_dict["lidar"]["points"] # times = input_dict["lidar"]["times"] # points = np.hstack([points_only, times]) try: points = input_dict["current_frame"]["lidar"]["combined"] except Exception: points = input_dict["current_frame"]["lidar"]["points"] keyframe_points = input_dict["keyframe"]["lidar"]["combined"] if training: anno_dict = input_dict["current_frame"]["lidar"]["annotations"] gt_dict = { "gt_boxes": anno_dict["boxes"], "gt_names": np.array(anno_dict["names"]).reshape(-1), } if "difficulty" not in anno_dict: difficulty = np.zeros([anno_dict["boxes"].shape[0]], dtype=np.int32) gt_dict["difficulty"] = difficulty else: gt_dict["difficulty"] = anno_dict["difficulty"] # if use_group_id and "group_ids" in anno_dict: # group_ids = anno_dict["group_ids"] # gt_dict["group_ids"] = group_ids calib = None if "calib" in input_dict: calib = input_dict["current_frame"]["calib"] if reference_detections is not None: assert calib is not None and "image" in input_dict["current_frame"] C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2) frustums = box_np_ops.get_frustum_v2(reference_detections, C) frustums -= T frustums = np.einsum("ij, akj->aki", np.linalg.inv(R), frustums) frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c) surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums) masks = points_in_convex_polygon_3d_jit(points, surfaces) points = points[masks.any(-1)] if remove_outside_points: assert calib is not None image_shape = input_dict["current_frame"]["image"]["image_shape"] points = box_np_ops.remove_outside_points(points, calib["rect"], calib["Trv2c"], calib["P2"], image_shape) if remove_environment is True and training: selected = kitti.keep_arrays_by_name(gt_names, target_assigner.classes) _dict_select(gt_dict, selected) masks = box_np_ops.points_in_rbbox(points, gt_dict["gt_boxes"]) points = points[masks.any(-1)] if training: # boxes_lidar = gt_dict["gt_boxes"] # cv2.imshow('pre-noise', bev_map) selected = kitti.drop_arrays_by_name(gt_dict["gt_names"], ["DontCare", "ignore"]) _dict_select(gt_dict, selected) if remove_unknown: remove_mask = gt_dict["difficulty"] == -1 """ gt_boxes_remove = gt_boxes[remove_mask] gt_boxes_remove[:, 3:6] += 0.25 points = prep.remove_points_in_boxes(points, gt_boxes_remove) """ keep_mask = np.logical_not(remove_mask) _dict_select(gt_dict, keep_mask) gt_dict.pop("difficulty") if min_points_in_gt > 0: # points_count_rbbox takes 10ms with 10 sweeps nuscenes data point_counts = box_np_ops.points_count_rbbox( points, gt_dict["gt_boxes"]) mask = point_counts >= min_points_in_gt _dict_select(gt_dict, mask) gt_boxes_mask = np.array( [n in class_names for n in gt_dict["gt_names"]], dtype=np.bool_) # db_sampler = None if db_sampler is not None: group_ids = None # if "group_ids" in gt_dict: # group_ids = gt_dict["group_ids"] sampled_dict = db_sampler.sample_all( root_path, gt_dict["gt_boxes"], gt_dict["gt_names"], num_point_features, random_crop, gt_group_ids=group_ids, calib=calib, ) if sampled_dict is not None: sampled_gt_names = sampled_dict["gt_names"] sampled_gt_boxes = sampled_dict["gt_boxes"] sampled_points = sampled_dict["points"] sampled_gt_masks = sampled_dict["gt_masks"] gt_dict["gt_names"] = np.concatenate( [gt_dict["gt_names"], sampled_gt_names], axis=0) gt_dict["gt_boxes"] = np.concatenate( [gt_dict["gt_boxes"], sampled_gt_boxes]) gt_boxes_mask = np.concatenate( [gt_boxes_mask, sampled_gt_masks], axis=0) # if group_ids is not None: # sampled_group_ids = sampled_dict["group_ids"] # gt_dict["group_ids"] = np.concatenate( # [gt_dict["group_ids"], sampled_group_ids]) if remove_points_after_sample: masks = box_np_ops.points_in_rbbox(points, sampled_gt_boxes) points = points[np.logical_not(masks.any(-1))] points = np.concatenate([sampled_points, points], axis=0) pc_range = voxel_generator.point_cloud_range # group_ids = None # if "group_ids" in gt_dict: # group_ids = gt_dict["group_ids"] # prep.noise_per_object_v3_( # gt_dict["gt_boxes"], # points, # gt_boxes_mask, # rotation_perturb=gt_rotation_noise, # center_noise_std=gt_loc_noise_std, # global_random_rot_range=global_random_rot_range, # group_ids=group_ids, # num_try=100) # should remove unrelated objects after noise per object # for k, v in gt_dict.items(): # print(k, v.shape) _dict_select(gt_dict, gt_boxes_mask) gt_classes = np.array( [class_names.index(n) + 1 for n in gt_dict["gt_names"]], dtype=np.int32) gt_dict["gt_classes"] = gt_classes # concatenate points_current = points.shape[0] points_keyframe = keyframe_points.shape[0] points = np.concatenate((points, keyframe_points), axis=0) # data aug gt_dict["gt_boxes"], points = prep.random_flip(gt_dict["gt_boxes"], points) gt_dict["gt_boxes"], points = prep.global_rotation( gt_dict["gt_boxes"], points, rotation=global_rotation_noise) gt_dict["gt_boxes"], points = prep.global_scaling_v2( gt_dict["gt_boxes"], points, *global_scaling_noise) prep.global_translate_(gt_dict["gt_boxes"], points, global_translate_noise_std) # slice points_keyframe = points[points_current:, :] points = points[:points_current, :] bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range(gt_dict["gt_boxes"], bv_range) _dict_select(gt_dict, mask) task_masks = [] flag = 0 for class_name in task_class_names: task_masks.append([ np.where(gt_dict["gt_classes"] == class_name.index(i) + 1 + flag) for i in class_name ]) flag += len(class_name) task_boxes = [] task_classes = [] task_names = [] flag2 = 0 for idx, mask in enumerate(task_masks): task_box = [] task_class = [] task_name = [] for m in mask: task_box.append(gt_dict["gt_boxes"][m]) task_class.append(gt_dict["gt_classes"][m] - flag2) task_name.append(gt_dict["gt_names"][m]) task_boxes.append(np.concatenate(task_box, axis=0)) task_classes.append(np.concatenate(task_class)) task_names.append(np.concatenate(task_name)) flag2 += len(mask) for task_box in task_boxes: # limit rad to [-pi, pi] task_box[:, -1] = box_np_ops.limit_period(task_box[:, -1], offset=0.5, period=2 * np.pi) # print(gt_dict.keys()) gt_dict["gt_classes"] = task_classes gt_dict["gt_names"] = task_names gt_dict["gt_boxes"] = task_boxes # if shuffle_points: # # shuffle is a little slow. # np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # [352, 400] # points = points[:int(points.shape[0] * 0.1), :] voxels, coordinates, num_points = voxel_generator.generate( points, max_voxels) # res = voxel_generator.generate( # points, max_voxels) # voxels = res["voxels"] # coordinates = res["coordinates"] # num_points = res["num_points_per_voxel"] num_voxels = np.array([voxels.shape[0]], dtype=np.int64) # key frame voxel keyframe_info = voxel_generator.generate(keyframe_points, max_voxels) keyframe_info = keyframe_voxels, keyframe_coordinates, keyframe_num_points keyframe_num_voxels = np.array([keyframe_voxels.shape[0]], dtype=np.int64) example = { "voxels": voxels, "num_points": num_points, "points": points, "coordinates": coordinates, "num_voxels": num_voxels, } example_keyframe = { "voxels": keyframe_voxels, "num_points": keyframe_num_points, "points": keyframe_points, "coordinates": keyframe_coordinates, "num_voxels": keyframe_num_voxels, } if training: example["gt_boxes"] = gt_dict["gt_boxes"] if calib is not None: example["calib"] = calib feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchorss = anchor_cache["anchors"] anchors_bvs = anchor_cache["anchors_bv"] anchors_dicts = anchor_cache["anchors_dict"] else: rets = [ target_assigner.generate_anchors(feature_map_size) for target_assigner in target_assigners ] anchorss = [ret["anchors"].reshape([-1, 7]) for ret in rets] anchors_dicts = [ target_assigner.generate_anchors_dict(feature_map_size) for target_assigner in target_assigners ] anchors_bvs = [ box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) for anchors in anchorss ] example["anchors"] = anchorss if anchor_area_threshold >= 0: example["anchors_mask"] = [] for idx, anchors_bv in enumerate(anchors_bvs): anchors_mask = None # slow with high resolution. recommend disable this forever. coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example["anchors_mask"].append(anchors_mask) example_sequences = {} example_sequences["current_frame"] = example example_sequences["keyframe"] = example_keyframe if not training: return example_sequences # voxel_labels = box_np_ops.assign_label_to_voxel(gt_boxes, coordinates, # voxel_size, coors_range) """ example.update({ 'gt_boxes': gt_boxes.astype(out_dtype), 'num_gt': np.array([gt_boxes.shape[0]]), # 'voxel_labels': voxel_labels, }) """ if create_targets: targets_dicts = [] for idx, target_assigner in enumerate(target_assigners): if "anchors_mask" in example: anchors_mask = example["anchors_mask"][idx] else: anchors_mask = None targets_dict = target_assigner.assign_v2( anchors_dicts[idx], gt_dict["gt_boxes"][idx], anchors_mask, gt_classes=gt_dict["gt_classes"][idx], gt_names=gt_dict["gt_names"][idx], ) targets_dicts.append(targets_dict) example_sequences["current_frame"].update({ "labels": [targets_dict["labels"] for targets_dict in targets_dicts], "reg_targets": [targets_dict["bbox_targets"] for targets_dict in targets_dicts], "reg_weights": [ targets_dict["bbox_outside_weights"] for targets_dict in targets_dicts ], }) return example_sequences