Пример #1
0
    def convert_detection_to_kitti_annos(self, detection):
        class_names = self._class_names
        det_image_idxes = [k for k in detection.keys()]
        gt_image_idxes = [
            str(info["image"]["image_idx"]) for info in self._kitti_infos
        ]
        # print(f"det_image_idxes: {det_image_idxes[:10]}")
        # print(f"gt_image_idxes: {gt_image_idxes[:10]}")
        annos = []
        # for i in range(len(detection)):
        for det_idx in gt_image_idxes:
            det = detection[det_idx]
            info = self._kitti_infos[gt_image_idxes.index(det_idx)]
            # info = self._kitti_infos[i]
            calib = info["calib"]
            rect = calib["R0_rect"]
            Trv2c = calib["Tr_velo_to_cam"]
            P2 = calib["P2"]
            final_box_preds = det["box3d_lidar"].detach().cpu().numpy()
            label_preds = det["label_preds"].detach().cpu().numpy()
            scores = det["scores"].detach().cpu().numpy()

            anno = get_start_result_anno()
            num_example = 0

            if final_box_preds.shape[0] != 0:
                final_box_preds[:, -1] = box_np_ops.limit_period(
                    final_box_preds[:, -1],
                    offset=0.5,
                    period=np.pi * 2,
                )
                final_box_preds[:, 2] -= final_box_preds[:, 5] / 2

                # aim: x, y, z, w, l, h, r -> -y, -z, x, h, w, l, r
                # (x, y, z, w, l, h r) in lidar -> (x', y', z', l, h, w, r) in camera
                box3d_camera = box_np_ops.box_lidar_to_camera(
                    final_box_preds, rect, Trv2c)
                camera_box_origin = [0.5, 1.0, 0.5]
                box_corners = box_np_ops.center_to_corner_box3d(
                    box3d_camera[:, :3],
                    box3d_camera[:, 3:6],
                    box3d_camera[:, 6],
                    camera_box_origin,
                    axis=1,
                )
                box_corners_in_image = box_np_ops.project_to_image(
                    box_corners, P2)
                # box_corners_in_image: [N, 8, 2]
                minxy = np.min(box_corners_in_image, axis=1)
                maxxy = np.max(box_corners_in_image, axis=1)
                bbox = np.concatenate([minxy, maxxy], axis=1)

                for j in range(box3d_camera.shape[0]):
                    image_shape = info["image"]["image_shape"]
                    if bbox[j, 0] > image_shape[1] or bbox[j,
                                                           1] > image_shape[0]:
                        continue
                    if bbox[j, 2] < 0 or bbox[j, 3] < 0:
                        continue
                    bbox[j, 2:] = np.minimum(bbox[j, 2:], image_shape[::-1])
                    bbox[j, :2] = np.maximum(bbox[j, :2], [0, 0])
                    anno["bbox"].append(bbox[j])

                    anno["alpha"].append(-np.arctan2(-final_box_preds[j, 1],
                                                     final_box_preds[j, 0]) +
                                         box3d_camera[j, 6])
                    # anno["dimensions"].append(box3d_camera[j, [4, 5, 3]])
                    anno["dimensions"].append(box3d_camera[j, 3:6])
                    anno["location"].append(box3d_camera[j, :3])
                    anno["rotation_y"].append(box3d_camera[j, 6])
                    anno["name"].append(class_names[int(label_preds[j])])
                    anno["truncated"].append(0.0)
                    anno["occluded"].append(0)
                    anno["score"].append(scores[j])

                    num_example += 1

            if num_example != 0:
                anno = {n: np.stack(v) for n, v in anno.items()}
                annos.append(anno)
            else:
                annos.append(empty_result_anno())
            num_example = annos[-1]["name"].shape[0]
            annos[-1]["metadata"] = det["metadata"]
        return annos
    def convert_detection_to_kitti_annos(self, detection):
        class_names = self._class_names
        det_image_idxes = [k for k in detection.keys()]
        gt_image_idxes = [
            str(info["image"]["image_idx"]) for info in self._kitti_infos
        ]

        annos = []

        for det_idx in gt_image_idxes:
            det = detection[det_idx]
            dim = det['box3d_lidar'][:, 3:6]
            l, w, h = dim[:, 0:1], dim[:, 1:2], dim[:, 2:3]
            det['box3d_lidar'][:, 2] = (det['box3d_lidar'][:, 2].T +
                                        (h / 2).T).reshape(-1)
            det['box3d_lidar'][:, -1] = det['box3d_lidar'][:, -1] * -1

            info = self._kitti_infos[gt_image_idxes.index(det_idx)]
            # info = self._kitti_infos[i]
            calib = info["calib"]
            rect = calib["R0_rect"]
            Trv2c = calib["Tr_velo_to_cam"]
            P2 = calib["P2"]

            # final_box_preds = det["box3d_lidar"].detach().cpu().numpy()
            # label_preds = det["label_preds"].detach().cpu().numpy()
            # scores = det["scores"].detach().cpu().numpy()
            final_box_preds = det["box3d_lidar"]
            label_preds = det["label_preds"]
            scores = det["scores"]
            anno = get_start_result_anno()
            num_example = 0

            if final_box_preds.shape[0] != 0:
                final_box_preds[:, -1] = box_np_ops.limit_period(
                    final_box_preds[:, -1],
                    offset=0.5,
                    period=np.pi * 2,
                )

                box3d_camera = final_box_preds
                camera_box_origin = [0.5, 0.5, 0.5]
                box_corners = box_np_ops.center_to_corner_box3d(
                    box3d_camera[:, :3],
                    box3d_camera[:, 3:6],
                    box3d_camera[:, 6],
                    camera_box_origin,
                    axis=2,
                )
                box_corners_in_image = box_np_ops.project_to_image(
                    box_corners, P2)
                # box_corners_in_image: [N, 8, 2]
                minxy = np.min(box_corners_in_image, axis=1)
                maxxy = np.max(box_corners_in_image, axis=1)
                bbox = np.concatenate([minxy, maxxy], axis=1)

                for j in range(box3d_camera.shape[0]):
                    anno["bbox"].append([-1, -1, -1, -1])

                    anno["alpha"].append(0)
                    # anno["dimensions"].append(box3d_camera[j, [4, 5, 3]])
                    anno["dimensions"].append(box3d_camera[j, 3:6])
                    anno["location"].append(box3d_camera[j, :3])
                    anno["rotation_y"].append(box3d_camera[j, 6])
                    anno["name"].append(class_names[int(label_preds[j] - 1)])
                    anno["truncated"].append(0.0)
                    anno["occluded"].append(0)
                    anno["score"].append(scores[j])

                    num_example += 1

            if num_example != 0:
                anno = {n: np.stack(v) for n, v in anno.items()}
                annos.append(anno)
            else:
                annos.append(empty_result_anno())
            num_example = annos[-1]["name"].shape[0]
            annos[-1]["metadata"] = det["metadata"]
        return annos
Пример #3
0
def prep_pointcloud_rpn(
    input_dict,
    root_path,
    task_class_names=[],
    prep_cfg=None,
    db_sampler=None,
    remove_outside_points=False,
    training=True,
    num_point_features=4,
    random_crop=False,
    reference_detections=None,
    out_dtype=np.float32,
    min_points_in_gt=-1,
    logger=None,
):
    """
    convert point cloud to voxels, create targets if ground truths exists.
    input_dict format: dataset.get_sensor_data format
    """
    assert prep_cfg is not None

    remove_environment = prep_cfg.REMOVE_UNKOWN_EXAMPLES

    if training:
        remove_unknown = prep_cfg.REMOVE_UNKOWN_EXAMPLES
        gt_rotation_noise = prep_cfg.GT_ROT_NOISE
        gt_loc_noise_std = prep_cfg.GT_LOC_NOISE
        global_rotation_noise = prep_cfg.GLOBAL_ROT_NOISE
        global_scaling_noise = prep_cfg.GLOBAL_SCALE_NOISE
        global_random_rot_range = prep_cfg.GLOBAL_ROT_PER_OBJ_RANGE
        global_translate_noise_std = prep_cfg.GLOBAL_TRANS_NOISE
        gt_points_drop = prep_cfg.GT_DROP_PERCENTAGE
        gt_drop_max_keep = prep_cfg.GT_DROP_MAX_KEEP_POINTS
        remove_points_after_sample = prep_cfg.REMOVE_POINTS_AFTER_SAMPLE

    class_names = list(itertools.chain(*task_class_names))

    # points_only = input_dict["lidar"]["points"]
    # times = input_dict["lidar"]["times"]
    # points = np.hstack([points_only, times])
    points = input_dict["lidar"]["points"]

    if training:
        anno_dict = input_dict["lidar"]["annotations"]
        gt_dict = {
            "gt_boxes": anno_dict["boxes"],
            "gt_names": np.array(anno_dict["names"]).reshape(-1),
        }

        if "difficulty" not in anno_dict:
            difficulty = np.zeros([anno_dict["boxes"].shape[0]],
                                  dtype=np.int32)
            gt_dict["difficulty"] = difficulty
        else:
            gt_dict["difficulty"] = anno_dict["difficulty"]
        # if use_group_id and "group_ids" in anno_dict:
        #     group_ids = anno_dict["group_ids"]
        #     gt_dict["group_ids"] = group_ids

    calib = None
    if "calib" in input_dict:
        calib = input_dict["calib"]

    if reference_detections is not None:
        assert calib is not None and "image" in input_dict
        C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2)
        frustums = box_np_ops.get_frustum_v2(reference_detections, C)
        frustums -= T
        frustums = np.einsum("ij, akj->aki", np.linalg.inv(R), frustums)
        frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c)
        surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums)
        masks = points_in_convex_polygon_3d_jit(points, surfaces)
        points = points[masks.any(-1)]

    if remove_outside_points:
        assert calib is not None
        image_shape = input_dict["image"]["image_shape"]
        points = box_np_ops.remove_outside_points(points, calib["rect"],
                                                  calib["Trv2c"], calib["P2"],
                                                  image_shape)
    if remove_environment is True and training:
        selected = kitti.keep_arrays_by_name(gt_names, target_assigner.classes)
        _dict_select(gt_dict, selected)
        masks = box_np_ops.points_in_rbbox(points, gt_dict["gt_boxes"])
        points = points[masks.any(-1)]

    if training:
        selected = kitti.drop_arrays_by_name(gt_dict["gt_names"],
                                             ["DontCare", "ignore"])
        _dict_select(gt_dict, selected)
        if remove_unknown:
            remove_mask = gt_dict["difficulty"] == -1
            """
            gt_boxes_remove = gt_boxes[remove_mask]
            gt_boxes_remove[:, 3:6] += 0.25
            points = prep.remove_points_in_boxes(points, gt_boxes_remove)
            """
            keep_mask = np.logical_not(remove_mask)
            _dict_select(gt_dict, keep_mask)
        gt_dict.pop("difficulty")

        gt_boxes_mask = np.array(
            [n in class_names for n in gt_dict["gt_names"]], dtype=np.bool_)

        # db_sampler = None
        if db_sampler is not None:
            group_ids = None
            # if "group_ids" in gt_dict:
            #     group_ids = gt_dict["group_ids"]
            sampled_dict = db_sampler.sample_all(
                root_path,
                gt_dict["gt_boxes"],
                gt_dict["gt_names"],
                num_point_features,
                random_crop,
                gt_group_ids=group_ids,
                calib=calib,
            )

            if sampled_dict is not None:
                sampled_gt_names = sampled_dict["gt_names"]
                sampled_gt_boxes = sampled_dict["gt_boxes"]
                sampled_points = sampled_dict["points"]
                sampled_gt_masks = sampled_dict["gt_masks"]
                gt_dict["gt_names"] = np.concatenate(
                    [gt_dict["gt_names"], sampled_gt_names], axis=0)
                gt_dict["gt_boxes"] = np.concatenate(
                    [gt_dict["gt_boxes"], sampled_gt_boxes])
                gt_boxes_mask = np.concatenate(
                    [gt_boxes_mask, sampled_gt_masks], axis=0)

                # if group_ids is not None:
                #     sampled_group_ids = sampled_dict["group_ids"]
                #     gt_dict["group_ids"] = np.concatenate(
                #         [gt_dict["group_ids"], sampled_group_ids])

                if remove_points_after_sample:
                    masks = box_np_ops.points_in_rbbox(points,
                                                       sampled_gt_boxes)
                    points = points[np.logical_not(masks.any(-1))]

                points = np.concatenate([sampled_points, points], axis=0)

        # group_ids = None
        # if "group_ids" in gt_dict:
        #     group_ids = gt_dict["group_ids"]

        prep.noise_per_object_v3_(
            gt_dict["gt_boxes"],
            points,
            gt_boxes_mask,
            rotation_perturb=gt_rotation_noise,
            center_noise_std=gt_loc_noise_std,
            global_random_rot_range=global_random_rot_range,
            group_ids=None,
            num_try=100,
        )

        # should remove unrelated objects after noise per object
        # for k, v in gt_dict.items():
        #     print(k, v.shape)

        _dict_select(gt_dict, gt_boxes_mask)

        gt_classes = np.array(
            [class_names.index(n) + 1 for n in gt_dict["gt_names"]],
            dtype=np.int32)
        gt_dict["gt_classes"] = gt_classes

        gt_dict["gt_boxes"], points = prep.random_flip(gt_dict["gt_boxes"],
                                                       points)
        gt_dict["gt_boxes"], points = prep.global_rotation(
            gt_dict["gt_boxes"], points, rotation=global_rotation_noise)
        gt_dict["gt_boxes"], points = prep.global_scaling_v2(
            gt_dict["gt_boxes"], points, *global_scaling_noise)
        prep.global_translate_(gt_dict["gt_boxes"], points,
                               global_translate_noise_std)

        task_masks = []
        flag = 0
        for class_name in task_class_names:
            task_masks.append([
                np.where(gt_dict["gt_classes"] == class_name.index(i) + 1 +
                         flag) for i in class_name
            ])
            flag += len(class_name)

        task_boxes = []
        task_classes = []
        task_names = []
        flag2 = 0
        for idx, mask in enumerate(task_masks):
            task_box = []
            task_class = []
            task_name = []
            for m in mask:
                task_box.append(gt_dict["gt_boxes"][m])
                task_class.append(gt_dict["gt_classes"][m] - flag2)
                task_name.append(gt_dict["gt_names"][m])
            task_boxes.append(np.concatenate(task_box, axis=0))
            task_classes.append(np.concatenate(task_class))
            task_names.append(np.concatenate(task_name))
            flag2 += len(mask)

        for task_box in task_boxes:
            # limit rad to [-pi, pi]
            task_box[:, -1] = box_np_ops.limit_period(task_box[:, -1],
                                                      offset=0.5,
                                                      period=2 * np.pi)

        # print(gt_dict.keys())
        gt_dict["gt_classes"] = task_classes
        gt_dict["gt_names"] = task_names
        gt_dict["gt_boxes"] = task_boxes

        example = {
            "pts_input": points,
            "pts_rect": None,
            "pts_features": None,
            "gt_boxes3d": gt_dict["gt_boxes"],
            "rpn_cls_label": [],
            "rpn_reg_label": [],
        }

        if calib is not None:
            example["calib"] = calib

        return example
Пример #4
0
    def convert_detection_to_lvx_annos(self, detection):
        class_names = self._class_names
        lvx_infos = []
        for clips in self._start_idx:
            lvx_infos.extend(self._lvx_infos[clips[0] + 2:clips[1]])
        gt_image_idxes = [str(info["token"]) for info in lvx_infos]
        annos = []
        for det_idx in gt_image_idxes:
            det = detection[det_idx]
            final_box_preds = det["box3d_lidar"].detach().cpu().numpy()
            final_box_preds_1 = det["box3d_lidar_1"].detach().cpu().numpy()
            final_box_preds_2 = det["box3d_lidar_2"].detach().cpu().numpy()
            label_preds = det["label_preds"].detach().cpu().numpy()
            scores = det["scores"].detach().cpu().numpy()

            anno = get_start_result_anno()
            num_example = 0

            if final_box_preds.shape[0] != 0:
                final_box_preds[:, -1] = box_np_ops.limit_period(
                    final_box_preds[:, -1],
                    offset=0.5,
                    period=np.pi * 2,
                )
                final_box_preds_1[:, -1] = box_np_ops.limit_period(
                    final_box_preds_1[:, -1],
                    offset=0.5,
                    period=np.pi * 2,
                )
                final_box_preds_2[:, -1] = box_np_ops.limit_period(
                    final_box_preds_2[:, -1],
                    offset=0.5,
                    period=np.pi * 2,
                )
                bbox = np.asarray([0, 0, 500, 500])
                for j in range(final_box_preds.shape[0]):
                    anno["bbox"].append(bbox)
                    anno["alpha"].append(-10)
                    anno["dimensions"].append(final_box_preds[j, 3:6])
                    anno["location"].append(final_box_preds[j, :3])
                    anno["rotation_y"].append(final_box_preds[j, 6])
                    anno["dimensions_1"].append(final_box_preds_1[j, 3:6])
                    anno["location_1"].append(final_box_preds_1[j, :3])
                    anno["rotation_y_1"].append(final_box_preds_1[j, 6])
                    anno["dimensions_2"].append(final_box_preds_2[j, 3:6])
                    anno["location_2"].append(final_box_preds_2[j, :3])
                    anno["rotation_y_2"].append(final_box_preds_2[j, 6])
                    anno["name"].append(class_names[int(label_preds[j])])
                    anno["truncated"].append(0.0)
                    anno["occluded"].append(0)
                    anno["score"].append(scores[j])

                    num_example += 1

            if num_example != 0:
                anno = {n: np.stack(v) for n, v in anno.items()}
                annos.append(anno)
            else:
                annos.append(empty_result_anno())
            num_example = annos[-1]["name"].shape[0]
            annos[-1]["metadata"] = det["metadata"]
        return annos
Пример #5
0
def prep_sequence_pointcloud(
    input_dict,
    root_path,
    voxel_generator,
    target_assigners,
    prep_cfg=None,
    db_sampler=None,
    remove_outside_points=False,
    training=True,
    create_targets=True,
    num_point_features=4,
    anchor_cache=None,
    random_crop=False,
    reference_detections=None,
    out_size_factor=2,
    out_dtype=np.float32,
    min_points_in_gt=-1,
    logger=None,
):
    """
    convert point cloud to voxels, create targets if ground truths exists.
    input_dict format: dataset.get_sensor_data format
    """
    assert prep_cfg is not None

    remove_environment = prep_cfg.REMOVE_ENVIRONMENT
    max_voxels = prep_cfg.MAX_VOXELS_NUM
    shuffle_points = prep_cfg.SHUFFLE
    anchor_area_threshold = prep_cfg.ANCHOR_AREA_THRES

    if training:
        remove_unknown = prep_cfg.REMOVE_UNKOWN_EXAMPLES
        gt_rotation_noise = prep_cfg.GT_ROT_NOISE
        gt_loc_noise_std = prep_cfg.GT_LOC_NOISE
        global_rotation_noise = prep_cfg.GLOBAL_ROT_NOISE
        global_scaling_noise = prep_cfg.GLOBAL_SCALE_NOISE
        global_random_rot_range = prep_cfg.GLOBAL_ROT_PER_OBJ_RANGE
        global_translate_noise_std = prep_cfg.GLOBAL_TRANS_NOISE
        gt_points_drop = prep_cfg.GT_DROP_PERCENTAGE
        gt_drop_max_keep = prep_cfg.GT_DROP_MAX_KEEP_POINTS
        remove_points_after_sample = prep_cfg.REMOVE_POINTS_AFTER_SAMPLE
        min_points_in_gt = prep_cfg.get("MIN_POINTS_IN_GT", -1)

    task_class_names = [
        target_assigner.classes for target_assigner in target_assigners
    ]
    class_names = list(itertools.chain(*task_class_names))

    # points_only = input_dict["lidar"]["points"]
    # times = input_dict["lidar"]["times"]
    # points = np.hstack([points_only, times])
    try:
        points = input_dict["current_frame"]["lidar"]["combined"]
    except Exception:
        points = input_dict["current_frame"]["lidar"]["points"]
    keyframe_points = input_dict["keyframe"]["lidar"]["combined"]

    if training:
        anno_dict = input_dict["current_frame"]["lidar"]["annotations"]
        gt_dict = {
            "gt_boxes": anno_dict["boxes"],
            "gt_names": np.array(anno_dict["names"]).reshape(-1),
        }

        if "difficulty" not in anno_dict:
            difficulty = np.zeros([anno_dict["boxes"].shape[0]],
                                  dtype=np.int32)
            gt_dict["difficulty"] = difficulty
        else:
            gt_dict["difficulty"] = anno_dict["difficulty"]
        # if use_group_id and "group_ids" in anno_dict:
        #     group_ids = anno_dict["group_ids"]
        #     gt_dict["group_ids"] = group_ids

    calib = None
    if "calib" in input_dict:
        calib = input_dict["current_frame"]["calib"]

    if reference_detections is not None:
        assert calib is not None and "image" in input_dict["current_frame"]
        C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2)
        frustums = box_np_ops.get_frustum_v2(reference_detections, C)
        frustums -= T
        frustums = np.einsum("ij, akj->aki", np.linalg.inv(R), frustums)
        frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c)
        surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums)
        masks = points_in_convex_polygon_3d_jit(points, surfaces)
        points = points[masks.any(-1)]

    if remove_outside_points:
        assert calib is not None
        image_shape = input_dict["current_frame"]["image"]["image_shape"]
        points = box_np_ops.remove_outside_points(points, calib["rect"],
                                                  calib["Trv2c"], calib["P2"],
                                                  image_shape)
    if remove_environment is True and training:
        selected = kitti.keep_arrays_by_name(gt_names, target_assigner.classes)
        _dict_select(gt_dict, selected)
        masks = box_np_ops.points_in_rbbox(points, gt_dict["gt_boxes"])
        points = points[masks.any(-1)]

    if training:
        # boxes_lidar = gt_dict["gt_boxes"]
        # cv2.imshow('pre-noise', bev_map)
        selected = kitti.drop_arrays_by_name(gt_dict["gt_names"],
                                             ["DontCare", "ignore"])
        _dict_select(gt_dict, selected)
        if remove_unknown:
            remove_mask = gt_dict["difficulty"] == -1
            """
            gt_boxes_remove = gt_boxes[remove_mask]
            gt_boxes_remove[:, 3:6] += 0.25
            points = prep.remove_points_in_boxes(points, gt_boxes_remove)
            """
            keep_mask = np.logical_not(remove_mask)
            _dict_select(gt_dict, keep_mask)
        gt_dict.pop("difficulty")

        if min_points_in_gt > 0:
            # points_count_rbbox takes 10ms with 10 sweeps nuscenes data
            point_counts = box_np_ops.points_count_rbbox(
                points, gt_dict["gt_boxes"])
            mask = point_counts >= min_points_in_gt
            _dict_select(gt_dict, mask)

        gt_boxes_mask = np.array(
            [n in class_names for n in gt_dict["gt_names"]], dtype=np.bool_)

        # db_sampler = None
        if db_sampler is not None:
            group_ids = None
            # if "group_ids" in gt_dict:
            #     group_ids = gt_dict["group_ids"]
            sampled_dict = db_sampler.sample_all(
                root_path,
                gt_dict["gt_boxes"],
                gt_dict["gt_names"],
                num_point_features,
                random_crop,
                gt_group_ids=group_ids,
                calib=calib,
            )

            if sampled_dict is not None:
                sampled_gt_names = sampled_dict["gt_names"]
                sampled_gt_boxes = sampled_dict["gt_boxes"]
                sampled_points = sampled_dict["points"]
                sampled_gt_masks = sampled_dict["gt_masks"]
                gt_dict["gt_names"] = np.concatenate(
                    [gt_dict["gt_names"], sampled_gt_names], axis=0)
                gt_dict["gt_boxes"] = np.concatenate(
                    [gt_dict["gt_boxes"], sampled_gt_boxes])
                gt_boxes_mask = np.concatenate(
                    [gt_boxes_mask, sampled_gt_masks], axis=0)

                # if group_ids is not None:
                #     sampled_group_ids = sampled_dict["group_ids"]
                #     gt_dict["group_ids"] = np.concatenate(
                #         [gt_dict["group_ids"], sampled_group_ids])

                if remove_points_after_sample:
                    masks = box_np_ops.points_in_rbbox(points,
                                                       sampled_gt_boxes)
                    points = points[np.logical_not(masks.any(-1))]

                points = np.concatenate([sampled_points, points], axis=0)

        pc_range = voxel_generator.point_cloud_range

        # group_ids = None
        # if "group_ids" in gt_dict:
        #     group_ids = gt_dict["group_ids"]

        # prep.noise_per_object_v3_(
        #     gt_dict["gt_boxes"],
        #     points,
        #     gt_boxes_mask,
        #     rotation_perturb=gt_rotation_noise,
        #     center_noise_std=gt_loc_noise_std,
        #     global_random_rot_range=global_random_rot_range,
        #     group_ids=group_ids,
        #     num_try=100)

        # should remove unrelated objects after noise per object
        # for k, v in gt_dict.items():
        #     print(k, v.shape)

        _dict_select(gt_dict, gt_boxes_mask)

        gt_classes = np.array(
            [class_names.index(n) + 1 for n in gt_dict["gt_names"]],
            dtype=np.int32)
        gt_dict["gt_classes"] = gt_classes

        # concatenate
        points_current = points.shape[0]
        points_keyframe = keyframe_points.shape[0]
        points = np.concatenate((points, keyframe_points), axis=0)

        # data aug
        gt_dict["gt_boxes"], points = prep.random_flip(gt_dict["gt_boxes"],
                                                       points)
        gt_dict["gt_boxes"], points = prep.global_rotation(
            gt_dict["gt_boxes"], points, rotation=global_rotation_noise)
        gt_dict["gt_boxes"], points = prep.global_scaling_v2(
            gt_dict["gt_boxes"], points, *global_scaling_noise)
        prep.global_translate_(gt_dict["gt_boxes"], points,
                               global_translate_noise_std)

        # slice
        points_keyframe = points[points_current:, :]
        points = points[:points_current, :]

        bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]]
        mask = prep.filter_gt_box_outside_range(gt_dict["gt_boxes"], bv_range)
        _dict_select(gt_dict, mask)

        task_masks = []
        flag = 0
        for class_name in task_class_names:
            task_masks.append([
                np.where(gt_dict["gt_classes"] == class_name.index(i) + 1 +
                         flag) for i in class_name
            ])
            flag += len(class_name)

        task_boxes = []
        task_classes = []
        task_names = []
        flag2 = 0
        for idx, mask in enumerate(task_masks):
            task_box = []
            task_class = []
            task_name = []
            for m in mask:
                task_box.append(gt_dict["gt_boxes"][m])
                task_class.append(gt_dict["gt_classes"][m] - flag2)
                task_name.append(gt_dict["gt_names"][m])
            task_boxes.append(np.concatenate(task_box, axis=0))
            task_classes.append(np.concatenate(task_class))
            task_names.append(np.concatenate(task_name))
            flag2 += len(mask)

        for task_box in task_boxes:
            # limit rad to [-pi, pi]
            task_box[:, -1] = box_np_ops.limit_period(task_box[:, -1],
                                                      offset=0.5,
                                                      period=2 * np.pi)

        # print(gt_dict.keys())
        gt_dict["gt_classes"] = task_classes
        gt_dict["gt_names"] = task_names
        gt_dict["gt_boxes"] = task_boxes

    # if shuffle_points:
    #     # shuffle is a little slow.
    #     np.random.shuffle(points)

    # [0, -40, -3, 70.4, 40, 1]
    voxel_size = voxel_generator.voxel_size
    pc_range = voxel_generator.point_cloud_range
    grid_size = voxel_generator.grid_size
    # [352, 400]

    # points = points[:int(points.shape[0] * 0.1), :]
    voxels, coordinates, num_points = voxel_generator.generate(
        points, max_voxels)

    # res = voxel_generator.generate(
    #     points, max_voxels)
    # voxels = res["voxels"]
    # coordinates = res["coordinates"]
    # num_points = res["num_points_per_voxel"]

    num_voxels = np.array([voxels.shape[0]], dtype=np.int64)

    # key frame voxel
    keyframe_info = voxel_generator.generate(keyframe_points, max_voxels)
    keyframe_info = keyframe_voxels, keyframe_coordinates, keyframe_num_points

    keyframe_num_voxels = np.array([keyframe_voxels.shape[0]], dtype=np.int64)

    example = {
        "voxels": voxels,
        "num_points": num_points,
        "points": points,
        "coordinates": coordinates,
        "num_voxels": num_voxels,
    }

    example_keyframe = {
        "voxels": keyframe_voxels,
        "num_points": keyframe_num_points,
        "points": keyframe_points,
        "coordinates": keyframe_coordinates,
        "num_voxels": keyframe_num_voxels,
    }

    if training:
        example["gt_boxes"] = gt_dict["gt_boxes"]

    if calib is not None:
        example["calib"] = calib

    feature_map_size = grid_size[:2] // out_size_factor
    feature_map_size = [*feature_map_size, 1][::-1]

    if anchor_cache is not None:
        anchorss = anchor_cache["anchors"]
        anchors_bvs = anchor_cache["anchors_bv"]
        anchors_dicts = anchor_cache["anchors_dict"]
    else:
        rets = [
            target_assigner.generate_anchors(feature_map_size)
            for target_assigner in target_assigners
        ]
        anchorss = [ret["anchors"].reshape([-1, 7]) for ret in rets]
        anchors_dicts = [
            target_assigner.generate_anchors_dict(feature_map_size)
            for target_assigner in target_assigners
        ]
        anchors_bvs = [
            box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]])
            for anchors in anchorss
        ]

    example["anchors"] = anchorss

    if anchor_area_threshold >= 0:
        example["anchors_mask"] = []
        for idx, anchors_bv in enumerate(anchors_bvs):
            anchors_mask = None
            # slow with high resolution. recommend disable this forever.
            coors = coordinates
            dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask(
                coors, tuple(grid_size[::-1][1:]))
            dense_voxel_map = dense_voxel_map.cumsum(0)
            dense_voxel_map = dense_voxel_map.cumsum(1)
            anchors_area = box_np_ops.fused_get_anchors_area(
                dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size)
            anchors_mask = anchors_area > anchor_area_threshold
            # example['anchors_mask'] = anchors_mask.astype(np.uint8)
            example["anchors_mask"].append(anchors_mask)

    example_sequences = {}
    example_sequences["current_frame"] = example
    example_sequences["keyframe"] = example_keyframe

    if not training:
        return example_sequences

    # voxel_labels = box_np_ops.assign_label_to_voxel(gt_boxes, coordinates,
    #                                                 voxel_size, coors_range)
    """
    example.update({
        'gt_boxes': gt_boxes.astype(out_dtype),
        'num_gt': np.array([gt_boxes.shape[0]]),
        # 'voxel_labels': voxel_labels,
    })
    """
    if create_targets:
        targets_dicts = []
        for idx, target_assigner in enumerate(target_assigners):
            if "anchors_mask" in example:
                anchors_mask = example["anchors_mask"][idx]
            else:
                anchors_mask = None
            targets_dict = target_assigner.assign_v2(
                anchors_dicts[idx],
                gt_dict["gt_boxes"][idx],
                anchors_mask,
                gt_classes=gt_dict["gt_classes"][idx],
                gt_names=gt_dict["gt_names"][idx],
            )
            targets_dicts.append(targets_dict)

        example_sequences["current_frame"].update({
            "labels":
            [targets_dict["labels"] for targets_dict in targets_dicts],
            "reg_targets":
            [targets_dict["bbox_targets"] for targets_dict in targets_dicts],
            "reg_weights": [
                targets_dict["bbox_outside_weights"]
                for targets_dict in targets_dicts
            ],
        })
    return example_sequences