def points_in_rbbox(points, rbbox, lidar=True): if lidar: h_axis = 2 origin = [0.5, 0.5, 0] else: origin = [0.5, 1.0, 0.5] h_axis = 1 rbbox_corners = center_to_corner_box3d( rbbox[:, :3], rbbox[:, 3:6], rbbox[:, 6], origin=origin, axis=h_axis) surfaces = corner_to_surfaces_3d(rbbox_corners) indices = points_in_convex_polygon_3d_jit(points[:, :3], surfaces) return indices
def remove_outside_points(points, rect, Trv2c, P2, image_shape): # 5x faster than remove_outside_points_v1(2ms vs 10ms) C, R, T = projection_matrix_to_CRT_kitti(P2) image_bbox = [0, 0, image_shape[1], image_shape[0]] frustum = get_frustum(image_bbox, C) frustum -= T frustum = np.linalg.inv(R) @ frustum.T frustum = camera_to_lidar(frustum.T, rect, Trv2c) frustum_surfaces = corner_to_surfaces_3d_jit(frustum[np.newaxis, ...]) indices = points_in_convex_polygon_3d_jit(points[:, :3], frustum_surfaces) points = points[indices.reshape([-1])] return points
def assign_label_to_voxel(gt_boxes, coors, voxel_size, coors_range): """assign a 0/1 label to each voxel based on whether the center of voxel is in gt_box. LIDAR. """ voxel_size = np.array(voxel_size, dtype=gt_boxes.dtype) coors_range = np.array(coors_range, dtype=gt_boxes.dtype) shift = coors_range[:3] voxel_origins = coors[:, ::-1] * voxel_size + shift voxel_centers = voxel_origins + voxel_size * 0.5 gt_box_corners = center_to_corner_box3d( gt_boxes[:, :3] - voxel_size * 0.5, gt_boxes[:, 3:6] + voxel_size, gt_boxes[:, 6], origin=[0.5, 0.5, 0], axis=2) gt_surfaces = corner_to_surfaces_3d(gt_box_corners) ret = points_in_convex_polygon_3d_jit(voxel_centers, gt_surfaces) return np.any(ret, axis=1).astype(np.int64)
def noise_per_object_v2_(gt_boxes, points=None, valid_mask=None, rotation_perturb=np.pi / 4, center_noise_std=1.0, global_random_rot_range=np.pi / 4, num_try=100): """random rotate or remove each groundtrutn independently. use kitti viewer to test this function points_transform_ Args: gt_boxes: [N, 7], gt box in lidar.points_transform_ points: [M, 4], point cloud in lidar. """ num_boxes = gt_boxes.shape[0] if not isinstance(rotation_perturb, (list, tuple, np.ndarray)): rotation_perturb = [-rotation_perturb, rotation_perturb] if not isinstance(global_random_rot_range, (list, tuple, np.ndarray)): global_random_rot_range = [ -global_random_rot_range, global_random_rot_range ] if not isinstance(center_noise_std, (list, tuple, np.ndarray)): center_noise_std = [ center_noise_std, center_noise_std, center_noise_std ] if valid_mask is None: valid_mask = np.ones((num_boxes, ), dtype=np.bool_) center_noise_std = np.array(center_noise_std, dtype=gt_boxes.dtype) loc_noises = np.random.normal(scale=center_noise_std, size=[num_boxes, num_try, 3]) # loc_noises = np.random.uniform( # -center_noise_std, center_noise_std, size=[num_boxes, num_try, 3]) rot_noises = np.random.uniform(rotation_perturb[0], rotation_perturb[1], size=[num_boxes, num_try]) gt_grots = np.arctan2(gt_boxes[:, 0], gt_boxes[:, 1]) grot_lowers = global_random_rot_range[0] - gt_grots grot_uppers = global_random_rot_range[1] - gt_grots global_rot_noises = np.random.uniform(grot_lowers[..., np.newaxis], grot_uppers[..., np.newaxis], size=[num_boxes, num_try]) origin = [0.5, 0.5, 0] gt_box_corners = box_np_ops.center_to_corner_box3d(gt_boxes[:, :3], gt_boxes[:, 3:6], gt_boxes[:, 6], origin=origin, axis=2) if np.abs(global_random_rot_range[0] - global_random_rot_range[1]) < 1e-3: selected_noise = noise_per_box(gt_boxes[:, [0, 1, 3, 4, 6]], valid_mask, loc_noises, rot_noises) else: selected_noise = noise_per_box_v2_(gt_boxes[:, [0, 1, 3, 4, 6]], valid_mask, loc_noises, rot_noises, global_rot_noises) loc_transforms = _select_transform(loc_noises, selected_noise) rot_transforms = _select_transform(rot_noises, selected_noise) if points is not None: surfaces = box_np_ops.corner_to_surfaces_3d_jit(gt_box_corners) point_masks = points_in_convex_polygon_3d_jit(points[:, :3], surfaces) points_transform_(points, gt_boxes[:, :3], point_masks, loc_transforms, rot_transforms, valid_mask) box3d_transform_(gt_boxes, loc_transforms, rot_transforms, valid_mask)
def mask_points_in_corners(points, box_corners): surfaces = box_np_ops.corner_to_surfaces_3d(box_corners) mask = points_in_convex_polygon_3d_jit(points[:, :3], surfaces) return mask
def prep_pointcloud(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=20000, class_names=['Car'], remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, reduce_valid_area=False, remove_unknown=False, gt_rotation_noise=[-np.pi / 3, np.pi / 3], gt_loc_noise_std=[1.0, 1.0, 1.0], global_rotation_noise=[-np.pi / 4, np.pi / 4], global_scaling_noise=[0.95, 1.05], global_loc_noise_std=(0.2, 0.2, 0.2), global_random_rot_range=[0.78, 2.35], generate_bev=False, without_reflectivity=False, num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, add_rgb_to_points=False, lidar_input=False, unlabeled_db_sampler=None, out_size_factor=2, min_gt_point_dict=None, bev_only=False, use_group_id=False, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. """ points = input_dict["points"] if training: gt_boxes = input_dict["gt_boxes"] gt_names = input_dict["gt_names"] difficulty = input_dict["difficulty"] group_ids = None if use_group_id and "group_ids" in input_dict: group_ids = input_dict["group_ids"] rect = input_dict["rect"] Trv2c = input_dict["Trv2c"] P2 = input_dict["P2"] unlabeled_training = unlabeled_db_sampler is not None image_idx = input_dict["image_idx"] if reference_detections is not None: C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2) frustums = box_np_ops.get_frustum_v2(reference_detections, C) frustums -= T # frustums = np.linalg.inv(R) @ frustums.T frustums = np.einsum('ij, akj->aki', np.linalg.inv(R), frustums) frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c) surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums) masks = points_in_convex_polygon_3d_jit(points, surfaces) points = points[masks.any(-1)] if remove_outside_points and not lidar_input: image_shape = input_dict["image_shape"] points = box_np_ops.remove_outside_points(points, rect, Trv2c, P2, image_shape) if remove_environment is True and training: selected = kitti.keep_arrays_by_name(gt_names, class_names) gt_boxes = gt_boxes[selected] gt_names = gt_names[selected] difficulty = difficulty[selected] if group_ids is not None: group_ids = group_ids[selected] points = prep.remove_points_outside_boxes(points, gt_boxes) if training: # print(gt_names) selected = kitti.drop_arrays_by_name(gt_names, ["DontCare"]) gt_boxes = gt_boxes[selected] gt_names = gt_names[selected] difficulty = difficulty[selected] if group_ids is not None: group_ids = group_ids[selected] gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, rect, Trv2c) if remove_unknown: remove_mask = difficulty == -1 """ gt_boxes_remove = gt_boxes[remove_mask] gt_boxes_remove[:, 3:6] += 0.25 points = prep.remove_points_in_boxes(points, gt_boxes_remove) """ keep_mask = np.logical_not(remove_mask) gt_boxes = gt_boxes[keep_mask] gt_names = gt_names[keep_mask] difficulty = difficulty[keep_mask] if group_ids is not None: group_ids = group_ids[keep_mask] gt_boxes_mask = np.array([n in class_names for n in gt_names], dtype=np.bool_) if db_sampler is not None: sampled_dict = db_sampler.sample_all(root_path, gt_boxes, gt_names, num_point_features, random_crop, gt_group_ids=group_ids, rect=rect, Trv2c=Trv2c, P2=P2) if sampled_dict is not None: sampled_gt_names = sampled_dict["gt_names"] sampled_gt_boxes = sampled_dict["gt_boxes"] sampled_points = sampled_dict["points"] sampled_gt_masks = sampled_dict["gt_masks"] # gt_names = gt_names[gt_boxes_mask].tolist() gt_names = np.concatenate([gt_names, sampled_gt_names], axis=0) # gt_names += [s["name"] for s in sampled] gt_boxes = np.concatenate([gt_boxes, sampled_gt_boxes]) gt_boxes_mask = np.concatenate( [gt_boxes_mask, sampled_gt_masks], axis=0) if group_ids is not None: sampled_group_ids = sampled_dict["group_ids"] group_ids = np.concatenate([group_ids, sampled_group_ids]) if remove_points_after_sample: points = prep.remove_points_in_boxes( points, sampled_gt_boxes) points = np.concatenate([sampled_points, points], axis=0) # unlabeled_mask = np.zeros((gt_boxes.shape[0], ), dtype=np.bool_) if without_reflectivity: used_point_axes = list(range(num_point_features)) used_point_axes.pop(3) points = points[:, used_point_axes] pc_range = voxel_generator.point_cloud_range if bev_only: # set z and h to limits gt_boxes[:, 2] = pc_range[2] gt_boxes[:, 5] = pc_range[5] - pc_range[2] prep.noise_per_object_v3_( gt_boxes, points, gt_boxes_mask, rotation_perturb=gt_rotation_noise, center_noise_std=gt_loc_noise_std, global_random_rot_range=global_random_rot_range, group_ids=group_ids, num_try=100) # should remove unrelated objects after noise per object gt_boxes = gt_boxes[gt_boxes_mask] gt_names = gt_names[gt_boxes_mask] if group_ids is not None: group_ids = group_ids[gt_boxes_mask] gt_classes = np.array([class_names.index(n) + 1 for n in gt_names], dtype=np.int32) gt_boxes, points = prep.random_flip(gt_boxes, points) gt_boxes, points = prep.global_rotation(gt_boxes, points, rotation=global_rotation_noise) gt_boxes, points = prep.global_scaling_v2(gt_boxes, points, *global_scaling_noise) # Global translation gt_boxes, points = prep.global_translate(gt_boxes, points, global_loc_noise_std) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range(gt_boxes, bv_range) gt_boxes = gt_boxes[mask] gt_classes = gt_classes[mask] if group_ids is not None: group_ids = group_ids[mask] # limit rad to [-pi, pi] gt_boxes[:, 6] = box_np_ops.limit_period(gt_boxes[:, 6], offset=0.5, period=2 * np.pi) if shuffle_points: # shuffle is a little slow. np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # [352, 400] voxels, coordinates, num_points = voxel_generator.generate( points, max_voxels) example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": np.array([voxels.shape[0]], dtype=np.int64) } example.update({ 'rect': rect, 'Trv2c': Trv2c, 'P2': P2, }) # if not lidar_input: feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) example["anchors"] = anchors # print("debug", anchors.shape, matched_thresholds.shape) # anchors_bv = anchors_bv.reshape([-1, 4]) anchors_mask = None if anchor_area_threshold >= 0: coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask if generate_bev: bev_vxsize = voxel_size.copy() bev_vxsize[:2] /= 2 bev_vxsize[2] *= 2 bev_map = points_to_bev(points, bev_vxsize, pc_range, without_reflectivity) example["bev_map"] = bev_map if not training: return example if create_targets: targets_dict = target_assigner.assign( anchors, gt_boxes, anchors_mask, gt_classes=gt_classes, matched_thresholds=matched_thresholds, unmatched_thresholds=unmatched_thresholds) example.update({ 'labels': targets_dict['labels'], 'reg_targets': targets_dict['bbox_targets'], 'reg_weights': targets_dict['bbox_outside_weights'], }) return example