def _compare(self, boxes1, boxes2): """Compute matrix of (negated) sq distances. Args: boxlist1: BoxList holding N boxes. boxlist2: BoxList holding M boxes. Returns: A tensor with shape [N, M] representing negated pairwise squared distance. """ boxes1_bv = box_np_ops.rbbox2d_to_near_bbox(boxes1) boxes2_bv = box_np_ops.rbbox2d_to_near_bbox(boxes2) ret = box_np_ops.iou_jit(boxes1_bv, boxes2_bv, eps=0.0) return ret
def __init__(self, info_path, root_path, num_point_features, target_assigner, feature_map_size, prep_func): with open(info_path, 'rb') as f: infos = pickle.load(f) #self._kitti_infos = kitti.filter_infos_by_used_classes(infos, class_names) self._root_path = root_path self._kitti_infos = infos # kitti_infos_train.pkl self._num_point_features = num_point_features print("remain number of infos:", len(self._kitti_infos)) # generate anchors cache # feature_map_size:(1,248,216) ret = target_assigner.generate_anchors( feature_map_size) # 生成训练用锚框,包括匹配的锚框与未匹配的锚框 anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) anchor_cache = { # [z_nums*y_nums*x_nums*rotation_nums,[xyz_center+sizes+rad]] (xyz,wlh,rad,前后左右上下) "anchors": anchors, # array[107136, 7] "anchors_bv": anchors_bv, # array[107136, 4] [x_min,y_min,x_max,y_max] "matched_thresholds": matched_thresholds, # array[107136] "unmatched_thresholds": unmatched_thresholds, # array[107136] } self._prep_func = partial(prep_func, anchor_cache=anchor_cache)
def __init__(self, info_path, root_path, class_names, num_point_features, target_assigner, feature_map_size, prep_func): #with open(info_path, 'rb') as f: # infos = pickle.load(f) self._root_path = root_path self.argoverse_loader = ArgoverseTrackingLoader(root_path) self.am = ArgoverseMap() self.inform = info_path self._num_point_features = num_point_features self.class_names = class_names #print("remain number of infos:", len(self._kitti_infos)) # generate anchors cache # [352, 400] ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, } self._prep_func = partial(prep_func, anchor_cache=anchor_cache)
def __init__(self, info_path, root_path, num_point_features, target_assigner, feature_map_size, prep_func): with open(info_path, 'rb') as f: infos = pickle.load(f) #self._kitti_infos = kitti.filter_infos_by_used_classes(infos, class_names) self._root_path = root_path self._kitti_infos = infos self._num_point_features = num_point_features print("remain number of infos:", len(self._kitti_infos)) # generate anchors cache # [352, 400] ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] if target_assigner._anchor_generators[0]._anchor_strides[1]<0.1: anchors_bv = box_np_ops.polarbbox2d_to_near_bbox( anchors[:, [0, 1, 3, 4, 6]]) else: anchors_bv = box_np_ops.rbbox2d_to_near_bbox( anchors[:, [0, 1, 3, 4, 6]]) anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, } self._prep_func = partial(prep_func, anchor_cache=anchor_cache)
def generate_example(net, model_cfg, points, device): target_assigner = net.target_assigner voxel_generator = net.voxel_generator grid_size = voxel_generator.grid_size feature_map_size = grid_size[:2] // config_tool.get_downsample_factor( model_cfg) feature_map_size = [*feature_map_size, 1][::-1] anchors_np = target_assigner.generate_anchors(feature_map_size)["anchors"] anchors = torch.tensor(anchors_np, dtype=torch.float32, device=device) anchors = anchors.view(1, -1, 7) anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors_np[:, [0, 1, 3, 4, 6]]) res = voxel_generator.generate(points, max_voxels=12000) voxels = res["voxels"] coordinates = res["coordinates"] num_points = res["num_points_per_voxel"] # add batch idx to coords coords = np.pad(coordinates, ((0, 0), (1, 0)), mode='constant', constant_values=0) voxels = torch.tensor(voxels, dtype=torch.float32, device=device) coords = torch.tensor(coords, dtype=torch.float32, device=device) num_points = torch.tensor(num_points, dtype=torch.float32, device=device) # generate anchor mask # slow with high resolution. recommend disable this forever. voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area(dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > 1 anchors_mask = torch.tensor(anchors_mask, dtype=torch.uint8, device=device) # example['anchors_mask'] = anchors_mask.astype(np.uint8) example = { "anchors": anchors, "voxels": voxels, "num_points": num_points, "coordinates": coords, 'anchors_mask': anchors_mask, } return example
def _build(self): config = self.config input_cfg = config.eval_input_reader model_cfg = config.model.second train_cfg = config.train_config batch_size = 1 voxel_generator = voxel_builder.build(model_cfg.voxel_generator) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] grid_size = voxel_generator.grid_size self.voxel_generator = voxel_generator vfe_num_filters = list(model_cfg.voxel_feature_extractor.num_filters) box_coder = box_coder_builder.build(model_cfg.box_coder) target_assigner_cfg = model_cfg.target_assigner target_assigner = target_assigner_builder.build( target_assigner_cfg, bv_range, box_coder) self.target_assigner = target_assigner out_size_factor = model_cfg.rpn.layer_strides[ 0] / model_cfg.rpn.upsample_strides[0] out_size_factor *= model_cfg.middle_feature_extractor.downsample_factor out_size_factor = int(out_size_factor) assert out_size_factor > 0 self.net = second_builder.build(model_cfg, voxel_generator, target_assigner) self.net.cuda().eval() if train_cfg.enable_mixed_precision: self.net.half() self.net.metrics_to_float() self.net.convert_norm_to_float(self.net) feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] ret = target_assigner.generate_anchors(feature_map_size) anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) #print("feature_map_size is ",feature_map_size) #print("generated_anchors shape is",ret['anchors'].shape) anchors = ret["anchors"] #print("",ret['anchors'][0,79,79,1,:]) anchors = anchors.reshape([-1, 7]) #anchors_reshape = anchors.reshape([1,200,176,14]) #print("",anchors_reshape[0,79,79,7:]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) self.anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, "anchors_dict": anchors_dict, }
def generate_anchor_mask(self, anchors, anchor_area_threshold=-1): anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) anchors_mask = None if anchor_area_threshold >= 0: # slow with high resolution. recommend disable this forever. coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) return anchors_mask
def create_preprocess_fn(self): prep_cfg = self.config.eval_input_reader.preprocess out_size_factor=self.get_downsample_factor() assert out_size_factor > 0 preprocess_fn = partial(prep_main, voxel_generator=self.net.voxel_generator, target_assigner=self.net.target_assigner, max_voxels=prep_cfg.max_number_of_voxels, shuffle_points=prep_cfg.shuffle_points, anchor_area_threshold=prep_cfg.anchor_area_threshold, out_size_factor=out_size_factor, multi_gpu=False, calib=None) grid_size = self.net.voxel_generator.grid_size feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] assert all([n != '' for n in self.net.target_assigner.classes]), \ "you must specify class_name in anchor_generators." print("feature_map_size", feature_map_size) ret = self.net.target_assigner.generate_anchors(feature_map_size) class_names = self.net.target_assigner.classes anchors_dict = self.net.target_assigner.generate_anchors_dict(feature_map_size) anchors_list = [] for k, v in anchors_dict.items(): anchors_list.append(v["anchors"]) # anchors = ret["anchors"] anchors = np.concatenate(anchors_list, axis=0) anchors = anchors.reshape([-1, self.net.target_assigner.box_ndim]) assert np.allclose(anchors, ret["anchors"].reshape(-1, self.net.target_assigner.box_ndim)) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox( anchors[:, [0, 1, 3, 4, 6]]) anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, "anchors_dict": anchors_dict, } preprocess_fn = partial(preprocess_fn, anchor_cache=anchor_cache) return preprocess_fn
def __init__(self, points, root_path, num_point_features, target_assigner, feature_map_size, prep_func): #self._kitti_infos = kitti.filter_infos_by_used_classes(infos, class_names) self._root_path = root_path self._num_point_features = num_point_features self.points = points # generate anchors cache # [352, 400] ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, } self._prep_func = partial(prep_func, anchor_cache=anchor_cache)
def prep_pointcloud_ikg(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=20000, class_names=['Pedestrian'], remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, reduce_valid_area=False, remove_unknown=False, gt_rotation_noise=[-np.pi / 3, np.pi / 3], gt_loc_noise_std=[1.0, 1.0, 1.0], global_rotation_noise=[-np.pi / 4, np.pi / 4], global_scaling_noise=[0.95, 1.05], global_loc_noise_std=(0.2, 0.2, 0.2), global_random_rot_range=[0.78, 2.35], generate_bev=False, without_reflectivity=False, num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, add_rgb_to_points=False, lidar_input=False, unlabeled_db_sampler=None, out_size_factor=2, min_gt_point_dict=None, bev_only=False, use_group_id=False, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. """ points = input_dict["points"] if training: gt_boxes = input_dict["gt_boxes"] gt_names = input_dict["gt_names"] image_idx = input_dict["image_idx"] if training: # print(gt_names) selected = kitti.drop_arrays_by_name(gt_names, ["DontCare"]) gt_boxes = gt_boxes[selected] gt_names = gt_names[selected] # unlabeled_mask = np.zeros((gt_boxes.shape[0], ), dtype=np.bool_) gt_boxes_mask = np.array([n in class_names for n in gt_names], dtype=np.bool_) pc_range = voxel_generator.point_cloud_range if bev_only: # set z and h to limits gt_boxes[:, 2] = pc_range[2] gt_boxes[:, 5] = pc_range[5] - pc_range[2] # should remove unrelated objects after noise per object gt_boxes = gt_boxes[gt_boxes_mask] gt_names = gt_names[gt_boxes_mask] gt_classes = np.array([class_names.index(n) + 1 for n in gt_names], dtype=np.int32) # gt_boxes, points = prep.random_flip(gt_boxes, points) # gt_boxes, points = prep.global_rotation( # gt_boxes, points, rotation=global_rotation_noise) # gt_boxes, points = prep.global_scaling_v2(gt_boxes, points, # *global_scaling_noise) # Global translation #gt_boxes, points = prep.global_translate(gt_boxes, points, global_loc_noise_std) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range(gt_boxes, bv_range) gt_boxes = gt_boxes[mask] gt_classes = gt_classes[mask] # limit rad to [-pi, pi] gt_boxes[:, 6] = box_np_ops.limit_period(gt_boxes[:, 6], offset=0.5, period=2 * np.pi) #打乱点的顺序 # if shuffle_points: # # shuffle is a little slow. # np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # [352, 400] voxels, coordinates, num_points = voxel_generator.generate( points, max_voxels) example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": np.array([voxels.shape[0]], dtype=np.int64) } # if not lidar_input: feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) example["anchors"] = anchors # print("debug", anchors.shape, matched_thresholds.shape) # anchors_bv = anchors_bv.reshape([-1, 4]) anchors_mask = None if anchor_area_threshold >= 0: coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask if generate_bev: bev_vxsize = voxel_size.copy() bev_vxsize[:2] /= 2 bev_vxsize[2] *= 2 bev_map = points_to_bev(points, bev_vxsize, pc_range, without_reflectivity) example["bev_map"] = bev_map #if testing(without labels),return example = { #voxels,num_points,coordinates,num_voxels} if not training: return example if create_targets: targets_dict = target_assigner.assign( anchors, gt_boxes, anchors_mask, gt_classes=gt_classes, matched_thresholds=matched_thresholds, unmatched_thresholds=unmatched_thresholds) example.update({ 'labels': targets_dict['labels'], 'reg_targets': targets_dict['bbox_targets'], 'reg_weights': targets_dict['bbox_outside_weights'], }) return example
def prep_main(points, calib, voxel_generator, target_assigner, max_voxels=20000, shuffle_points=False, anchor_area_threshold=1, anchor_cache=None, out_size_factor=2, multi_gpu=False): """ Main preprocessing: coverting points to voxels. Main args: points: np.ndarray(dtype=float32, shape=(N, 3+)) TODO: confirm this. calib: calibration Returns: example: dict{'voxels', 'num_points', 'coordinates', 'num_voxels', 'metrics', 'calib', 'anchors', 'anchors_mask', 'anchors_dict', 'matched_thresholds', 'unmatched_thresholds' } """ metrics = {} if shuffle_points: # shuffle is a little slow. np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # [352, 400] t1 = time.time() if not multi_gpu: res = voxel_generator.generate( points, max_voxels) voxels = res["voxels"] coordinates = res["coordinates"] num_points = res["num_points_per_voxel"] num_voxels = np.array([voxels.shape[0]], dtype=np.int64) else: res = voxel_generator.generate_multi_gpu( points, max_voxels) voxels = res["voxels"] coordinates = res["coordinates"] num_points = res["num_points_per_voxel"] num_voxels = np.array([res["voxel_num"]], dtype=np.int64) metrics["voxel_gene_time"] = time.time() - t1 example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": num_voxels, "metrics": metrics, } if calib is not None: example["calib"] = calib feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] anchors_dict = anchor_cache["anchors_dict"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, target_assigner.box_ndim]) anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) anchors_bv = box_np_ops.rbbox2d_to_near_bbox( anchors[:, [0, 1, 3, 4, 6]]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] example["anchors"] = anchors example["anchors_dict"] = anchors_dict example["matched_thresholds"] = matched_thresholds example["unmatched_thresholds"] = unmatched_thresholds anchors_mask = None if anchor_area_threshold >= 0: # slow with high resolution. recommend disable this forever. coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask return example
def test(config_path=args.config_path, model_dir=args.model_dir, result_path=None, create_folder=False, pickle_result=True, include_roadmap=False, device=1): """train a VoxelNet model specified by a config file. """ if create_folder: if pathlib.Path(model_dir).exists(): model_dir = torchplus.train.create_folder(model_dir) model_dir = pathlib.Path(model_dir) model_dir.mkdir(parents=True, exist_ok=True) eval_checkpoint_dir = model_dir / 'eval_checkpoints' eval_checkpoint_dir.mkdir(parents=True, exist_ok=True) if result_path is None: result_path = model_dir / 'results' config_file_bkp = "pipeline.config" config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) shutil.copyfile(config_path, str(model_dir / config_file_bkp)) input_cfg = config.train_input_reader eval_input_cfg = config.eval_input_reader model_cfg = config.model.second train_cfg = config.train_config batch_size = 1 class_names = list(input_cfg.class_names) ###################### # BUILD VOXEL GENERATOR ###################### voxel_generator = voxel_builder.build(model_cfg.voxel_generator) grid_size = voxel_generator.grid_size ###################### # BUILD TARGET ASSIGNER ###################### bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] box_coder = box_coder_builder.build(model_cfg.box_coder) target_assigner_cfg = model_cfg.target_assigner target_assigner = target_assigner_builder.build(target_assigner_cfg, bv_range, box_coder) ###################### # BUILD NET ###################### center_limit_range = model_cfg.post_center_limit_range net = second_builder.build(model_cfg, voxel_generator, target_assigner, include_roadmap) net.cuda().eval() print("num_trainable parameters:", len(list(net.parameters()))) # for n, p in net.named_parameters(): # print(n, p.shape) #torchplus.train.try_restore_latest_checkpoints(model_dir, [net]) torchplus.train.restore(args.model_path, net) #torchplus.train.restore("./ped_models_56/voxelnet-275130.tckpt",net) out_size_factor = model_cfg.rpn.layer_strides[ 0] / model_cfg.rpn.upsample_strides[0] print(out_size_factor) #out_size_factor *= model_cfg.middle_feature_extractor.downsample_factor out_size_factor = int(out_size_factor) feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] print(feature_map_size) ret = target_assigner.generate_anchors(feature_map_size) #anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, #"anchors_dict": anchors_dict, } am = ArgoverseMap() dt_annos = [] root_dir = os.path.join('./../../argodataset/argoverse-tracking/', args.set) argoverse_loader = ArgoverseTrackingLoader(root_dir) prog_cnt = 0 for seq in range(len(argoverse_loader)): argoverse_data = argoverse_loader[seq] nlf = argoverse_data.num_lidar_frame for frame in range(nlf): prog_cnt += 1 if prog_cnt % 50 == 0: print(prog_cnt) points = argoverse_data.get_lidar(frame) roi_pts = copy.deepcopy(points) city_name = argoverse_data.city_name city_to_egovehicle_se3 = argoverse_data.get_pose(frame) ''' roi_pts = city_to_egovehicle_se3.transform_point_cloud(roi_pts) # put into city coords #non roi roi_pts_flag = am.remove_non_roi_points(roi_pts, city_name) # remove non-driveable region roi_pts = roi_pts[roi_pts_flag] roi_pts = am.remove_ground_surface(roi_pts, city_name) # remove ground surface # convert city to lidar co-ordinates roi_pts = city_to_egovehicle_se3.inverse_transform_point_cloud(roi_pts) ''' if args.include_roi or args.dr_area or not args.include_road_points: roi_pts = city_to_egovehicle_se3.transform_point_cloud( roi_pts) # put into city coords if args.include_roi: roi_pts_flag = am.remove_non_roi_points( roi_pts, city_name) # remove non-driveable region roi_pts = roi_pts[roi_pts_flag] if not args.include_roi and args.dr_area: roi_pts_flag = am.remove_non_driveable_area_points( roi_pts, city_name) # remove non-driveable region roi_pts = roi_pts[roi_pts_flag] if not args.include_road_points: roi_pts = am.remove_ground_surface( roi_pts, city_name) # remove ground surface # convert city to lidar co-ordinates if args.include_roi or args.dr_area or not args.include_road_points: roi_pts = city_to_egovehicle_se3.inverse_transform_point_cloud( roi_pts) roi_pts[:, 2] = roi_pts[:, 2] - 1.73 pts_x, pts_y, pts_z = roi_pts[:, 0], roi_pts[:, 1], roi_pts[:, 2] input_dict = { 'points': roi_pts, 'pointcloud_num_features': 3, } out_size_factor = model_cfg.rpn.layer_strides[ 0] // model_cfg.rpn.upsample_strides[0] example = prep_pointcloud( input_dict=input_dict, root_path=None, voxel_generator=voxel_generator, target_assigner=target_assigner, max_voxels=input_cfg.max_number_of_voxels, class_names=list(input_cfg.class_names), training=False, create_targets=False, shuffle_points=input_cfg.shuffle_points, generate_bev=False, without_reflectivity=model_cfg.without_reflectivity, num_point_features=model_cfg.num_point_features, anchor_area_threshold=input_cfg.anchor_area_threshold, anchor_cache=anchor_cache, out_size_factor=out_size_factor, out_dtype=np.float32) if "anchors_mask" in example: example["anchors_mask"] = example["anchors_mask"].astype( np.uint8) example["image_idx"] = str(seq) + "_" + str(frame) example["image_shape"] = np.array([400, 400], dtype=np.int32) example["road_map"] = None example["include_roadmap"] = False example["points"] = roi_pts #torch.save(example,"./network_input_examples/" + info) example = merge_second_batch([example]) example_torch = example_convert_to_torch(example, device=args.device) try: result_annos = predict_kitti_to_anno( net, example_torch, input_cfg.class_names, model_cfg.post_center_limit_range, model_cfg.lidar_input) except: print(seq, frame) continue dt_annos += result_annos if pickle_result: sdi = args.save_path.rfind('/') save_dir = args.save_path[:sdi] if not os.path.exists(save_dir): os.mkdir(save_dir) with open(args.save_path, 'wb') as f: pickle.dump(dt_annos, f)
def prep_pointcloud(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=20000, remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, remove_unknown=False, gt_rotation_noise=(-np.pi / 3, np.pi / 3), gt_loc_noise_std=(1.0, 1.0, 1.0), global_rotation_noise=(-np.pi / 4, np.pi / 4), global_scaling_noise=(0.95, 1.05), global_random_rot_range=(0.78, 2.35), global_translate_noise_std=(0, 0, 0), num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, out_size_factor=2, use_group_id=False, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. input_dict format: dataset.get_sensor_data format """ # t = time.time() class_names = target_assigner.classes points = input_dict["lidar"]["points"] if training: anno_dict = input_dict["lidar"]["annotations"] gt_dict = { "gt_boxes": anno_dict["boxes"], "gt_names": anno_dict["names"], } if "difficulty" not in anno_dict: difficulty = np.zeros([anno_dict["boxes"].shape[0]], dtype=np.int32) gt_dict["difficulty"] = difficulty else: gt_dict["difficulty"] = anno_dict["difficulty"] if use_group_id and "group_ids" in anno_dict: group_ids = anno_dict["group_ids"] gt_dict["group_ids"] = group_ids calib = None if "calib" in input_dict: calib = input_dict["calib"] if reference_detections is not None: assert calib is not None and "image" in input_dict C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2) frustums = box_np_ops.get_frustum_v2(reference_detections, C) frustums -= T frustums = np.einsum('ij, akj->aki', np.linalg.inv(R), frustums) frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c) surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums) masks = points_in_convex_polygon_3d_jit(points, surfaces) points = points[masks.any(-1)] if remove_outside_points: assert calib is not None image_shape = input_dict["image"]["image_shape"] points = box_np_ops.remove_outside_points(points, calib["rect"], calib["Trv2c"], calib["P2"], image_shape) if remove_environment is True and training: selected = kitti.keep_arrays_by_name(gt_names, target_assigner.classes) _dict_select(gt_dict, selected) masks = box_np_ops.points_in_rbbox(points, gt_dict["gt_boxes"]) points = points[masks.any(-1)] if training: # boxes_lidar = gt_dict["gt_boxes"] # bev_map = simplevis.nuscene_vis(points, boxes_lidar) # cv2.imshow('pre-noise', bev_map) selected = kitti.drop_arrays_by_name(gt_dict["gt_names"], ["DontCare"]) _dict_select(gt_dict, selected) if remove_unknown: remove_mask = gt_dict["difficulty"] == -1 """ gt_boxes_remove = gt_boxes[remove_mask] gt_boxes_remove[:, 3:6] += 0.25 points = prep.remove_points_in_boxes(points, gt_boxes_remove) """ keep_mask = np.logical_not(remove_mask) _dict_select(gt_dict, keep_mask) gt_dict.pop("difficulty") gt_boxes_mask = np.array( [n in class_names for n in gt_dict["gt_names"]], dtype=np.bool_) if db_sampler is not None: group_ids = None if "group_ids" in gt_dict: group_ids = gt_dict["group_ids"] sampled_dict = db_sampler.sample_all(root_path, gt_dict["gt_boxes"], gt_dict["gt_names"], num_point_features, random_crop, gt_group_ids=group_ids, calib=calib) if sampled_dict is not None: sampled_gt_names = sampled_dict["gt_names"] sampled_gt_boxes = sampled_dict["gt_boxes"] sampled_points = sampled_dict["points"] sampled_gt_masks = sampled_dict["gt_masks"] gt_dict["gt_names"] = np.concatenate( [gt_dict["gt_names"], sampled_gt_names], axis=0) gt_dict["gt_boxes"] = np.concatenate( [gt_dict["gt_boxes"], sampled_gt_boxes]) gt_boxes_mask = np.concatenate( [gt_boxes_mask, sampled_gt_masks], axis=0) if group_ids is not None: sampled_group_ids = sampled_dict["group_ids"] gt_dict["group_ids"] = np.concatenate( [gt_dict["group_ids"], sampled_group_ids]) if remove_points_after_sample: masks = box_np_ops.points_in_rbbox(points, sampled_gt_boxes) points = points[np.logical_not(masks.any(-1))] points = np.concatenate([sampled_points, points], axis=0) pc_range = voxel_generator.point_cloud_range group_ids = None if "group_ids" in gt_dict: group_ids = gt_dict["group_ids"] prep.noise_per_object_v3_( gt_dict["gt_boxes"], points, gt_boxes_mask, rotation_perturb=gt_rotation_noise, center_noise_std=gt_loc_noise_std, global_random_rot_range=global_random_rot_range, group_ids=group_ids, num_try=100) # should remove unrelated objects after noise per object # for k, v in gt_dict.items(): # print(k, v.shape) _dict_select(gt_dict, gt_boxes_mask) gt_classes = np.array( [class_names.index(n) + 1 for n in gt_dict["gt_names"]], dtype=np.int32) gt_dict["gt_classes"] = gt_classes gt_dict["gt_boxes"], points = prep.random_flip(gt_dict["gt_boxes"], points) gt_dict["gt_boxes"], points = prep.global_rotation( gt_dict["gt_boxes"], points, rotation=global_rotation_noise) gt_dict["gt_boxes"], points = prep.global_scaling_v2( gt_dict["gt_boxes"], points, *global_scaling_noise) prep.global_translate_(gt_dict["gt_boxes"], points, global_translate_noise_std) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range(gt_dict["gt_boxes"], bv_range) _dict_select(gt_dict, mask) # limit rad to [-pi, pi] gt_dict["gt_boxes"][:, 6] = box_np_ops.limit_period( gt_dict["gt_boxes"][:, 6], offset=0.5, period=2 * np.pi) # boxes_lidar = gt_dict["gt_boxes"] # bev_map = simplevis.nuscene_vis(points, boxes_lidar) # cv2.imshow('post-noise', bev_map) # cv2.waitKey(0) if shuffle_points: # shuffle is a little slow. np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # [352, 400] voxels, coordinates, num_points = voxel_generator.generate( points, max_voxels) example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": np.array([voxels.shape[0]], dtype=np.int64) } if calib is not None: example["calib"] = calib feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] anchors_dict = anchor_cache["anchors_dict"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) example["anchors"] = anchors anchors_mask = None if anchor_area_threshold >= 0: # slow with high resolution. recommend disable this forever. coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask if not training: return example # voxel_labels = box_np_ops.assign_label_to_voxel(gt_boxes, coordinates, # voxel_size, coors_range) """ example.update({ 'gt_boxes': gt_boxes.astype(out_dtype), 'num_gt': np.array([gt_boxes.shape[0]]), # 'voxel_labels': voxel_labels, }) """ if create_targets: targets_dict = target_assigner.assign_v2( anchors_dict, gt_dict["gt_boxes"], anchors_mask, gt_classes=gt_dict["gt_classes"], gt_names=gt_dict["gt_names"]) example.update({ 'labels': targets_dict['labels'], 'reg_targets': targets_dict['bbox_targets'], 'reg_weights': targets_dict['bbox_outside_weights'], }) return example
def prep_pointcloud2(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=20000, class_names=['Car'], remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, reduce_valid_area=False, remove_unknown=False, gt_rotation_noise=[-np.pi / 3, np.pi / 3], gt_loc_noise_std=[1.0, 1.0, 1.0], global_rotation_noise=[-np.pi / 4, np.pi / 4], global_scaling_noise=[0.95, 1.05], global_loc_noise_std=(0.2, 0.2, 0.2), global_random_rot_range=[0.78, 2.35], generate_bev=False, without_reflectivity=False, num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, add_rgb_to_points=False, lidar_input=False, unlabeled_db_sampler=None, out_size_factor=2, min_gt_point_dict=None, bev_only=False, use_group_id=False, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. """ points = input_dict["points"] rect = input_dict["rect"] Trv2c = input_dict["Trv2c"] P2 = input_dict["P2"] unlabeled_training = unlabeled_db_sampler is not None image_idx = input_dict["image_idx"] if reference_detections is not None: C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2) frustums = box_np_ops.get_frustum_v2(reference_detections, C) frustums -= T # frustums = np.linalg.inv(R) @ frustums.T frustums = np.einsum('ij, akj->aki', np.linalg.inv(R), frustums) frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c) surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums) masks = points_in_convex_polygon_3d_jit(points, surfaces) points = points[masks.any(-1)] now = rospy.get_rostime() print('before shuffle TIME:' + str(now.to_sec())) if shuffle_points: # shuffle is a little slow. np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # [352, 400] now = rospy.get_rostime() print('before voxel generate:' + str(now.to_sec())) voxels, coordinates, num_points = voxel_generator.generate( points, max_voxels) now = rospy.get_rostime() print('before encording TIME:' + str(now.to_sec())) example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": np.array([voxels.shape[0]], dtype=np.int64) } now = rospy.get_rostime() print('before encording2 TIME:' + str(now.to_sec())) example.update({ 'rect': rect, 'Trv2c': Trv2c, 'P2': P2, }) now = rospy.get_rostime() print('before anchor_cache TIME:' + str(now.to_sec())) # if not lidar_input: feature_map_size = grid_size[:2] // out_size_factor now = rospy.get_rostime() print('before anchor_fm TIME:' + str(now.to_sec())) feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) example["anchors"] = anchors # print("debug", anchors.shape, matched_thresholds.shape) # anchors_bv = anchors_bv.reshape([-1, 4]) anchors_mask = None now = rospy.get_rostime() print('before create_anchor_mask TIME:' + str(now.to_sec())) #costs time if anchor_area_threshold >= 0: coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask now = rospy.get_rostime() print('before create_bev TIME:' + str(now.to_sec())) if generate_bev: bev_vxsize = voxel_size.copy() bev_vxsize[:2] /= 2 bev_vxsize[2] *= 2 bev_map = points_to_bev(points, bev_vxsize, pc_range, without_reflectivity) example["bev_map"] = bev_map now = rospy.get_rostime() print('after create bev TIME:' + str(now.to_sec())) if not training: return example
def prep_pointcloud(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=20000, remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, remove_unknown=False, gt_rotation_noise=(-np.pi / 3, np.pi / 3), gt_loc_noise_std=(1.0, 1.0, 1.0), global_rotation_noise=(-np.pi / 4, np.pi / 4), global_scaling_noise=(0.95, 1.05), global_random_rot_range=(0.78, 2.35), global_translate_noise_std=(0, 0, 0), num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, out_size_factor=2, use_group_id=False, multi_gpu=False, min_points_in_gt=-1, random_flip_x=True, random_flip_y=True, sample_importance=1.0, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. input_dict format: dataset.get_sensor_data format """ t = time.time() class_names = target_assigner.classes points = input_dict["lidar"]["points"] if training: anno_dict = input_dict["lidar"]["annotations"] gt_dict = { "gt_boxes": anno_dict["boxes"], "gt_names": anno_dict["names"], "gt_importance": np.ones([anno_dict["boxes"].shape[0]], dtype=anno_dict["boxes"].dtype), } if "difficulty" not in anno_dict: difficulty = np.zeros([anno_dict["boxes"].shape[0]], dtype=np.int32) gt_dict["difficulty"] = difficulty else: gt_dict["difficulty"] = anno_dict["difficulty"] if use_group_id and "group_ids" in anno_dict: group_ids = anno_dict["group_ids"] gt_dict["group_ids"] = group_ids calib = None if "calib" in input_dict: calib = input_dict["calib"] if reference_detections is not None: assert calib is not None and "image" in input_dict C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2) frustums = box_np_ops.get_frustum_v2(reference_detections, C) frustums -= T frustums = np.einsum('ij, akj->aki', np.linalg.inv(R), frustums) frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c) surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums) masks = points_in_convex_polygon_3d_jit(points, surfaces) points = points[masks.any(-1)] if remove_outside_points: assert calib is not None image_shape = input_dict["image"]["image_shape"] points = box_np_ops.remove_outside_points( points, calib["rect"], calib["Trv2c"], calib["P2"], image_shape) if remove_environment is True and training: selected = kitti.keep_arrays_by_name(gt_names, target_assigner.classes) _dict_select(gt_dict, selected) masks = box_np_ops.points_in_rbbox(points, gt_dict["gt_boxes"]) points = points[masks.any(-1)] metrics = {} if training: """ boxes_lidar = gt_dict["gt_boxes"] bev_map = simplevis.nuscene_vis(points, boxes_lidar) cv2.imshow('pre-noise', bev_map) """ selected = kitti.drop_arrays_by_name(gt_dict["gt_names"], ["DontCare"]) _dict_select(gt_dict, selected) if remove_unknown: remove_mask = gt_dict["difficulty"] == -1 """ gt_boxes_remove = gt_boxes[remove_mask] gt_boxes_remove[:, 3:6] += 0.25 points = prep.remove_points_in_boxes(points, gt_boxes_remove) """ keep_mask = np.logical_not(remove_mask) _dict_select(gt_dict, keep_mask) gt_dict.pop("difficulty") if min_points_in_gt > 0: # points_count_rbbox takes 10ms with 10 sweeps nuscenes data point_counts = box_np_ops.points_count_rbbox(points, gt_dict["gt_boxes"]) mask = point_counts >= min_points_in_gt _dict_select(gt_dict, mask) gt_boxes_mask = np.array( [n in class_names for n in gt_dict["gt_names"]], dtype=np.bool_) if db_sampler is not None: group_ids = None if "group_ids" in gt_dict: group_ids = gt_dict["group_ids"] sampled_dict = db_sampler.sample_all( root_path, gt_dict["gt_boxes"], gt_dict["gt_names"], num_point_features, random_crop, gt_group_ids=group_ids, calib=calib) if sampled_dict is not None: sampled_gt_names = sampled_dict["gt_names"] sampled_gt_boxes = sampled_dict["gt_boxes"] sampled_points = sampled_dict["points"] sampled_gt_masks = sampled_dict["gt_masks"] gt_dict["gt_names"] = np.concatenate( [gt_dict["gt_names"], sampled_gt_names], axis=0) gt_dict["gt_boxes"] = np.concatenate( [gt_dict["gt_boxes"], sampled_gt_boxes]) gt_boxes_mask = np.concatenate( [gt_boxes_mask, sampled_gt_masks], axis=0) sampled_gt_importance = np.full([sampled_gt_boxes.shape[0]], sample_importance, dtype=sampled_gt_boxes.dtype) gt_dict["gt_importance"] = np.concatenate( [gt_dict["gt_importance"], sampled_gt_importance]) if group_ids is not None: sampled_group_ids = sampled_dict["group_ids"] gt_dict["group_ids"] = np.concatenate( [gt_dict["group_ids"], sampled_group_ids]) if remove_points_after_sample: masks = box_np_ops.points_in_rbbox(points, sampled_gt_boxes) points = points[np.logical_not(masks.any(-1))] points = np.concatenate([sampled_points, points], axis=0) pc_range = voxel_generator.point_cloud_range group_ids = None if "group_ids" in gt_dict: group_ids = gt_dict["group_ids"] prep.noise_per_object_v3_( gt_dict["gt_boxes"], points, gt_boxes_mask, rotation_perturb=gt_rotation_noise, center_noise_std=gt_loc_noise_std, global_random_rot_range=global_random_rot_range, group_ids=group_ids, num_try=100) # should remove unrelated objects after noise per object # for k, v in gt_dict.items(): # print(k, v.shape) _dict_select(gt_dict, gt_boxes_mask) gt_classes = np.array( [class_names.index(n) + 1 for n in gt_dict["gt_names"]], dtype=np.int32) gt_dict["gt_classes"] = gt_classes gt_dict["gt_boxes"], points = prep.random_flip(gt_dict["gt_boxes"], points, 0.5, random_flip_x, random_flip_y) gt_dict["gt_boxes"], points = prep.global_rotation_v2( gt_dict["gt_boxes"], points, *global_rotation_noise) gt_dict["gt_boxes"], points = prep.global_scaling_v2( gt_dict["gt_boxes"], points, *global_scaling_noise) prep.global_translate_(gt_dict["gt_boxes"], points, global_translate_noise_std) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range_by_center(gt_dict["gt_boxes"], bv_range) _dict_select(gt_dict, mask) # limit rad to [-pi, pi] gt_dict["gt_boxes"][:, 6] = box_np_ops.limit_period( gt_dict["gt_boxes"][:, 6], offset=0.5, period=2 * np.pi) # boxes_lidar = gt_dict["gt_boxes"] # bev_map = simplevis.nuscene_vis(points, boxes_lidar) # cv2.imshow('post-noise', bev_map) # cv2.waitKey(0) if shuffle_points: # shuffle is a little slow. np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # [352, 400] t1 = time.time() if not multi_gpu: res = voxel_generator.generate( points, max_voxels) voxels = res["voxels"] coordinates = res["coordinates"] num_points = res["num_points_per_voxel"] num_voxels = np.array([voxels.shape[0]], dtype=np.int64) else: res = voxel_generator.generate_multi_gpu( points, max_voxels) voxels = res["voxels"] coordinates = res["coordinates"] num_points = res["num_points_per_voxel"] num_voxels = np.array([res["voxel_num"]], dtype=np.int64) metrics["voxel_gene_time"] = time.time() - t1 example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": num_voxels, "metrics": metrics, } if calib is not None: example["calib"] = calib feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] anchors_dict = anchor_cache["anchors_dict"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, target_assigner.box_ndim]) anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) anchors_bv = box_np_ops.rbbox2d_to_near_bbox( anchors[:, [0, 1, 3, 4, 6]]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] example["anchors"] = anchors anchors_mask = None if anchor_area_threshold >= 0: # slow with high resolution. recommend disable this forever. coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask # print("prep time", time.time() - t) metrics["prep_time"] = time.time() - t if not training: return example example["gt_names"] = gt_dict["gt_names"] # voxel_labels = box_np_ops.assign_label_to_voxel(gt_boxes, coordinates, # voxel_size, coors_range) if create_targets: t1 = time.time() targets_dict = target_assigner.assign( anchors, anchors_dict, gt_dict["gt_boxes"], anchors_mask, gt_classes=gt_dict["gt_classes"], gt_names=gt_dict["gt_names"], matched_thresholds=matched_thresholds, unmatched_thresholds=unmatched_thresholds, importance=gt_dict["gt_importance"]) """ boxes_lidar = gt_dict["gt_boxes"] bev_map = simplevis.nuscene_vis(points, boxes_lidar, gt_dict["gt_names"]) assigned_anchors = anchors[targets_dict['labels'] > 0] ignored_anchors = anchors[targets_dict['labels'] == -1] bev_map = simplevis.draw_box_in_bev(bev_map, [-50, -50, 3, 50, 50, 1], ignored_anchors, [128, 128, 128], 2) bev_map = simplevis.draw_box_in_bev(bev_map, [-50, -50, 3, 50, 50, 1], assigned_anchors, [255, 0, 0]) cv2.imshow('anchors', bev_map) cv2.waitKey(0) boxes_lidar = gt_dict["gt_boxes"] pp_map = np.zeros(grid_size[:2], dtype=np.float32) voxels_max = np.max(voxels[:, :, 2], axis=1, keepdims=False) voxels_min = np.min(voxels[:, :, 2], axis=1, keepdims=False) voxels_height = voxels_max - voxels_min voxels_height = np.minimum(voxels_height, 4) # sns.distplot(voxels_height) # plt.show() pp_map[coordinates[:, 1], coordinates[:, 2]] = voxels_height / 4 pp_map = (pp_map * 255).astype(np.uint8) pp_map = cv2.cvtColor(pp_map, cv2.COLOR_GRAY2RGB) pp_map = simplevis.draw_box_in_bev(pp_map, [-50, -50, 3, 50, 50, 1], boxes_lidar, [128, 0, 128], 1) cv2.imshow('heights', pp_map) cv2.waitKey(0) """ example.update({ 'labels': targets_dict['labels'], 'reg_targets': targets_dict['bbox_targets'], # 'reg_weights': targets_dict['bbox_outside_weights'], 'importance': targets_dict['importance'], }) return example
def prep_pointcloud(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=20000, class_names=['PEDESTRIAN'], remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, reduce_valid_area=False, remove_unknown=False, gt_rotation_noise=[-np.pi / 3, np.pi / 3], gt_loc_noise_std=[1.0, 1.0, 1.0], global_rotation_noise=[-np.pi / 4, np.pi / 4], global_scaling_noise=[0.95, 1.05], global_loc_noise_std=(0.2, 0.2, 0.2), global_random_rot_range=[0.78, 2.35], generate_bev=False, without_reflectivity=False, num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, add_rgb_to_points=False, lidar_input=False, unlabeled_db_sampler=None, out_size_factor=2, min_gt_point_dict=None, bev_only=False, use_group_id=False, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. """ points = input_dict["points"] pc_range = voxel_generator.point_cloud_range pts_x, pts_y, pts_z = points[:, 0], points[:, 1], points[:, 2] range_flag = ((pts_x >= pc_range[0]) & (pts_x <= pc_range[3]) & (pts_y >= pc_range[1]) & (pts_y <= pc_range[4]) & (pts_z >= pc_range[2]) & (pts_z <= pc_range[5])) points = points[range_flag] if training: gt_boxes = input_dict["gt_boxes"] gt_names = input_dict["gt_names"] ## group_ids ? np.arange(num_gt,dtype=np.int32) num_gt - number of objects (of all categories) in annotated lidar frame group_ids = None if use_group_id and "group_ids" in input_dict: group_ids = input_dict["group_ids"] #unlabeled_training = unlabeled_db_sampler is not None if training: gt_boxes_mask = np.array([n in class_names for n in gt_names], dtype=np.bool_) #print(gt_boxes_mask.shape,gt_boxes.shape,"before") prep.noise_per_object_v3_( gt_boxes, points, gt_boxes_mask, rotation_perturb=gt_rotation_noise, center_noise_std=gt_loc_noise_std, global_random_rot_range=global_random_rot_range, group_ids=group_ids, num_try=100) #print(gt_boxes_mask.shape,gt_boxes.shape,"after") # should remove unrelated objects after noise per object gt_boxes = gt_boxes[gt_boxes_mask] gt_names = gt_names[gt_boxes_mask] if group_ids is not None: group_ids = group_ids[gt_boxes_mask] gt_classes = np.array([class_names.index(n) + 1 for n in gt_names], dtype=np.int32) #need to check the output gt_boxes, points = prep.random_flip(gt_boxes, points) gt_boxes, points = prep.global_rotation(gt_boxes, points, rotation=global_rotation_noise) gt_boxes, points = prep.global_scaling_v2(gt_boxes, points, *global_scaling_noise) # Global translation gt_boxes, points = prep.global_translate(gt_boxes, points, global_loc_noise_std) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range(gt_boxes, bv_range) gt_boxes = gt_boxes[mask] gt_classes = gt_classes[mask] if group_ids is not None: group_ids = group_ids[mask] # limit rad to [-pi, pi] gt_boxes[:, 6] = box_np_ops.limit_period(gt_boxes[:, 6], offset=0.5, period=2 * np.pi) #assert -np.pi/2 <= g <= np.pi/2 if shuffle_points: # shuffle is a little slow. np.random.shuffle(points) voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size voxels, coordinates, num_points = voxel_generator.generate( points, max_voxels) example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": np.array([voxels.shape[0]], dtype=np.int64) } # if not lidar_input: feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) example["anchors"] = anchors anchors_mask = None if anchor_area_threshold >= 0: coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold example['anchors_mask'] = anchors_mask if generate_bev: bev_vxsize = voxel_size.copy() bev_vxsize[:2] /= 2 bev_vxsize[2] *= 2 bev_map = points_to_bev(points, bev_vxsize, pc_range, without_reflectivity) example["bev_map"] = bev_map if not training: return example if create_targets: targets_dict = target_assigner.assign( anchors, gt_boxes, anchors_mask, gt_classes=gt_classes, matched_thresholds=matched_thresholds, unmatched_thresholds=unmatched_thresholds) example.update({ 'labels': targets_dict['labels'], 'reg_targets': targets_dict['bbox_targets'], 'reg_weights': targets_dict['bbox_outside_weights'], }) return example
def __init__(self, model_dir, filter=True, tensorrt=True, bg_dir=None, anchors_area=0): self.model_dir = model_dir self.config_path = os.path.join('./configs', "xyres_28_huituo.config") self.anchors_area = anchors_area self.bg_filter = None self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") config = pipeline_pb2.TrainEvalPipelineConfig() with open(self.config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) model_cfg = config.model.second classes_cfg = model_cfg.target_assigner.class_settings if filter: assert bg_dir is not None, "如果采用背景滤波,必须输入背景表目录" self.bg_filter = bg_filter(model_cfg.voxel_generator, num_path=bg_dir + '/num_table.txt', var_path=bg_dir + '/var_table.txt', num_point=2, diff=0.03, is_statistic=False) net = self.build_network(model_cfg).to(self.device) self.anchor_generator = anchor_generator_builder.build(classes_cfg[0]) self.voxel_generator = net.voxel_generator self.voxel_size = self.voxel_generator.voxel_size self.grid_size = self.voxel_generator.grid_size self.pc_range = self.voxel_generator.point_cloud_range self.max_voxel_num = 12000 if filter: self.max_voxel_num = 2000 ckpt_path = os.path.join(model_dir, 'voxelnet.tckpt') state_dict = torch.load(ckpt_path) for key in [ "global_step", "rpn_acc.total", "rpn_acc.count", "rpn_precision.total", "rpn_precision.count", "rpn_recall.total", "rpn_recall.count", "rpn_metrics.prec_total", "rpn_metrics.prec_count", "rpn_metrics.rec_total", "rpn_metrics.rec_count", "rpn_cls_loss.total", "rpn_cls_loss.count", "rpn_loc_loss.total", "rpn_loc_loss.count", "rpn_total_loss.total", "rpn_total_loss.count" ]: if key in state_dict.keys(): state_dict.pop(key) net.load_state_dict(state_dict) if tensorrt: pfe_trt = TRTModule() pfe_trt.load_state_dict( torch.load(os.path.join(model_dir, 'pfe.trt'))) rpn_trt = TRTModule() rpn_trt.load_state_dict( torch.load(os.path.join(model_dir, 'rpn.trt'))) net.voxel_feature_extractor = pfe_trt net.rpn = rpn_trt grid_size = self.voxel_generator.grid_size feature_map_size = grid_size[:2] // get_downsample_factor(model_cfg) feature_map_size = [*feature_map_size, 1][::-1] anchors = self.anchor_generator.generate_anchors(feature_map_size) self.anchors = anchors.reshape((1, -1, 7)) self.anchors_bv = box_np_ops.rbbox2d_to_near_bbox( anchors[:, [0, 1, 3, 4, 6]]) # self.anchors_bv=anchors_bv.reshape((1,-1,4)) if tensorrt: self.float_dtype = torch.float32 else: self.float_dtype = torch.float32 self.net = net.eval()
def build(input_reader_config, model_config, training, voxel_generator, target_assigner, multi_gpu=False): """Builds a tensor dictionary based on the InputReader config. Args: input_reader_config: A input_reader_pb2.InputReader object. Returns: A tensor dict based on the input_reader_config. Raises: ValueError: On invalid input reader proto. ValueError: If no input paths are specified. """ if not isinstance(input_reader_config, input_reader_pb2.InputReader): raise ValueError('input_reader_config not of type ' 'input_reader_pb2.InputReader.') prep_cfg = input_reader_config.preprocess dataset_cfg = input_reader_config.dataset #kitti_info_path: "/home/lichao/v1.0-mini/infos_train.pkl" # kitti_root_path: "/home/lichao/v1.0-mini" # dataset_class_name: "NuScenesDataset" num_point_features = model_config.num_point_features #4 out_size_factor = get_downsample_factor(model_config) #8 assert out_size_factor > 0 cfg = input_reader_config db_sampler_cfg = prep_cfg.database_sampler #database_info_path: "/home/lichao/v1.0-mini/kitti_dbinfos_train.pkl" # sample_groups { name_to_max_num key: "car" value: 30 } } global_random_rotation_range_per_object: 0.0# global_random_rotation_range_per_object: 0.0# rate: 1.0 db_sampler = None if len(db_sampler_cfg.sample_groups ) > 0 or db_sampler_cfg.database_info_path != "": # enable sample db_sampler = dbsampler_builder.build(db_sampler_cfg) #加载了gt_base的一些东西 grid_size = voxel_generator.grid_size #[400,400] feature_map_size = grid_size[:2] // out_size_factor #[50,50] feature_map_size = [*feature_map_size, 1][::-1] #[50,50] print("feature_map_size", feature_map_size) assert all([n != '' for n in target_assigner.classes ]), "you must specify class_name in anchor_generators." dataset_cls = get_dataset_class( dataset_cfg.dataset_class_name) # NuScenesDataset assert dataset_cls.NumPointFeatures >= 3, "you must set this to correct value" assert dataset_cls.NumPointFeatures == num_point_features, "currently you need keep them same" prep_func = partial( #pre_func partial 的功能:固定函数参数,返回一个新的函数。 prep_pointcloud, #data\preprocess.py root_path=dataset_cfg.kitti_root_path, voxel_generator=voxel_generator, #VoxelGeneratorV2 target_assigner=target_assigner, training=training, max_voxels=prep_cfg.max_number_of_voxels, #25000 eval 30000 remove_outside_points=False, remove_unknown=prep_cfg.remove_unknown_examples, create_targets=training, shuffle_points=prep_cfg.shuffle_points, gt_rotation_noise=list(prep_cfg.groundtruth_rotation_uniform_noise), gt_loc_noise_std=list(prep_cfg.groundtruth_localization_noise_std), global_rotation_noise=list(prep_cfg.global_rotation_uniform_noise), global_scaling_noise=list(prep_cfg.global_scaling_uniform_noise), global_random_rot_range=list( prep_cfg.global_random_rotation_range_per_object), global_translate_noise_std=list(prep_cfg.global_translate_noise_std), db_sampler=db_sampler, num_point_features=dataset_cls.NumPointFeatures, anchor_area_threshold=prep_cfg.anchor_area_threshold, gt_points_drop=prep_cfg.groundtruth_points_drop_percentage, gt_drop_max_keep=prep_cfg.groundtruth_drop_max_keep_points, remove_points_after_sample=prep_cfg.remove_points_after_sample, remove_environment=prep_cfg.remove_environment, use_group_id=prep_cfg.use_group_id, out_size_factor=out_size_factor, #8 multi_gpu=multi_gpu, min_points_in_gt=prep_cfg.min_num_of_points_in_gt, random_flip_x=prep_cfg.random_flip_x, random_flip_y=prep_cfg.random_flip_y, sample_importance=prep_cfg.sample_importance) ret = target_assigner.generate_anchors(feature_map_size) class_names = target_assigner.classes # ['car', 'bicycle', 'bus', 'construction_vehicle', 'motorcycle', 'pedestrian', 'traffic_cone', 'trailer', 'truck', 'barrier'] anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) anchors_list = [] for k, v in anchors_dict.items(): anchors_list.append(v["anchors"]) #50000个 # anchors = ret["anchors"] anchors = np.concatenate(anchors_list, axis=0) anchors = anchors.reshape([-1, target_assigner.box_ndim]) assert np.allclose(anchors, ret["anchors"].reshape(-1, target_assigner.box_ndim)) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox( anchors[:, [0, 1, 3, 4, 6]]) #bev_anchor 4维 anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, "anchors_dict": anchors_dict, } prep_func = partial(prep_func, anchor_cache=anchor_cache) dataset = dataset_cls( info_path=dataset_cfg.kitti_info_path, #数据的路径 root_path=dataset_cfg.kitti_root_path, class_names=class_names, #10个类 prep_func=prep_func) return dataset # _nusc_infos <class 'dict'>: {'lidar_path': '/home/lichao/v1.0-mini/samples/LIDAR_TOP/n015-2018-07-24-11-22-45+0800__LIDAR_TOP__1532402927647951.pcd.bin', 'cam_front_path': '/home/lichao/v1.0-mini/samples/CAM_FRONT/n015-2018-07-24-11-22-45+0800__CAM_FRONT__1532402927612460.jpg', 'token': 'ca9a282c9e77460f8360f564131a8af5', 'sweeps': [], 'lidar2ego_translation': [0.943713, 0.0, 1.84023], 'lidar2ego_rotation': [0.7077955119163518, -0.006492242056004365, 0.010646214713995808, -0.7063073142877817], 'ego2global_translation': [411.3039349319818, 1180.8903791765097, 0.0], 'ego2global_rotation': [0.5720320396729045, -0.0016977771610471074, 0.011798001930183783, -0.8201446642457809], 'timestamp': 1532402927647951, 'gt_boxes': array([[ 1.84143850e+01, 5.95160251e+01, 7.69634574e-01,
def draw_anchors(self, gt_boxes_lidar, points=None, image_idx=0, gt_names=None): # print(gt_names) voxel_size = np.array(self._voxel_size, dtype=np.float32) # voxel_size = np.array([0.2, 0.2, 0.4], dtype=np.float32) coors_range = np.array(self._coors_range, dtype=np.float32) # coors_range = np.array([0, -40, -3, 70.4, 40, 1], dtype=np.float32) grid_size = (coors_range[3:] - coors_range[:3]) / voxel_size grid_size = np.round(grid_size).astype(np.int64) # print(grid_size) bv_range = coors_range[[0, 1, 3, 4]] anchor_generator = AnchorGeneratorStride( # sizes=[0.6, 0.8, 1.73, 0.6, 1.76, 1.73], sizes=[0.6, 1.76, 1.73], anchor_strides=[0.4, 0.4, 0.0], anchor_offsets=[0.2, -39.8, -1.465], rotations=[0, 1.5707963267948966], match_threshold=0.5, unmatch_threshold=0.35, ) anchor_generator1 = AnchorGeneratorStride( # sizes=[0.6, 0.8, 1.73, 0.6, 1.76, 1.73], sizes=[0.6, 0.8, 1.73], anchor_strides=[0.4, 0.4, 0.0], anchor_offsets=[0.2, -39.8, -1.465], rotations=[0, 1.5707963267948966], match_threshold=0.5, unmatch_threshold=0.35, ) anchor_generator2 = AnchorGeneratorStride( # sizes=[0.6, 0.8, 1.73, 0.6, 1.76, 1.73], sizes=[1.6, 3.9, 1.56], anchor_strides=[0.4, 0.4, 0.0], anchor_offsets=[0.2, -39.8, -1.55442884], rotations=[0, 1.5707963267948966], # rotations=[0], match_threshold=0.6, unmatch_threshold=0.45, ) anchor_generators = [anchor_generator2] box_coder = GroundBox3dCoder() # similarity_calc = DistanceSimilarity(1.0) similarity_calc = NearestIouSimilarity() target_assigner = TargetAssigner(box_coder, anchor_generators, similarity_calc) # anchors = box_np_ops.create_anchors_v2( # bv_range, grid_size[:2] // 2, sizes=anchor_dims) # matched_thresholds = [0.45, 0.45, 0.6] # unmatched_thresholds = [0.3, 0.3, 0.45] t = time.time() feature_map_size = grid_size[:2] // 2 feature_map_size = [*feature_map_size, 1][::-1] # print(feature_map_size) # """ ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] print(f"num_anchors_ {len(anchors)}") if points is not None: voxels, coors, num_points = points_to_voxel( points, self._voxel_size, # self._coors_range, coors_range, self._max_num_points, reverse_index=True, max_voxels=self._max_voxels) # print(np.min(coors, 0), np.max(coors, 0)) dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_mask = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, coors_range, grid_size) > 1 # print(np.sum(anchors_mask), anchors_mask.shape) class_names = [ 'Car', "Pedestrian", "Cyclist", 'Van', 'Truck', "Tram", 'Misc', 'Person_sitting' ] gt_classes = np.array([class_names.index(n) + 1 for n in gt_names], dtype=np.int32) t = time.time() target_dict = target_assigner.assign( anchors, gt_boxes_lidar, anchors_mask, gt_classes=gt_classes, matched_thresholds=matched_thresholds, unmatched_thresholds=unmatched_thresholds) labels = target_dict["labels"] reg_targets = target_dict["bbox_targets"] reg_weights = target_dict["bbox_outside_weights"] # print(labels[labels > 0]) # decoded_reg_targets = box_np_ops.second_box_decode(reg_targets, anchors) # print(decoded_reg_targets.reshape(-1, 7)[labels > 0]) print("target time", (time.time() - t)) print(f"num_pos={np.sum(labels > 0)}") colors = np.zeros([anchors.shape[0], 4]) ignored_color = bbox_plot.gl_color(GLColor.Gray, 0.5) pos_color = bbox_plot.gl_color(GLColor.Cyan, 0.5) colors[labels == -1] = ignored_color colors[labels > 0] = pos_color cared_anchors_mask = np.logical_and(labels != 0, anchors_mask) colors = colors[cared_anchors_mask] anchors_not_neg = box_np_ops.rbbox3d_to_corners( anchors)[cared_anchors_mask] self.boxes3d("anchors", anchors_not_neg, colors=colors)
def prep_pointcloud(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=70000, class_names=['Car'], remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, reduce_valid_area=False, remove_unknown=False, gt_rotation_noise=[-np.pi / 3, np.pi / 3], gt_loc_noise_std=[1.0, 1.0, 1.0], global_rotation_noise=[-np.pi / 4, np.pi / 4], global_scaling_noise=[0.95, 1.05], global_loc_noise_std=(0.2, 0.2, 0.2), global_random_rot_range=[0.78, 2.35], generate_bev=False, without_reflectivity=False, num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, add_rgb_to_points=False, lidar_input=False, unlabeled_db_sampler=None, out_size_factor=2, min_gt_point_dict=None, bev_only=False, use_group_id=False, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. """ points = input_dict["lidar"]["points"] if training: gt_boxes = input_dict['lidar']['annotations']["gt_boxes"] gt_names = input_dict['lidar']['annotations']["gt_names"] # difficulty = input_dict["difficulty"] group_ids = None if use_group_id and "group_ids" in input_dict: group_ids = input_dict["group_ids"] # rect = input_dict["rect"] # Trv2c = input_dict["Trv2c"] # P2 = input_dict["P2"] unlabeled_training = unlabeled_db_sampler is not None calib = None # print(gt_dict) # print("+++++++++++++++1111111+++++++++++++++") if "calib" in input_dict: calib = input_dict["calib"] if remove_environment is True and training: selected = kitti.keep_arrays_by_name(gt_names, class_names) gt_boxes = gt_boxes[selected] gt_names = gt_names[selected] # difficulty = difficulty[selected] if group_ids is not None: group_ids = group_ids[selected] points = prep.remove_points_outside_boxes(points, gt_boxes) if training: # print(gt_names) selected = kitti.drop_arrays_by_name(gt_names, ["DontCare"]) gt_boxes = gt_boxes[selected] gt_names = gt_names[selected] # difficulty = difficulty[selected] if group_ids is not None: group_ids = group_ids[selected] # if remove_unknown: # remove_mask = difficulty == -1 # """ # gt_boxes_remove = gt_boxes[remove_mask] # gt_boxes_remove[:, 3:6] += 0.25 # points = prep.remove_points_in_boxes(points, gt_boxes_remove) # """ # keep_mask = np.logical_not(remove_mask) # gt_boxes = gt_boxes[keep_mask] # gt_names = gt_names[keep_mask] # difficulty = difficulty[keep_mask] # if group_ids is not None: # group_ids = group_ids[keep_mask] gt_boxes_mask = np.array( [n in class_names for n in gt_names], dtype=np.bool_) if db_sampler is not None: sampled_dict = db_sampler.sample_all( root_path, gt_boxes, gt_names, num_point_features, random_crop, gt_group_ids=group_ids ) if sampled_dict is not None: sampled_gt_names = sampled_dict["gt_names"] sampled_gt_boxes = sampled_dict["gt_boxes"] sampled_points = sampled_dict["points"] sampled_gt_masks = sampled_dict["gt_masks"] # gt_names = gt_names[gt_boxes_mask].tolist() gt_names = np.concatenate([gt_names, sampled_gt_names], axis=0) # gt_names += [s["name"] for s in sampled] gt_boxes = np.concatenate([gt_boxes, sampled_gt_boxes]) gt_boxes_mask = np.concatenate( [gt_boxes_mask, sampled_gt_masks], axis=0) if group_ids is not None: sampled_group_ids = sampled_dict["group_ids"] group_ids = np.concatenate([group_ids, sampled_group_ids]) if remove_points_after_sample: points = prep.remove_points_in_boxes( points, sampled_gt_boxes) points = np.concatenate([sampled_points, points], axis=0) # unlabeled_mask = np.zeros((gt_boxes.shape[0], ), dtype=np.bool_) if without_reflectivity: used_point_axes = list(range(num_point_features)) used_point_axes.pop(3) points = points[:, used_point_axes] pc_range = voxel_generator.point_cloud_range if bev_only: # set z and h to limits gt_boxes[:, 2] = pc_range[2] gt_boxes[:, 5] = pc_range[5] - pc_range[2] prep.noise_per_object_v3_( gt_boxes, points, gt_boxes_mask, rotation_perturb=gt_rotation_noise, center_noise_std=gt_loc_noise_std, global_random_rot_range=global_random_rot_range, group_ids=group_ids, num_try=100) # should remove unrelated objects after noise per object gt_boxes = gt_boxes[gt_boxes_mask] gt_names = gt_names[gt_boxes_mask] if group_ids is not None: group_ids = group_ids[gt_boxes_mask] gt_classes = np.array( [class_names.index(n) + 1 for n in gt_names], dtype=np.int32) gt_boxes, points = prep.random_flip(gt_boxes, points) gt_boxes, points = prep.global_rotation( gt_boxes, points, rotation=global_rotation_noise) gt_boxes, points = prep.global_scaling_v2(gt_boxes, points, *global_scaling_noise) # Global translation gt_boxes, points = prep.global_translate(gt_boxes, points, global_loc_noise_std) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range(gt_boxes, bv_range) gt_boxes = gt_boxes[mask] gt_classes = gt_classes[mask] if group_ids is not None: group_ids = group_ids[mask] # limit rad to [-pi, pi] gt_boxes[:, 6] = box_np_ops.limit_period( gt_boxes[:, 6], offset=0.5, period=2 * np.pi) if shuffle_points: # shuffle is a little slow. np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # [352, 400] # max_voxels: maximum number of voxels voxels, coordinates, num_points = voxel_generator.generate( points, max_voxels) example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": np.array([voxels.shape[0]], dtype=np.int64), } # if not lidar_input: feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox( anchors[:, [0, 1, 3, 4, 6]]) example["anchors"] = anchors # print("debug", anchors.shape, matched_thresholds.shape) # anchors_bv = anchors_bv.reshape([-1, 4]) anchors_mask = None if anchor_area_threshold >= 0: coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask if generate_bev: bev_vxsize = voxel_size.copy() bev_vxsize[:2] /= 2 bev_vxsize[2] *= 2 bev_map = points_to_bev(points, bev_vxsize, pc_range, without_reflectivity) example["bev_map"] = bev_map if not training: return example if create_targets: targets_dict = target_assigner.assign( anchors, gt_boxes, anchors_mask, gt_classes=gt_classes, matched_thresholds=matched_thresholds, unmatched_thresholds=unmatched_thresholds) example.update({ 'labels': targets_dict['labels'], 'reg_targets': targets_dict['bbox_targets'], 'reg_weights': targets_dict['bbox_outside_weights'], }) example["points"]= input_dict["lidar"]["points"] example['gt_boxes'] = input_dict['lidar']['annotations']["gt_boxes"] example['gt_names'] = input_dict['lidar']['annotations']["gt_names"] return example
def construct_example_for_predict(self, img, pc, rect, tr, p2): """ using params and input to construct an example :param img: :param pc: :param rect: :param tr: :param p2: :return: """ generate_bev = self.model_cfg.use_bev without_reflectivity = self.model_cfg.without_reflectivity num_point_features = self.model_cfg.num_point_features out_size_factor = self.model_cfg.rpn.layer_strides[ 0] // self.model_cfg.rpn.upsample_strides[0] cfg = self.input_cfg db_sampler_cfg = self.input_cfg.database_sampler db_sampler = None if len(db_sampler_cfg.sample_groups) > 0: # enable sample db_sampler = dbsampler_builder.build(db_sampler_cfg) u_db_sampler_cfg = self.input_cfg.unlabeled_database_sampler u_db_sampler = None if len(u_db_sampler_cfg.sample_groups) > 0: # enable sample u_db_sampler = dbsampler_builder.build(u_db_sampler_cfg) voxel_generator = voxel_builder.build(self.model_cfg.voxel_generator) grid_size = voxel_generator.grid_size # [352, 400] feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] ret = self.target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, } # preparing point cloud prep_func = partial( prep_pointcloud, root_path=cfg.kitti_root_path, class_names=list(cfg.class_names), voxel_generator=voxel_generator, target_assigner=self.target_assigner, training=False, max_voxels=cfg.max_number_of_voxels, remove_outside_points=False, remove_unknown=cfg.remove_unknown_examples, create_targets=False, shuffle_points=cfg.shuffle_points, gt_rotation_noise=list(cfg.groundtruth_rotation_uniform_noise), gt_loc_noise_std=list(cfg.groundtruth_localization_noise_std), global_rotation_noise=list(cfg.global_rotation_uniform_noise), global_scaling_noise=list(cfg.global_scaling_uniform_noise), global_random_rot_range=list( cfg.global_random_rotation_range_per_object), db_sampler=db_sampler, unlabeled_db_sampler=u_db_sampler, generate_bev=generate_bev, without_reflectivity=without_reflectivity, num_point_features=num_point_features, anchor_area_threshold=cfg.anchor_area_threshold, gt_points_drop=cfg.groundtruth_points_drop_percentage, gt_drop_max_keep=cfg.groundtruth_drop_max_keep_points, remove_points_after_sample=cfg.remove_points_after_sample, remove_environment=cfg.remove_environment, use_group_id=cfg.use_group_id, out_size_factor=out_size_factor, anchor_cache=anchor_cache, ) example = prepare_v9_for_predict(img, pc, num_point_features, r0_rect=rect, tr_velo_2_cam=tr, p2=p2, prep_func=prep_func) return example
def prep_pointcloud(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=20000, class_names=['Car'], remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, reduce_valid_area=False, remove_unknown=False, gt_rotation_noise=[-np.pi / 3, np.pi / 3], gt_loc_noise_std=[1.0, 1.0, 1.0], global_rotation_noise=[-np.pi / 4, np.pi / 4], global_scaling_noise=[0.95, 1.05], global_random_rot_range=[0.78, 2.35], generate_bev=False, without_reflectivity=False, num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, add_rgb_to_points=False, lidar_input=False, unlabeled_db_sampler=None, out_size_factor=2, min_gt_point_dict=None, bev_only=False, use_group_id=False, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. """ points = input_dict["points"] if training: gt_boxes = input_dict["gt_boxes"] gt_names = input_dict["gt_names"] difficulty = input_dict["difficulty"] group_ids = None if use_group_id and "group_ids" in input_dict: group_ids = input_dict["group_ids"] rect = input_dict["rect"] Trv2c = input_dict["Trv2c"] P2 = input_dict["P2"] unlabeled_training = unlabeled_db_sampler is not None #image_idx = input_dict["image_idx"] if reference_detections is not None: C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2) frustums = box_np_ops.get_frustum_v2(reference_detections, C) frustums -= T # frustums = np.linalg.inv(R) @ frustums.T frustums = np.einsum('ij, akj->aki', np.linalg.inv(R), frustums) frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c) surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums) masks = points_in_convex_polygon_3d_jit(points, surfaces) points = points[masks.any(-1)] if remove_outside_points and not lidar_input: image_shape = input_dict["image_shape"] points = box_np_ops.remove_outside_points(points, rect, Trv2c, P2, image_shape) if remove_environment is True and training: selected = kitti.keep_arrays_by_name(gt_names, class_names) gt_boxes = gt_boxes[selected] gt_names = gt_names[selected] difficulty = difficulty[selected] if group_ids is not None: group_ids = group_ids[selected] points = prep.remove_points_outside_boxes(points, gt_boxes) if training: # print(gt_names) selected = kitti.drop_arrays_by_name(gt_names, ["DontCare"]) gt_boxes = gt_boxes[selected] gt_names = gt_names[selected] difficulty = difficulty[selected] if group_ids is not None: group_ids = group_ids[selected] gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, rect, Trv2c) if remove_unknown: remove_mask = difficulty == -1 """ gt_boxes_remove = gt_boxes[remove_mask] gt_boxes_remove[:, 3:6] += 0.25 points = prep.remove_points_in_boxes(points, gt_boxes_remove) """ keep_mask = np.logical_not(remove_mask) gt_boxes = gt_boxes[keep_mask] gt_names = gt_names[keep_mask] difficulty = difficulty[keep_mask] if group_ids is not None: group_ids = group_ids[keep_mask] gt_boxes_mask = np.array([n in class_names for n in gt_names], dtype=np.bool_) if db_sampler is not None: sampled_dict = db_sampler.sample_all(root_path, gt_boxes, gt_names, num_point_features, random_crop, gt_group_ids=group_ids, rect=rect, Trv2c=Trv2c, P2=P2) if sampled_dict is not None: sampled_gt_names = sampled_dict["gt_names"] sampled_gt_boxes = sampled_dict["gt_boxes"] sampled_points = sampled_dict["points"] sampled_gt_masks = sampled_dict["gt_masks"] # gt_names = gt_names[gt_boxes_mask].tolist() gt_names = np.concatenate([gt_names, sampled_gt_names], axis=0) # gt_names += [s["name"] for s in sampled] gt_boxes = np.concatenate([gt_boxes, sampled_gt_boxes]) gt_boxes_mask = np.concatenate( [gt_boxes_mask, sampled_gt_masks], axis=0) if group_ids is not None: sampled_group_ids = sampled_dict["group_ids"] group_ids = np.concatenate([group_ids, sampled_group_ids]) if remove_points_after_sample: points = prep.remove_points_in_boxes( points, sampled_gt_boxes) points = np.concatenate([sampled_points, points], axis=0) # unlabeled_mask = np.zeros((gt_boxes.shape[0], ), dtype=np.bool_) if without_reflectivity: used_point_axes = list(range(num_point_features)) used_point_axes.pop(3) points = points[:, used_point_axes] pc_range = voxel_generator.point_cloud_range if bev_only: # set z and h to limits gt_boxes[:, 2] = pc_range[2] gt_boxes[:, 5] = pc_range[5] - pc_range[2] prep.noise_per_object_v3_( gt_boxes, points, gt_boxes_mask, rotation_perturb=gt_rotation_noise, center_noise_std=gt_loc_noise_std, global_random_rot_range=global_random_rot_range, group_ids=group_ids, num_try=100) # should remove unrelated objects after noise per object gt_boxes = gt_boxes[gt_boxes_mask] gt_names = gt_names[gt_boxes_mask] if group_ids is not None: group_ids = group_ids[gt_boxes_mask] gt_classes = np.array([class_names.index(n) + 1 for n in gt_names], dtype=np.int32) gt_boxes, points = prep.random_flip(gt_boxes, points) gt_boxes, points = prep.global_rotation(gt_boxes, points, rotation=global_rotation_noise) gt_boxes, points = prep.global_scaling_v2(gt_boxes, points, *global_scaling_noise) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range(gt_boxes, bv_range) gt_boxes = gt_boxes[mask] gt_classes = gt_classes[mask] gt_names = gt_names[mask] if group_ids is not None: group_ids = group_ids[mask] # limit rad to [-pi, pi] gt_boxes[:, 6] = box_np_ops.limit_period(gt_boxes[:, 6], offset=0.5, period=2 * np.pi) if shuffle_points: # shuffle is a little slow. np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # [352, 400] voxels, coordinates, num_points = voxel_generator.generate( points, max_voxels) example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": np.array([voxels.shape[0]], dtype=np.int64) } example.update({ 'rect': rect, 'Trv2c': Trv2c, 'P2': P2, }) # if not lidar_input: feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] anchors_dict = anchor_cache["anchors_dict"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) example["anchors"] = anchors # print("debug", anchors.shape, matched_thresholds.shape) # anchors_bv = anchors_bv.reshape([-1, 4]) anchors_mask = None if anchor_area_threshold >= 0: coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask if not training: return example if create_targets: targets_dict = target_assigner.assign_v2(anchors_dict, gt_boxes, anchors_mask, gt_classes=gt_classes, gt_names=gt_names) example.update({ 'labels': targets_dict['labels'], 'reg_targets': targets_dict['bbox_targets'], 'reg_weights': targets_dict['bbox_outside_weights'], }) return example
def build(input_reader_config, model_config, training, voxel_generator, target_assigner, multi_gpu=False): """Builds a tensor dictionary based on the InputReader config. Args: input_reader_config: A input_reader_pb2.InputReader object. Returns: A tensor dict based on the input_reader_config. Raises: ValueError: On invalid input reader proto. ValueError: If no input paths are specified. """ if not isinstance(input_reader_config, input_reader_pb2.InputReader): raise ValueError('input_reader_config not of type ' 'input_reader_pb2.InputReader.') prep_cfg = input_reader_config.preprocess dataset_cfg = input_reader_config.dataset num_point_features = model_config.num_point_features out_size_factor = get_downsample_factor(model_config) assert out_size_factor > 0 cfg = input_reader_config db_sampler_cfg = prep_cfg.database_sampler db_sampler = None if len(db_sampler_cfg.sample_groups) > 0: # enable sample db_sampler = dbsampler_builder.build(db_sampler_cfg) grid_size = voxel_generator.grid_size # [352, 400] feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] print("feature_map_size", feature_map_size) assert all([n != '' for n in target_assigner.classes ]), "you must specify class_name in anchor_generators." dataset_cls = get_dataset_class(dataset_cfg.dataset_class_name) assert dataset_cls.NumPointFeatures >= 3, "you must set this to correct value" assert dataset_cls.NumPointFeatures == num_point_features, "currently you need keep them same" prep_func = partial( prep_pointcloud, root_path=dataset_cfg.kitti_root_path, voxel_generator=voxel_generator, target_assigner=target_assigner, training=training, max_voxels=prep_cfg.max_number_of_voxels, remove_outside_points=False, remove_unknown=prep_cfg.remove_unknown_examples, create_targets=training, shuffle_points=prep_cfg.shuffle_points, gt_rotation_noise=list(prep_cfg.groundtruth_rotation_uniform_noise), gt_loc_noise_std=list(prep_cfg.groundtruth_localization_noise_std), global_rotation_noise=list(prep_cfg.global_rotation_uniform_noise), global_scaling_noise=list(prep_cfg.global_scaling_uniform_noise), global_random_rot_range=list( prep_cfg.global_random_rotation_range_per_object), global_translate_noise_std=list(prep_cfg.global_translate_noise_std), db_sampler=db_sampler, num_point_features=dataset_cls.NumPointFeatures, anchor_area_threshold=prep_cfg.anchor_area_threshold, gt_points_drop=prep_cfg.groundtruth_points_drop_percentage, gt_drop_max_keep=prep_cfg.groundtruth_drop_max_keep_points, remove_points_after_sample=prep_cfg.remove_points_after_sample, remove_environment=prep_cfg.remove_environment, use_group_id=prep_cfg.use_group_id, out_size_factor=out_size_factor, multi_gpu=multi_gpu) ret = target_assigner.generate_anchors(feature_map_size) class_names = target_assigner.classes anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, "anchors_dict": anchors_dict, } prep_func = partial(prep_func, anchor_cache=anchor_cache) dataset = dataset_cls(info_path=dataset_cfg.kitti_info_path, root_path=dataset_cfg.kitti_root_path, class_names=class_names, prep_func=prep_func) return dataset
def build( input_reader_config, model_config, training, voxel_generator, target_assigner, multi_gpu=False, generate_anchors_cachae=True, #True for pillar and second segmentation=False, bcl_keep_voxels=None, seg_keep_points=None, points_per_voxel=None): """Builds a tensor dictionary based on the InputReader config. Args: input_reader_config: A input_reader_pb2.InputReader object. Returns: A tensor dict based on the input_reader_config. Raises: ValueError: On invalid input reader proto. ValueError: If no input paths are specified. """ if not isinstance(input_reader_config, input_reader_pb2.InputReader): raise ValueError('input_reader_config not of type ' 'input_reader_pb2.InputReader.') prep_cfg = input_reader_config.preprocess dataset_cfg = input_reader_config.dataset num_point_features = model_config.num_point_features out_size_factor = get_downsample_factor(model_config) assert out_size_factor > 0 cfg = input_reader_config db_sampler_cfg = prep_cfg.database_sampler db_sampler = None if len(db_sampler_cfg.sample_groups ) > 0 or db_sampler_cfg.database_info_path != "": # enable sample db_sampler = dbsampler_builder.build(db_sampler_cfg) grid_size = voxel_generator.grid_size feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] print("feature_map_size", feature_map_size) assert all([n != '' for n in target_assigner.classes ]), "you must specify class_name in anchor_generators." dataset_cls = get_dataset_class(dataset_cfg.dataset_class_name) assert dataset_cls.NumPointFeatures >= 3, "you must set this to correct value" assert dataset_cls.NumPointFeatures == num_point_features, "currently you need keep them same" prep_func = partial( prep_pointcloud, root_path=dataset_cfg.kitti_root_path, voxel_generator=voxel_generator, target_assigner=target_assigner, training=training, max_voxels=prep_cfg.max_number_of_voxels, remove_outside_points=False, remove_unknown=prep_cfg.remove_unknown_examples, create_targets=training, shuffle_points=prep_cfg.shuffle_points, gt_rotation_noise=list(prep_cfg.groundtruth_rotation_uniform_noise), gt_loc_noise_std=list(prep_cfg.groundtruth_localization_noise_std), global_rotation_noise=list(prep_cfg.global_rotation_uniform_noise), global_scaling_noise=list(prep_cfg.global_scaling_uniform_noise), global_random_rot_range=list( prep_cfg.global_random_rotation_range_per_object), global_translate_noise_std=list(prep_cfg.global_translate_noise_std), db_sampler=db_sampler, num_point_features=dataset_cls.NumPointFeatures, anchor_area_threshold=prep_cfg.anchor_area_threshold, gt_points_drop=prep_cfg.groundtruth_points_drop_percentage, gt_drop_max_keep=prep_cfg.groundtruth_drop_max_keep_points, remove_points_after_sample=prep_cfg.remove_points_after_sample, remove_environment=prep_cfg.remove_environment, use_group_id=prep_cfg.use_group_id, out_size_factor=out_size_factor, multi_gpu=multi_gpu, min_points_in_gt=prep_cfg.min_num_of_points_in_gt, random_flip_x=prep_cfg.random_flip_x, random_flip_y=prep_cfg.random_flip_y, sample_importance=prep_cfg.sample_importance) """#leo add condition for catch is understand plz delete the comment""" anchor_cache = None class_names = target_assigner.classes if generate_anchors_cachae: ret = target_assigner.generate_anchors( feature_map_size) #use bcl as voxel comment this line anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) anchors_list = [] for k, v in anchors_dict.items(): anchors_list.append(v["anchors"]) # anchor_cache = None #if wants generate anchors from voxels anchors = np.concatenate(anchors_list, axis=0) anchors = anchors.reshape([-1, target_assigner.box_ndim]) assert np.allclose( anchors, ret["anchors"].reshape(-1, target_assigner.box_ndim)) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) anchor_cache = { "anchors": anchors, "anchors_bv": anchors_bv, "matched_thresholds": matched_thresholds, "unmatched_thresholds": unmatched_thresholds, "anchors_dict": anchors_dict, } prep_func = partial(prep_func, anchor_cache=anchor_cache, segmentation=segmentation, bcl_keep_voxels=bcl_keep_voxels, seg_keep_points=seg_keep_points, points_per_voxel=points_per_voxel) dataset = dataset_cls(info_path=dataset_cfg.kitti_info_path, root_path=dataset_cfg.kitti_root_path, class_names=class_names, prep_func=prep_func) return dataset
def prep_pointcloud(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=20000, class_names=['Car'], remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, reduce_valid_area=False, remove_unknown=False, gt_rotation_noise=[-np.pi / 3, np.pi / 3], gt_loc_noise_std=[1.0, 1.0, 1.0], global_rotation_noise=[-np.pi / 4, np.pi / 4], global_scaling_noise=[0.95, 1.05], global_loc_noise_std=(0.2, 0.2, 0.2), global_random_rot_range=[0.78, 2.35], generate_bev=False, without_reflectivity=False, num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, add_rgb_to_points=False, lidar_input=False, unlabeled_db_sampler=None, out_size_factor=2, min_gt_point_dict=None, bev_only=False, use_group_id=False, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. """ # 这部分用来读取某一帧数据 points = input_dict["points"] # velodyne_reduced, array(N*4) if training: gt_boxes = input_dict["gt_boxes"] # 真值框,位置,尺寸,绝对转角,N*1,一个真值框一行 gt_names = input_dict["gt_names"] difficulty = input_dict["difficulty"] group_ids = None if use_group_id and "group_ids" in input_dict: # False group_ids = input_dict["group_ids"] rect = input_dict["rect"] Trv2c = input_dict["Trv2c"] P2 = input_dict["P2"] unlabeled_training = unlabeled_db_sampler is not None image_idx = input_dict["image_idx"] if reference_detections is not None: # None C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2) frustums = box_np_ops.get_frustum_v2(reference_detections, C) frustums -= T # frustums = np.linalg.inv(R) @ frustums.T frustums = np.einsum('ij, akj->aki', np.linalg.inv(R), frustums) frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c) surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums) masks = points_in_convex_polygon_3d_jit(points, surfaces) points = points[masks.any(-1)] if remove_outside_points and not lidar_input: # False image_shape = input_dict["image_shape"] points = box_np_ops.remove_outside_points(points, rect, Trv2c, P2, image_shape) if remove_environment is True and training: # False selected = kitti.keep_arrays_by_name(gt_names, class_names) gt_boxes = gt_boxes[selected] gt_names = gt_names[selected] difficulty = difficulty[selected] if group_ids is not None: # None group_ids = group_ids[selected] points = prep.remove_points_outside_boxes(points, gt_boxes) if training: # 先去掉真值内的DontCare selected = kitti.drop_arrays_by_name(gt_names, ["DontCare"]) # 去掉DontCare gt_boxes = gt_boxes[selected] gt_names = gt_names[selected] difficulty = difficulty[selected] if group_ids is not None: # None group_ids = group_ids[selected] gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, rect, Trv2c) # 相机坐标下的真值框转换成激光雷达坐标下[xyz_lidar,w,l,h,r],一个对象一行 if remove_unknown: # False remove_mask = difficulty == -1 """ gt_boxes_remove = gt_boxes[remove_mask] gt_boxes_remove[:, 3:6] += 0.25 points = prep.remove_points_in_boxes(points, gt_boxes_remove) """ keep_mask = np.logical_not(remove_mask) gt_boxes = gt_boxes[keep_mask] gt_names = gt_names[keep_mask] difficulty = difficulty[keep_mask] if group_ids is not None: # None group_ids = group_ids[keep_mask] gt_boxes_mask = np.array( # 目标类别的对象标签,布尔类型,同样一行一个对象 [n in class_names for n in gt_names], dtype=np.bool_) # 下面用来对去掉DontCare的真值进行采样补充 if db_sampler is not None: # not None # 数据库预处理类里的方法,返回的是该帧数据内各类别用来补充的采样真值数据,包括真值内出现的非目标类别 sampled_dict = db_sampler.sample_all( root_path, gt_boxes, gt_names, num_point_features, random_crop, gt_group_ids=group_ids, # None rect=rect, Trv2c=Trv2c, P2=P2) if sampled_dict is not None: # 下面将原始数据与补充采样数据合并 sampled_gt_names = sampled_dict["gt_names"] sampled_gt_boxes = sampled_dict["gt_boxes"] sampled_points = sampled_dict["points"] sampled_gt_masks = sampled_dict["gt_masks"] # gt_names = gt_names[gt_boxes_mask].tolist() gt_names = np.concatenate([gt_names, sampled_gt_names], axis=0) # gt_names += [s["name"] for s in sampled] gt_boxes = np.concatenate([gt_boxes, sampled_gt_boxes]) gt_boxes_mask = np.concatenate( [gt_boxes_mask, sampled_gt_masks], axis=0) # 真值框是目标类别的标志 if group_ids is not None: # None sampled_group_ids = sampled_dict["group_ids"] group_ids = np.concatenate([group_ids, sampled_group_ids]) if remove_points_after_sample: # False,将采样框所占位置点云去除 points = prep.remove_points_in_boxes( points, sampled_gt_boxes) points = np.concatenate([sampled_points, points], axis=0) # 合并原始点云与采样点云 # unlabeled_mask = np.zeros((gt_boxes.shape[0], ), dtype=np.bool_) if without_reflectivity: # False used_point_axes = list(range(num_point_features)) used_point_axes.pop(3) points = points[:, used_point_axes] pc_range = voxel_generator.point_cloud_range if bev_only: # set z and h to limits, False gt_boxes[:, 2] = pc_range[2] gt_boxes[:, 5] = pc_range[5] - pc_range[2] prep.noise_per_object_v3_( # 每一个对象添加扰动,实际上就是修改每个对象真值框坐标与内部点云坐标 gt_boxes, # 补充过采样数据的真值框 points, # 补充过采样数据的点云 gt_boxes_mask, # 真值框标志(不区分原始数据与采样数据) rotation_perturb=gt_rotation_noise, # 旋转角度范围,平均分布 center_noise_std=gt_loc_noise_std, # 位置正态分布方差 global_random_rot_range=global_random_rot_range, # 全局旋转[0.0,0.0] group_ids=group_ids, # None num_try=100) # should remove unrelated objects after noise per object gt_boxes = gt_boxes[gt_boxes_mask] # 添加扰动后筛选目标类别真值框 gt_names = gt_names[gt_boxes_mask] if group_ids is not None: # Fasle group_ids = group_ids[gt_boxes_mask] gt_classes = np.array( # (gt_num) [class_names.index(n) + 1 for n in gt_names], dtype=np.int32) gt_boxes, points = prep.random_flip(gt_boxes, points) # 随机翻转 gt_boxes, points = prep.global_rotation( # 全局旋转 gt_boxes, points, rotation=global_rotation_noise) gt_boxes, points = prep.global_scaling_v2(gt_boxes, points, # 全局缩放 *global_scaling_noise) # Global translation # 全局定位扰动 gt_boxes, points = prep.global_translate(gt_boxes, points, global_loc_noise_std) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range(gt_boxes, bv_range) # 过滤超出鸟瞰图范围的真值框 gt_boxes = gt_boxes[mask] gt_classes = gt_classes[mask] if group_ids is not None: # None group_ids = group_ids[mask] # limit rad to [-pi, pi] gt_boxes[:, 6] = box_np_ops.limit_period( # 调整真值转角至目标范围 gt_boxes[:, 6], offset=0.5, period=2 * np.pi) if shuffle_points: # True,打乱全局点云数据 # shuffle is a little slow. np.random.shuffle(points) voxel_size = voxel_generator.voxel_size # [0.16, 0.16, 4] pc_range = voxel_generator.point_cloud_range # [0, -39.68, -3, 69.12, 39.68, 1] grid_size = voxel_generator.grid_size # [432, 496, 1] x,y,z # 生成体素 """ Returns: voxels:[num_voxels, 100, 4] 体素索引映射全局体素特征 coordinates:[num_voxels, 3] 体素索引映射体素坐标 num_points:(num_voxels,) 体素索引映射体素内的点数 """ voxels, coordinates, num_points = voxel_generator.generate( points, max_voxels) example = { 'voxels': voxels, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": np.array([voxels.shape[0]], dtype=np.int64) } example.update({ 'rect': rect, 'Trv2c': Trv2c, 'P2': P2, }) # if not lidar_input: feature_map_size = grid_size[:2] // out_size_factor # [216 248] feature_map_size = [*feature_map_size, 1][::-1] # [1,248,216] if anchor_cache is not None: anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] else: ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, 7]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] anchors_bv = box_np_ops.rbbox2d_to_near_bbox( anchors[:, [0, 1, 3, 4, 6]]) example["anchors"] = anchors # print("debug", anchors.shape, matched_thresholds.shape) # anchors_bv = anchors_bv.reshape([-1, 4]) anchors_mask = None if anchor_area_threshold >= 0: # True coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( # 某处体素是否被采样,否为0,是为1.array[496,432] y,x coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) # 元素值为按列累加到元素的和 dense_voxel_map = dense_voxel_map.cumsum(1) # 元素值为按行累加到元素的和,相当于统计了元素两个维度上之前元素的和 anchors_area = box_np_ops.fused_get_anchors_area( # 统计每个锚框内的体素数量 dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # 标记超过一个体素的锚框 # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask if generate_bev: # False bev_vxsize = voxel_size.copy() bev_vxsize[:2] /= 2 bev_vxsize[2] *= 2 bev_map = points_to_bev(points, bev_vxsize, pc_range, without_reflectivity) example["bev_map"] = bev_map if not training: return example # 测试数据集不需要创建训练目标 if create_targets: # True targets_dict = target_assigner.assign( anchors, # (248*216*2,7) gt_boxes, # (gt_num, 7) anchors_mask, # (248*216*2,) gt_classes=gt_classes, # (gt_num,) matched_thresholds=matched_thresholds, # (248*216*2,) unmatched_thresholds=unmatched_thresholds) # (248*216*2,) example.update({ 'labels': targets_dict['labels'], # (total_anchors,),所有锚框对应真值类别(1,2...),无对应真值设为0,dontcare设为-1 'reg_targets': targets_dict['bbox_targets'], # (total_anchors, 7),所有锚框对应真值相对锚框的偏差编码,无对应真值设为[0,0,0,0,0,0,0] 'reg_weights': targets_dict['bbox_outside_weights'], # (total_anchors,),所有锚框的外部权重,有对应真值设为1,无对应真值设为0 }) return example
def prep_pointcloud(input_dict, root_path, voxel_generator, target_assigner, db_sampler=None, max_voxels=20000, max_sweeps=10, remove_outside_points=False, training=True, create_targets=True, shuffle_points=False, remove_unknown=False, gt_rotation_noise=(-np.pi / 3, np.pi / 3), gt_loc_noise_std=(1.0, 1.0, 1.0), global_rotation_noise=(-np.pi / 4, np.pi / 4), global_scaling_noise=(0.95, 1.05), global_random_rot_range=(0.78, 2.35), global_translate_noise_std=(0, 0, 0), num_point_features=4, anchor_area_threshold=1, gt_points_drop=0.0, gt_drop_max_keep=10, remove_points_after_sample=True, anchor_cache=None, remove_environment=False, random_crop=False, reference_detections=None, out_size_factor=2, use_group_id=False, multi_gpu=False, min_points_in_gt=-1, random_flip_x=True, random_flip_y=True, sample_importance=1.0, out_dtype=np.float32): """convert point cloud to voxels, create targets if ground truths exists. input_dict format: dataset.get_sensor_data format """ t = time.time() class_names = target_assigner.classes points = input_dict["lidar"]["points"] indices = input_dict["lidar"]["indices"] origins = input_dict["lidar"]["origins"] if training: anno_dict = input_dict["lidar"]["annotations"] gt_dict = { "gt_boxes": anno_dict["boxes"], "gt_names": anno_dict["names"], "gt_importance": np.ones([anno_dict["boxes"].shape[0]], dtype=anno_dict["boxes"].dtype), } if "difficulty" not in anno_dict: difficulty = np.zeros([anno_dict["boxes"].shape[0]], dtype=np.int32) gt_dict["difficulty"] = difficulty else: gt_dict["difficulty"] = anno_dict["difficulty"] if use_group_id and "group_ids" in anno_dict: group_ids = anno_dict["group_ids"] gt_dict["group_ids"] = group_ids calib = None if "calib" in input_dict: calib = input_dict["calib"] # # Disable these two since we do not do this for NuScenes # if reference_detections is not None: # assert calib is not None and "image" in input_dict # C, R, T = box_np_ops.projection_matrix_to_CRT_kitti(P2) # frustums = box_np_ops.get_frustum_v2(reference_detections, C) # frustums -= T # frustums = np.einsum('ij, akj->aki', np.linalg.inv(R), frustums) # frustums = box_np_ops.camera_to_lidar(frustums, rect, Trv2c) # surfaces = box_np_ops.corner_to_surfaces_3d_jit(frustums) # masks = points_in_convex_polygon_3d_jit(points, surfaces) # points = points[masks.any(-1)] # if remove_outside_points: # assert calib is not None # image_shape = input_dict["image"]["image_shape"] # points = box_np_ops.remove_outside_points( # points, calib["rect"], calib["Trv2c"], calib["P2"], image_shape) # # Very interesting attempt # # I have tried the same and found it doesn't really work # if remove_environment is True and training: # selected = kitti.keep_arrays_by_name(gt_names, target_assigner.classes) # _dict_select(gt_dict, selected) # masks = box_np_ops.points_in_rbbox(points, gt_dict["gt_boxes"]) # points = points[masks.any(-1)] metrics = {} point_indices_to_remove = None if training: """ boxes_lidar = gt_dict["gt_boxes"] bev_map = simplevis.nuscene_vis(points, boxes_lidar) cv2.imshow('pre-noise', bev_map) """ selected = kitti.drop_arrays_by_name(gt_dict["gt_names"], ["Denture"]) _dict_select(gt_dict, selected) if remove_unknown: remove_mask = gt_dict["difficulty"] == -1 """ gt_boxes_remove = gt_boxes[remove_mask] gt_boxes_remove[:, 3:6] += 0.25 points = prep.remove_points_in_boxes(points, gt_boxes_remove) """ keep_mask = np.logical_not(remove_mask) _dict_select(gt_dict, keep_mask) gt_dict.pop("difficulty") # This part is interesting - we will need to do the same if min_points_in_gt > 0: # points_count_rbbox takes 10ms with 10 sweeps nuscenes data point_counts = box_np_ops.points_count_rbbox( points, gt_dict["gt_boxes"]) mask = point_counts >= min_points_in_gt _dict_select(gt_dict, mask) gt_boxes_mask = np.array( [n in class_names for n in gt_dict["gt_names"]], dtype=np.bool_) if db_sampler is not None: group_ids = None if "group_ids" in gt_dict: group_ids = gt_dict["group_ids"] sampled_dict = db_sampler.sample_all(root_path, gt_dict["gt_boxes"], gt_dict["gt_names"], num_point_features, random_crop, gt_group_ids=group_ids, calib=calib) if sampled_dict is not None: sampled_gt_names = sampled_dict["gt_names"] sampled_gt_boxes = sampled_dict["gt_boxes"] sampled_points = sampled_dict["points"] sampled_gt_masks = sampled_dict["gt_masks"] gt_dict["gt_names"] = np.concatenate( [gt_dict["gt_names"], sampled_gt_names], axis=0) gt_dict["gt_boxes"] = np.concatenate( [gt_dict["gt_boxes"], sampled_gt_boxes]) gt_boxes_mask = np.concatenate( [gt_boxes_mask, sampled_gt_masks], axis=0) sampled_gt_importance = np.full([sampled_gt_boxes.shape[0]], sample_importance, dtype=sampled_gt_boxes.dtype) gt_dict["gt_importance"] = np.concatenate( [gt_dict["gt_importance"], sampled_gt_importance]) if group_ids is not None: sampled_group_ids = sampled_dict["group_ids"] gt_dict["group_ids"] = np.concatenate( [gt_dict["group_ids"], sampled_group_ids]) # # Commented out because we have a new way of removing points # if remove_points_after_sample: # masks = box_np_ops.points_in_rbbox(points, sampled_gt_boxes) # point_indices_to_remove = np.flatnonzero(masks.any(-1)) # # # Delay this process so we can use the full point cloud # # # when we do the ray stopping algorithm # # points = points[np.logical_not(masks.any(-1))] # # Paste objects behind so that we don't have to update indices # points = np.concatenate([sampled_points, points], axis=0) points = np.concatenate([points, sampled_points], axis=0) pc_range = voxel_generator.point_cloud_range group_ids = None if "group_ids" in gt_dict: group_ids = gt_dict["group_ids"] # # Disable this one for now (not used in PointPillars anyways) # prep.noise_per_object_v3_( # gt_dict["gt_boxes"], # points, # gt_boxes_mask, # rotation_perturb=gt_rotation_noise, # center_noise_std=gt_loc_noise_std, # global_random_rot_range=global_random_rot_range, # group_ids=group_ids, # num_try=100) # should remove unrelated objects after noise per object # for k, v in gt_dict.items(): # print(k, v.shape) _dict_select(gt_dict, gt_boxes_mask) gt_classes = np.array( [class_names.index(n) + 1 for n in gt_dict["gt_names"]], dtype=np.int32) gt_dict["gt_classes"] = gt_classes gt_dict["gt_boxes"], points, origins = prep.random_flip( gt_dict["gt_boxes"], points, origins, 0.5, random_flip_x, random_flip_y) gt_dict["gt_boxes"], points, origins = prep.global_rotation_v2( gt_dict["gt_boxes"], points, origins, *global_rotation_noise) gt_dict["gt_boxes"], points, origins = prep.global_scaling_v2( gt_dict["gt_boxes"], points, origins, *global_scaling_noise) prep.global_translate_(gt_dict["gt_boxes"], points, origins, global_translate_noise_std) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] mask = prep.filter_gt_box_outside_range_by_center( gt_dict["gt_boxes"], bv_range) _dict_select(gt_dict, mask) # limit rad to [-pi, pi] gt_dict["gt_boxes"][:, 6] = box_np_ops.limit_period( gt_dict["gt_boxes"][:, 6], offset=0.5, period=2 * np.pi) # boxes_lidar = gt_dict["gt_boxes"] # bev_map = simplevis.nuscene_vis(points, boxes_lidar) # cv2.imshow('post-noise', bev_map) # cv2.waitKey(0) # # Disable this for now (not used in PointPillars anyways) # if shuffle_points: # # shuffle is a little slow. # np.random.shuffle(points) # [0, -40, -3, 70.4, 40, 1] voxel_size = voxel_generator.voxel_size pc_range = voxel_generator.point_cloud_range grid_size = voxel_generator.grid_size # organize points into lists based on timestamps time_stamps = points[ indices[:-1], -1] # counting on the fact we do not miss points from any intermediate time_stamps time_stamps = (time_stamps[:-1] + time_stamps[1:]) / 2 time_stamps = [-1000.0] + time_stamps.tolist() + [1000.0] # add boundaries time_stamps = np.array(time_stamps) # # LL_OCCUPIED, LL_FREE = 0.85, -0.4 # lo_occupied = np.log(0.7 / (1 - 0.7)) # lo_free = np.log(0.4 / (1 - 0.4)) # is there are additional points (from database sampling) num_original = indices[-1] if len(points) > num_original: # split data into two half (indexed and un-indexed) original_points, sampled_points = points[:num_original], points[ num_original:] # compute occupancy and masks # visibility, original_mask, sampled_mask = mapping.compute_visibility_and_masks( # original_points, sampled_points, origins, time_stamps, pc_range, min(voxel_size) # ) logodds, original_mask, sampled_mask = mapping.compute_logodds_and_masks( original_points, sampled_points, origins, time_stamps, pc_range, min(voxel_size) # , lo_occupied, lo_free ) # apply visible mask points = np.concatenate( (original_points[original_mask], sampled_points[sampled_mask])) else: # visibility = mapping.compute_visibility( # points, origins, time_stamps, pc_range, min(voxel_size) # ) logodds = mapping.compute_logodds( points, origins, time_stamps, pc_range, min(voxel_size) #, lo_occupied, lo_free ) # T = len(time_stamps)-1 # visibility = visibility.reshape(T, -1) # if T < (1 + max_sweeps): # visibility = np.pad(visibility, ((0, (1+max_sweeps)-T), (0,0)), 'edge') # with open(f'./utils/mapping/examples/{time.time()}.pkl', 'wb') as f: # ## # pickle.dump(original_points, f) # pickle.dump(sampled_points, f) # pickle.dump(origins, f) # pickle.dump(time_stamps, f) # pickle.dump(pc_range, f) # pickle.dump(voxel_size, f) # ## # pickle.dump(occupancy, f) # pickle.dump(original_mask, f) # pickle.dump(sampled_mask, f) if training: if min_points_in_gt > 0: # points_count_rbbox takes 10ms with 10 sweeps nuscenes data point_counts = box_np_ops.points_count_rbbox( points, gt_dict["gt_boxes"]) mask = point_counts >= min_points_in_gt _dict_select(gt_dict, mask) # [352, 400] t1 = time.time() if not multi_gpu: res = voxel_generator.generate(points, max_voxels) voxels = res["voxels"] coordinates = res["coordinates"] num_points = res["num_points_per_voxel"] num_voxels = np.array([voxels.shape[0]], dtype=np.int64) else: res = voxel_generator.generate_multi_gpu(points, max_voxels) voxels = res["voxels"] coordinates = res["coordinates"] num_points = res["num_points_per_voxel"] num_voxels = np.array([res["voxel_num"]], dtype=np.int64) metrics["voxel_gene_time"] = time.time() - t1 example = { 'voxels': voxels, # 'visibility': visibility, 'logodds': logodds, 'num_points': num_points, 'coordinates': coordinates, "num_voxels": num_voxels, "metrics": metrics, } if calib is not None: example["calib"] = calib feature_map_size = grid_size[:2] // out_size_factor feature_map_size = [*feature_map_size, 1][::-1] # print(f'feature_map_size in prep_pointcloud(): {feature_map_size}') if anchor_cache is not None: # print('having anchor cache') anchors = anchor_cache["anchors"] anchors_bv = anchor_cache["anchors_bv"] anchors_dict = anchor_cache["anchors_dict"] matched_thresholds = anchor_cache["matched_thresholds"] unmatched_thresholds = anchor_cache["unmatched_thresholds"] else: # print('NOT having anchor cache') ret = target_assigner.generate_anchors(feature_map_size) anchors = ret["anchors"] anchors = anchors.reshape([-1, target_assigner.box_ndim]) anchors_dict = target_assigner.generate_anchors_dict(feature_map_size) anchors_bv = box_np_ops.rbbox2d_to_near_bbox(anchors[:, [0, 1, 3, 4, 6]]) matched_thresholds = ret["matched_thresholds"] unmatched_thresholds = ret["unmatched_thresholds"] # print(f'anchors.shape: {anchors.shape}') example["anchors"] = anchors anchors_mask = None if anchor_area_threshold >= 0: # slow with high resolution. recommend disable this forever. coors = coordinates dense_voxel_map = box_np_ops.sparse_sum_for_anchors_mask( coors, tuple(grid_size[::-1][1:])) dense_voxel_map = dense_voxel_map.cumsum(0) dense_voxel_map = dense_voxel_map.cumsum(1) anchors_area = box_np_ops.fused_get_anchors_area( dense_voxel_map, anchors_bv, voxel_size, pc_range, grid_size) anchors_mask = anchors_area > anchor_area_threshold # example['anchors_mask'] = anchors_mask.astype(np.uint8) example['anchors_mask'] = anchors_mask # print("prep time", time.time() - t) metrics["prep_time"] = time.time() - t if not training: return example example["gt_names"] = gt_dict["gt_names"] # voxel_labels = box_np_ops.assign_label_to_voxel(gt_boxes, coordinates, # voxel_size, coors_range) if create_targets: t1 = time.time() targets_dict = target_assigner.assign( anchors, anchors_dict, gt_dict["gt_boxes"], anchors_mask, gt_classes=gt_dict["gt_classes"], gt_names=gt_dict["gt_names"], matched_thresholds=matched_thresholds, unmatched_thresholds=unmatched_thresholds, importance=gt_dict["gt_importance"]) """ boxes_lidar = gt_dict["gt_boxes"] bev_map = simplevis.nuscene_vis(points, boxes_lidar, gt_dict["gt_names"]) assigned_anchors = anchors[targets_dict['labels'] > 0] ignored_anchors = anchors[targets_dict['labels'] == -1] bev_map = simplevis.draw_box_in_bev(bev_map, [-50, -50, 3, 50, 50, 1], ignored_anchors, [128, 128, 128], 2) bev_map = simplevis.draw_box_in_bev(bev_map, [-50, -50, 3, 50, 50, 1], assigned_anchors, [255, 0, 0]) cv2.imshow('anchors', bev_map) cv2.waitKey(0) boxes_lidar = gt_dict["gt_boxes"] pp_map = np.zeros(grid_size[:2], dtype=np.float32) voxels_max = np.max(voxels[:, :, 2], axis=1, keepdims=False) voxels_min = np.min(voxels[:, :, 2], axis=1, keepdims=False) voxels_height = voxels_max - voxels_min voxels_height = np.minimum(voxels_height, 4) # sns.distplot(voxels_height) # plt.show() pp_map[coordinates[:, 1], coordinates[:, 2]] = voxels_height / 4 pp_map = (pp_map * 255).astype(np.uint8) pp_map = cv2.cvtColor(pp_map, cv2.COLOR_GRAY2RGB) pp_map = simplevis.draw_box_in_bev(pp_map, [-50, -50, 3, 50, 50, 1], boxes_lidar, [128, 0, 128], 1) cv2.imshow('heights', pp_map) cv2.waitKey(0) """ example.update({ 'labels': targets_dict['labels'], 'reg_targets': targets_dict['bbox_targets'], # 'reg_weights': targets_dict['bbox_outside_weights'], 'importance': targets_dict['importance'], }) return example