def _read_and_prep_v9(info, root_path, num_point_features, prep_func): """read data from KITTI-format infos, then call prep function. """ # velodyne_path = str(pathlib.Path(root_path) / info['velodyne_path']) # velodyne_path += '_reduced' v_path = pathlib.Path(root_path) / info['velodyne_path'] v_path = v_path.parent.parent / (v_path.parent.stem + "_reduced") / v_path.name i_path = pathlib.Path(root_path) / info['img_path'] points = np.fromfile(str(v_path), dtype=np.float32, count=-1).reshape([-1, num_point_features]) image_idx = info['image_idx'] rect = info['calib/R0_rect'].astype(np.float32) Trv2c = info['calib/Tr_velo_to_cam'].astype(np.float32) P2 = info['calib/P2'].astype(np.float32) #image_pang_bgr = cv2.imread(str(i_path)) #image_pang = image_pang_bgr[..., :: -1] pil2tensor = transforms.ToTensor() pil_image = Image.open(str(i_path)).convert("RGB") image_pang_bgr = np.array(pil_image)[:, :, [2, 1, 0]] image_pang = image_pang_bgr #image_pang = pil2tensor(image_pang_bgr) input_dict = { 'points': points, 'rect': rect, 'Trv2c': Trv2c, 'P2': P2, 'image_shape': np.array(info["img_shape"], dtype=np.int32), 'image_idx': image_idx, 'image_path': info['img_path'], 'images': image_pang # 'pointcloud_num_features': num_point_features, } if 'annos' in info: annos = info['annos'] # we need other objects to avoid collision when sample annos = kitti.remove_dontcare(annos) loc = annos["location"] dims = annos["dimensions"] rots = annos["rotation_y"] gt_names = annos["name"] gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]], axis=1).astype(np.float32) difficulty = annos["difficulty"] input_dict.update({ 'gt_boxes': gt_boxes, 'gt_names': gt_names, 'difficulty': difficulty, }) if 'group_ids' in annos: input_dict['group_ids'] = annos["group_ids"] example = prep_func(input_dict=input_dict) example["image_idx"] = image_idx example["image_shape"] = input_dict["image_shape"] #example["images"] = input_dict["image"] if "anchors_mask" in example: example["anchors_mask"] = example["anchors_mask"].astype(np.uint8) return example
def _read_and_prep_v9(info, root_path, num_point_features, prep_func): """read data from KITTI-format infos, then call prep function. """ read_image = True # velodyne_path = str(pathlib.Path(root_path) / info['velodyne_path']) # velodyne_path += '_reduced' v_path = pathlib.Path(root_path) / info['velodyne_path'] v_path = v_path.parent.parent / (v_path.parent.stem + "_reduced") / v_path.name points = np.fromfile(str(v_path), dtype=np.float32, count=-1).reshape([-1, num_point_features]) image_idx = info['image_idx'] rect = info['calib/R0_rect'].astype(np.float32) Trv2c = info['calib/Tr_velo_to_cam'].astype(np.float32) P2 = info['calib/P2'].astype(np.float32) image_path = info['img_path'] if read_image: image_path = Path(root_path) / image_path image_str = Image.open(os.path.join(image_path)) input_dict = { 'points': points, 'rect': rect, 'Trv2c': Trv2c, 'P2': P2, 'image_shape': np.array(info["img_shape"], dtype=np.int32), 'image_idx': image_idx, 'cam': image_str, 'image_path': info['img_path'], # 'pointcloud_num_features': num_point_features, } if 'annos' in info: annos = info['annos'] # we need other objects to avoid collision when sample annos = kitti.remove_dontcare(annos) loc = annos["location"] dims = annos["dimensions"] rots = annos["rotation_y"] gt_names = annos["name"] # print(gt_names, len(loc)) gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]], axis=1).astype(np.float32) # gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, rect, Trv2c) difficulty = annos["difficulty"] input_dict.update({ 'gt_boxes': gt_boxes, 'gt_names': gt_names, 'difficulty': difficulty, }) if 'group_ids' in annos: input_dict['group_ids'] = annos["group_ids"] example = prep_func(input_dict=input_dict) example["image_idx"] = image_idx example["image_shape"] = input_dict["image_shape"] if "anchors_mask" in example: example["anchors_mask"] = example["anchors_mask"].astype(np.uint8) return example
def _read_and_prep_v9(info, root_path, num_point_features, prep_func): # info: # kitti_infos_train.pkl内的某一个列表元素(某一帧数据) """read data from KITTI-format infos, then call prep function. """ # velodyne_path = str(pathlib.Path(root_path) / info['velodyne_path']) # velodyne_path += '_reduced' v_path = pathlib.Path(root_path) / info['velodyne_path'] # e.g. root_path/training/velodyne/000010.bin v_path = v_path.parent.parent / ( v_path.parent.stem + "_reduced") / v_path.name # e.g. root_path/training/velodyne_reduced/000010.bin points = np.fromfile( str(v_path), dtype=np.float32, count=-1).reshape([-1, num_point_features]) # 将点云数据转化为N*4的数组 image_idx = info['image_idx'] rect = info['calib/R0_rect'].astype(np.float32) Trv2c = info['calib/Tr_velo_to_cam'].astype(np.float32) P2 = info['calib/P2'].astype(np.float32) input_dict = { 'points': points, # velodyne_reduced 'rect': rect, 'Trv2c': Trv2c, 'P2': P2, 'image_shape': np.array(info["img_shape"], dtype=np.int32), # 图像尺寸 'image_idx': image_idx, # 图像编号 'image_path': info['img_path'], # 图像路径 # 'pointcloud_num_features': num_point_features, } if 'annos' in info: # True annos = info['annos'] # we need other objects to avoid collision when sample annos = kitti.remove_dontcare(annos) loc = annos["location"] dims = annos["dimensions"] rots = annos["rotation_y"] gt_names = annos["name"] # print(gt_names, len(loc)) gt_boxes = np.concatenate( [loc, dims, rots[..., np.newaxis]], axis=1).astype(np.float32) # 真值框,位置,尺寸,绝对转角,N*1,一个真值框一行 # gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, rect, Trv2c) difficulty = annos["difficulty"] input_dict.update({ 'gt_boxes': gt_boxes, # 真值框 'gt_names': gt_names, # 真值类别 'difficulty': difficulty, # 难度 }) if 'group_ids' in annos: # True input_dict['group_ids'] = annos["group_ids"] example = prep_func(input_dict=input_dict) # 预处理传参 example["image_idx"] = image_idx # 数据帧索引 example["image_shape"] = input_dict["image_shape"] # 图像尺寸 if "anchors_mask" in example: # True example["anchors_mask"] = example["anchors_mask"].astype(np.uint8) # 标记超过一个体素的锚框 return example
def get_sensor_data(self, query): read_image = False idx = query if isinstance(query, dict): read_image = "cam" in query assert "lidar" in query idx = query["lidar"]["idx"] info = self._kitti_infos[idx] res = { "lidar": { "type": "lidar", "points": None, }, "metadata": { "image_idx": info["image"]["image_idx"], "image_shape": info["image"]["image_shape"], }, "calib": None, "cam": {} } pc_info = info["point_cloud"] velo_path = Path(pc_info['velodyne_path']) if not velo_path.is_absolute(): velo_path = Path(self._root_path) / pc_info['velodyne_path'] velo_reduced_path = velo_path.parent.parent / ( velo_path.parent.stem + '_reduced') / velo_path.name if velo_reduced_path.exists(): velo_path = velo_reduced_path points = np.fromfile(str(velo_path), dtype=np.float32, count=-1).reshape([-1, self.NumPointFeatures]) res["lidar"]["points"] = points image_info = info["image"] image_path = image_info['image_path'] if read_image: image_path = self._root_path / image_path with open(str(image_path), 'rb') as f: image_str = f.read() res["cam"] = { "type": "camera", "data": image_str, "datatype": image_path.suffix[1:], } calib = info["calib"] calib_dict = { 'rect': calib['R0_rect'], 'Trv2c': calib['Tr_velo_to_cam'], 'P2': calib['P2'], } res["calib"] = calib_dict if 'annos' in info: annos = info['annos'] # we need other objects to avoid collision when sample annos = kitti.remove_dontcare(annos) locs = annos["location"] dims = annos["dimensions"] rots = annos["rotation_y"] gt_names = annos["name"] # rots = np.concatenate([np.zeros([locs.shape[0], 2], dtype=np.float32), rots], axis=1) gt_boxes = np.concatenate([locs, dims, rots[..., np.newaxis]], axis=1).astype(np.float32) calib = info["calib"] gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, calib["R0_rect"], calib["Tr_velo_to_cam"]) # only center format is allowed. so we need to convert # kitti [0.5, 0.5, 0] center to [0.5, 0.5, 0.5] box_np_ops.change_box3d_center_(gt_boxes, [0.5, 0.5, 0], [0.5, 0.5, 0.5]) res["lidar"]["annotations"] = { 'boxes': gt_boxes, 'names': gt_names, } res["cam"]["annotations"] = { 'boxes': annos["bbox"], 'names': gt_names, } return res
def __getitem__(self, idx): """ you need to create a input dict in this function for network inference. format: { anchors voxels num_points coordinates ground_truth: { gt_boxes gt_names [optional]difficulty [optional]group_ids } [optional]anchors_mask, slow in SECOND v1.5, don't use this. [optional]metadata, in kitti, image index is saved in metadata } """ info = self._kitti_infos[idx] kitti.convert_to_kitti_info_version2(info) pc_info = info["point_cloud"] if "points" not in pc_info: velo_path = pathlib.Path(pc_info['velodyne_path']) if not velo_path.is_absolute(): velo_path = pathlib.Path( self._root_path) / pc_info['velodyne_path'] velo_reduced_path = velo_path.parent.parent / ( velo_path.parent.stem + '_reduced') / velo_path.name if velo_reduced_path.exists(): velo_path = velo_reduced_path points = np.fromfile(str(velo_path), dtype=np.float32, count=-1).reshape( [-1, self._num_point_features]) input_dict = { 'points': points, } if "image" in info: input_dict["image"] = info["image"] if "calib" in info: calib = info["calib"] calib_dict = { 'rect': calib['R0_rect'], 'Trv2c': calib['Tr_velo_to_cam'], 'P2': calib['P2'], } input_dict["calib"] = calib_dict if 'annos' in info: annos = info['annos'] annos_dict = {} # we need other objects to avoid collision when sample annos = kitti.remove_dontcare(annos) loc = annos["location"] dims = annos["dimensions"] rots = annos["rotation_y"] gt_names = annos["name"] gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]], axis=1).astype(np.float32) if "calib" in info: calib = info["calib"] gt_boxes = box_np_ops.box_camera_to_lidar( gt_boxes, calib["R0_rect"], calib["Tr_velo_to_cam"]) # only center format is allowed. so we need to convert # kitti [0.5, 0.5, 0] center to [0.5, 0.5, 0.5] box_np_ops.change_box3d_center_(gt_boxes, [0.5, 0.5, 0], [0.5, 0.5, 0.5]) gt_dict = { 'gt_boxes': gt_boxes, 'gt_names': gt_names, } if 'difficulty' in annos: gt_dict['difficulty'] = annos["difficulty"] if 'group_ids' in annos: gt_dict['group_ids'] = annos["group_ids"] input_dict["ground_truth"] = gt_dict example = self._prep_func(input_dict=input_dict) example["metadata"] = {} if "image" in info: example["metadata"]["image"] = input_dict["image"] if "anchors_mask" in example: example["anchors_mask"] = example["anchors_mask"].astype(np.uint8) return example
def get_inference_input_dict(self, info, points): # assert self.anchor_cache is not None # assert self.target_assigner is not None # assert self.voxel_generator is not None assert self.fv_generator is not None assert self.config is not None assert self.built is True rect = info['calib/R0_rect'] P2 = info['calib/P2'] Trv2c = info['calib/Tr_velo_to_cam'] input_cfg = self.config.eval_input_reader model_cfg = self.config.model.second root_path = '/home/js/data/KITTI/object' input_dict = { 'points': points, 'rect': rect, 'Trv2c': Trv2c, 'P2': P2, 'image_shape': np.array(info["img_shape"], dtype=np.int32), 'image_idx': info['image_idx'], 'image_path': root_path + '/' + info['img_path'], # 'pointcloud_num_features': num_point_features, } if 'annos' in info: annos = info['annos'] # we need other objects to avoid collision when sample annos = kitti.remove_dontcare(annos) loc = annos["location"] dims = annos["dimensions"] rots = annos["rotation_y"] # alpha = annos["alpha"] gt_names = annos["name"] # print(gt_names, len(loc)) gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]], axis=1).astype(np.float32) # gt_boxes = np.concatenate( # [loc, dims, alpha[..., np.newaxis]], axis=1).astype(np.float32) # gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, rect, Trv2c) difficulty = annos["difficulty"] input_dict.update({ 'gt_boxes': gt_boxes, 'gt_names': gt_names, 'difficulty': difficulty, }) if 'group_ids' in annos: input_dict['group_ids'] = annos["group_ids"] out_size_factor = model_cfg.rpn.layer_strides[ 0] // model_cfg.rpn.upsample_strides[0] print("RGB_embedding: ", self.RGB_embedding) example = prep_pointcloud( input_dict=input_dict, root_path=str(self.root_path), # voxel_generator=self.voxel_generator, fv_generator=self.fv_generator, target_assigner=self.target_assigner, max_voxels=input_cfg.max_number_of_voxels, class_names=list(input_cfg.class_names), training=False, create_targets=False, shuffle_points=input_cfg.shuffle_points, generate_bev=False, remove_outside_points=False, without_reflectivity=model_cfg.without_reflectivity, num_point_features=model_cfg.num_point_features, anchor_area_threshold=input_cfg.anchor_area_threshold, anchor_cache=self.anchor_cache, out_size_factor=out_size_factor, out_dtype=np.float32, num_classes=model_cfg.num_class, RGB_embedding=self.RGB_embedding) example["image_idx"] = info['image_idx'] example["image_shape"] = input_dict["image_shape"] example["points"] = points if "anchors_mask" in example: example["anchors_mask"] = example["anchors_mask"].astype(np.uint8) ############# # convert example to batched example ############# example = merge_second_batch([example]) return example