def _read_and_prep_v9(info, root_path, num_point_features, prep_func):
"""read data from KITTI-format infos, then call prep function.
"""
# velodyne_path = str(pathlib.Path(root_path) / info['velodyne_path'])
# velodyne_path += '_reduced'
v_path = pathlib.Path(root_path) / info['velodyne_path']
v_path = v_path.parent.parent / (v_path.parent.stem +
"_reduced") / v_path.name
i_path = pathlib.Path(root_path) / info['img_path']
points = np.fromfile(str(v_path), dtype=np.float32,
count=-1).reshape([-1, num_point_features])
image_idx = info['image_idx']
rect = info['calib/R0_rect'].astype(np.float32)
Trv2c = info['calib/Tr_velo_to_cam'].astype(np.float32)
P2 = info['calib/P2'].astype(np.float32)
#image_pang_bgr = cv2.imread(str(i_path))
#image_pang = image_pang_bgr[..., :: -1]
pil2tensor = transforms.ToTensor()
pil_image = Image.open(str(i_path)).convert("RGB")
image_pang_bgr = np.array(pil_image)[:, :, [2, 1, 0]]
image_pang = image_pang_bgr
#image_pang = pil2tensor(image_pang_bgr)
input_dict = {
'points': points,
'rect': rect,
'Trv2c': Trv2c,
'P2': P2,
'image_shape': np.array(info["img_shape"], dtype=np.int32),
'image_idx': image_idx,
'image_path': info['img_path'],
'images': image_pang
# 'pointcloud_num_features': num_point_features,
}
if 'annos' in info:
annos = info['annos']
# we need other objects to avoid collision when sample
annos = kitti.remove_dontcare(annos)
loc = annos["location"]
dims = annos["dimensions"]
rots = annos["rotation_y"]
gt_names = annos["name"]
gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]],
axis=1).astype(np.float32)
difficulty = annos["difficulty"]
input_dict.update({
'gt_boxes': gt_boxes,
'gt_names': gt_names,
'difficulty': difficulty,
})
if 'group_ids' in annos:
input_dict['group_ids'] = annos["group_ids"]
example = prep_func(input_dict=input_dict)
example["image_idx"] = image_idx
example["image_shape"] = input_dict["image_shape"]
#example["images"] = input_dict["image"]
if "anchors_mask" in example:
example["anchors_mask"] = example["anchors_mask"].astype(np.uint8)
return example
def _read_and_prep_v9(info, root_path, num_point_features, prep_func):
"""read data from KITTI-format infos, then call prep function.
"""
read_image = True
# velodyne_path = str(pathlib.Path(root_path) / info['velodyne_path'])
# velodyne_path += '_reduced'
v_path = pathlib.Path(root_path) / info['velodyne_path']
v_path = v_path.parent.parent / (v_path.parent.stem +
"_reduced") / v_path.name
points = np.fromfile(str(v_path), dtype=np.float32,
count=-1).reshape([-1, num_point_features])
image_idx = info['image_idx']
rect = info['calib/R0_rect'].astype(np.float32)
Trv2c = info['calib/Tr_velo_to_cam'].astype(np.float32)
P2 = info['calib/P2'].astype(np.float32)
image_path = info['img_path']
if read_image:
image_path = Path(root_path) / image_path
image_str = Image.open(os.path.join(image_path))
input_dict = {
'points': points,
'rect': rect,
'Trv2c': Trv2c,
'P2': P2,
'image_shape': np.array(info["img_shape"], dtype=np.int32),
'image_idx': image_idx,
'cam': image_str,
'image_path': info['img_path'],
# 'pointcloud_num_features': num_point_features,
}
if 'annos' in info:
annos = info['annos']
# we need other objects to avoid collision when sample
annos = kitti.remove_dontcare(annos)
loc = annos["location"]
dims = annos["dimensions"]
rots = annos["rotation_y"]
gt_names = annos["name"]
# print(gt_names, len(loc))
gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]],
axis=1).astype(np.float32)
# gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, rect, Trv2c)
difficulty = annos["difficulty"]
input_dict.update({
'gt_boxes': gt_boxes,
'gt_names': gt_names,
'difficulty': difficulty,
})
if 'group_ids' in annos:
input_dict['group_ids'] = annos["group_ids"]
example = prep_func(input_dict=input_dict)
example["image_idx"] = image_idx
example["image_shape"] = input_dict["image_shape"]
if "anchors_mask" in example:
example["anchors_mask"] = example["anchors_mask"].astype(np.uint8)
return example
def _read_and_prep_v9(info, root_path, num_point_features, prep_func):
# info: # kitti_infos_train.pkl内的某一个列表元素（某一帧数据）
"""read data from KITTI-format infos, then call prep function.
"""
# velodyne_path = str(pathlib.Path(root_path) / info['velodyne_path'])
# velodyne_path += '_reduced'
v_path = pathlib.Path(root_path) / info['velodyne_path'] # e.g. root_path/training/velodyne/000010.bin
v_path = v_path.parent.parent / (
v_path.parent.stem + "_reduced") / v_path.name # e.g. root_path/training/velodyne_reduced/000010.bin
points = np.fromfile(
str(v_path), dtype=np.float32,
count=-1).reshape([-1, num_point_features]) # 将点云数据转化为N*4的数组
image_idx = info['image_idx']
rect = info['calib/R0_rect'].astype(np.float32)
Trv2c = info['calib/Tr_velo_to_cam'].astype(np.float32)
P2 = info['calib/P2'].astype(np.float32)
input_dict = {
'points': points, # velodyne_reduced
'rect': rect,
'Trv2c': Trv2c,
'P2': P2,
'image_shape': np.array(info["img_shape"], dtype=np.int32), # 图像尺寸
'image_idx': image_idx, # 图像编号
'image_path': info['img_path'], # 图像路径
# 'pointcloud_num_features': num_point_features,
}
if 'annos' in info: # True
annos = info['annos']
# we need other objects to avoid collision when sample
annos = kitti.remove_dontcare(annos)
loc = annos["location"]
dims = annos["dimensions"]
rots = annos["rotation_y"]
gt_names = annos["name"]
# print(gt_names, len(loc))
gt_boxes = np.concatenate(
[loc, dims, rots[..., np.newaxis]], axis=1).astype(np.float32) # 真值框，位置，尺寸，绝对转角，N*1，一个真值框一行
# gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, rect, Trv2c)
difficulty = annos["difficulty"]
input_dict.update({
'gt_boxes': gt_boxes, # 真值框
'gt_names': gt_names, # 真值类别
'difficulty': difficulty, # 难度
})
if 'group_ids' in annos: # True
input_dict['group_ids'] = annos["group_ids"]
example = prep_func(input_dict=input_dict) # 预处理传参
example["image_idx"] = image_idx # 数据帧索引
example["image_shape"] = input_dict["image_shape"] # 图像尺寸
if "anchors_mask" in example: # True
example["anchors_mask"] = example["anchors_mask"].astype(np.uint8) # 标记超过一个体素的锚框
return example
def get_sensor_data(self, query):
read_image = False
idx = query
if isinstance(query, dict):
read_image = "cam" in query
assert "lidar" in query
idx = query["lidar"]["idx"]
info = self._kitti_infos[idx]
res = {
"lidar": {
"type": "lidar",
"points": None,
},
"metadata": {
"image_idx": info["image"]["image_idx"],
"image_shape": info["image"]["image_shape"],
},
"calib": None,
"cam": {}
}
pc_info = info["point_cloud"]
velo_path = Path(pc_info['velodyne_path'])
if not velo_path.is_absolute():
velo_path = Path(self._root_path) / pc_info['velodyne_path']
velo_reduced_path = velo_path.parent.parent / (
velo_path.parent.stem + '_reduced') / velo_path.name
if velo_reduced_path.exists():
velo_path = velo_reduced_path
points = np.fromfile(str(velo_path), dtype=np.float32,
count=-1).reshape([-1, self.NumPointFeatures])
res["lidar"]["points"] = points
image_info = info["image"]
image_path = image_info['image_path']
if read_image:
image_path = self._root_path / image_path
with open(str(image_path), 'rb') as f:
image_str = f.read()
res["cam"] = {
"type": "camera",
"data": image_str,
"datatype": image_path.suffix[1:],
}
calib = info["calib"]
calib_dict = {
'rect': calib['R0_rect'],
'Trv2c': calib['Tr_velo_to_cam'],
'P2': calib['P2'],
}
res["calib"] = calib_dict
if 'annos' in info:
annos = info['annos']
# we need other objects to avoid collision when sample
annos = kitti.remove_dontcare(annos)
locs = annos["location"]
dims = annos["dimensions"]
rots = annos["rotation_y"]
gt_names = annos["name"]
# rots = np.concatenate([np.zeros([locs.shape[0], 2], dtype=np.float32), rots], axis=1)
gt_boxes = np.concatenate([locs, dims, rots[..., np.newaxis]],
axis=1).astype(np.float32)
calib = info["calib"]
gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes,
calib["R0_rect"],
calib["Tr_velo_to_cam"])
# only center format is allowed. so we need to convert
# kitti [0.5, 0.5, 0] center to [0.5, 0.5, 0.5]
box_np_ops.change_box3d_center_(gt_boxes, [0.5, 0.5, 0],
[0.5, 0.5, 0.5])
res["lidar"]["annotations"] = {
'boxes': gt_boxes,
'names': gt_names,
}
res["cam"]["annotations"] = {
'boxes': annos["bbox"],
'names': gt_names,
}
return res
def __getitem__(self, idx):
"""
you need to create a input dict in this function for network inference.
format: {
anchors
voxels
num_points
coordinates
ground_truth: {
gt_boxes
gt_names
[optional]difficulty
[optional]group_ids
}
[optional]anchors_mask, slow in SECOND v1.5, don't use this.
[optional]metadata, in kitti, image index is saved in metadata
}
"""
info = self._kitti_infos[idx]
kitti.convert_to_kitti_info_version2(info)
pc_info = info["point_cloud"]
if "points" not in pc_info:
velo_path = pathlib.Path(pc_info['velodyne_path'])
if not velo_path.is_absolute():
velo_path = pathlib.Path(
self._root_path) / pc_info['velodyne_path']
velo_reduced_path = velo_path.parent.parent / (
velo_path.parent.stem + '_reduced') / velo_path.name
if velo_reduced_path.exists():
velo_path = velo_reduced_path
points = np.fromfile(str(velo_path), dtype=np.float32,
count=-1).reshape(
[-1, self._num_point_features])
input_dict = {
'points': points,
}
if "image" in info:
input_dict["image"] = info["image"]
if "calib" in info:
calib = info["calib"]
calib_dict = {
'rect': calib['R0_rect'],
'Trv2c': calib['Tr_velo_to_cam'],
'P2': calib['P2'],
}
input_dict["calib"] = calib_dict
if 'annos' in info:
annos = info['annos']
annos_dict = {}
# we need other objects to avoid collision when sample
annos = kitti.remove_dontcare(annos)
loc = annos["location"]
dims = annos["dimensions"]
rots = annos["rotation_y"]
gt_names = annos["name"]
gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]],
axis=1).astype(np.float32)
if "calib" in info:
calib = info["calib"]
gt_boxes = box_np_ops.box_camera_to_lidar(
gt_boxes, calib["R0_rect"], calib["Tr_velo_to_cam"])
# only center format is allowed. so we need to convert
# kitti [0.5, 0.5, 0] center to [0.5, 0.5, 0.5]
box_np_ops.change_box3d_center_(gt_boxes, [0.5, 0.5, 0],
[0.5, 0.5, 0.5])
gt_dict = {
'gt_boxes': gt_boxes,
'gt_names': gt_names,
}
if 'difficulty' in annos:
gt_dict['difficulty'] = annos["difficulty"]
if 'group_ids' in annos:
gt_dict['group_ids'] = annos["group_ids"]
input_dict["ground_truth"] = gt_dict
example = self._prep_func(input_dict=input_dict)
example["metadata"] = {}
if "image" in info:
example["metadata"]["image"] = input_dict["image"]
if "anchors_mask" in example:
example["anchors_mask"] = example["anchors_mask"].astype(np.uint8)
return example
def get_inference_input_dict(self, info, points):
# assert self.anchor_cache is not None
# assert self.target_assigner is not None
# assert self.voxel_generator is not None
assert self.fv_generator is not None
assert self.config is not None
assert self.built is True
rect = info['calib/R0_rect']
P2 = info['calib/P2']
Trv2c = info['calib/Tr_velo_to_cam']
input_cfg = self.config.eval_input_reader
model_cfg = self.config.model.second
root_path = '/home/js/data/KITTI/object'
input_dict = {
'points': points,
'rect': rect,
'Trv2c': Trv2c,
'P2': P2,
'image_shape': np.array(info["img_shape"], dtype=np.int32),
'image_idx': info['image_idx'],
'image_path': root_path + '/' + info['img_path'],
# 'pointcloud_num_features': num_point_features,
}
if 'annos' in info:
annos = info['annos']
# we need other objects to avoid collision when sample
annos = kitti.remove_dontcare(annos)
loc = annos["location"]
dims = annos["dimensions"]
rots = annos["rotation_y"]
# alpha = annos["alpha"]
gt_names = annos["name"]
# print(gt_names, len(loc))
gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]],
axis=1).astype(np.float32)
# gt_boxes = np.concatenate(
# [loc, dims, alpha[..., np.newaxis]], axis=1).astype(np.float32)
# gt_boxes = box_np_ops.box_camera_to_lidar(gt_boxes, rect, Trv2c)
difficulty = annos["difficulty"]
input_dict.update({
'gt_boxes': gt_boxes,
'gt_names': gt_names,
'difficulty': difficulty,
})
if 'group_ids' in annos:
input_dict['group_ids'] = annos["group_ids"]
out_size_factor = model_cfg.rpn.layer_strides[
0] // model_cfg.rpn.upsample_strides[0]
print("RGB_embedding: ", self.RGB_embedding)
example = prep_pointcloud(
input_dict=input_dict,
root_path=str(self.root_path),
# voxel_generator=self.voxel_generator,
fv_generator=self.fv_generator,
target_assigner=self.target_assigner,
max_voxels=input_cfg.max_number_of_voxels,
class_names=list(input_cfg.class_names),
training=False,
create_targets=False,
shuffle_points=input_cfg.shuffle_points,
generate_bev=False,
remove_outside_points=False,
without_reflectivity=model_cfg.without_reflectivity,
num_point_features=model_cfg.num_point_features,
anchor_area_threshold=input_cfg.anchor_area_threshold,
anchor_cache=self.anchor_cache,
out_size_factor=out_size_factor,
out_dtype=np.float32,
num_classes=model_cfg.num_class,
RGB_embedding=self.RGB_embedding)
example["image_idx"] = info['image_idx']
example["image_shape"] = input_dict["image_shape"]
example["points"] = points
if "anchors_mask" in example:
example["anchors_mask"] = example["anchors_mask"].astype(np.uint8)
#############
# convert example to batched example
#############
example = merge_second_batch([example])
return example
