def test_config_build_pipeline():
"""Test that all detection models defined in the configs can be
initialized."""
from mmcv import Config
from mmdet3d.datasets.pipelines import Compose
config_dpath = _get_config_directory()
print('Found config_dpath = {!r}'.format(config_dpath))
# Other configs needs database sampler.
config_names = [
'pointpillars/hv_pointpillars_secfpn_sbn-all_4x8_2x_nus-3d.py',
]
print('Using {} config files'.format(len(config_names)))
for config_fname in config_names:
config_fpath = join(config_dpath, config_fname)
config_mod = Config.fromfile(config_fpath)
# build train_pipeline
train_pipeline = Compose(config_mod.train_pipeline)
test_pipeline = Compose(config_mod.test_pipeline)
assert train_pipeline is not None
assert test_pipeline is not None
def __init__(
self,
sf_path='/public/MARS/datasets/nuScenes-SF/trainval-camera',
img_path='data/nuscenes/',
pose_path='/public/MARS/datasets/nuScenes-SF/meta/cam_pose_intrinsic_v3.json',
pipeline=None,
training=True,
**kwargs):
self.sf_path = sf_path
self.img_path = img_path
self.pose_path = pose_path
self.training = training
if self.training:
self.depth_pred_path = '/public/MARS/datasets/nuScenes-SF/meta/two_cam_meta/sf_inp_train_meta.json'
self.meta_path = '/public/MARS/datasets/nuScenes-SF/meta/spatial_temp_train_v2.json'
self.merged_file_path = '/public/MARS/datasets/nuScenes-SF/meta/two_cam_meta/spatial_temp_merged_path_train.json'
else:
self.depth_pred_path = '/public/MARS/datasets/nuScenes-SF/meta/two_cam_meta/sf_inp_val_meta.json' # change need
self.meta_path = '/public/MARS/datasets/nuScenes-SF/meta/spatial_temp_val_v2.json'
self.merged_file_path = '/public/MARS/datasets/nuScenes-SF/meta/two_cam_meta/spatial_temp_merged_path_val.json'
self.data_infos = self.load_annotations() #[:200]
if pipeline is not None:
self.pipeline = Compose(pipeline)
# not important
self.flag = np.zeros(len(self), dtype=np.uint8)
def inference_mono_3d_detector(model, image, ann_file):
"""Inference image with the monocular 3D detector.
Args:
model (nn.Module): The loaded detector.
image (str): Image files.
ann_file (str): Annotation files.
Returns:
tuple: Predicted results and data from pipeline.
"""
cfg = model.cfg
device = next(model.parameters()).device # model device
# build the data pipeline
test_pipeline = deepcopy(cfg.data.test.pipeline)
test_pipeline = Compose(test_pipeline)
box_type_3d, box_mode_3d = get_box_type(cfg.data.test.box_type_3d)
# get data info containing calib
data_infos = mmcv.load(ann_file)
# find the info corresponding to this image
for x in data_infos['images']:
if osp.basename(x['file_name']) != osp.basename(image):
continue
img_info = x
break
data = dict(
img_prefix=osp.dirname(image),
img_info=dict(filename=osp.basename(image)),
box_type_3d=box_type_3d,
box_mode_3d=box_mode_3d,
img_fields=[],
bbox3d_fields=[],
pts_mask_fields=[],
pts_seg_fields=[],
bbox_fields=[],
mask_fields=[],
seg_fields=[])
# camera points to image conversion
if box_mode_3d == Box3DMode.CAM:
data['img_info'].update(dict(cam_intrinsic=img_info['cam_intrinsic']))
data = test_pipeline(data)
data = collate([data], samples_per_gpu=1)
if next(model.parameters()).is_cuda:
# scatter to specified GPU
data = scatter(data, [device.index])[0]
else:
# this is a workaround to avoid the bug of MMDataParallel
data['img_metas'] = data['img_metas'][0].data
data['img'] = data['img'][0].data
# forward the model
with torch.no_grad():
result = model(return_loss=False, rescale=True, **data)
return result, data
def datapipelinefrompcd(self, config, pcd):
# build the data pipeline
point_cloud_range = [0, -39.68, -3, 69.12, 39.68, 1]
class_names = ['Pedestrian', 'Cyclist', 'Car']
my_pipeline = [
#dict(type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4),
dict(type='MultiScaleFlipAug3D',
img_scale=(1333, 800),
pts_scale_ratio=1,
flip=False,
transforms=[
dict(type='GlobalRotScaleTrans',
rot_range=[0, 0],
scale_ratio_range=[1., 1.],
translation_std=[0, 0, 0]),
dict(type='RandomFlip3D'),
dict(type='PointsRangeFilter',
point_cloud_range=point_cloud_range),
dict(type='DefaultFormatBundle3D',
class_names=class_names,
with_label=False),
dict(type='Collect3D', keys=['points'])
])
]
#lidarpoints=LiDARPoints(torch.from_numpy(pcd[1,:]))
lidarpoints = LiDARPoints(pcd[:, :4], points_dim=4)
self.test_pipeline = my_pipeline #deepcopy(config.data.test.pipeline)
self.test_pipeline = Compose(self.test_pipeline)
box_type_3d, box_mode_3d = get_box_type(config.data.test.box_type_3d)
data = dict(
pts_filename=[], #pcd,
box_type_3d=box_type_3d,
box_mode_3d=box_mode_3d,
sweeps=[],
# set timestamp = 0
timestamp=[0],
img_fields=[],
bbox3d_fields=[],
pts_mask_fields=[],
pts_seg_fields=[],
bbox_fields=[],
mask_fields=[],
seg_fields=[],
points=lidarpoints)
data = self.test_pipeline(data)
#data['points']=pcd
data = collate([data], samples_per_gpu=1)
return data
def inference_detector(model, pcd):
"""Inference point cloud with the detector.
Args:
model (nn.Module): The loaded detector.
pcd (str): Point cloud files.
Returns:
tuple: Predicted results and data from pipeline.
"""
cfg = model.cfg
device = next(model.parameters()).device # model device
# build the data pipeline
test_pipeline = deepcopy(cfg.data.test.pipeline)
test_pipeline = Compose(test_pipeline)
box_type_3d, box_mode_3d = get_box_type(cfg.data.test.box_type_3d)
data = dict(
pts_filename=pcd,
box_type_3d=box_type_3d,
box_mode_3d=box_mode_3d,
# for ScanNet demo we need axis_align_matrix
ann_info=dict(axis_align_matrix=np.eye(4)),
sweeps=[],
# set timestamp = 0
timestamp=[0],
img_fields=[],
bbox3d_fields=[],
pts_mask_fields=[],
pts_seg_fields=[],
bbox_fields=[],
mask_fields=[],
seg_fields=[])
data = test_pipeline(data)
data = collate([data], samples_per_gpu=1)
if next(model.parameters()).is_cuda:
# scatter to specified GPU
data = scatter(data, [device.index])[0]
else:
# this is a workaround to avoid the bug of MMDataParallel
data['img_metas'] = data['img_metas'][0].data
data['points'] = data['points'][0].data
# forward the model
with torch.no_grad():
result = model(return_loss=False, rescale=True, **data)
return result, data
def __init__(self,
data_path='data/nuscenes/depth_maps/train/detpth_map',
pipeline=None,
training=True,
**kwargs):
self.depth_root = os.path.join(data_path, 'depth_data')
self.meta_path = os.path.join(data_path, 'meta.json')
self.training = training
#from IPython import embed
#embed()
if training:
#self.data_infos = self.load_annotations()[:-20000]
self.data_infos = self.load_annotations()[:-20000]
else:
self.data_infos = self.load_annotations()[-20000:]
if pipeline is not None:
self.pipeline = Compose(pipeline)
# not important
self.flag = np.zeros(len(self), dtype=np.uint8)
def datapipelinefromlidarfile(self, config, pts_filename):
# build the data pipeline
self.test_pipeline = deepcopy(config.data.test.pipeline)
self.test_pipeline = Compose(self.test_pipeline)
box_type_3d, box_mode_3d = get_box_type(config.data.test.box_type_3d)
data = dict(
pts_filename=pts_filename, #pcd,
box_type_3d=box_type_3d,
box_mode_3d=box_mode_3d,
sweeps=[],
# set timestamp = 0
timestamp=[0],
img_fields=[],
bbox3d_fields=[],
pts_mask_fields=[],
pts_seg_fields=[],
bbox_fields=[],
mask_fields=[],
seg_fields=[])
return data
def inference_detector(model, pcd):
"""Inference point cloud with the detector.
Args:
model (nn.Module): The loaded detector.
pcd (str): Point cloud files.
Returns:
tuple: Predicted results and data from pipeline.
"""
cfg = model.cfg
device = next(model.parameters()).device # model device
# build the data pipeline
test_pipeline = deepcopy(cfg.data.test.pipeline)
test_pipeline = Compose(test_pipeline)
box_type_3d, box_mode_3d = get_box_type(cfg.data.test.box_type_3d)
data = dict(pts_filename=pcd,
box_type_3d=box_type_3d,
box_mode_3d=box_mode_3d,
img_fields=[],
bbox3d_fields=[],
pts_mask_fields=[],
pts_seg_fields=[],
bbox_fields=[],
mask_fields=[],
seg_fields=[])
data = test_pipeline(data)
data = collate([data], samples_per_gpu=1)
if next(model.parameters()).is_cuda:
# scatter to specified GPU
data = scatter(data, [device.index])[0]
else:
raise NotImplementedError('Not support cpu-only currently')
# forward the model
with torch.no_grad():
result = model(return_loss=False, rescale=True, **data)
return result, data
def inference_segmentor(model, pcd):
"""Inference point cloud with the segmentor.
Args:
model (nn.Module): The loaded segmentor.
pcd (str): Point cloud files.
Returns:
tuple: Predicted results and data from pipeline.
"""
cfg = model.cfg
device = next(model.parameters()).device # model device
# build the data pipeline
test_pipeline = deepcopy(cfg.data.test.pipeline)
test_pipeline = Compose(test_pipeline)
data = dict(
pts_filename=pcd,
img_fields=[],
bbox3d_fields=[],
pts_mask_fields=[],
pts_seg_fields=[],
bbox_fields=[],
mask_fields=[],
seg_fields=[])
data = test_pipeline(data)
data = collate([data], samples_per_gpu=1)
if next(model.parameters()).is_cuda:
# scatter to specified GPU
data = scatter(data, [device.index])[0]
else:
# this is a workaround to avoid the bug of MMDataParallel
data['img_metas'] = data['img_metas'][0].data
data['points'] = data['points'][0].data
# forward the model
with torch.no_grad():
result = model(return_loss=False, rescale=True, **data)
return result, data
def test_outdoor_aug_pipeline():
point_cloud_range = [0, -40, -3, 70.4, 40, 1]
class_names = ['Car']
np.random.seed(0)
train_pipeline = [
dict(type='LoadPointsFromFile',
coord_type='LIDAR',
load_dim=4,
use_dim=4),
dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True),
dict(type='ObjectNoise',
num_try=100,
translation_std=[1.0, 1.0, 0.5],
global_rot_range=[0.0, 0.0],
rot_range=[-0.78539816, 0.78539816]),
dict(type='RandomFlip3D', flip_ratio_bev_horizontal=0.5),
dict(type='GlobalRotScaleTrans',
rot_range=[-0.78539816, 0.78539816],
scale_ratio_range=[0.95, 1.05]),
dict(type='PointsRangeFilter', point_cloud_range=point_cloud_range),
dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
dict(type='PointShuffle'),
dict(type='DefaultFormatBundle3D', class_names=class_names),
dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
]
pipeline = Compose(train_pipeline)
gt_bboxes_3d = LiDARInstance3DBoxes(
torch.tensor([
[
2.16902428e+01, -4.06038128e-02, -1.61906636e+00,
1.65999997e+00, 3.20000005e+00, 1.61000001e+00, -1.53999996e+00
],
[
7.05006886e+00, -6.57459593e+00, -1.60107934e+00,
2.27999997e+00, 1.27799997e+01, 3.66000009e+00, 1.54999995e+00
],
[
2.24698811e+01, -6.69203758e+00, -1.50118136e+00,
2.31999993e+00, 1.47299995e+01, 3.64000010e+00, 1.59000003e+00
],
[
3.48291969e+01, -7.09058380e+00, -1.36622977e+00,
2.31999993e+00, 1.00400000e+01, 3.60999990e+00, 1.61000001e+00
],
[
4.62394600e+01, -7.75838804e+00, -1.32405007e+00,
2.33999991e+00, 1.28299999e+01, 3.63000011e+00, 1.63999999e+00
],
[
2.82966995e+01, -5.55755794e-01, -1.30332506e+00,
1.47000003e+00, 2.23000002e+00, 1.48000002e+00, -1.57000005e+00
],
[
2.66690197e+01, 2.18230209e+01, -1.73605704e+00,
1.55999994e+00, 3.48000002e+00, 1.39999998e+00, -1.69000006e+00
],
[
3.13197803e+01, 8.16214371e+00, -1.62177873e+00,
1.74000001e+00, 3.76999998e+00, 1.48000002e+00, 2.78999996e+00
],
[
4.34395561e+01, -1.95209332e+01, -1.20757008e+00,
1.69000006e+00, 4.09999990e+00, 1.40999997e+00, -1.53999996e+00
],
[
3.29882965e+01, -3.79360509e+00, -1.69245458e+00,
1.74000001e+00, 4.09000015e+00, 1.49000001e+00, -1.52999997e+00
],
[
3.85469360e+01, 8.35060215e+00, -1.31423414e+00,
1.59000003e+00, 4.28000021e+00, 1.45000005e+00, 1.73000002e+00
],
[
2.22492104e+01, -1.13536005e+01, -1.38272512e+00,
1.62000000e+00, 3.55999994e+00, 1.71000004e+00, 2.48000002e+00
],
[
3.36115799e+01, -1.97708054e+01, -4.92827654e-01,
1.64999998e+00, 3.54999995e+00, 1.79999995e+00, -1.57000005e+00
],
[
9.85029602e+00, -1.51294518e+00, -1.66834795e+00,
1.59000003e+00, 3.17000008e+00, 1.38999999e+00, -8.39999974e-01
]
],
dtype=torch.float32))
gt_labels_3d = np.array([0, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
results = dict(pts_filename='tests/data/kitti/a.bin',
ann_info=dict(gt_bboxes_3d=gt_bboxes_3d,
gt_labels_3d=gt_labels_3d),
bbox3d_fields=[],
img_fields=[])
output = pipeline(results)
expected_tensor = torch.tensor(
[[20.6514, -8.8250, -1.0816, 1.5893, 3.0637, 1.5414, -1.9216],
[7.9374, 4.9457, -1.2008, 2.1829, 12.2357, 3.5041, 1.6629],
[20.8115, -2.0273, -1.8893, 2.2212, 14.1026, 3.4850, 2.6513],
[32.3850, -5.2135, -1.1321, 2.2212, 9.6124, 3.4562, 2.6498],
[43.7022, -7.8316, -0.5090, 2.2403, 12.2836, 3.4754, 2.0146],
[25.3300, -9.6670, -1.0855, 1.4074, 2.1350, 1.4170, -0.7141],
[16.5414, -29.0583, -0.9768, 1.4936, 3.3318, 1.3404, -0.7153],
[24.6548, -18.9226, -1.3567, 1.6659, 3.6094, 1.4170, 1.3970],
[45.8403, 1.8183, -1.1626, 1.6180, 3.9254, 1.3499, -0.6886],
[30.6288, -8.4497, -1.4881, 1.6659, 3.9158, 1.4265, -0.7241],
[32.3316, -22.4611, -1.3131, 1.5223, 4.0977, 1.3882, 2.4186],
[22.4492, 3.2944, -2.1674, 1.5510, 3.4084, 1.6372, 0.3928],
[37.3824, 5.0472, -0.6579, 1.5797, 3.3988, 1.7233, -1.4862],
[8.9259, -1.2578, -1.6081, 1.5223, 3.0350, 1.3308, -1.7212]])
assert torch.allclose(output['gt_bboxes_3d']._data.tensor,
expected_tensor,
atol=1e-3)
def test_outdoor_velocity_aug_pipeline():
point_cloud_range = [-50, -50, -5, 50, 50, 3]
class_names = [
'car', 'truck', 'trailer', 'bus', 'construction_vehicle', 'bicycle',
'motorcycle', 'pedestrian', 'traffic_cone', 'barrier'
]
np.random.seed(0)
train_pipeline = [
dict(type='LoadPointsFromFile',
coord_type='LIDAR',
load_dim=4,
use_dim=4),
dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True),
dict(type='GlobalRotScaleTrans',
rot_range=[-0.3925, 0.3925],
scale_ratio_range=[0.95, 1.05],
translation_std=[0, 0, 0]),
dict(type='RandomFlip3D', flip_ratio_bev_horizontal=0.5),
dict(type='PointsRangeFilter', point_cloud_range=point_cloud_range),
dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
dict(type='PointShuffle'),
dict(type='DefaultFormatBundle3D', class_names=class_names),
dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
]
pipeline = Compose(train_pipeline)
gt_bboxes_3d = LiDARInstance3DBoxes(torch.tensor(
[[
-5.2422e+00, 4.0021e+01, -4.7643e-01, 2.0620e+00, 4.4090e+00,
1.5480e+00, -1.4880e+00, 8.5338e-03, 4.4934e-02
],
[
-2.6675e+01, 5.5950e+00, -1.3053e+00, 3.4300e-01, 4.5800e-01,
7.8200e-01, -4.6276e+00, -4.3284e-04, -1.8543e-03
],
[
-5.8098e+00, 3.5409e+01, -6.6511e-01, 2.3960e+00, 3.9690e+00,
1.7320e+00, -4.6520e+00, 0.0000e+00, 0.0000e+00
],
[
-3.1309e+01, 1.0901e+00, -1.0561e+00, 1.9440e+00, 3.8570e+00,
1.7230e+00, -2.8143e+00, -2.7606e-02, -8.0573e-02
],
[
-4.5642e+01, 2.0136e+01, -2.4681e-02, 1.9870e+00, 4.4400e+00,
1.9420e+00, 2.8336e-01, 0.0000e+00, 0.0000e+00
],
[
-5.1617e+00, 1.8305e+01, -1.0879e+00, 2.3230e+00, 4.8510e+00,
1.3710e+00, -1.5803e+00, 0.0000e+00, 0.0000e+00
],
[
-2.5285e+01, 4.1442e+00, -1.2713e+00, 1.7550e+00, 1.9890e+00,
2.2200e+00, -4.4900e+00, -3.1784e-02, -1.5291e-01
],
[
-2.2611e+00, 1.9170e+01, -1.1452e+00, 9.1900e-01, 1.1230e+00,
1.9310e+00, 4.7790e-02, 6.7684e-02, -1.7537e+00
],
[
-6.5878e+01, 1.3500e+01, -2.2528e-01, 1.8200e+00, 3.8520e+00,
1.5450e+00, -2.8757e+00, 0.0000e+00, 0.0000e+00
],
[
-5.4490e+00, 2.8363e+01, -7.7275e-01, 2.2360e+00, 3.7540e+00,
1.5590e+00, -4.6520e+00, -7.9736e-03, 7.7207e-03
]],
dtype=torch.float32),
box_dim=9)
gt_labels_3d = np.array([0, 8, 0, 0, 0, 0, -1, 7, 0, 0])
results = dict(pts_filename='tests/data/kitti/a.bin',
ann_info=dict(gt_bboxes_3d=gt_bboxes_3d,
gt_labels_3d=gt_labels_3d),
bbox3d_fields=[],
img_fields=[])
output = pipeline(results)
expected_tensor = torch.tensor(
[[
-3.7849e+00, -4.1057e+01, -4.8668e-01, 2.1064e+00, 4.5039e+00,
1.5813e+00, -1.6919e+00, 1.0469e-02, -4.5533e-02
],
[
-2.7010e+01, -6.7551e+00, -1.3334e+00, 3.5038e-01, 4.6786e-01,
7.9883e-01, 1.4477e+00, -5.1440e-04, 1.8758e-03
],
[
-4.5448e+00, -3.6372e+01, -6.7942e-01, 2.4476e+00, 4.0544e+00,
1.7693e+00, 1.4721e+00, 0.0000e+00, -0.0000e+00
],
[
-3.1916e+01, -2.3379e+00, -1.0788e+00, 1.9858e+00, 3.9400e+00,
1.7601e+00, -3.6564e-01, -3.1333e-02, 8.1166e-02
],
[
-4.5802e+01, -2.2340e+01, -2.5213e-02, 2.0298e+00, 4.5355e+00,
1.9838e+00, 2.8199e+00, 0.0000e+00, -0.0000e+00
],
[
-4.5526e+00, -1.8887e+01, -1.1114e+00, 2.3730e+00, 4.9554e+00,
1.4005e+00, -1.5997e+00, 0.0000e+00, -0.0000e+00
],
[
-2.5648e+01, -5.2197e+00, -1.2987e+00, 1.7928e+00, 2.0318e+00,
2.2678e+00, 1.3100e+00, -3.8428e-02, 1.5485e-01
],
[
-1.5578e+00, -1.9657e+01, -1.1699e+00, 9.3878e-01, 1.1472e+00,
1.9726e+00, 3.0555e+00, 4.5907e-04, 1.7928e+00
],
[
-4.4522e+00, -2.9166e+01, -7.8938e-01, 2.2841e+00, 3.8348e+00,
1.5925e+00, 1.4721e+00, -7.8371e-03, -8.1931e-03
]])
assert torch.allclose(output['gt_bboxes_3d']._data.tensor,
expected_tensor,
atol=1e-3)
def inference_multi_modality_detector(model, pcd, image, ann_file):
"""Inference point cloud with the multi-modality detector.
Args:
model (nn.Module): The loaded detector.
pcd (str): Point cloud files.
image (str): Image files.
ann_file (str): Annotation files.
Returns:
tuple: Predicted results and data from pipeline.
"""
cfg = model.cfg
device = next(model.parameters()).device # model device
# build the data pipeline
test_pipeline = deepcopy(cfg.data.test.pipeline)
test_pipeline = Compose(test_pipeline)
box_type_3d, box_mode_3d = get_box_type(cfg.data.test.box_type_3d)
# get data info containing calib
data_infos = mmcv.load(ann_file)
image_idx = int(re.findall(r'\d+', image)[-1]) # xxx/sunrgbd_000017.jpg
for x in data_infos:
if int(x['image']['image_idx']) != image_idx:
continue
info = x
break
data = dict(pts_filename=pcd,
img_prefix=osp.dirname(image),
img_info=dict(filename=osp.basename(image)),
box_type_3d=box_type_3d,
box_mode_3d=box_mode_3d,
img_fields=[],
bbox3d_fields=[],
pts_mask_fields=[],
pts_seg_fields=[],
bbox_fields=[],
mask_fields=[],
seg_fields=[])
data = test_pipeline(data)
# TODO: this code is dataset-specific. Move lidar2img and
# depth2img to .pkl annotations in the future.
# LiDAR to image conversion
if box_mode_3d == Box3DMode.LIDAR:
rect = info['calib']['R0_rect'].astype(np.float32)
Trv2c = info['calib']['Tr_velo_to_cam'].astype(np.float32)
P2 = info['calib']['P2'].astype(np.float32)
lidar2img = P2 @ rect @ Trv2c
data['img_metas'][0].data['lidar2img'] = lidar2img
# Depth to image conversion
elif box_mode_3d == Box3DMode.DEPTH:
rt_mat = info['calib']['Rt']
# follow Coord3DMode.convert_point
rt_mat = np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]
]) @ rt_mat.transpose(1, 0)
depth2img = info['calib']['K'] @ rt_mat
data['img_metas'][0].data['depth2img'] = depth2img
data = collate([data], samples_per_gpu=1)
if next(model.parameters()).is_cuda:
# scatter to specified GPU
data = scatter(data, [device.index])[0]
else:
# this is a workaround to avoid the bug of MMDataParallel
data['img_metas'] = data['img_metas'][0].data
data['points'] = data['points'][0].data
data['img'] = data['img'][0].data
# forward the model
with torch.no_grad():
result = model(return_loss=False, rescale=True, **data)
return result, data
def test_scannet_pipeline():
class_names = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
'window', 'bookshelf', 'picture', 'counter', 'desk',
'curtain', 'refrigerator', 'showercurtrain', 'toilet',
'sink', 'bathtub', 'garbagebin')
np.random.seed(0)
pipelines = [
dict(type='LoadPointsFromFile',
coord_type='DEPTH',
shift_height=True,
load_dim=6,
use_dim=[0, 1, 2]),
dict(type='LoadAnnotations3D',
with_bbox_3d=True,
with_label_3d=True,
with_mask_3d=True,
with_seg_3d=True),
dict(type='GlobalAlignment', rotation_axis=2),
dict(type='PointSegClassMapping',
valid_cat_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28,
33, 34, 36, 39)),
dict(type='IndoorPointSample', num_points=5),
dict(type='RandomFlip3D',
sync_2d=False,
flip_ratio_bev_horizontal=1.0,
flip_ratio_bev_vertical=1.0),
dict(type='GlobalRotScaleTrans',
rot_range=[-0.087266, 0.087266],
scale_ratio_range=[1.0, 1.0],
shift_height=True),
dict(type='DefaultFormatBundle3D', class_names=class_names),
dict(type='Collect3D',
keys=[
'points', 'gt_bboxes_3d', 'gt_labels_3d', 'pts_semantic_mask',
'pts_instance_mask'
]),
]
pipeline = Compose(pipelines)
info = mmcv.load('./tests/data/scannet/scannet_infos.pkl')[0]
results = dict()
data_path = './tests/data/scannet'
results['pts_filename'] = osp.join(data_path, info['pts_path'])
if info['annos']['gt_num'] != 0:
scannet_gt_bboxes_3d = info['annos']['gt_boxes_upright_depth'].astype(
np.float32)
scannet_gt_labels_3d = info['annos']['class'].astype(np.long)
else:
scannet_gt_bboxes_3d = np.zeros((1, 6), dtype=np.float32)
scannet_gt_labels_3d = np.zeros((1, ), dtype=np.long)
results['ann_info'] = dict()
results['ann_info']['pts_instance_mask_path'] = osp.join(
data_path, info['pts_instance_mask_path'])
results['ann_info']['pts_semantic_mask_path'] = osp.join(
data_path, info['pts_semantic_mask_path'])
results['ann_info']['gt_bboxes_3d'] = DepthInstance3DBoxes(
scannet_gt_bboxes_3d, box_dim=6, with_yaw=False)
results['ann_info']['gt_labels_3d'] = scannet_gt_labels_3d
results['ann_info']['axis_align_matrix'] = \
info['annos']['axis_align_matrix']
results['img_fields'] = []
results['bbox3d_fields'] = []
results['pts_mask_fields'] = []
results['pts_seg_fields'] = []
results = pipeline(results)
points = results['points']._data
gt_bboxes_3d = results['gt_bboxes_3d']._data
gt_labels_3d = results['gt_labels_3d']._data
pts_semantic_mask = results['pts_semantic_mask']._data
pts_instance_mask = results['pts_instance_mask']._data
expected_points = torch.tensor(
[[1.8339e+00, 2.1093e+00, 2.2900e+00, 2.3895e+00],
[3.6079e+00, 1.4592e-01, 2.0687e+00, 2.1682e+00],
[4.1886e+00, 5.0614e+00, -1.0841e-01, -8.8736e-03],
[6.8790e+00, 1.5086e+00, -9.3154e-02, 6.3816e-03],
[4.8253e+00, 2.6668e-01, 1.4917e+00, 1.5912e+00]])
expected_gt_bboxes_3d = torch.tensor(
[[-1.1835, -3.6317, 1.8565, 1.7577, 0.3761, 0.5724, 0.0000],
[-3.1832, 3.2269, 1.5268, 0.6727, 0.2251, 0.6715, 0.0000],
[-0.9598, -2.2864, 0.6165, 0.7506, 2.5709, 1.2145, 0.0000],
[-2.6988, -2.7354, 0.9722, 0.7680, 1.8877, 0.2870, 0.0000],
[3.2989, 0.2885, 1.0712, 0.7600, 3.8814, 2.1603, 0.0000]])
expected_gt_labels_3d = np.array([
6, 6, 4, 9, 11, 11, 10, 0, 15, 17, 17, 17, 3, 12, 4, 4, 14, 1, 0, 0, 0,
0, 0, 0, 5, 5, 5
])
expected_pts_semantic_mask = np.array([0, 18, 18, 18, 18])
expected_pts_instance_mask = np.array([44, 22, 10, 10, 57])
assert torch.allclose(points, expected_points, 1e-2)
assert torch.allclose(gt_bboxes_3d.tensor[:5, :], expected_gt_bboxes_3d,
1e-2)
assert np.all(gt_labels_3d.numpy() == expected_gt_labels_3d)
assert np.all(pts_semantic_mask.numpy() == expected_pts_semantic_mask)
assert np.all(pts_instance_mask.numpy() == expected_pts_instance_mask)
def test_scannet_seg_pipeline():
class_names = ('wall', 'floor', 'cabinet', 'bed', 'chair', 'sofa', 'table',
'door', 'window', 'bookshelf', 'picture', 'counter', 'desk',
'curtain', 'refrigerator', 'showercurtrain', 'toilet',
'sink', 'bathtub', 'otherfurniture')
np.random.seed(0)
pipelines = [
dict(type='LoadPointsFromFile',
coord_type='DEPTH',
shift_height=False,
use_color=True,
load_dim=6,
use_dim=[0, 1, 2, 3, 4, 5]),
dict(type='LoadAnnotations3D',
with_bbox_3d=False,
with_label_3d=False,
with_mask_3d=False,
with_seg_3d=True),
dict(type='PointSegClassMapping',
valid_cat_ids=(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24,
28, 33, 34, 36, 39),
max_cat_id=40),
dict(type='IndoorPatchPointSample',
num_points=5,
block_size=1.5,
sample_rate=1.0,
ignore_index=len(class_names),
use_normalized_coord=True),
dict(type='NormalizePointsColor', color_mean=None),
dict(type='DefaultFormatBundle3D', class_names=class_names),
dict(type='Collect3D', keys=['points', 'pts_semantic_mask'])
]
pipeline = Compose(pipelines)
info = mmcv.load('./tests/data/scannet/scannet_infos.pkl')[0]
results = dict()
data_path = './tests/data/scannet'
results['pts_filename'] = osp.join(data_path, info['pts_path'])
results['ann_info'] = dict()
results['ann_info']['pts_semantic_mask_path'] = osp.join(
data_path, info['pts_semantic_mask_path'])
results['pts_seg_fields'] = []
results = pipeline(results)
points = results['points']._data
pts_semantic_mask = results['pts_semantic_mask']._data
# build sampled points
scannet_points = np.fromfile(osp.join(data_path, info['pts_path']),
dtype=np.float32).reshape((-1, 6))
scannet_choices = np.array([87, 34, 58, 9, 18])
scannet_center = np.array([-2.1772466, -3.4789145, 1.242711])
scannet_center[2] = 0.0
scannet_coord_max = np.amax(scannet_points[:, :3], axis=0)
expected_points = np.concatenate([
scannet_points[scannet_choices, :3] - scannet_center,
scannet_points[scannet_choices, 3:] / 255.,
scannet_points[scannet_choices, :3] / scannet_coord_max
],
axis=1)
expected_pts_semantic_mask = np.array([13, 13, 12, 2, 0])
assert np.allclose(points.numpy(), expected_points, atol=1e-6)
assert np.all(pts_semantic_mask.numpy() == expected_pts_semantic_mask)
def test_sunrgbd_pipeline():
class_names = ('bed', 'table', 'sofa', 'chair', 'toilet', 'desk',
'dresser', 'night_stand', 'bookshelf', 'bathtub')
np.random.seed(0)
pipelines = [
dict(type='LoadPointsFromFile',
coord_type='DEPTH',
shift_height=True,
load_dim=6,
use_dim=[0, 1, 2]),
dict(type='LoadAnnotations3D'),
dict(
type='RandomFlip3D',
sync_2d=False,
flip_ratio_bev_horizontal=1.0,
),
dict(type='GlobalRotScaleTrans',
rot_range=[-0.523599, 0.523599],
scale_ratio_range=[0.85, 1.15],
shift_height=True),
dict(type='IndoorPointSample', num_points=5),
dict(type='DefaultFormatBundle3D', class_names=class_names),
dict(type='Collect3D', keys=['points', 'gt_bboxes_3d',
'gt_labels_3d']),
]
pipeline = Compose(pipelines)
results = dict()
info = mmcv.load('./tests/data/sunrgbd/sunrgbd_infos.pkl')[0]
data_path = './tests/data/sunrgbd'
results['pts_filename'] = osp.join(data_path, info['pts_path'])
if info['annos']['gt_num'] != 0:
gt_bboxes_3d = info['annos']['gt_boxes_upright_depth'].astype(
np.float32)
gt_labels_3d = info['annos']['class'].astype(np.long)
else:
gt_bboxes_3d = np.zeros((1, 7), dtype=np.float32)
gt_labels_3d = np.zeros((1, ), dtype=np.long)
# prepare input of pipeline
results['ann_info'] = dict()
results['ann_info']['gt_bboxes_3d'] = DepthInstance3DBoxes(gt_bboxes_3d)
results['ann_info']['gt_labels_3d'] = gt_labels_3d
results['img_fields'] = []
results['bbox3d_fields'] = []
results['pts_mask_fields'] = []
results['pts_seg_fields'] = []
results = pipeline(results)
points = results['points']._data
gt_bboxes_3d = results['gt_bboxes_3d']._data
gt_labels_3d = results['gt_labels_3d']._data
expected_points = torch.tensor([[0.8678, 1.3470, 0.1105, 0.0905],
[0.8707, 1.3635, 0.0437, 0.0238],
[0.8636, 1.3511, 0.0504, 0.0304],
[0.8690, 1.3461, 0.1265, 0.1065],
[0.8668, 1.3434, 0.1216, 0.1017]])
expected_gt_bboxes_3d = torch.tensor(
[[-1.2136, 4.0206, -0.2412, 2.2493, 1.8444, 1.9245, 1.3989],
[-2.7420, 4.5777, -0.7686, 0.5718, 0.8629, 0.9510, 1.4446],
[0.9729, 1.9087, -0.1443, 0.6965, 1.5273, 2.0563, 2.9924]])
expected_gt_labels_3d = np.array([0, 7, 6])
assert torch.allclose(gt_bboxes_3d.tensor, expected_gt_bboxes_3d, 1e-3)
assert np.allclose(gt_labels_3d.flatten(), expected_gt_labels_3d)
assert torch.allclose(points, expected_points, 1e-2)
def test_s3dis_seg_pipeline():
class_names = ('ceiling', 'floor', 'wall', 'beam', 'column', 'window',
'door', 'table', 'chair', 'sofa', 'bookcase', 'board',
'clutter')
np.random.seed(0)
pipelines = [
dict(type='LoadPointsFromFile',
coord_type='DEPTH',
shift_height=False,
use_color=True,
load_dim=6,
use_dim=[0, 1, 2, 3, 4, 5]),
dict(type='LoadAnnotations3D',
with_bbox_3d=False,
with_label_3d=False,
with_mask_3d=False,
with_seg_3d=True),
dict(type='PointSegClassMapping',
valid_cat_ids=tuple(range(len(class_names))),
max_cat_id=13),
dict(type='IndoorPatchPointSample',
num_points=5,
block_size=1.0,
sample_rate=1.0,
ignore_index=len(class_names),
use_normalized_coord=True),
dict(type='NormalizePointsColor', color_mean=None),
dict(type='DefaultFormatBundle3D', class_names=class_names),
dict(type='Collect3D', keys=['points', 'pts_semantic_mask'])
]
pipeline = Compose(pipelines)
info = mmcv.load('./tests/data/s3dis/s3dis_infos.pkl')[0]
results = dict()
data_path = './tests/data/s3dis'
results['pts_filename'] = osp.join(data_path, info['pts_path'])
results['ann_info'] = dict()
results['ann_info']['pts_semantic_mask_path'] = osp.join(
data_path, info['pts_semantic_mask_path'])
results['pts_seg_fields'] = []
results = pipeline(results)
points = results['points']._data
pts_semantic_mask = results['pts_semantic_mask']._data
# build sampled points
s3dis_points = np.fromfile(osp.join(data_path, info['pts_path']),
dtype=np.float32).reshape((-1, 6))
s3dis_choices = np.array([87, 37, 60, 18, 31])
s3dis_center = np.array([2.691, 2.231, 3.172])
s3dis_center[2] = 0.0
s3dis_coord_max = np.amax(s3dis_points[:, :3], axis=0)
expected_points = np.concatenate([
s3dis_points[s3dis_choices, :3] - s3dis_center,
s3dis_points[s3dis_choices, 3:] / 255.,
s3dis_points[s3dis_choices, :3] / s3dis_coord_max
],
axis=1)
expected_pts_semantic_mask = np.array([0, 1, 0, 8, 0])
assert np.allclose(points.numpy(), expected_points, atol=1e-6)
assert np.all(pts_semantic_mask.numpy() == expected_pts_semantic_mask)
def test_scannet_pipeline():
class_names = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door',
'window', 'bookshelf', 'picture', 'counter', 'desk',
'curtain', 'refrigerator', 'showercurtrain', 'toilet',
'sink', 'bathtub', 'garbagebin')
np.random.seed(0)
pipelines = [
dict(type='LoadPointsFromFile',
coord_type='DEPTH',
shift_height=True,
load_dim=6,
use_dim=[0, 1, 2]),
dict(type='LoadAnnotations3D',
with_bbox_3d=True,
with_label_3d=True,
with_mask_3d=True,
with_seg_3d=True),
dict(type='IndoorPointSample', num_points=5),
dict(type='RandomFlip3D',
sync_2d=False,
flip_ratio_bev_horizontal=1.0,
flip_ratio_bev_vertical=1.0),
dict(type='GlobalRotScaleTrans',
rot_range=[-0.087266, 0.087266],
scale_ratio_range=[1.0, 1.0],
shift_height=True),
dict(type='DefaultFormatBundle3D', class_names=class_names),
dict(type='Collect3D',
keys=[
'points', 'gt_bboxes_3d', 'gt_labels_3d', 'pts_semantic_mask',
'pts_instance_mask'
]),
]
pipeline = Compose(pipelines)
info = mmcv.load('./tests/data/scannet/scannet_infos.pkl')[0]
results = dict()
data_path = './tests/data/scannet'
results['pts_filename'] = osp.join(data_path, info['pts_path'])
if info['annos']['gt_num'] != 0:
scannet_gt_bboxes_3d = info['annos']['gt_boxes_upright_depth'].astype(
np.float32)
scannet_gt_labels_3d = info['annos']['class'].astype(np.long)
else:
scannet_gt_bboxes_3d = np.zeros((1, 6), dtype=np.float32)
scannet_gt_labels_3d = np.zeros((1, ), dtype=np.long)
results['ann_info'] = dict()
results['ann_info']['pts_instance_mask_path'] = osp.join(
data_path, info['pts_instance_mask_path'])
results['ann_info']['pts_semantic_mask_path'] = osp.join(
data_path, info['pts_semantic_mask_path'])
results['ann_info']['gt_bboxes_3d'] = DepthInstance3DBoxes(
scannet_gt_bboxes_3d, box_dim=6, with_yaw=False)
results['ann_info']['gt_labels_3d'] = scannet_gt_labels_3d
results['img_fields'] = []
results['bbox3d_fields'] = []
results['pts_mask_fields'] = []
results['pts_seg_fields'] = []
results = pipeline(results)
points = results['points']._data
gt_bboxes_3d = results['gt_bboxes_3d']._data
gt_labels_3d = results['gt_labels_3d']._data
pts_semantic_mask = results['pts_semantic_mask']._data
pts_instance_mask = results['pts_instance_mask']._data
expected_points = torch.tensor([[-2.7231, -2.2068, 2.3543, 2.3895],
[-0.4065, -3.4857, 2.1330, 2.1682],
[-1.4578, 1.3510, -0.0441, -0.0089],
[2.2428, -1.1323, -0.0288, 0.0064],
[0.7052, -2.9752, 1.5560, 1.5912]])
expected_gt_bboxes_3d = torch.tensor(
[[-1.1835, -3.6317, 1.8565, 1.7577, 0.3761, 0.5724, 0.0000],
[-3.1832, 3.2269, 1.5268, 0.6727, 0.2251, 0.6715, 0.0000],
[-0.9598, -2.2864, 0.6165, 0.7506, 2.5709, 1.2145, 0.0000],
[-2.6988, -2.7354, 0.9722, 0.7680, 1.8877, 0.2870, 0.0000],
[3.2989, 0.2885, 1.0712, 0.7600, 3.8814, 2.1603, 0.0000]])
expected_gt_labels_3d = np.array([
6, 6, 4, 9, 11, 11, 10, 0, 15, 17, 17, 17, 3, 12, 4, 4, 14, 1, 0, 0, 0,
0, 0, 0, 5, 5, 5
])
expected_pts_semantic_mask = np.array([3, 1, 2, 2, 15])
expected_pts_instance_mask = np.array([44, 22, 10, 10, 57])
assert torch.allclose(points, expected_points, 1e-2)
assert torch.allclose(gt_bboxes_3d.tensor[:5, :], expected_gt_bboxes_3d,
1e-2)
assert np.all(gt_labels_3d.numpy() == expected_gt_labels_3d)
assert np.all(pts_semantic_mask.numpy() == expected_pts_semantic_mask)
assert np.all(pts_instance_mask.numpy() == expected_pts_instance_mask)
def test_config_data_pipeline():
"""Test whether the data pipeline is valid and can process corner cases.
CommandLine:
xdoctest -m tests/test_config.py test_config_build_data_pipeline
"""
import numpy as np
from mmcv import Config
from mmdet3d.datasets.pipelines import Compose
config_dpath = _get_config_directory()
print('Found config_dpath = {!r}'.format(config_dpath))
# Only tests a representative subset of configurations
config_names = [
'mvxnet/faster_rcnn_r50_fpn_caffe_2x8_1x_nus.py',
'mvxnet/'
'faster_rcnn_r50_fpn_caffe_1x_kitti-2d-3class_coco-3x-pretrain.py',
]
def dummy_masks(h, w, num_obj=3, mode='bitmap'):
assert mode in ('polygon', 'bitmap')
if mode == 'bitmap':
masks = np.random.randint(0, 2, (num_obj, h, w), dtype=np.uint8)
masks = BitmapMasks(masks, h, w)
else:
masks = []
for i in range(num_obj):
masks.append([])
masks[-1].append(
np.random.uniform(0, min(h - 1, w - 1), (8 + 4 * i, )))
masks[-1].append(
np.random.uniform(0, min(h - 1, w - 1), (10 + 4 * i, )))
masks = PolygonMasks(masks, h, w)
return masks
print('Using {} config files'.format(len(config_names)))
for config_fname in config_names:
config_fpath = join(config_dpath, config_fname)
config_mod = Config.fromfile(config_fpath)
# remove loading pipeline
loading_pipeline = config_mod.train_pipeline.pop(0)
loading_ann_pipeline = config_mod.train_pipeline.pop(0)
config_mod.test_pipeline.pop(0)
train_pipeline = Compose(config_mod.train_pipeline)
test_pipeline = Compose(config_mod.test_pipeline)
print(
'Building data pipeline, config_fpath = {!r}'.format(config_fpath))
print('Test training data pipeline: \n{!r}'.format(train_pipeline))
img = np.random.randint(0, 255, size=(888, 666, 3), dtype=np.uint8)
if loading_pipeline.get('to_float32', False):
img = img.astype(np.float32)
mode = 'bitmap' if loading_ann_pipeline.get('poly2mask',
True) else 'polygon'
results = dict(
filename='test_img.png',
ori_filename='test_img.png',
img=img,
img_shape=img.shape,
ori_shape=img.shape,
gt_bboxes=np.array([[35.2, 11.7, 39.7, 15.7]], dtype=np.float32),
gt_labels=np.array([1], dtype=np.int64),
gt_masks=dummy_masks(img.shape[0], img.shape[1], mode=mode),
)
results['img_fields'] = ['img']
results['bbox_fields'] = ['gt_bboxes']
results['mask_fields'] = ['gt_masks']
output_results = train_pipeline(results)
assert output_results is not None
print('Test testing data pipeline: \n{!r}'.format(test_pipeline))
results = dict(
filename='test_img.png',
ori_filename='test_img.png',
img=img,
img_shape=img.shape,
ori_shape=img.shape,
gt_bboxes=np.array([[35.2, 11.7, 39.7, 15.7]], dtype=np.float32),
gt_labels=np.array([1], dtype=np.int64),
gt_masks=dummy_masks(img.shape[0], img.shape[1], mode=mode),
)
results['img_fields'] = ['img']
results['bbox_fields'] = ['gt_bboxes']
results['mask_fields'] = ['gt_masks']
output_results = test_pipeline(results)
assert output_results is not None
# test empty GT
print('Test empty GT with training data pipeline: \n{!r}'.format(
train_pipeline))
results = dict(
filename='test_img.png',
ori_filename='test_img.png',
img=img,
img_shape=img.shape,
ori_shape=img.shape,
gt_bboxes=np.zeros((0, 4), dtype=np.float32),
gt_labels=np.array([], dtype=np.int64),
gt_masks=dummy_masks(img.shape[0],
img.shape[1],
num_obj=0,
mode=mode),
)
results['img_fields'] = ['img']
results['bbox_fields'] = ['gt_bboxes']
results['mask_fields'] = ['gt_masks']
output_results = train_pipeline(results)
assert output_results is not None
print('Test empty GT with testing data pipeline: \n{!r}'.format(
test_pipeline))
results = dict(
filename='test_img.png',
ori_filename='test_img.png',
img=img,
img_shape=img.shape,
ori_shape=img.shape,
gt_bboxes=np.zeros((0, 4), dtype=np.float32),
gt_labels=np.array([], dtype=np.int64),
gt_masks=dummy_masks(img.shape[0],
img.shape[1],
num_obj=0,
mode=mode),
)
results['img_fields'] = ['img']
results['bbox_fields'] = ['gt_bboxes']
results['mask_fields'] = ['gt_masks']
output_results = test_pipeline(results)
assert output_results is not None