def __init__(self,
nusc: NuScenes,
config: DetectionConfig,
result_path: str,
eval_set: str,
output_dir: str = None,
verbose: bool = True):
"""
Initialize a DetectionEval object.
:param nusc: A NuScenes object.
:param config: A DetectionConfig object.
:param result_path: Path of the nuScenes JSON result file.
:param eval_set: The dataset split to evaluate on, e.g. train, val or test.
:param output_dir: Folder to save plots and results to.
:param verbose: Whether to print to stdout.
"""
self.nusc = nusc
self.result_path = result_path
self.eval_set = eval_set
self.output_dir = output_dir
self.verbose = verbose
self.cfg = config
# Check result file exists.
assert os.path.exists(result_path), 'Error: The result file does not exist!'
# Make dirs.
self.plot_dir = os.path.join(self.output_dir, 'plots')
if not os.path.isdir(self.output_dir):
os.makedirs(self.output_dir)
if not os.path.isdir(self.plot_dir):
os.makedirs(self.plot_dir)
# Load data.
if verbose:
print('Initializing nuScenes detection evaluation')
self.pred_boxes, self.meta = load_prediction(self.result_path, self.cfg.max_boxes_per_sample, DetectionBox,
verbose=verbose)
self.gt_boxes = load_gt(self.nusc, self.eval_set, DetectionBox, verbose=verbose)
assert set(self.pred_boxes.sample_tokens) == set(self.gt_boxes.sample_tokens), \
"Samples in split doesn't match samples in predictions."
# Add center distances.
self.pred_boxes = add_center_dist(nusc, self.pred_boxes)
self.gt_boxes = add_center_dist(nusc, self.gt_boxes)
# Filter boxes (distance, points per box, etc.).
if verbose:
print('Filtering predictions')
self.pred_boxes = filter_eval_boxes(nusc, self.pred_boxes, self.cfg.class_range, verbose=verbose)
if verbose:
print('Filtering ground truth annotations')
self.gt_boxes = filter_eval_boxes(nusc, self.gt_boxes, self.cfg.class_range, verbose=verbose)
self.sample_tokens = self.gt_boxes.sample_tokens
def generate_perfect(version, eval_set, dataroot='/data/nuscenes'):
"""Generate perfect detections.
"""
nusc = NuScenes(version='v1.0-{}'.format(version),
dataroot=dataroot,
verbose=True)
cfg = config_factory('detection_cvpr_2019')
gt_boxes = load_gt(nusc, eval_set, DetectionBox, verbose=True)
gt_boxes = add_center_dist(nusc, gt_boxes)
gt_boxes = filter_eval_boxes(nusc, gt_boxes, cfg.class_range, verbose=True)
submission = synthetic_noise_trunk(gt_boxes)
outname = f'perfect_{version}_{eval_set}.json'
print('saving', outname)
with open(outname, 'w') as writer:
json.dump(submission, writer)
def generate_drop_noise(version, eval_set, drop_p, dataroot='/data/nuscenes'):
"""Generate pseudo submissions where every box is dropped with
probability p.
"""
nusc = NuScenes(version='v1.0-{}'.format(version),
dataroot=dataroot,
verbose=True)
cfg = config_factory('detection_cvpr_2019')
gt_boxes = load_gt(nusc, eval_set, DetectionBox, verbose=True)
gt_boxes = add_center_dist(nusc, gt_boxes)
gt_boxes = filter_eval_boxes(nusc, gt_boxes, cfg.class_range, verbose=True)
for drop_p in [drop_p]:
submission = synthetic_noise_trunk(gt_boxes, drop_p=drop_p)
outname = f'perfect_{version}_{eval_set}_{drop_p}.json'
print('saving', outname)
with open(outname, 'w') as writer:
json.dump(submission, writer)
def test_filter_eval_boxes(self):
"""
This tests runs the evaluation for an arbitrary random set of predictions.
This score is then captured in this very test such that if we change the eval code,
this test will trigger if the results changed.
"""
# Get the maximum distance from the config
cfg = config_factory('detection_cvpr_2019')
max_dist = cfg.class_range
assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'
nusc = NuScenes(version='v1.0-mini',
dataroot=os.environ['NUSCENES'],
verbose=False)
sample_token = '0af0feb5b1394b928dd13d648de898f5'
# This sample has a bike rack instance 'bfe685042aa34ab7b2b2f24ee0f1645f' with these parameters
# 'translation': [683.681, 1592.002, 0.809],
# 'size': [1.641, 14.465, 1.4],
# 'rotation': [0.3473693995546558, 0.0, 0.0, 0.9377283723195315]
# Test bicycle filtering by creating a box at the same position as the bike rack.
box1 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
0) # box1 should be filtered.
# Test motorcycle filtering by creating a box at the same position as the bike rack.
box2 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='motorcycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
0) # both box1 and box2 should be filtered.
# Now create a car at the same position as the bike rack.
box3 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
1) # box1 and box2 to be filtered. box3 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
# Now add a bike outside the bike rack.
box4 = DetectionBox(sample_token=sample_token,
translation=(68.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2 to be filtered. box3, box4 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Add another bike on the bike rack center,
# but set the ego_dist (derived from ego_translation) higher than what's defined in max_dist
box5 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
ego_translation=(100.0, 0.0, 0.0))
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4, box5])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2, box5 filtered. box3, box4 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Add another bike on the bike rack center but set the num_pts to be zero so that it gets filtered.
box6 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=0)
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token,
[box1, box2, box3, box4, box5, box6])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2, box5, box6 filtered. box3, box4 stay
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Check for a sample where there are no bike racks. Everything should be filtered correctly.
sample_token = 'ca9a282c9e77460f8360f564131a8af5' # This sample has no bike-racks.
box1 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
ego_translation=(25.0, 0.0, 0.0))
box2 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='motorcycle',
ego_translation=(45.0, 0.0, 0.0))
box3 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car',
ego_translation=(45.0, 0.0, 0.0))
box4 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car',
ego_translation=(55.0, 0.0, 0.0))
box5 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=1)
box6 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=0)
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token,
[box1, box2, box3, box4, box5, box6])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
3) # box2, box4, box6 filtered. box1, box3, box5 stay
self.assertEqual(filtered_boxes.boxes[sample_token][0].ego_dist, 25.0)
self.assertEqual(filtered_boxes.boxes[sample_token][1].ego_dist, 45.0)
self.assertEqual(filtered_boxes.boxes[sample_token][2].num_pts, 1)
def __init__(self,
config: TrackingConfig,
result_path: str,
eval_set: str,
output_dir: str,
nusc_version: str,
nusc_dataroot: str,
verbose: bool = True,
render_classes: List[str] = None):
"""
Initialize a TrackingEval object.
:param config: A TrackingConfig object.
:param result_path: Path of the nuScenes JSON result file.
:param eval_set: The dataset split to evaluate on, e.g. train, val or test.
:param output_dir: Folder to save plots and results to.
:param nusc_version: The version of the NuScenes dataset.
:param nusc_dataroot: Path of the nuScenes dataset on disk.
:param verbose: Whether to print to stdout.
:param render_classes: Classes to render to disk or None.
"""
self.cfg = config
self.result_path = result_path
self.eval_set = eval_set
self.output_dir = output_dir
self.verbose = verbose
self.render_classes = render_classes
# Check result file exists.
assert os.path.exists(
result_path), 'Error: The result file does not exist!'
# Make dirs.
self.plot_dir = os.path.join(self.output_dir, 'plots')
if not os.path.isdir(self.output_dir):
os.makedirs(self.output_dir)
if not os.path.isdir(self.plot_dir):
os.makedirs(self.plot_dir)
# Initialize NuScenes object.
# We do not store it in self to let garbage collection take care of it and save memory.
nusc = NuScenes(version=nusc_version,
verbose=verbose,
dataroot=nusc_dataroot)
self.nusc = nusc
# Load data.
if verbose:
print('Initializing nuScenes tracking evaluation')
pred_boxes, self.meta = load_prediction(self.result_path,
self.cfg.max_boxes_per_sample,
TrackingBox,
verbose=verbose)
gt_boxes = load_gt(nusc, self.eval_set, TrackingBox, verbose=verbose)
assert set(pred_boxes.sample_tokens) == set(gt_boxes.sample_tokens), \
"Samples in split don't match samples in predicted tracks."
# Add center distances.
pred_boxes = add_center_dist(nusc, pred_boxes)
gt_boxes = add_center_dist(nusc, gt_boxes)
# Filter boxes (distance, points per box, etc.).
if verbose:
print('Filtering tracks')
pred_boxes = filter_eval_boxes(nusc,
pred_boxes,
self.cfg.class_range,
verbose=verbose)
if verbose:
print('Filtering ground truth tracks')
gt_boxes = filter_eval_boxes(nusc,
gt_boxes,
self.cfg.class_range,
verbose=verbose)
self.sample_tokens = gt_boxes.sample_tokens #len():6019
# Convert boxes to tracks format.
self.tracks_gt = create_tracks(gt_boxes, nusc, self.eval_set, gt=True)
self.tracks_pred = create_tracks(pred_boxes,
nusc,
self.eval_set,
gt=False)
