def basic_test(self,
eval_set: str = 'mini_val',
add_errors: bool = False,
render_curves: bool = False) -> Dict[str, Any]:
"""
Run the evaluation with fixed randomness on the specified subset, with or without introducing errors in the
submission.
:param eval_set: Which split to evaluate on.
:param add_errors: Whether to use GT as submission or introduce additional errors.
:param render_curves: Whether to render stats curves to disk.
:return: The metrics returned by the evaluation.
"""
random.seed(42)
np.random.seed(42)
assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'
if eval_set.startswith('mini'):
version = 'v1.0-mini'
elif eval_set == 'test':
version = 'v1.0-test'
else:
version = 'v1.0-trainval'
nusc = NuScenes(version=version, dataroot=os.environ['NUSCENES'], verbose=False)
with open(self.res_mockup, 'w') as f:
mock = self._mock_submission(nusc, eval_set, add_errors=add_errors)
json.dump(mock, f, indent=2)
cfg = config_factory('tracking_nips_2019')
nusc_eval = TrackingEval(cfg, self.res_mockup, eval_set=eval_set, output_dir=self.res_eval_folder,
nusc_version=version, nusc_dataroot=os.environ['NUSCENES'], verbose=False)
metrics = nusc_eval.main(render_curves=render_curves)
return metrics
def test_gt_submission(self):
""" Test with GT submission. """
# Get config.
cfg = config_factory('tracking_nips_2019')
# Define inputs.
class_name, tracks_gt = TestAlgo.single_scene()
verbose = False
# Remove one prediction.
timestamp_boxes_pred = copy.deepcopy(tracks_gt['scene-1'])
tracks_pred = {'scene-1': timestamp_boxes_pred}
# Accumulate metrics.
ev = TrackingEvaluation(tracks_gt,
tracks_pred,
class_name,
cfg.dist_fcn_callable,
cfg.dist_th_tp,
cfg.min_recall,
num_thresholds=TrackingMetricData.nelem,
metric_worst=cfg.metric_worst,
verbose=verbose)
md = ev.accumulate()
# Check outputs.
assert np.all(md.tp == 4)
assert np.all(md.fn == 0)
assert np.all(md.fp == 0)
assert np.all(md.lgd == 0)
assert np.all(md.tid == 0)
assert np.all(md.frag == 0)
assert np.all(md.ids == 0)
def test_delta(self):
"""
This tests runs the evaluation for an arbitrary random set of predictions.
This score is then captured in this very test such that if we change the eval code,
this test will trigger if the results changed.
"""
random.seed(42)
np.random.seed(42)
assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'
nusc = NuScenes(version='v1.0-mini', dataroot=os.environ['NUSCENES'], verbose=False)
with open(self.res_mockup, 'w') as f:
json.dump(self._mock_submission(nusc, 'mini_val'), f, indent=2)
cfg = config_factory('detection_cvpr_2019')
nusc_eval = DetectionEval(nusc, cfg, self.res_mockup, eval_set='mini_val', output_dir=self.res_eval_folder,
verbose=False)
metrics, md_list = nusc_eval.evaluate()
# 1. Score = 0.22082865720221012. Measured on the branch "release_v0.2" on March 7 2019.
# 2. Score = 0.2199307290627096. Changed to measure center distance from the ego-vehicle.
# 3. Score = 0.24954451673961747. Changed to 1.0-mini and cleaned up build script.
# 4. Score = 0.20478832626986893. Updated treatment of cones, barriers, and other algo tunings.
# 5. Score = 0.2043569666105005. AP calculation area is changed from >=min_recall to >min_recall.
# 6. Score = 0.20636954644294506. After bike-rack filtering.
# 7. Score = 0.20237925145690996. After TP reversion bug.
# 8. Score = 0.24047129251302665. After bike racks bug.
# 9. Score = 0.24104572227466886. After bug fix in calc_tp. Include the max recall and exclude the min recall.
# 10. Score = 0.19449091580477748. Changed to use v1.0 mini_val split.
self.assertAlmostEqual(metrics.nd_score, 0.19449091580477748)
def test_scenarios(self):
""" More flexible scenario test structure. """
def create_tracks(_scenario, tag=None):
tracks = {}
for entry_id, entry in enumerate(_scenario['input']['pos_' + tag]):
tracking_id = 'tag_{}'.format(entry_id)
for timestamp, pos in enumerate(entry):
if timestamp not in tracks.keys():
tracks[timestamp] = []
box = TrackingBox(translation=(pos[0], pos[1], 0.0),
tracking_id=tracking_id,
tracking_name='car',
tracking_score=0.5)
tracks[timestamp].append(box)
return tracks
# Get config.
cfg = config_factory('tracking_nips_2019')
for scenario in get_scenarios():
tracks_gt = {'scene-1': create_tracks(scenario, tag='gt')}
tracks_pred = {'scene-1': create_tracks(scenario, tag='pred')}
# Accumulate metrics.
ev = TrackingEvaluation(tracks_gt,
tracks_pred,
'car',
cfg.dist_fcn_callable,
cfg.dist_th_tp,
cfg.min_recall,
num_thresholds=TrackingMetricData.nelem,
metric_worst=cfg.metric_worst,
verbose=False)
md = ev.accumulate()
for key, value in scenario['output'].items():
metric_values = getattr(md, key)
metric_values = metric_values[np.logical_not(
np.isnan(metric_values))]
assert np.all(metric_values == value)
def test_empty_submission(self):
""" Test a submission with no predictions. """
# Get config.
cfg = config_factory('tracking_nips_2019')
# Define inputs.
class_name, tracks_gt = TestAlgo.single_scene()
verbose = False
# Remove all predictions.
timestamp_boxes_pred = copy.deepcopy(tracks_gt['scene-1'])
for timestamp, box in timestamp_boxes_pred.items():
timestamp_boxes_pred[timestamp] = []
tracks_pred = {'scene-1': timestamp_boxes_pred}
# Accumulate metrics.
ev = TrackingEvaluation(tracks_gt,
tracks_pred,
class_name,
cfg.dist_fcn_callable,
cfg.dist_th_tp,
cfg.min_recall,
num_thresholds=TrackingMetricData.nelem,
metric_worst=cfg.metric_worst,
verbose=verbose)
md = ev.accumulate()
# Check outputs.
assert np.all(md.mota == 0)
assert np.all(md.motar == 0)
assert np.all(np.isnan(md.recall_hypo))
assert np.all(md.tp == 0)
assert np.all(md.fn == 4)
assert np.all(np.isnan(
md.fp)) # FP/Frag/IDS are nan as we there were no predictions.
assert np.all(md.lgd == 20)
assert np.all(md.tid == 20)
assert np.all(np.isnan(md.frag))
assert np.all(np.isnan(md.ids))
def test_drop_prediction_multiple(self):
""" Drop the first three predictions from the GT submission. """
# Get config.
cfg = config_factory('tracking_nips_2019')
# Define inputs.
class_name, tracks_gt = TestAlgo.single_scene()
verbose = False
# Remove one predicted box.
timestamp_boxes_pred = copy.deepcopy(tracks_gt['scene-1'])
timestamp_boxes_pred[0] = []
timestamp_boxes_pred[1] = []
timestamp_boxes_pred[2] = []
tracks_pred = {'scene-1': timestamp_boxes_pred}
# Accumulate metrics.
ev = TrackingEvaluation(tracks_gt,
tracks_pred,
class_name,
cfg.dist_fcn_callable,
cfg.dist_th_tp,
cfg.min_recall,
num_thresholds=TrackingMetricData.nelem,
metric_worst=cfg.metric_worst,
verbose=verbose)
md = ev.accumulate()
# Check outputs.
# Recall values above 0.75 (3/4 correct) are not achieved and therefore nan.
first_achieved = np.where(md.recall_hypo <= 0.25)[0][0]
assert np.all(np.isnan(md.confidence[:first_achieved]))
assert md.tp[first_achieved] == 1
assert md.fp[first_achieved] == 0
assert md.fn[first_achieved] == 3
assert md.lgd[first_achieved] == 3 * 0.5
assert md.tid[first_achieved] == 3 * 0.5
assert md.frag[first_achieved] == 0
assert md.ids[first_achieved] == 0
def test_drop_gt_interpolate(self):
""" Drop one box from the GT and interpolate the results to fill in that box. """
# Get config.
cfg = config_factory('tracking_nips_2019')
# Define inputs.
class_name, tracks_gt = TestAlgo.single_scene()
verbose = False
# Remove one GT box.
timestamp_boxes_pred = copy.deepcopy(tracks_gt['scene-1'])
tracks_gt['scene-1'][1] = []
tracks_pred = {'scene-1': timestamp_boxes_pred}
# Interpolate to "restore" dropped GT.
tracks_gt['scene-1'] = interpolate_tracks(
defaultdict(list, tracks_gt['scene-1']))
# Accumulate metrics.
ev = TrackingEvaluation(tracks_gt,
tracks_pred,
class_name,
cfg.dist_fcn_callable,
cfg.dist_th_tp,
cfg.min_recall,
num_thresholds=TrackingMetricData.nelem,
metric_worst=cfg.metric_worst,
verbose=verbose)
md = ev.accumulate()
# Check outputs.
assert np.all(md.tp == 4)
assert np.all(md.fp == 0)
assert np.all(md.fn == 0)
assert np.all(md.lgd == 0)
assert np.all(md.tid == 0)
assert np.all(md.frag == 0)
assert np.all(md.ids == 0)
def test_identity_switch(self):
""" Change the tracking_id of one frame from the GT submission. """
# Get config.
cfg = config_factory('tracking_nips_2019')
# Define inputs.
class_name, tracks_gt = TestAlgo.single_scene()
verbose = False
# Remove one predicted box.
timestamp_boxes_pred = copy.deepcopy(tracks_gt['scene-1'])
timestamp_boxes_pred[2][0].tracking_id = 'tb'
tracks_pred = {'scene-1': timestamp_boxes_pred}
# Accumulate metrics.
ev = TrackingEvaluation(tracks_gt,
tracks_pred,
class_name,
cfg.dist_fcn_callable,
cfg.dist_th_tp,
cfg.min_recall,
num_thresholds=TrackingMetricData.nelem,
metric_worst=cfg.metric_worst,
verbose=verbose)
md = ev.accumulate()
# Check outputs.
first_achieved = np.where(md.recall_hypo <= 0.5)[0][0]
assert md.tp[first_achieved] == 2
assert md.fp[first_achieved] == 0
assert md.fn[first_achieved] == 0
assert md.lgd[first_achieved] == 0
assert md.tid[first_achieved] == 0
assert md.frag[first_achieved] == 0
assert md.ids[
first_achieved] == 2 # One wrong id leads to 2 identity switches.
class TestAlgo(unittest.TestCase):
cfg = config_factory('detection_cvpr_2019')
@staticmethod
def _mock_results(nsamples, ngt, npred, detection_name):
def random_attr():
"""
This is the most straight-forward way to generate a random attribute.
Not currently used b/c we want the test fixture to be back-wards compatible.
"""
# Get relevant attributes.
rel_attributes = detection_name_to_rel_attributes(detection_name)
if len(rel_attributes) == 0:
# Empty string for classes without attributes.
return ''
else:
# Pick a random attribute otherwise.
return rel_attributes[np.random.randint(0, len(rel_attributes))]
pred = EvalBoxes()
gt = EvalBoxes()
for sample_itt in range(nsamples):
this_gt = []
for box_itt in range(ngt):
translation_xy = tuple(np.random.rand(2) * 15)
this_gt.append(DetectionBox(
sample_token=str(sample_itt),
translation=(translation_xy[0], translation_xy[1], 0.0),
size=tuple(np.random.rand(3)*4),
rotation=tuple(np.random.rand(4)),
velocity=tuple(np.random.rand(3)[:2]*4),
detection_name=detection_name,
detection_score=random.random(),
attribute_name=random_attr(),
ego_translation=(random.random() * 10, 0, 0),
))
gt.add_boxes(str(sample_itt), this_gt)
for sample_itt in range(nsamples):
this_pred = []
for box_itt in range(npred):
translation_xy = tuple(np.random.rand(2) * 10)
this_pred.append(DetectionBox(
sample_token=str(sample_itt),
translation=(translation_xy[0], translation_xy[1], 0.0),
size=tuple(np.random.rand(3) * 4),
rotation=tuple(np.random.rand(4)),
velocity=tuple(np.random.rand(3)[:2] * 4),
detection_name=detection_name,
detection_score=random.random(),
attribute_name=random_attr(),
ego_translation=(random.random() * 10, 0, 0),
))
pred.add_boxes(str(sample_itt), this_pred)
return gt, pred
def test_nd_score(self):
"""
This tests runs the full evaluation for an arbitrary random set of predictions.
"""
random.seed(42)
np.random.seed(42)
mdl = DetectionMetricDataList()
for class_name in self.cfg.class_names:
gt, pred = self._mock_results(30, 3, 25, class_name)
for dist_th in self.cfg.dist_ths:
mdl.set(class_name, dist_th, accumulate(gt, pred, class_name, center_distance, 2))
metrics = DetectionMetrics(self.cfg)
for class_name in self.cfg.class_names:
for dist_th in self.cfg.dist_ths:
ap = calc_ap(mdl[(class_name, dist_th)], self.cfg.min_recall, self.cfg.min_precision)
metrics.add_label_ap(class_name, dist_th, ap)
for metric_name in TP_METRICS:
metric_data = mdl[(class_name, self.cfg.dist_th_tp)]
if class_name in ['traffic_cone'] and metric_name in ['attr_err', 'vel_err', 'orient_err']:
tp = np.nan
elif class_name in ['barrier'] and metric_name in ['attr_err', 'vel_err']:
tp = np.nan
else:
tp = calc_tp(metric_data, self.cfg.min_recall, metric_name)
metrics.add_label_tp(class_name, metric_name, tp)
self.assertEqual(0.08606662159639042, metrics.nd_score)
def test_calc_tp(self):
"""Test for calc_tp()."""
random.seed(42)
np.random.seed(42)
md = DetectionMetricData.random_md()
# min_recall greater than 1.
self.assertEqual(1.0, calc_tp(md, min_recall=1, metric_name='trans_err'))
def test_calc_ap(self):
"""Test for calc_ap()."""
random.seed(42)
np.random.seed(42)
md = DetectionMetricData.random_md()
# Negative min_recall and min_precision
self.assertRaises(AssertionError, calc_ap, md, -0.5, 0.4)
self.assertRaises(AssertionError, calc_ap, md, 0.5, -0.8)
# More than 1 min_precision/min_recall
self.assertRaises(AssertionError, calc_ap, md, 0.7, 1)
self.assertRaises(AssertionError, calc_ap, md, 1.2, 0)
parser.add_argument('--verbose',
type=int,
default=1,
help='Whether to print to stdout.')
args = parser.parse_args()
result_path_ = os.path.expanduser(args.result_path)
output_dir_ = os.path.expanduser(args.output_dir)
eval_set_ = args.eval_set
dataroot_ = args.dataroot
version_ = args.version
config_path = args.config_path
plot_examples_ = args.plot_examples
render_curves_ = bool(args.render_curves)
verbose_ = bool(args.verbose)
if config_path == '':
cfg_ = config_factory('detection_cvpr_2019')
else:
with open(config_path, 'r') as _f:
cfg_ = DetectionConfig.deserialize(json.load(_f))
nusc_ = NuScenes(version=version_, verbose=verbose_, dataroot=dataroot_)
nusc_eval = DetectionEval(nusc_,
config=cfg_,
result_path=result_path_,
eval_set=eval_set_,
output_dir=output_dir_,
verbose=verbose_)
nusc_eval.main(plot_examples=plot_examples_, render_curves=render_curves_)
help='Path to the configuration file.'
'If no path given, the NIPS 2019 configuration will be used.')
parser.add_argument('--render_curves', type=int, default=1,
help='Whether to render statistic curves to disk.')
parser.add_argument('--verbose', type=int, default=1,
help='Whether to print to stdout.')
parser.add_argument('--render_classes', type=str, default='', nargs='+',
help='For which classes we render tracking results to disk.')
args = parser.parse_args()
result_path_ = os.path.expanduser(args.result_path)
output_dir_ = os.path.expanduser(args.output_dir)
eval_set_ = args.eval_set
dataroot_ = args.dataroot
version_ = args.version
config_path = args.config_path
render_curves_ = bool(args.render_curves)
verbose_ = bool(args.verbose)
render_classes_ = args.render_classes
if config_path == '':
cfg_ = config_factory('tracking_nips_2019')
else:
with open(config_path, 'r') as _f:
cfg_ = TrackingConfig.deserialize(json.load(_f))
nusc_eval = TrackingEval(config=cfg_, result_path=result_path_, eval_set=eval_set_, output_dir=output_dir_,
nusc_version=version_, nusc_dataroot=dataroot_, verbose=verbose_,
render_classes=render_classes_)
nusc_eval.main(render_curves=render_curves_)
def test_filter_eval_boxes(self):
"""
This tests runs the evaluation for an arbitrary random set of predictions.
This score is then captured in this very test such that if we change the eval code,
this test will trigger if the results changed.
"""
# Get the maximum distance from the config
cfg = config_factory('detection_cvpr_2019')
max_dist = cfg.class_range
assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'
nusc = NuScenes(version='v1.0-mini',
dataroot=os.environ['NUSCENES'],
verbose=False)
sample_token = '0af0feb5b1394b928dd13d648de898f5'
# This sample has a bike rack instance 'bfe685042aa34ab7b2b2f24ee0f1645f' with these parameters
# 'translation': [683.681, 1592.002, 0.809],
# 'size': [1.641, 14.465, 1.4],
# 'rotation': [0.3473693995546558, 0.0, 0.0, 0.9377283723195315]
# Test bicycle filtering by creating a box at the same position as the bike rack.
box1 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
0) # box1 should be filtered.
# Test motorcycle filtering by creating a box at the same position as the bike rack.
box2 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='motorcycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
0) # both box1 and box2 should be filtered.
# Now create a car at the same position as the bike rack.
box3 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
1) # box1 and box2 to be filtered. box3 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
# Now add a bike outside the bike rack.
box4 = DetectionBox(sample_token=sample_token,
translation=(68.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2 to be filtered. box3, box4 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Add another bike on the bike rack center,
# but set the ego_dist (derived from ego_translation) higher than what's defined in max_dist
box5 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
ego_translation=(100.0, 0.0, 0.0))
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4, box5])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2, box5 filtered. box3, box4 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Add another bike on the bike rack center but set the num_pts to be zero so that it gets filtered.
box6 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=0)
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token,
[box1, box2, box3, box4, box5, box6])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2, box5, box6 filtered. box3, box4 stay
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Check for a sample where there are no bike racks. Everything should be filtered correctly.
sample_token = 'ca9a282c9e77460f8360f564131a8af5' # This sample has no bike-racks.
box1 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
ego_translation=(25.0, 0.0, 0.0))
box2 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='motorcycle',
ego_translation=(45.0, 0.0, 0.0))
box3 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car',
ego_translation=(45.0, 0.0, 0.0))
box4 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car',
ego_translation=(55.0, 0.0, 0.0))
box5 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=1)
box6 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=0)
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token,
[box1, box2, box3, box4, box5, box6])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
3) # box2, box4, box6 filtered. box1, box3, box5 stay
self.assertEqual(filtered_boxes.boxes[sample_token][0].ego_dist, 25.0)
self.assertEqual(filtered_boxes.boxes[sample_token][1].ego_dist, 45.0)
self.assertEqual(filtered_boxes.boxes[sample_token][2].num_pts, 1)