def nusc_eval_kit(
result_path:
str = 'data/demo/vmetrics/submission2.json', # The submission as a JSON file.
groundtruth_path:
str = 'data/demo/vmetrics/gt2.json', # The ground truth as a JSON file.
output_dir:
str = 'data/eval_out/vmetrics/', # Folder to store result metrics.
config_path:
str = '', #'Path to the configuration file. If no path given, the CVPR 2019 configuration will be used.
render_curves: bool = True, # Whether to render PR and TP curves to disk.
verbose: bool = True, # Whether to print to stdout.
is_average_delay=False
): # Whether to calculate average delay metric. Keep it False for now.
if config_path == '':
cfg = config_factory('detection_cvpr_2019')
else:
with open(config_path, 'r') as _f:
cfg = DetectionConfig.deserialize(json.load(_f))
nusc_eval = DetectionEval(config=cfg,
result_path=result_path,
groundtruth_path=groundtruth_path,
output_dir=output_dir,
verbose=verbose,
is_average_delay=is_average_delay)
nusc_eval.main(render_curves=render_curves)
def test_DetectionConfig_serialization(self):
""" Test serialization round trip """
cfg = {
'class_range': {
'car': 1,
'truck': 1,
'bus': 1,
'trailer': 1,
'construction_vehicle': 1,
'pedestrian': 1,
'motorcycle': 1,
'bicycle': 1,
'traffic_cone': 1,
'barrier': 1
},
'dist_fcn': 'distance',
'dist_ths': [0.0, 1.0],
'dist_th_tp': 1.0,
'min_recall': 0.0,
'min_precision': 0.0,
'max_boxes_per_sample': 1,
'mean_ap_weight': 1
}
detect_config = DetectionConfig.deserialize(cfg)
cfg_output = detect_config.serialize()
self.assertEqual(cfg, cfg_output)
def config_factory(
configuration_name: str
) -> Union[DetectionConfig]: #, TrackingConfig]:
"""
Creates a *Config instance that can be used to initialize a *Eval instance, where * stands for Detection/Tracking.
Note that this only works if the config file is located in the nuscenes/eval/common/configs folder.
:param configuration_name: Name of desired configuration in eval_detection_configs.
:return: *Config instance.
"""
# Check if config exists.
tokens = configuration_name.split('_')
assert len(
tokens
) > 1, 'Error: Configuration name must be have prefix "detection_"!' # or "tracking_"!'
task = tokens[0]
this_dir = os.path.dirname(os.path.abspath(__file__))
cfg_path = os.path.join(this_dir, '..', task, 'configs',
'%s.json' % configuration_name)
assert os.path.exists(
cfg_path), 'Requested unknown configuration {}'.format(
configuration_name)
# Load config file and deserialize it.
with open(cfg_path, 'r') as f:
data = json.load(f)
if task == 'detection':
cfg = DetectionConfig.deserialize(data)
# elif task == 'tracking':
# cfg = TrackingConfig.deserialize(data)
else:
raise Exception('Error: Invalid config file name: %s' %
configuration_name)
return cfg
def test_serialization(self):
""" test that instance serialization protocol works with json encoding """
this_dir = os.path.dirname(os.path.abspath(__file__))
cfg_name = 'cvpr_2019.json'
config_path = os.path.join(this_dir, '..', 'configs', cfg_name)
with open(config_path) as f:
cfg = json.load(f)
detect_cfg = DetectionConfig.deserialize(cfg)
self.assertEqual(cfg, detect_cfg.serialize())
recovered = DetectionConfig.deserialize(
json.loads(json.dumps(detect_cfg.serialize())))
self.assertEqual(detect_cfg, recovered)
def config_factory(configuration_name: str) -> DetectionConfig:
"""
Creates a DetectionConfig instance that can be used to initialize a NuScenesEval instance.
:param configuration_name: Name of desired configuration in eval_detection_configs.
:return: DetectionConfig instance.
"""
assert configuration_name in eval_detection_configs.keys(), \
'Requested unknown configuration {}'.format(configuration_name)
return DetectionConfig.deserialize(
eval_detection_configs[configuration_name])
def config_factory() -> DetectionConfig:
"""
Creates a DetectionConfig instance that can be used to initialize a NuScenesEval instance.
Note that this only works if the config file is located in the nuscenes/eval/detection/configs folder.
:param configuration_name: Name of desired configuration in eval_detection_configs.
:return: DetectionConfig instance.
"""
# Check if config exists.
this_dir = os.path.dirname(os.path.abspath(__file__))
cfg_path = os.path.join(this_dir, 'UDI_2020.json')
assert os.path.exists(cfg_path), \
'Wrong configuration file'
# Load config file and deserialize it.
with open(cfg_path, 'r') as f:
data = json.load(f)
cfg = DetectionConfig.deserialize(data)
return cfg
def test_serialization(self):
""" Test that instance serialization protocol works with json encoding. """
cfg = {
'class_range': {
'car': 1.0,
'truck': 1.0,
'bus': 1.0,
'trailer': 1.0,
'construction_vehicle': 1.0,
'pedestrian': 1.0,
'motorcycle': 1.0,
'bicycle': 1.0,
'traffic_cone': 1.0,
'barrier': 1.0
},
'dist_fcn': 'distance',
'dist_ths': [0.0, 1.0],
'dist_th_tp': 1.0,
'min_recall': 0.0,
'min_precision': 0.0,
'max_boxes_per_sample': 1,
'mean_ap_weight': 1.0
}
detect_config = DetectionConfig.deserialize(cfg)
metrics = DetectionMetrics(cfg=detect_config)
for i, name in enumerate(cfg['class_range'].keys()):
metrics.add_label_ap(name, 1.0, float(i))
for j, tp_name in enumerate(TP_METRICS):
metrics.add_label_tp(name, tp_name, float(j))
serialized = json.dumps(metrics.serialize())
deserialized = DetectionMetrics.deserialize(json.loads(serialized))
self.assertEqual(metrics, deserialized)
def test_delta(self):
"""
This tests runs the evaluation for an arbitrary random set of predictions.
This score is then captured in this very test such that if we change the eval code,
this test will trigger if the results changed.
"""
random.seed(42)
np.random.seed(42)
assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'
nusc = NuScenes(version='v1.0-mini', dataroot=os.environ['NUSCENES'], verbose=False)
with open(self.res_mockup, 'w') as f:
json.dump(self._mock_submission(nusc, 'mini_val'), f, indent=2)
this_dir = os.path.dirname(os.path.abspath(__file__))
cfg_name = 'cvpr_2019.json'
cfg_path = os.path.join(this_dir, '..', 'configs', cfg_name)
with open(cfg_path, 'r') as f:
cfg = DetectionConfig.deserialize(json.load(f))
nusc_eval = NuScenesEval(nusc, cfg, self.res_mockup, eval_set='mini_val', output_dir=self.res_eval_folder,
verbose=False)
metrics, md_list = nusc_eval.evaluate()
# 1. Score = 0.22082865720221012. Measured on the branch "release_v0.2" on March 7 2019.
# 2. Score = 0.2199307290627096. Changed to measure center distance from the ego-vehicle.
# 3. Score = 0.24954451673961747. Changed to 1.0-mini and cleaned up build script.
# 4. Score = 0.20478832626986893. Updated treatment of cones, barriers, and other algo tunings.
# 5. Score = 0.2043569666105005. AP calculation area is changed from >=min_recall to >min_recall.
# 6. Score = 0.20636954644294506. After bike-rack filtering.
# 7. Score = 0.20237925145690996. After TP reversion bug.
# 8. Score = 0.24047129251302665. After bike racks bug.
# 9. Score = 0.24104572227466886. After bug fix in calc_tp. Include the max recall and exclude the min recall.
# 10. Score = 0.19449091580477748. Changed to use v1.0 mini_val split.
self.assertAlmostEqual(metrics.nd_score, 0.19449091580477748)
parser.add_argument('--verbose',
type=int,
default=1,
help='Whether to print to stdout.')
args = parser.parse_args()
result_path_ = os.path.expanduser(args.result_path)
output_dir_ = os.path.expanduser(args.output_dir)
eval_set_ = args.eval_set
dataroot_ = args.dataroot
version_ = args.version
config_path = args.config_path
plot_examples_ = args.plot_examples
render_curves_ = bool(args.render_curves)
verbose_ = bool(args.verbose)
if config_path == '':
cfg_ = config_factory('cvpr_2019')
else:
with open(config_path, 'r') as f:
cfg_ = DetectionConfig.deserialize(json.load(f))
nusc_ = NuScenes(version=version_, verbose=verbose_, dataroot=dataroot_)
nusc_eval = NuScenesEval(nusc_,
config=cfg_,
result_path=result_path_,
eval_set=eval_set_,
output_dir=output_dir_,
verbose=verbose_)
nusc_eval.main(plot_examples=plot_examples_, render_curves=render_curves_)
def test_filter_eval_boxes(self):
"""
This tests runs the evaluation for an arbitrary random set of predictions.
This score is then captured in this very test such that if we change the eval code,
this test will trigger if the results changed.
"""
# Get the maximum distance from the config
this_dir = os.path.dirname(os.path.abspath(__file__))
cfg_name = 'cvpr_2019.json'
cfg_path = os.path.join(this_dir, '..', 'configs', cfg_name)
with open(cfg_path, 'r') as f:
cfg = DetectionConfig.deserialize(json.load(f))
max_dist = cfg.class_range
assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'
nusc = NuScenes(version='v1.0-mini',
dataroot=os.environ['NUSCENES'],
verbose=False)
sample_token = '0af0feb5b1394b928dd13d648de898f5'
# This sample has a bike rack instance 'bfe685042aa34ab7b2b2f24ee0f1645f' with these parameters
# 'translation': [683.681, 1592.002, 0.809],
# 'size': [1.641, 14.465, 1.4],
# 'rotation': [0.3473693995546558, 0.0, 0.0, 0.9377283723195315]
# Test bicycle filtering by creating a box at the same position as the bike rack.
box1 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
0) # box1 should be filtered.
# Test motorcycle filtering by creating a box at the same position as the bike rack.
box2 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='motorcycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
0) # both box1 and box2 should be filtered.
# Now create a car at the same position as the bike rack.
box3 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
1) # box1 and box2 to be filtered. box3 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
# Now add a bike outside the bike rack.
box4 = EvalBox(sample_token=sample_token,
translation=(68.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2 to be filtered. box3, box4 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Add another bike on the bike rack center but set the ego_dist higher than what's defined in max_dist
box5 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
ego_dist=100.0)
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4, box5])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2, box5 filtered. box3, box4 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Add another bike on the bike rack center but set the num_pts to be zero so that it gets filtered.
box6 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=0)
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token,
[box1, box2, box3, box4, box5, box6])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2, box5, box6 filtered. box3, box4 stay
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Check for a sample where there are no bike racks. Everything should be filtered correctly.
sample_token = 'ca9a282c9e77460f8360f564131a8af5' # This sample has no bike-racks.
box1 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
ego_dist=25.0)
box2 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='motorcycle',
ego_dist=45.0)
box3 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car',
ego_dist=45.0)
box4 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car',
ego_dist=55.0)
box5 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=1)
box6 = EvalBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=0)
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token,
[box1, box2, box3, box4, box5, box6])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
3) # box2, box4, box6 filtered. box1, box3, box5 stay
self.assertEqual(filtered_boxes.boxes[sample_token][0].ego_dist, 25.0)
self.assertEqual(filtered_boxes.boxes[sample_token][1].ego_dist, 45.0)
self.assertEqual(filtered_boxes.boxes[sample_token][2].num_pts, 1)
class TestAlgo(unittest.TestCase):
this_dir = os.path.dirname(os.path.abspath(__file__))
cfg_name = 'cvpr_2019.json'
cfg_path = os.path.join(this_dir, '..', 'configs', cfg_name)
with open(cfg_path, 'r') as f:
cfg = DetectionConfig.deserialize(json.load(f))
@staticmethod
def _mock_results(nsamples, ngt, npred, detection_name):
def random_attr():
"""
This is the most straight-forward way to generate a random attribute.
Not currently used b/c we want the test fixture to be back-wards compatible.
"""
# Get relevant attributes.
rel_attributes = detection_name_to_rel_attributes(detection_name)
if len(rel_attributes) == 0:
# Empty string for classes without attributes.
return ''
else:
# Pick a random attribute otherwise.
return rel_attributes[np.random.randint(
0, len(rel_attributes))]
pred = EvalBoxes()
gt = EvalBoxes()
for sample_itt in range(nsamples):
this_gt = []
for box_itt in range(ngt):
this_gt.append(
EvalBox(
sample_token=str(sample_itt),
translation=tuple(
list(np.random.rand(2) * 15) + [0.0]),
size=tuple(np.random.rand(3) * 4),
rotation=tuple(np.random.rand(4)),
velocity=tuple(np.random.rand(3)[:2] * 4),
detection_name=detection_name,
detection_score=random.random(),
attribute_name=random_attr(),
ego_dist=random.random() * 10,
))
gt.add_boxes(str(sample_itt), this_gt)
for sample_itt in range(nsamples):
this_pred = []
for box_itt in range(npred):
this_pred.append(
EvalBox(
sample_token=str(sample_itt),
translation=tuple(
list(np.random.rand(2) * 10) + [0.0]),
size=tuple(np.random.rand(3) * 4),
rotation=tuple(np.random.rand(4)),
velocity=tuple(np.random.rand(3)[:2] * 4),
detection_name=detection_name,
detection_score=random.random(),
attribute_name=random_attr(),
ego_dist=random.random() * 10,
))
pred.add_boxes(str(sample_itt), this_pred)
return gt, pred
def test_nd_score(self):
"""
This tests runs the full evaluation for an arbitrary random set of predictions.
"""
random.seed(42)
np.random.seed(42)
mdl = MetricDataList()
for class_name in self.cfg.class_names:
gt, pred = self._mock_results(30, 3, 25, class_name)
for dist_th in self.cfg.dist_ths:
mdl.set(class_name, dist_th,
accumulate(gt, pred, class_name, 'center_distance', 2))
metrics = DetectionMetrics(self.cfg)
for class_name in self.cfg.class_names:
for dist_th in self.cfg.dist_ths:
ap = calc_ap(mdl[(class_name, dist_th)], self.cfg.min_recall,
self.cfg.min_precision)
metrics.add_label_ap(class_name, dist_th, ap)
for metric_name in TP_METRICS:
metric_data = mdl[(class_name, self.cfg.dist_th_tp)]
if class_name in ['traffic_cone'] and metric_name in [
'attr_err', 'vel_err', 'orient_err'
]:
tp = np.nan
elif class_name in ['barrier'] and metric_name in [
'attr_err', 'vel_err'
]:
tp = np.nan
else:
tp = calc_tp(metric_data, self.cfg.min_recall, metric_name)
metrics.add_label_tp(class_name, metric_name, tp)
self.assertEqual(0.08606662159639042, metrics.nd_score)
def test_calc_tp(self):
"""Test for calc_tp()."""
random.seed(42)
np.random.seed(42)
md = MetricData.random_md()
# min_recall greater than 1.
self.assertEqual(1.0, calc_tp(md,
min_recall=1,
metric_name='trans_err'))
def test_calc_ap(self):
"""Test for calc_ap()."""
random.seed(42)
np.random.seed(42)
md = MetricData.random_md()
# Negative min_recall and min_precision
self.assertRaises(AssertionError, calc_ap, md, -0.5, 0.4)
self.assertRaises(AssertionError, calc_ap, md, 0.5, -0.8)
# More than 1 min_precision/min_recall
self.assertRaises(AssertionError, calc_ap, md, 0.7, 1)
self.assertRaises(AssertionError, calc_ap, md, 1.2, 0)