def test_OffRoadRate(self):
with patch.object(metrics.OffRoadRate, 'load_drivable_area_masks'):
helper = MagicMock(spec=PredictHelper)
off_road_rate = metrics.OffRoadRate(helper, [metrics.RowMean()])
self.assertDictEqual(off_road_rate.serialize(), {
'name': 'OffRoadRate',
'aggregators': [{
'name': 'RowMean'
}]
})
def test_MinFDEK(self):
min_fde = metrics.MinFDEK([1, 5, 10], [metrics.RowMean()])
self.assertDictEqual(
min_fde.serialize(), {
'name': 'MinFDEK',
'k_to_report': [1, 5, 10],
'aggregators': [{
'name': 'RowMean'
}]
})
def test_MissRateTopK(self):
hit_rate = metrics.MissRateTopK([1, 5, 10], [metrics.RowMean()], 2)
self.assertDictEqual(
hit_rate.serialize(), {
'k_to_report': [1, 5, 10],
'name': 'MissRateTopK',
'aggregators': [{
'name': 'RowMean'
}],
'tolerance': 2
})
def test_flatten_metrics(self):
results = {
"MinFDEK": {
"RowMean": [5.92, 6.1, 7.2]
},
"MinADEK": {
"RowMean": [2.48, 3.29, 3.79]
},
"MissRateTopK_2": {
"RowMean": [0.37, 0.45, 0.55]
}
}
metric_functions = [
metrics.MinFDEK([1, 5, 10], aggregators=[metrics.RowMean()]),
metrics.MinADEK([1, 5, 10], aggregators=[metrics.RowMean()]),
metrics.MissRateTopK([1, 5, 10],
tolerance=2,
aggregators=[metrics.RowMean()])
]
flattened = metrics.flatten_metrics(results, metric_functions)
answer = {
'MinFDEK_1': 5.92,
'MinFDEK_5': 6.1,
'MinFDEK_10': 7.2,
'MinADEK_1': 2.48,
'MinADEK_5': 3.29,
'MinADEK_10': 3.79,
'MissRateTopK_2_1': 0.37,
'MissRateTopK_2_5': 0.45,
'MissRateTopK_2_10': 0.55
}
self.assertDictEqual(flattened, answer)
def _do_test(self, map_name, predictions, answer):
with patch.object(PredictHelper,
'get_map_name_from_sample_token') as get_map_name:
get_map_name.return_value = map_name
nusc = NuScenes('v1.0-mini', dataroot=os.environ['NUSCENES'])
helper = PredictHelper(nusc)
off_road_rate = metrics.OffRoadRate(helper, [metrics.RowMean()])
probabilities = np.array([1 / 3] * predictions.shape[0])
prediction = Prediction('foo-instance', 'foo-sample', predictions,
probabilities)
# Two violations out of three trajectories
np.testing.assert_allclose(off_road_rate(np.array([]), prediction),
np.array([answer]))
def test_RowMean(self):
rm = metrics.RowMean()
value = rm(np.arange(20).reshape(2, 10))
self.assertListEqual(list(value), [5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
self.assertDictEqual(rm.serialize(), {'name': 'RowMean'})
