def test_raises_error_when_seconds_negative(self) -> None:
mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1', '1b']}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
with self.assertRaises(ValueError):
helper.get_future_for_agent('1', '1', -1, False)
with self.assertRaises(ValueError):
helper.get_past_for_agent('1', '1', -1, False)
with self.assertRaises(ValueError):
helper.get_past_for_sample('1', -1, False)
with self.assertRaises(ValueError):
helper.get_future_for_sample('1', -1, False)
def test_get_future_for_last_returns_nothing(self) -> None:
mock_samples = [{'token': '6', 'timestamp': 0}]
# Testing we get nothing if we're at the last annotation
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '6', 3, False)
np.testing.assert_equal(future, np.array([]))
def test_get_no_data_when_seconds_0(self):
mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1']}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
np.testing.assert_equal(helper.get_future_for_agent('1', '1', 0, False), np.array([]))
np.testing.assert_equal(helper.get_past_for_agent('1', '1', 0, False), np.array([]))
np.testing.assert_equal(helper.get_future_for_sample('1', 0, False), np.array([]))
np.testing.assert_equal(helper.get_past_for_sample('1', 0, False), np.array([]))
def test_get_future_for_agent_no_data_to_get(self,):
mock_samples = [{'token': '1', 'timestamp': 0},
{'token': '2', 'timestamp': 3.5e6}]
# Testing we get nothing if the first sample annotation is past our threshold
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, False)
np.testing.assert_equal(future, np.array([]))
def test_get_future_for_agent_in_agent_frame(self):
mock_samples = [{'token': '1', 'timestamp': 0},
{'token': '2', 'timestamp': 1e6},
{'token': '3', 'timestamp': 2e6},
{'token': '4', 'timestamp': 3e6},
{'token': '5', 'timestamp': 4e6}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, True)
np.testing.assert_allclose(future, np.array([[-1, 1], [-2, 2], [-3, 3]]))
def test_get_future_for_agent_within_buffer(self,):
mock_samples = [{'token': '1', 'timestamp': 0},
{'token': '2', 'timestamp': 1e6},
{'token': '3', 'timestamp': 2.6e6},
{'token': '4', 'timestamp': 3.05e6},
{'token': '5', 'timestamp': 3.5e6}]
# Testing we get data if it is after future seconds but within buffer
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, False)
np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))
def test_get_future_for_agent_less_amount(self,):
mock_samples = [{'token': '1', 'timestamp': 0},
{'token': '2', 'timestamp': 1e6},
{'token': '3', 'timestamp': 2.6e6},
{'token': '4', 'timestamp': 4e6},
{'token': '5', 'timestamp': 5.5e6}]
# Testing we do not include data after the future seconds
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, False)
np.testing.assert_equal(future, np.array([[1, 1], [2, 2]]))
def test_get_future_for_agent_exact_amount(self,):
mock_samples = [{'token': '1', 'timestamp': 0},
{'token': '2', 'timestamp': 1e6},
{'token': '3', 'timestamp': 2e6},
{'token': '4', 'timestamp': 3e6},
{'token': '5', 'timestamp': 4e6}]
# Testing we can get the exact amount of future seconds available
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, False)
np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))
def compute_metrics(predictions: List[Dict[str, Any]],
helper: PredictHelper, config: PredictionConfig) -> Dict[str, Any]:
"""
Computes metrics from a set of predictions.
:param predictions: List of prediction JSON objects.
:param helper: Instance of PredictHelper that wraps the nuScenes val set.
:param config: Config file.
:return: Metrics. Nested dictionary where keys are metric names and value is a dictionary
mapping the Aggregator name to the results.
"""
n_preds = len(predictions)
containers = {metric.name: np.zeros((n_preds, metric.shape)) for metric in config.metrics}
for i, prediction_str in enumerate(predictions):
prediction = Prediction.deserialize(prediction_str)
ground_truth = helper.get_future_for_agent(prediction.instance, prediction.sample,
config.seconds, in_agent_frame=False)
for metric in config.metrics:
containers[metric.name][i] = metric(ground_truth, prediction)
aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict)
for metric in config.metrics:
for agg in metric.aggregators:
aggregations[metric.name][agg.name] = agg(containers[metric.name])
return aggregations
