def test_raises_error_when_seconds_negative(self) -> None: mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1', '1b']}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) with self.assertRaises(ValueError): helper.get_future_for_agent('1', '1', -1, False) with self.assertRaises(ValueError): helper.get_past_for_agent('1', '1', -1, False) with self.assertRaises(ValueError): helper.get_past_for_sample('1', -1, False) with self.assertRaises(ValueError): helper.get_future_for_sample('1', -1, False)
def test_get_future_for_last_returns_nothing(self) -> None: mock_samples = [{'token': '6', 'timestamp': 0}] # Testing we get nothing if we're at the last annotation nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '6', 3, False) np.testing.assert_equal(future, np.array([]))
def test_get_no_data_when_seconds_0(self): mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1']}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) np.testing.assert_equal(helper.get_future_for_agent('1', '1', 0, False), np.array([])) np.testing.assert_equal(helper.get_past_for_agent('1', '1', 0, False), np.array([])) np.testing.assert_equal(helper.get_future_for_sample('1', 0, False), np.array([])) np.testing.assert_equal(helper.get_past_for_sample('1', 0, False), np.array([]))
def test_get_future_for_agent_no_data_to_get(self,): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 3.5e6}] # Testing we get nothing if the first sample annotation is past our threshold nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([]))
def test_get_future_for_agent_in_agent_frame(self): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 1e6}, {'token': '3', 'timestamp': 2e6}, {'token': '4', 'timestamp': 3e6}, {'token': '5', 'timestamp': 4e6}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, True) np.testing.assert_allclose(future, np.array([[-1, 1], [-2, 2], [-3, 3]]))
def test_get_future_for_agent_within_buffer(self,): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 1e6}, {'token': '3', 'timestamp': 2.6e6}, {'token': '4', 'timestamp': 3.05e6}, {'token': '5', 'timestamp': 3.5e6}] # Testing we get data if it is after future seconds but within buffer nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))
def test_get_future_for_agent_less_amount(self,): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 1e6}, {'token': '3', 'timestamp': 2.6e6}, {'token': '4', 'timestamp': 4e6}, {'token': '5', 'timestamp': 5.5e6}] # Testing we do not include data after the future seconds nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([[1, 1], [2, 2]]))
def test_get_future_for_agent_exact_amount(self,): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 1e6}, {'token': '3', 'timestamp': 2e6}, {'token': '4', 'timestamp': 3e6}, {'token': '5', 'timestamp': 4e6}] # Testing we can get the exact amount of future seconds available nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))
def compute_metrics(predictions: List[Dict[str, Any]], helper: PredictHelper, config: PredictionConfig) -> Dict[str, Any]: """ Computes metrics from a set of predictions. :param predictions: List of prediction JSON objects. :param helper: Instance of PredictHelper that wraps the nuScenes val set. :param config: Config file. :return: Metrics. Nested dictionary where keys are metric names and value is a dictionary mapping the Aggregator name to the results. """ n_preds = len(predictions) containers = {metric.name: np.zeros((n_preds, metric.shape)) for metric in config.metrics} for i, prediction_str in enumerate(predictions): prediction = Prediction.deserialize(prediction_str) ground_truth = helper.get_future_for_agent(prediction.instance, prediction.sample, config.seconds, in_agent_frame=False) for metric in config.metrics: containers[metric.name][i] = metric(ground_truth, prediction) aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict) for metric in config.metrics: for agg in metric.aggregators: aggregations[metric.name][agg.name] = agg(containers[metric.name]) return aggregations