def test_raises_error_when_seconds_negative(self): mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1', '1b']}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) with self.assertRaises(ValueError): helper.get_future_for_agent('1', '1', -1, False) with self.assertRaises(ValueError): helper.get_past_for_agent('1', '1', -1, False) with self.assertRaises(ValueError): helper.get_past_for_sample('1', -1, False) with self.assertRaises(ValueError): helper.get_future_for_sample('1', -1, False)
def compute_metrics(predictions: List[Dict[str, Any]], helper: PredictHelper, config: PredictionConfig) -> Dict[str, Any]: """ Computes metrics from a set of predictions. :param predictions: List of prediction JSON objects. :param helper: Instance of PredictHelper that wraps the nuScenes val set. :param config: Config file. :return: Metrics. Nested dictionary where keys are metric names and value is a dictionary mapping the Aggregator name to the results. """ n_preds = len(predictions) containers = { metric.name: np.zeros((n_preds, metric.shape)) for metric in config.metrics } for i, prediction_str in enumerate(predictions): prediction = Prediction.deserialize(prediction_str) ground_truth = helper.get_future_for_agent(prediction.instance, prediction.sample, config.seconds, in_agent_frame=False) for metric in config.metrics: containers[metric.name][i] = metric(ground_truth, prediction) aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict) for metric in config.metrics: for agg in metric.aggregators: aggregations[metric.name][agg.name] = agg(containers[metric.name]) return aggregations
def test_get_future_for_last_returns_nothing(self): mock_samples = [{'token': '6', 'timestamp': 0}] # Testing we get nothing if we're at the last annotation nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '6', 3, False) np.testing.assert_equal(future, np.array([]))
def test_get_no_data_when_seconds_0(self): mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1']}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) np.testing.assert_equal( helper.get_future_for_agent('1', '1', 0, False), np.array([])) np.testing.assert_equal(helper.get_past_for_agent('1', '1', 0, False), np.array([])) np.testing.assert_equal(helper.get_future_for_sample('1', 0, False), np.array([])) np.testing.assert_equal(helper.get_past_for_sample('1', 0, False), np.array([]))
def test_get_future_for_agent_no_data_to_get(self, ): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 3.5e6 }] # Testing we get nothing if the first sample annotation is past our threshold nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([]))
def test_get_future_for_agent_in_agent_frame(self): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 1e6 }, { 'token': '3', 'timestamp': 2e6 }, { 'token': '4', 'timestamp': 3e6 }, { 'token': '5', 'timestamp': 4e6 }] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, True) np.testing.assert_allclose(future, np.array([[-1, 1], [-2, 2], [-3, 3]]))
def test_get_future_for_agent_within_buffer(self, ): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 1e6 }, { 'token': '3', 'timestamp': 2.6e6 }, { 'token': '4', 'timestamp': 3.05e6 }, { 'token': '5', 'timestamp': 3.5e6 }] # Testing we get data if it is after future seconds but within buffer nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))
def test_get_future_for_agent_less_amount(self, ): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 1e6 }, { 'token': '3', 'timestamp': 2.6e6 }, { 'token': '4', 'timestamp': 4e6 }, { 'token': '5', 'timestamp': 5.5e6 }] # Testing we do not include data after the future seconds nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([[1, 1], [2, 2]]))
def test_get_future_for_agent_exact_amount(self, ): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 1e6 }, { 'token': '3', 'timestamp': 2e6 }, { 'token': '4', 'timestamp': 3e6 }, { 'token': '5', 'timestamp': 4e6 }] # Testing we can get the exact amount of future seconds available nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))