def test_acceleration_nan_not_enough_data(self): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 0.5e6}, {'token': '3', 'timestamp': 1e6}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertTrue(np.isnan(helper.get_acceleration_for_agent('1', '2')))
def _kinematics_from_tokens(helper: PredictHelper, instance: str, sample: str) -> KinematicsData: """ Returns the 2D position, velocity and acceleration vectors from the given track records, along with the speed, yaw rate, (scalar) acceleration (magnitude), and heading. :param helper: Instance of PredictHelper. :instance: Token of instance. :sample: Token of sample. :return: KinematicsData. """ annotation = helper.get_sample_annotation(instance, sample) x, y, _ = annotation['translation'] yaw = quaternion_yaw(Quaternion(annotation['rotation'])) velocity = helper.get_velocity_for_agent(instance, sample) acceleration = helper.get_acceleration_for_agent(instance, sample) yaw_rate = helper.get_heading_change_rate_for_agent(instance, sample) if np.isnan(velocity): velocity = 0.0 if np.isnan(acceleration): acceleration = 0.0 if np.isnan(yaw_rate): yaw_rate = 0.0 hx, hy = np.cos(yaw), np.sin(yaw) vx, vy = velocity * hx, velocity * hy ax, ay = acceleration * hx, acceleration * hy return x, y, vx, vy, ax, ay, velocity, yaw_rate, acceleration, yaw
def test_get_past_for_agent_in_frame(self) -> None: mock_samples = [{ 'token': '5', 'timestamp': 0 }, { 'token': '4', 'timestamp': -1e6 }, { 'token': '3', 'timestamp': -2e6 }, { 'token': '2', 'timestamp': -3e6 }, { 'token': '1', 'timestamp': -4e6 }] # Testing we can get the exact amount of past seconds available nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, True) np.testing.assert_allclose(past, np.array([[1., -1.], [2., -2.], [3., -3.]]))
def test_get_future_for_sample(self): mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1', '1b']}, {'token': '2', 'timestamp': 1e6}, {'token': '3', 'timestamp': 2e6}, {'token': '4', 'timestamp': 3e6}, {'token': '5', 'timestamp': 4e6}] nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_sample("1", 3, False) answer = {'1': np.array([[1, 1], [2, 2], [3, 3]]), '2': np.array([[7, 7], [8, 8], [9, 9]])} for k in answer: np.testing.assert_equal(answer[k], future[k]) future_in_sample = helper.get_future_for_sample("1", 3, True) answer_in_sample = {'1': np.array([[-1, 1], [-2, 2], [-3, 3]]), '2': np.array([[-1, 1], [-2, 2], [-3, 3]])} for k in answer_in_sample: np.testing.assert_allclose(answer_in_sample[k], future_in_sample[k])
def test_acceleration_zero(self): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 0.5e6}, {'token': '3', 'timestamp': 1e6}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertEqual(helper.get_acceleration_for_agent('1', '3'), 0)
def test_velocity_return_nan_one_obs(self) -> None: mock_samples = [{'token': '1', 'timestamp': 0}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertTrue(np.isnan(helper.get_velocity_for_agent('1', '1')))
def test_get_past_for_sample(self) -> None: mock_samples = [{ 'token': '5', 'timestamp': 0, 'anns': ['5', '5b'] }, { 'token': '4', 'timestamp': -1e6 }, { 'token': '3', 'timestamp': -2e6 }, { 'token': '2', 'timestamp': -3e6 }, { 'token': '1', 'timestamp': -4e6 }] nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_sample('5', 3, True) answer = { '1': np.array([[1, -1], [2, -2], [3, -3]]), '2': np.array([[1, -1], [2, -2], [3, -3]]) } for k in answer: np.testing.assert_allclose(past[k], answer[k])
def test_get_annotations_for_sample(self) -> None: mock_samples = [{'token': '1', 'timestamp': -4e6, 'anns': ['1', '1b']}] nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples) helper = PredictHelper(nusc) annotations = helper.get_annotations_for_sample('1') answer = [{ 'token': '1', 'instance_token': '1', 'sample_token': '1', 'translation': [0, 0, 0], 'rotation': [1, 0, 0, 0], 'prev': '', 'next': '2' }, { 'token': '1b', 'instance_token': '2', 'sample_token': '1', 'translation': [6, 6, 6], 'rotation': [1, 0, 0, 0], 'prev': '', 'next': '2b' }] self.assertListEqual(annotations, answer)
def test_heading_change_rate_near_pi(self): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 0.5e6}] mock_annotations = copy.copy(self.mock_annotations) mock_annotations[0]['rotation'] = [np.cos((np.pi - 0.05)/2), 0, 0, np.sin((np.pi - 0.05) / 2)] mock_annotations[1]['rotation'] = [np.cos((-np.pi + 0.05)/2), 0, 0, np.sin((-np.pi + 0.05) / 2)] nusc = MockNuScenes(mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertAlmostEqual(helper.get_heading_change_rate_for_agent('1', '2'), 0.2)
def test_get_past_for_last_returns_nothing(self) -> None: mock_samples = [{'token': '1', 'timestamp': 0}] # Testing we get nothing if we're at the last annotation nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '1', 3, False) np.testing.assert_equal(past, np.array([]))
def test_acceleration_nonzero(self): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 0.5e6}, {'token': '3', 'timestamp': 1e6}] mock_annotations = copy.copy(self.mock_annotations) mock_annotations[2]['translation'] = [3, 3, 3] nusc = MockNuScenes(mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertAlmostEqual(helper.get_acceleration_for_agent('1', '3'), 2 * (np.sqrt(32) - np.sqrt(8)))
def test_get_past_for_agent_no_data_to_get(self,): mock_samples = [{'token': '5', 'timestamp': 0}, {'token': '4', 'timestamp': -3.5e6}] # Testing we get nothing if the first sample annotation is past our threshold nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, False) np.testing.assert_equal(past, np.array([]))
def test_velocity(self): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 0.5e6}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertEqual(helper.get_velocity_for_agent("1", "2"), np.sqrt(8))
def test_get_sample_annotation(self,): mock_annotation = {'token': '1', 'instance_token': 'instance_1', 'sample_token': 'sample_1'} mock_sample = {'token': 'sample_1', 'timestamp': 0} nusc = MockNuScenes([mock_annotation], [mock_sample]) helper = PredictHelper(nusc) self.assertDictEqual(mock_annotation, helper.get_sample_annotation('instance_1', 'sample_1'))
def test_get_future_for_agent_in_agent_frame(self): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 1e6}, {'token': '3', 'timestamp': 2e6}, {'token': '4', 'timestamp': 3e6}, {'token': '5', 'timestamp': 4e6}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, True) np.testing.assert_allclose(future, np.array([[-1, 1], [-2, 2], [-3, 3]]))
def test_heading_change_rate(self) -> None: mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 0.5e6 }] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertEqual(helper.get_heading_change_rate_for_agent('1', '2'), np.pi)
def test_get_past_for_agent_within_buffer(self,): mock_samples = [{'token': '5', 'timestamp': 0}, {'token': '4', 'timestamp': -1e6}, {'token': '3', 'timestamp': -3.05e6}, {'token': '2', 'timestamp': -3.2e6}] # Testing we get data if it is after future seconds but within buffer nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, False) np.testing.assert_equal(past, np.array([[3, 3], [2, 2]]))
def test_get_past_for_agent_exact_amount(self,): mock_samples = [{'token': '5', 'timestamp': 0}, {'token': '4', 'timestamp': -1e6}, {'token': '3', 'timestamp': -2e6}, {'token': '2', 'timestamp': -3e6}, {'token': '1', 'timestamp': -4e6}] # Testing we can get the exact amount of past seconds available nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, False) np.testing.assert_equal(past, np.array([[3, 3], [2, 2], [1, 1]]))
def test_get_past_for_agent_less_amount(self,): mock_samples = [{'token': '5', 'timestamp': 0}, {'token': '4', 'timestamp': -1e6}, {'token': '3', 'timestamp': -2.6e6}, {'token': '2', 'timestamp': -4e6}, {'token': '1', 'timestamp': -5.5e6}] # Testing we do not include data after the past seconds nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, False) np.testing.assert_equal(past, np.array([[3, 3], [2, 2]]))
def test_get_future_for_agent_less_amount(self,): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 1e6}, {'token': '3', 'timestamp': 2.6e6}, {'token': '4', 'timestamp': 4e6}, {'token': '5', 'timestamp': 5.5e6}] # Testing we do not include data after the future seconds nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([[1, 1], [2, 2]]))
def test_raises_error_when_seconds_negative(self) -> None: mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1', '1b']}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) with self.assertRaises(ValueError): helper.get_future_for_agent('1', '1', -1, False) with self.assertRaises(ValueError): helper.get_past_for_agent('1', '1', -1, False) with self.assertRaises(ValueError): helper.get_past_for_sample('1', -1, False) with self.assertRaises(ValueError): helper.get_future_for_sample('1', -1, False)
def main(version: str, data_root: str, split_name: str, output_dir: str, config_name: str = 'predict_2020_icra.json') -> None: """ Performs inference for all of the baseline models defined in models.py. :param version: NuScenes data set version. :param data_root: Directory where the NuScenes data is stored. :param split_name: NuScenes data split name, e.g. train, val, mini_train, etc. :param output_dir: Directory where predictions should be stored. :param config_name: Name of config file. """ nusc = NuScenes(version=version, dataroot=data_root) helper = PredictHelper(nusc) dataset = get_prediction_challenge_split(split_name) config = load_prediction_config(helper, config_name) oracle = PhysicsOracle(config.seconds, helper) cv_heading = ConstantVelocityHeading(config.seconds, helper) cv_preds = [] oracle_preds = [] for token in dataset: cv_preds.append(cv_heading(token).serialize()) oracle_preds.append(oracle(token).serialize()) json.dump(cv_preds, open(os.path.join(output_dir, "cv_preds.json"), "w")) json.dump(oracle_preds, open(os.path.join(output_dir, "oracle_preds.json"), "w"))
def test_get_no_data_when_seconds_0(self): mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1']}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) np.testing.assert_equal(helper.get_future_for_agent('1', '1', 0, False), np.array([])) np.testing.assert_equal(helper.get_past_for_agent('1', '1', 0, False), np.array([])) np.testing.assert_equal(helper.get_future_for_sample('1', 0, False), np.array([])) np.testing.assert_equal(helper.get_past_for_sample('1', 0, False), np.array([]))
def test_get_future_for_agent_exact_amount(self) -> None: mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 1e6 }, { 'token': '3', 'timestamp': 2e6 }, { 'token': '4', 'timestamp': 3e6 }, { 'token': '5', 'timestamp': 4e6 }] # Testing we can get the exact amount of future seconds available nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))
def test_get_future_for_agent_within_buffer(self) -> None: mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 1e6 }, { 'token': '3', 'timestamp': 2.6e6 }, { 'token': '4', 'timestamp': 3.05e6 }, { 'token': '5', 'timestamp': 3.5e6 }] # Testing we get data if it is after future seconds but within buffer nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, False) np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))
def main(version: str, data_root: str, submission_path: str, config_name: str = 'predict_2020_icra.json') -> None: """ Computes metrics for a submission stored in submission_path with a given submission_name with the metrics specified by the config_name. :param version: NuScenes data set version. :param data_root: Directory storing NuScenes data. :param submission_path: Directory storing submission. :param config_name: Name of config file. """ predictions = json.load(open(submission_path, "r")) nusc = NuScenes(version=version, dataroot=data_root) helper = PredictHelper(nusc) config = load_prediction_config(helper, config_name) results = compute_metrics(predictions, helper, config) json.dump(results, open(submission_path.replace('.json', '_metrics.json'), "w"), indent=2)
def compute_metrics(predictions: List[Dict[str, Any]], helper: PredictHelper, config: PredictionConfig) -> Dict[str, Any]: """ Computes metrics from a set of predictions. :param predictions: List of prediction JSON objects. :param helper: Instance of PredictHelper that wraps the nuScenes val set. :param config: Config file. :return: Metrics. Nested dictionary where keys are metric names and value is a dictionary mapping the Aggregator name to the results. """ n_preds = len(predictions) containers = {metric.name: np.zeros((n_preds, metric.shape)) for metric in config.metrics} for i, prediction_str in enumerate(predictions): prediction = Prediction.deserialize(prediction_str) ground_truth = helper.get_future_for_agent(prediction.instance, prediction.sample, config.seconds, in_agent_frame=False) for metric in config.metrics: containers[metric.name][i] = metric(ground_truth, prediction) aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict) for metric in config.metrics: for agg in metric.aggregators: aggregations[metric.name][agg.name] = agg(containers[metric.name]) return aggregations
def test_velocity_return_nan_big_diff(self): mock_samples = [{'token': '1', 'timestamp': 0}, {'token': '2', 'timestamp': 2.5e6}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertTrue(np.isnan(helper.get_velocity_for_agent('1', '2')))
'4080c30aa7104d91ad005a50b18f6108_ca9a282c9e77460f8360f564131a8af5', 'c1958768d48640948f6053d04cffd35b_ca9a282c9e77460f8360f564131a8af5', '4005437c730645c2b628dc1da999e06a_39586f9d59004284a7114a68825e8eec', 'a017fe4e9c3d445784aae034b1322006_356d81f38dd9473ba590f39e266f54e5', 'a0049f95375044b8987fbcca8fda1e2b_c923fe08b2ff4e27975d2bf30934383b', '61dd7d03d7ad466d89f901ed64e2c0dd_e0845f5322254dafadbbed75aaa07969', '86ed8530809d4b1b8fbc53808f599339_39586f9d59004284a7114a68825e8eec', '2a80b29c0281435ca4893e158a281ce0_2afb9d32310e4546a71cbe432911eca2', '8ce4fe54af77467d90c840465f69677f_de7593d76648450e947ba0c203dee1b0', 'f4af7fd215ee47aa8b64bac0443d7be8_9ee4020153674b9e9943d395ff8cfdf3' ] tokens = tokens * 32 nusc = NuScenes('v1.0-trainval', dataroot=args.data_root) helper = PredictHelper(nusc) dataset = TestDataset(tokens, helper) dataloader = DataLoader(dataset, batch_size=16, num_workers=16) backbone = ResNetBackbone('resnet18') model = MTP(backbone, NUM_MODES) model = model.to(device) loss_function = MTPLoss(NUM_MODES, 1, 5) current_loss = 10000 optimizer = optim.SGD(model.parameters(), lr=0.1) n_iter = 0