def test_velocity_return_nan_one_obs(self):
mock_samples = [{'token': '1', 'timestamp': 0}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertTrue(np.isnan(helper.get_velocity_for_agent('1', '1')))
def test_get_past_for_sample(self):
mock_samples = [{
'token': '5',
'timestamp': 0,
'anns': ['5', '5b']
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -2e6
}, {
'token': '2',
'timestamp': -3e6
}, {
'token': '1',
'timestamp': -4e6
}]
nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_sample('5', 3, True)
answer = {
'1': np.array([[-1, -1], [-2, -2], [-3, -3]]),
'2': np.array([[-1, -1], [-2, -2], [-3, -3]])
}
for k in answer:
np.testing.assert_equal(answer[k], answer[k])
def test_get_annotations_for_sample(self) -> None:
mock_samples = [{'token': '1', 'timestamp': -4e6, 'anns': ['1', '1b']}]
nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples)
helper = PredictHelper(nusc)
annotations = helper.get_annotations_for_sample('1')
answer = [{
'token': '1',
'instance_token': '1',
'sample_token': '1',
'translation': [0, 0, 0],
'rotation': [1, 0, 0, 0],
'prev': '',
'next': '2'
}, {
'token': '1b',
'instance_token': '2',
'sample_token': '1',
'translation': [6, 6, 6],
'rotation': [1, 0, 0, 0],
'prev': '',
'next': '2b'
}]
self.assertListEqual(annotations, answer)
def _kinematics_from_tokens(helper: PredictHelper, instance: str,
sample: str) -> KinematicsData:
"""
Returns the 2D position, velocity and acceleration vectors from the given track records,
along with the speed, yaw rate, (scalar) acceleration (magnitude), and heading.
:param helper: Instance of PredictHelper.
:instance: Token of instance.
:sample: Token of sample.
:return: KinematicsData.
"""
annotation = helper.get_sample_annotation(instance, sample)
x, y, _ = annotation['translation']
yaw = quaternion_yaw(Quaternion(annotation['rotation']))
velocity = helper.get_velocity_for_agent(instance, sample)
acceleration = helper.get_acceleration_for_agent(instance, sample)
yaw_rate = helper.get_heading_change_rate_for_agent(instance, sample)
if np.isnan(velocity):
velocity = 0.0
if np.isnan(acceleration):
acceleration = 0.0
if np.isnan(yaw_rate):
yaw_rate = 0.0
hx, hy = np.cos(yaw), np.sin(yaw)
vx, vy = velocity * hx, velocity * hy
ax, ay = acceleration * hx, acceleration * hy
return x, y, vx, vy, ax, ay, velocity, yaw_rate, acceleration, yaw
def test_get_past_for_agent_in_frame(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -2e6
}, {
'token': '2',
'timestamp': -3e6
}, {
'token': '1',
'timestamp': -4e6
}]
# Testing we can get the exact amount of past seconds available
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, True)
np.testing.assert_allclose(past,
np.array([[1., -1.], [2., -2.], [3., -3.]]))
def test_get_past_for_last_returns_nothing(self):
mock_samples = [{'token': '1', 'timestamp': 0}]
# Testing we get nothing if we're at the last annotation
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '1', 3, False)
np.testing.assert_equal(past, np.array([]))
def test_velocity_return_nan_big_diff(self) -> None:
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 2.5e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertTrue(np.isnan(helper.get_velocity_for_agent('1', '2')))
def test_heading_change_rate(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertEqual(helper.get_heading_change_rate_for_agent('1', '2'),
np.pi)
def test_acceleration_zero(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}, {
'token': '3',
'timestamp': 1e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertEqual(helper.get_acceleration_for_agent('1', '3'), 0)
def test_velocity(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertEqual(helper.get_velocity_for_agent("1", "2"), np.sqrt(8))
def test_get_past_for_agent_no_data_to_get(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -3.5e6
}]
# Testing we get nothing if the first sample annotation is past our threshold
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, False)
np.testing.assert_equal(past, np.array([]))
def test_acceleration_nan_not_enough_data(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}, {
'token': '3',
'timestamp': 1e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertTrue(np.isnan(helper.get_acceleration_for_agent('1', '2')))
def test_raises_error_when_seconds_negative(self):
mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1', '1b']}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
with self.assertRaises(ValueError):
helper.get_future_for_agent('1', '1', -1, False)
with self.assertRaises(ValueError):
helper.get_past_for_agent('1', '1', -1, False)
with self.assertRaises(ValueError):
helper.get_past_for_sample('1', -1, False)
with self.assertRaises(ValueError):
helper.get_future_for_sample('1', -1, False)
def test_get_sample_annotation(self, ):
mock_annotation = {
'token': '1',
'instance_token': 'instance_1',
'sample_token': 'sample_1'
}
mock_sample = {'token': 'sample_1', 'timestamp': 0}
nusc = MockNuScenes([mock_annotation], [mock_sample])
helper = PredictHelper(nusc)
self.assertDictEqual(
mock_annotation,
helper.get_sample_annotation('instance_1', 'sample_1'))
def compute_metrics(predictions: List[Dict[str, Any]], helper: PredictHelper,
config: PredictionConfig) -> Dict[str, Any]:
"""
Computes metrics from a set of predictions.
:param predictions: List of prediction JSON objects.
:param helper: Instance of PredictHelper that wraps the nuScenes val set.
:param config: Config file.
:return: Metrics. Nested dictionary where keys are metric names and value is a dictionary
mapping the Aggregator name to the results.
"""
n_preds = len(predictions)
containers = {
metric.name: np.zeros((n_preds, metric.shape))
for metric in config.metrics
}
for i, prediction_str in enumerate(predictions):
prediction = Prediction.deserialize(prediction_str)
ground_truth = helper.get_future_for_agent(prediction.instance,
prediction.sample,
config.seconds,
in_agent_frame=False)
for metric in config.metrics:
containers[metric.name][i] = metric(ground_truth, prediction)
aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict)
for metric in config.metrics:
for agg in metric.aggregators:
aggregations[metric.name][agg.name] = agg(containers[metric.name])
return aggregations
def test_acceleration_nonzero(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}, {
'token': '3',
'timestamp': 1e6
}]
mock_annotations = copy.copy(self.mock_annotations)
mock_annotations[2]['translation'] = [3, 3, 3]
nusc = MockNuScenes(mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertAlmostEqual(helper.get_acceleration_for_agent('1', '3'),
2 * (np.sqrt(32) - np.sqrt(8)))
def test_get_future_for_sample(self):
mock_samples = [{
'token': '1',
'timestamp': 0,
'anns': ['1', '1b']
}, {
'token': '2',
'timestamp': 1e6
}, {
'token': '3',
'timestamp': 2e6
}, {
'token': '4',
'timestamp': 3e6
}, {
'token': '5',
'timestamp': 4e6
}]
nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_sample("1", 3, False)
answer = {
'1': np.array([[1, 1], [2, 2], [3, 3]]),
'2': np.array([[7, 7], [8, 8], [9, 9]])
}
for k in answer:
np.testing.assert_equal(answer[k], future[k])
future_in_sample = helper.get_future_for_sample("1", 3, True)
answer_in_sample = {
'1': np.array([[-1, 1], [-2, 2], [-3, 3]]),
'2': np.array([[-1, 1], [-2, 2], [-3, 3]])
}
for k in answer_in_sample:
np.testing.assert_allclose(answer_in_sample[k],
future_in_sample[k])
def test_get_past_for_agent_within_buffer(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -3.05e6
}, {
'token': '2',
'timestamp': -3.2e6
}]
# Testing we get data if it is after future seconds but within buffer
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, False)
np.testing.assert_equal(past, np.array([[3, 3], [2, 2]]))
def test_heading_change_rate_near_pi(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}]
mock_annotations = copy.copy(self.mock_annotations)
mock_annotations[0]['rotation'] = [
np.cos((np.pi - 0.05) / 2), 0, 0,
np.sin((np.pi - 0.05) / 2)
]
mock_annotations[1]['rotation'] = [
np.cos((-np.pi + 0.05) / 2), 0, 0,
np.sin((-np.pi + 0.05) / 2)
]
nusc = MockNuScenes(mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertAlmostEqual(
helper.get_heading_change_rate_for_agent('1', '2'), 0.2)
def test_get_future_for_agent_in_agent_frame(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 1e6
}, {
'token': '3',
'timestamp': 2e6
}, {
'token': '4',
'timestamp': 3e6
}, {
'token': '5',
'timestamp': 4e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, True)
np.testing.assert_allclose(future, np.array([[-1, 1], [-2, 2], [-3,
3]]))
def test_get_future_for_agent_exact_amount(self) -> None:
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 1e6
}, {
'token': '3',
'timestamp': 2e6
}, {
'token': '4',
'timestamp': 3e6
}, {
'token': '5',
'timestamp': 4e6
}]
# Testing we can get the exact amount of future seconds available
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, False)
np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))
def test_get_future_for_agent_within_buffer(self) -> None:
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 1e6
}, {
'token': '3',
'timestamp': 2.6e6
}, {
'token': '4',
'timestamp': 3.05e6
}, {
'token': '5',
'timestamp': 3.5e6
}]
# Testing we get data if it is after future seconds but within buffer
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, False)
np.testing.assert_equal(future, np.array([[1, 1], [2, 2], [3, 3]]))
def test_get_past_for_agent_less_amount(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -2.6e6
}, {
'token': '2',
'timestamp': -4e6
}, {
'token': '1',
'timestamp': -5.5e6
}]
# Testing we do not include data after the past seconds
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, False)
np.testing.assert_equal(past, np.array([[3, 3], [2, 2]]))
def test_get_past_for_agent_exact_amount(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -2e6
}, {
'token': '2',
'timestamp': -3e6
}, {
'token': '1',
'timestamp': -4e6
}]
# Testing we can get the exact amount of past seconds available
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, False)
np.testing.assert_equal(past, np.array([[3, 3], [2, 2], [1, 1]]))
def test_get_future_for_agent_less_amount(self, ):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 1e6
}, {
'token': '3',
'timestamp': 2.6e6
}, {
'token': '4',
'timestamp': 4e6
}, {
'token': '5',
'timestamp': 5.5e6
}]
# Testing we do not include data after the future seconds
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, False)
np.testing.assert_equal(future, np.array([[1, 1], [2, 2]]))
def _do_test(self, map_name, predictions, answer):
with patch.object(PredictHelper,
'get_map_name_from_sample_token') as get_map_name:
get_map_name.return_value = map_name
nusc = NuScenes('v1.0-mini', dataroot=os.environ['NUSCENES'])
helper = PredictHelper(nusc)
off_road_rate = metrics.OffRoadRate(helper, [metrics.RowMean()])
probabilities = np.array([1 / 3] * predictions.shape[0])
prediction = Prediction('foo-instance', 'foo-sample', predictions,
probabilities)
# Two violations out of three trajectories
np.testing.assert_allclose(off_road_rate(np.array([]), prediction),
np.array([answer]))
def test_get_no_data_when_seconds_0(self):
mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1']}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
np.testing.assert_equal(
helper.get_future_for_agent('1', '1', 0, False), np.array([]))
np.testing.assert_equal(helper.get_past_for_agent('1', '1', 0, False),
np.array([]))
np.testing.assert_equal(helper.get_future_for_sample('1', 0, False),
np.array([]))
np.testing.assert_equal(helper.get_past_for_sample('1', 0, False),
np.array([]))
def main(version: str, data_root: str, split_name: str, output_dir: str, submission_name: str, config_name: str) \
-> None:
"""
Makes predictions for a submission to the nuScenes prediction challenge.
:param version: NuScenes version.
:param data_root: Directory storing NuScenes data.
:param split_name: Data split to run inference on.
:param output_dir: Directory to store the output file.
:param submission_name: Name of the submission to use for the results file.
:param config_name: Name of config file to use.
"""
nusc = NuScenes(version=version, dataroot=data_root)
helper = PredictHelper(nusc)
dataset = get_prediction_challenge_split(split_name)
config = load_prediction_config(helper, config_name)
predictions = do_inference_for_submission(helper, config, dataset)
predictions = [prediction.serialize() for prediction in predictions]
json.dump(predictions, open(os.path.join(output_dir, f"{submission_name}_inference.json"), "w"))
def main(version: str,
data_root: str,
submission_path: str,
config_name: str = 'predict_2020_icra.json') -> None:
"""
Computes metrics for a submission stored in submission_path with a given submission_name with the metrics
specified by the config_name.
:param version: nuScenes data set version.
:param data_root: Directory storing NuScenes data.
:param submission_path: Directory storing submission.
:param config_name: Name of config file.
"""
predictions = json.load(open(submission_path, "r"))
nusc = NuScenes(version=version, dataroot=data_root)
helper = PredictHelper(nusc)
config = load_prediction_config(helper, config_name)
results = compute_metrics(predictions, helper, config)
json.dump(results,
open(submission_path.replace('.json', '_metrics.json'), "w"),
indent=2)
'4080c30aa7104d91ad005a50b18f6108_ca9a282c9e77460f8360f564131a8af5',
'c1958768d48640948f6053d04cffd35b_ca9a282c9e77460f8360f564131a8af5',
'4005437c730645c2b628dc1da999e06a_39586f9d59004284a7114a68825e8eec',
'a017fe4e9c3d445784aae034b1322006_356d81f38dd9473ba590f39e266f54e5',
'a0049f95375044b8987fbcca8fda1e2b_c923fe08b2ff4e27975d2bf30934383b',
'61dd7d03d7ad466d89f901ed64e2c0dd_e0845f5322254dafadbbed75aaa07969',
'86ed8530809d4b1b8fbc53808f599339_39586f9d59004284a7114a68825e8eec',
'2a80b29c0281435ca4893e158a281ce0_2afb9d32310e4546a71cbe432911eca2',
'8ce4fe54af77467d90c840465f69677f_de7593d76648450e947ba0c203dee1b0',
'f4af7fd215ee47aa8b64bac0443d7be8_9ee4020153674b9e9943d395ff8cfdf3'
]
tokens = tokens * 32
nusc = NuScenes('v1.0-trainval', dataroot=args.data_root)
helper = PredictHelper(nusc)
dataset = TestDataset(tokens, helper)
dataloader = DataLoader(dataset, batch_size=16, num_workers=16)
backbone = ResNetBackbone('resnet18')
model = MTP(backbone, NUM_MODES)
model = model.to(device)
loss_function = MTPLoss(NUM_MODES, 1, 5)
current_loss = 10000
optimizer = optim.SGD(model.parameters(), lr=0.1)
n_iter = 0
