def __init__(self):
DATAROOT = '/home/patrick/datasets/nuscenes' # This is the path where you stored your copy of the nuScenes dataset.
self.nuscenes = NuScenes('v1.0-mini', dataroot=DATAROOT)
self.mini_train = get_prediction_challenge_split("mini_train",
dataroot=DATAROOT)
self.helper = PredictHelper(self.nuscenes)
self.physics_oracle = PhysicsOracle(sec_from_now=6, helper=self.helper)
self.map_rasterizer = StaticLayerRasterizer(self.helper,
meters_ahead=60,
meters_behind=10,
meters_left=35,
meters_right=35)
self.agent_rasterizer = FutureAgentBoxesWithFadedHistory(
self.helper,
meters_ahead=60,
meters_behind=10,
meters_left=35,
meters_right=35)
self.json_path = 'manual_results.json'
self.annotations = []
if os.path.exists(self.json_path):
with open(self.json_path) as json_file:
self.annotations = json.load(
json_file) # Load existing JSON file
def test_velocity_return_nan_one_obs(self):
mock_samples = [{'token': '1', 'timestamp': 0}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertTrue(np.isnan(helper.get_velocity_for_agent('1', '1')))
def test_get_past_for_sample(self):
mock_samples = [{
'token': '5',
'timestamp': 0,
'anns': ['5', '5b']
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -2e6
}, {
'token': '2',
'timestamp': -3e6
}, {
'token': '1',
'timestamp': -4e6
}]
nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_sample('5', 3, True)
answer = {
'1': np.array([[-1, -1], [-2, -2], [-3, -3]]),
'2': np.array([[-1, -1], [-2, -2], [-3, -3]])
}
for k in answer:
np.testing.assert_equal(answer[k], answer[k])
def _kinematics_from_tokens(helper: PredictHelper, instance: str,
sample: str) -> KinematicsData:
"""
Returns the 2D position, velocity and acceleration vectors from the given track records,
along with the speed, yaw rate, (scalar) acceleration (magnitude), and heading.
:param helper: Instance of PredictHelper.
:instance: Token of instance.
:sample: Token of sample.
:return: KinematicsData.
"""
annotation = helper.get_sample_annotation(instance, sample)
x, y, _ = annotation['translation']
yaw = quaternion_yaw(Quaternion(annotation['rotation']))
velocity = helper.get_velocity_for_agent(instance, sample)
acceleration = helper.get_acceleration_for_agent(instance, sample)
yaw_rate = helper.get_heading_change_rate_for_agent(instance, sample)
if np.isnan(velocity):
velocity = 0.0
if np.isnan(acceleration):
acceleration = 0.0
if np.isnan(yaw_rate):
yaw_rate = 0.0
hx, hy = np.cos(yaw), np.sin(yaw)
vx, vy = velocity * hx, velocity * hy
ax, ay = acceleration * hx, acceleration * hy
return x, y, vx, vy, ax, ay, velocity, yaw_rate, acceleration, yaw
def test_get_past_for_agent_in_frame(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -2e6
}, {
'token': '2',
'timestamp': -3e6
}, {
'token': '1',
'timestamp': -4e6
}]
# Testing we can get the exact amount of past seconds available
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, True)
np.testing.assert_allclose(past,
np.array([[1., -1.], [2., -2.], [3., -3.]]))
def __init__(self,
DATAROOT='./data/sets/nuscenes',
dataset_version='v1.0-mini'):
self.DATAROOT = DATAROOT
self.dataset_version = dataset_version
self.nuscenes = NuScenes(dataset_version, dataroot=self.DATAROOT)
self.helper = PredictHelper(self.nuscenes)
def test_get_annotations_for_sample(self) -> None:
mock_samples = [{'token': '1', 'timestamp': -4e6, 'anns': ['1', '1b']}]
nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples)
helper = PredictHelper(nusc)
annotations = helper.get_annotations_for_sample('1')
answer = [{
'token': '1',
'instance_token': '1',
'sample_token': '1',
'translation': [0, 0, 0],
'rotation': [1, 0, 0, 0],
'prev': '',
'next': '2'
}, {
'token': '1b',
'instance_token': '2',
'sample_token': '1',
'translation': [6, 6, 6],
'rotation': [1, 0, 0, 0],
'prev': '',
'next': '2b'
}]
self.assertListEqual(annotations, answer)
def populate_agent_states(center_agent_annotation: Dict[str, Any],
center_agent_pixels: Tuple[float, float],
annotations: List[Dict[str, Any]],
base_image: np.ndarray,
helper: PredictHelper,
resolution: float = 0.1) -> None:
"""
Adds agent states to 4 channel base_image
:param center_agent_annotation: Annotation record for the agent
that is in the center of the image.
:param center_agent_pixels: Pixel location of the agent in the
center of the image.
:param annotations: Annotation records for other agents
:param base_image: 4 channel image to populate with agent states.
:param helper: Predict helper
:param resolution: Size of the image in pixels / meter.
:return: None.
"""
agent_x, agent_y = center_agent_annotation['translation'][:2]
for i, annotation in enumerate(annotations):
if annotation['instance_token'] != center_agent_annotation[
'instance_token']:
location = annotation['translation'][:2]
row_pixel, column_pixel = convert_to_pixel_coords(
location, (agent_x, agent_y), center_agent_pixels, resolution)
if 0 <= row_pixel < base_image.shape[
0] and 0 <= column_pixel < base_image.shape[1]:
v = helper.get_velocity_for_agent(annotation['instance_token'],
annotation['sample_token'])
a = helper.get_acceleration_for_agent(
annotation['instance_token'], annotation['sample_token'])
omega = helper.get_heading_change_rate_for_agent(
annotation['instance_token'], annotation['sample_token'])
if base_image[row_pixel, column_pixel, 0] == 0:
base_image[row_pixel, column_pixel, 0] = 1
if not np.isnan(v):
base_image[row_pixel, column_pixel, 1] = v
if not np.isnan(a):
base_image[row_pixel, column_pixel, 2] = a
if not np.isnan(omega):
base_image[row_pixel, column_pixel, 3] = omega
else:
base_image[row_pixel, column_pixel, 0] += 1
if not np.isnan(v):
base_image[row_pixel, column_pixel, 1] = min(
v, base_image[row_pixel, column_pixel, 1])
if not np.isnan(a):
base_image[row_pixel, column_pixel, 2] = min(
a, base_image[row_pixel, column_pixel, 2])
if not np.isnan(omega):
base_image[row_pixel, column_pixel, 3] = min(
omega, base_image[row_pixel, column_pixel, 3])
def __init__(self, set_name="mini_train"):
#assert statements
set_paths = [
'train', 'val', 'test', 'mini_train', 'mini_val', 'train_detect',
'train_track'
]
assert set_name in set_paths, "Incorrect set_name"
#Initialize data and Prediction Helper classes
self.data_path = DATA_PATH
self.nusc = NuScenes(version=DATA_VERSION,
dataroot=self.data_path,
verbose=True)
self.helper = PredictHelper(self.nusc)
#get all the scenes
self.scenes = create_splits_scenes()
#get all the scenes in the trainset
self.set_name = set_name
self.trainset = self.scenes[
self.set_name] #List of scenes as part of training set
self.prediction_scenes = json.load(
open(self.data_path + "maps/prediction_scenes.json", "r")
) #Dictionary containing list of instance and sample tokens for each scene
print("Number of samples in train set: %d" % (len(self.trainset)))
def test_get_past_for_last_returns_nothing(self):
mock_samples = [{'token': '1', 'timestamp': 0}]
# Testing we get nothing if we're at the last annotation
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '1', 3, False)
np.testing.assert_equal(past, np.array([]))
def test_velocity_return_nan_big_diff(self) -> None:
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 2.5e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertTrue(np.isnan(helper.get_velocity_for_agent('1', '2')))
def test_heading_change_rate(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertEqual(helper.get_heading_change_rate_for_agent('1', '2'),
np.pi)
def test_velocity(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertEqual(helper.get_velocity_for_agent("1", "2"), np.sqrt(8))
def test_acceleration_zero(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}, {
'token': '3',
'timestamp': 1e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertEqual(helper.get_acceleration_for_agent('1', '3'), 0)
def test_get_past_for_agent_no_data_to_get(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -3.5e6
}]
# Testing we get nothing if the first sample annotation is past our threshold
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, False)
np.testing.assert_equal(past, np.array([]))
def test_acceleration_nan_not_enough_data(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}, {
'token': '3',
'timestamp': 1e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertTrue(np.isnan(helper.get_acceleration_for_agent('1', '2')))
def test_get_sample_annotation(self, ):
mock_annotation = {
'token': '1',
'instance_token': 'instance_1',
'sample_token': 'sample_1'
}
mock_sample = {'token': 'sample_1', 'timestamp': 0}
nusc = MockNuScenes([mock_annotation], [mock_sample])
helper = PredictHelper(nusc)
self.assertDictEqual(
mock_annotation,
helper.get_sample_annotation('instance_1', 'sample_1'))
def test_raises_error_when_seconds_negative(self):
mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1', '1b']}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
with self.assertRaises(ValueError):
helper.get_future_for_agent('1', '1', -1, False)
with self.assertRaises(ValueError):
helper.get_past_for_agent('1', '1', -1, False)
with self.assertRaises(ValueError):
helper.get_past_for_sample('1', -1, False)
with self.assertRaises(ValueError):
helper.get_future_for_sample('1', -1, False)
def main(version: str,
data_root: str,
split_name: str,
output_dir: str,
config_name: str = 'predict_2020_icra.json') -> None:
"""
Performs inference for all of the baseline models defined in the physics model module.
:param version: nuScenes data set version.
:param data_root: Directory where the NuScenes data is stored.
:param split_name: nuScenes data split name, e.g. train, val, mini_train, etc.
:param output_dir: Directory where predictions should be stored.
:param config_name: Name of config file.
"""
nusc = NuScenes(version=version, dataroot=data_root)
helper = PredictHelper(nusc)
dataset = get_prediction_challenge_split(split_name)
config = load_prediction_config(helper, config_name)
oracle = PhysicsOracle(config.seconds, helper)
cv_heading = ConstantVelocityHeading(config.seconds, helper)
cv_preds = []
oracle_preds = []
for token in dataset:
cv_preds.append(cv_heading(token).serialize())
oracle_preds.append(oracle(token).serialize())
json.dump(cv_preds, open(os.path.join(output_dir, "cv_preds.json"), "w"))
json.dump(oracle_preds,
open(os.path.join(output_dir, "oracle_preds.json"), "w"))
def main(args):
print("Running with args:")
print(vars(args))
print("Device:")
print(device)
# load data
nusc = NuScenes(version=args.version, dataroot=args.data_root)
helper = PredictHelper(nusc)
data_tokens = get_prediction_challenge_split(args.split_name, dataroot=args.data_root)
if args.key == "covernet":
dataset = CoverNetDataset(data_tokens, helper)
elif args.key == "mtp":
dataset = MTPDataset(data_tokens, helper)
dataloader = DataLoader(dataset, batch_size=16, num_workers=0, shuffle=False)
print(f"Loaded split {args.split_name}, length {len(dataset)}, in {len(dataloader)} batches.")
# prepare model
model = get_model(args)
model.load_state_dict(
torch.load(os.path.join(args.experiment_dir, 'weights', args.weights)))
model.eval()
predictions = get_predictions(args, dataloader, model)
json.dump(predictions,
open(os.path.join(args.experiment_dir, f'{args.key}_preds_{datetime.datetime.now():%Y-%m-%d %Hh%Mm%Ss}_{args.suffix}.json'), "w"))
def compute_metrics(predictions: List[Dict[str, Any]], helper: PredictHelper,
config: PredictionConfig) -> Dict[str, Any]:
"""
Computes metrics from a set of predictions.
:param predictions: List of prediction JSON objects.
:param helper: Instance of PredictHelper that wraps the nuScenes val set.
:param config: Config file.
:return: Metrics. Nested dictionary where keys are metric names and value is a dictionary
mapping the Aggregator name to the results.
"""
n_preds = len(predictions)
containers = {
metric.name: np.zeros((n_preds, metric.shape))
for metric in config.metrics
}
for i, prediction_str in enumerate(predictions):
prediction = Prediction.deserialize(prediction_str)
ground_truth = helper.get_future_for_agent(prediction.instance,
prediction.sample,
config.seconds,
in_agent_frame=False)
for metric in config.metrics:
containers[metric.name][i] = metric(ground_truth, prediction)
aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict)
for metric in config.metrics:
for agg in metric.aggregators:
aggregations[metric.name][agg.name] = agg(containers[metric.name])
return aggregations
def __init__(self,
DATAROOT='./data/sets/nuscenes',
dataset_version='v1.0-mini'):
self.DATAROOT = DATAROOT
self.dataset_version = dataset_version
self.nuscenes = NuScenes(dataset_version, dataroot=self.DATAROOT)
self.helper = PredictHelper(self.nuscenes)
# ['vehicle.car', 'vehicle.truck', 'vehicle.bus.rigid', 'vehicle.bus.bendy', 'vehicle.construction']
self.category_token_to_id = {
"fd69059b62a3469fbaef25340c0eab7f": 1, # 'vehicle.car'
"6021b5187b924d64be64a702e5570edf": 1, # 'vehicle.truck'
"fedb11688db84088883945752e480c2c": 2, # 'vehicle.bus.rigid'
"003edbfb9ca849ee8a7496e9af3025d4": 2, # 'vehicle.bus.bendy'
"5b3cd6f2bca64b83aa3d0008df87d0e4": 3, # 'vehicle.construction'
"7b2ff083a64e4d53809ae5d9be563504": 1
} # vehicle.emergency.police
def test_acceleration_nonzero(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}, {
'token': '3',
'timestamp': 1e6
}]
mock_annotations = copy.copy(self.mock_annotations)
mock_annotations[2]['translation'] = [3, 3, 3]
nusc = MockNuScenes(mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertAlmostEqual(helper.get_acceleration_for_agent('1', '3'),
2 * (np.sqrt(32) - np.sqrt(8)))
def main(args):
print("Args:")
print(vars(args))
print("Device:")
print(device)
# prepare output directories
if not os.path.exists(args.experiment_dir):
os.mkdir(args.experiment_dir)
if not os.path.exists(os.path.join(args.experiment_dir, 'weights')):
os.mkdir(os.path.join(args.experiment_dir, 'weights'))
# store the arguments for reference
config_fname = f'config_for_runtime_{RUN_TIME:%Y-%m-%d %Hh%Mm%Ss}.json'
with open(os.path.join(args.experiment_dir, config_fname),
'w') as json_file:
json.dump(vars(args), json_file)
# load data
nusc = NuScenes(version=args.version, dataroot=args.data_root)
helper = PredictHelper(nusc)
train_tokens = get_prediction_challenge_split(args.train_split_name,
dataroot=args.data_root)
val_tokens = get_prediction_challenge_split(args.val_split_name,
dataroot=args.data_root)
# apply downsampling
train_tokens = np.random.choice(
train_tokens,
int(len(train_tokens) / args.train_downsample_factor),
replace=False)
val_tokens = np.random.choice(
val_tokens,
int(len(val_tokens) / args.val_downsample_factor),
replace=False)
# create data loaders
train_dataset = get_dataset(train_tokens, helper, args)
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=True)
val_dataset = get_dataset(val_tokens, helper, args)
val_dataloader = DataLoader(val_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=False)
# run training
train_epochs(train_dataloader=train_dataloader,
val_dataloader=val_dataloader,
args=args)
def test_get_future_for_sample(self):
mock_samples = [{
'token': '1',
'timestamp': 0,
'anns': ['1', '1b']
}, {
'token': '2',
'timestamp': 1e6
}, {
'token': '3',
'timestamp': 2e6
}, {
'token': '4',
'timestamp': 3e6
}, {
'token': '5',
'timestamp': 4e6
}]
nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_sample("1", 3, False)
answer = {
'1': np.array([[1, 1], [2, 2], [3, 3]]),
'2': np.array([[7, 7], [8, 8], [9, 9]])
}
for k in answer:
np.testing.assert_equal(answer[k], future[k])
future_in_sample = helper.get_future_for_sample("1", 3, True)
answer_in_sample = {
'1': np.array([[-1, 1], [-2, 2], [-3, 3]]),
'2': np.array([[-1, 1], [-2, 2], [-3, 3]])
}
for k in answer_in_sample:
np.testing.assert_allclose(answer_in_sample[k],
future_in_sample[k])
def test_get_past_for_agent_within_buffer(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -3.05e6
}, {
'token': '2',
'timestamp': -3.2e6
}]
# Testing we get data if it is after future seconds but within buffer
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, False)
np.testing.assert_equal(past, np.array([[3, 3], [2, 2]]))
def test_heading_change_rate_near_pi(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 0.5e6
}]
mock_annotations = copy.copy(self.mock_annotations)
mock_annotations[0]['rotation'] = [
np.cos((np.pi - 0.05) / 2), 0, 0,
np.sin((np.pi - 0.05) / 2)
]
mock_annotations[1]['rotation'] = [
np.cos((-np.pi + 0.05) / 2), 0, 0,
np.sin((-np.pi + 0.05) / 2)
]
nusc = MockNuScenes(mock_annotations, mock_samples)
helper = PredictHelper(nusc)
self.assertAlmostEqual(
helper.get_heading_change_rate_for_agent('1', '2'), 0.2)
def test_get_future_for_agent_in_agent_frame(self):
mock_samples = [{
'token': '1',
'timestamp': 0
}, {
'token': '2',
'timestamp': 1e6
}, {
'token': '3',
'timestamp': 2e6
}, {
'token': '4',
'timestamp': 3e6
}, {
'token': '5',
'timestamp': 4e6
}]
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
future = helper.get_future_for_agent('1', '1', 3, True)
np.testing.assert_allclose(future, np.array([[-1, 1], [-2, 2], [-3,
3]]))
def test_get_past_for_agent_less_amount(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -2.6e6
}, {
'token': '2',
'timestamp': -4e6
}, {
'token': '1',
'timestamp': -5.5e6
}]
# Testing we do not include data after the past seconds
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, False)
np.testing.assert_equal(past, np.array([[3, 3], [2, 2]]))
def test_get_past_for_agent_exact_amount(self, ):
mock_samples = [{
'token': '5',
'timestamp': 0
}, {
'token': '4',
'timestamp': -1e6
}, {
'token': '3',
'timestamp': -2e6
}, {
'token': '2',
'timestamp': -3e6
}, {
'token': '1',
'timestamp': -4e6
}]
# Testing we can get the exact amount of past seconds available
nusc = MockNuScenes(self.mock_annotations, mock_samples)
helper = PredictHelper(nusc)
past = helper.get_past_for_agent('1', '5', 3, False)
np.testing.assert_equal(past, np.array([[3, 3], [2, 2], [1, 1]]))
