def __init__(self): DATAROOT = '/home/patrick/datasets/nuscenes' # This is the path where you stored your copy of the nuScenes dataset. self.nuscenes = NuScenes('v1.0-mini', dataroot=DATAROOT) self.mini_train = get_prediction_challenge_split("mini_train", dataroot=DATAROOT) self.helper = PredictHelper(self.nuscenes) self.physics_oracle = PhysicsOracle(sec_from_now=6, helper=self.helper) self.map_rasterizer = StaticLayerRasterizer(self.helper, meters_ahead=60, meters_behind=10, meters_left=35, meters_right=35) self.agent_rasterizer = FutureAgentBoxesWithFadedHistory( self.helper, meters_ahead=60, meters_behind=10, meters_left=35, meters_right=35) self.json_path = 'manual_results.json' self.annotations = [] if os.path.exists(self.json_path): with open(self.json_path) as json_file: self.annotations = json.load( json_file) # Load existing JSON file
def test_velocity_return_nan_one_obs(self): mock_samples = [{'token': '1', 'timestamp': 0}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertTrue(np.isnan(helper.get_velocity_for_agent('1', '1')))
def test_get_past_for_sample(self): mock_samples = [{ 'token': '5', 'timestamp': 0, 'anns': ['5', '5b'] }, { 'token': '4', 'timestamp': -1e6 }, { 'token': '3', 'timestamp': -2e6 }, { 'token': '2', 'timestamp': -3e6 }, { 'token': '1', 'timestamp': -4e6 }] nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_sample('5', 3, True) answer = { '1': np.array([[-1, -1], [-2, -2], [-3, -3]]), '2': np.array([[-1, -1], [-2, -2], [-3, -3]]) } for k in answer: np.testing.assert_equal(answer[k], answer[k])
def _kinematics_from_tokens(helper: PredictHelper, instance: str, sample: str) -> KinematicsData: """ Returns the 2D position, velocity and acceleration vectors from the given track records, along with the speed, yaw rate, (scalar) acceleration (magnitude), and heading. :param helper: Instance of PredictHelper. :instance: Token of instance. :sample: Token of sample. :return: KinematicsData. """ annotation = helper.get_sample_annotation(instance, sample) x, y, _ = annotation['translation'] yaw = quaternion_yaw(Quaternion(annotation['rotation'])) velocity = helper.get_velocity_for_agent(instance, sample) acceleration = helper.get_acceleration_for_agent(instance, sample) yaw_rate = helper.get_heading_change_rate_for_agent(instance, sample) if np.isnan(velocity): velocity = 0.0 if np.isnan(acceleration): acceleration = 0.0 if np.isnan(yaw_rate): yaw_rate = 0.0 hx, hy = np.cos(yaw), np.sin(yaw) vx, vy = velocity * hx, velocity * hy ax, ay = acceleration * hx, acceleration * hy return x, y, vx, vy, ax, ay, velocity, yaw_rate, acceleration, yaw
def test_get_past_for_agent_in_frame(self, ): mock_samples = [{ 'token': '5', 'timestamp': 0 }, { 'token': '4', 'timestamp': -1e6 }, { 'token': '3', 'timestamp': -2e6 }, { 'token': '2', 'timestamp': -3e6 }, { 'token': '1', 'timestamp': -4e6 }] # Testing we can get the exact amount of past seconds available nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, True) np.testing.assert_allclose(past, np.array([[1., -1.], [2., -2.], [3., -3.]]))
def __init__(self, DATAROOT='./data/sets/nuscenes', dataset_version='v1.0-mini'): self.DATAROOT = DATAROOT self.dataset_version = dataset_version self.nuscenes = NuScenes(dataset_version, dataroot=self.DATAROOT) self.helper = PredictHelper(self.nuscenes)
def test_get_annotations_for_sample(self) -> None: mock_samples = [{'token': '1', 'timestamp': -4e6, 'anns': ['1', '1b']}] nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples) helper = PredictHelper(nusc) annotations = helper.get_annotations_for_sample('1') answer = [{ 'token': '1', 'instance_token': '1', 'sample_token': '1', 'translation': [0, 0, 0], 'rotation': [1, 0, 0, 0], 'prev': '', 'next': '2' }, { 'token': '1b', 'instance_token': '2', 'sample_token': '1', 'translation': [6, 6, 6], 'rotation': [1, 0, 0, 0], 'prev': '', 'next': '2b' }] self.assertListEqual(annotations, answer)
def populate_agent_states(center_agent_annotation: Dict[str, Any], center_agent_pixels: Tuple[float, float], annotations: List[Dict[str, Any]], base_image: np.ndarray, helper: PredictHelper, resolution: float = 0.1) -> None: """ Adds agent states to 4 channel base_image :param center_agent_annotation: Annotation record for the agent that is in the center of the image. :param center_agent_pixels: Pixel location of the agent in the center of the image. :param annotations: Annotation records for other agents :param base_image: 4 channel image to populate with agent states. :param helper: Predict helper :param resolution: Size of the image in pixels / meter. :return: None. """ agent_x, agent_y = center_agent_annotation['translation'][:2] for i, annotation in enumerate(annotations): if annotation['instance_token'] != center_agent_annotation[ 'instance_token']: location = annotation['translation'][:2] row_pixel, column_pixel = convert_to_pixel_coords( location, (agent_x, agent_y), center_agent_pixels, resolution) if 0 <= row_pixel < base_image.shape[ 0] and 0 <= column_pixel < base_image.shape[1]: v = helper.get_velocity_for_agent(annotation['instance_token'], annotation['sample_token']) a = helper.get_acceleration_for_agent( annotation['instance_token'], annotation['sample_token']) omega = helper.get_heading_change_rate_for_agent( annotation['instance_token'], annotation['sample_token']) if base_image[row_pixel, column_pixel, 0] == 0: base_image[row_pixel, column_pixel, 0] = 1 if not np.isnan(v): base_image[row_pixel, column_pixel, 1] = v if not np.isnan(a): base_image[row_pixel, column_pixel, 2] = a if not np.isnan(omega): base_image[row_pixel, column_pixel, 3] = omega else: base_image[row_pixel, column_pixel, 0] += 1 if not np.isnan(v): base_image[row_pixel, column_pixel, 1] = min( v, base_image[row_pixel, column_pixel, 1]) if not np.isnan(a): base_image[row_pixel, column_pixel, 2] = min( a, base_image[row_pixel, column_pixel, 2]) if not np.isnan(omega): base_image[row_pixel, column_pixel, 3] = min( omega, base_image[row_pixel, column_pixel, 3])
def __init__(self, set_name="mini_train"): #assert statements set_paths = [ 'train', 'val', 'test', 'mini_train', 'mini_val', 'train_detect', 'train_track' ] assert set_name in set_paths, "Incorrect set_name" #Initialize data and Prediction Helper classes self.data_path = DATA_PATH self.nusc = NuScenes(version=DATA_VERSION, dataroot=self.data_path, verbose=True) self.helper = PredictHelper(self.nusc) #get all the scenes self.scenes = create_splits_scenes() #get all the scenes in the trainset self.set_name = set_name self.trainset = self.scenes[ self.set_name] #List of scenes as part of training set self.prediction_scenes = json.load( open(self.data_path + "maps/prediction_scenes.json", "r") ) #Dictionary containing list of instance and sample tokens for each scene print("Number of samples in train set: %d" % (len(self.trainset)))
def test_get_past_for_last_returns_nothing(self): mock_samples = [{'token': '1', 'timestamp': 0}] # Testing we get nothing if we're at the last annotation nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '1', 3, False) np.testing.assert_equal(past, np.array([]))
def test_velocity_return_nan_big_diff(self) -> None: mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 2.5e6 }] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertTrue(np.isnan(helper.get_velocity_for_agent('1', '2')))
def test_heading_change_rate(self): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 0.5e6 }] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertEqual(helper.get_heading_change_rate_for_agent('1', '2'), np.pi)
def test_velocity(self): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 0.5e6 }] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertEqual(helper.get_velocity_for_agent("1", "2"), np.sqrt(8))
def test_acceleration_zero(self): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 0.5e6 }, { 'token': '3', 'timestamp': 1e6 }] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertEqual(helper.get_acceleration_for_agent('1', '3'), 0)
def test_get_past_for_agent_no_data_to_get(self, ): mock_samples = [{ 'token': '5', 'timestamp': 0 }, { 'token': '4', 'timestamp': -3.5e6 }] # Testing we get nothing if the first sample annotation is past our threshold nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, False) np.testing.assert_equal(past, np.array([]))
def test_acceleration_nan_not_enough_data(self): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 0.5e6 }, { 'token': '3', 'timestamp': 1e6 }] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertTrue(np.isnan(helper.get_acceleration_for_agent('1', '2')))
def test_get_sample_annotation(self, ): mock_annotation = { 'token': '1', 'instance_token': 'instance_1', 'sample_token': 'sample_1' } mock_sample = {'token': 'sample_1', 'timestamp': 0} nusc = MockNuScenes([mock_annotation], [mock_sample]) helper = PredictHelper(nusc) self.assertDictEqual( mock_annotation, helper.get_sample_annotation('instance_1', 'sample_1'))
def test_raises_error_when_seconds_negative(self): mock_samples = [{'token': '1', 'timestamp': 0, 'anns': ['1', '1b']}] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) with self.assertRaises(ValueError): helper.get_future_for_agent('1', '1', -1, False) with self.assertRaises(ValueError): helper.get_past_for_agent('1', '1', -1, False) with self.assertRaises(ValueError): helper.get_past_for_sample('1', -1, False) with self.assertRaises(ValueError): helper.get_future_for_sample('1', -1, False)
def main(version: str, data_root: str, split_name: str, output_dir: str, config_name: str = 'predict_2020_icra.json') -> None: """ Performs inference for all of the baseline models defined in the physics model module. :param version: nuScenes data set version. :param data_root: Directory where the NuScenes data is stored. :param split_name: nuScenes data split name, e.g. train, val, mini_train, etc. :param output_dir: Directory where predictions should be stored. :param config_name: Name of config file. """ nusc = NuScenes(version=version, dataroot=data_root) helper = PredictHelper(nusc) dataset = get_prediction_challenge_split(split_name) config = load_prediction_config(helper, config_name) oracle = PhysicsOracle(config.seconds, helper) cv_heading = ConstantVelocityHeading(config.seconds, helper) cv_preds = [] oracle_preds = [] for token in dataset: cv_preds.append(cv_heading(token).serialize()) oracle_preds.append(oracle(token).serialize()) json.dump(cv_preds, open(os.path.join(output_dir, "cv_preds.json"), "w")) json.dump(oracle_preds, open(os.path.join(output_dir, "oracle_preds.json"), "w"))
def main(args): print("Running with args:") print(vars(args)) print("Device:") print(device) # load data nusc = NuScenes(version=args.version, dataroot=args.data_root) helper = PredictHelper(nusc) data_tokens = get_prediction_challenge_split(args.split_name, dataroot=args.data_root) if args.key == "covernet": dataset = CoverNetDataset(data_tokens, helper) elif args.key == "mtp": dataset = MTPDataset(data_tokens, helper) dataloader = DataLoader(dataset, batch_size=16, num_workers=0, shuffle=False) print(f"Loaded split {args.split_name}, length {len(dataset)}, in {len(dataloader)} batches.") # prepare model model = get_model(args) model.load_state_dict( torch.load(os.path.join(args.experiment_dir, 'weights', args.weights))) model.eval() predictions = get_predictions(args, dataloader, model) json.dump(predictions, open(os.path.join(args.experiment_dir, f'{args.key}_preds_{datetime.datetime.now():%Y-%m-%d %Hh%Mm%Ss}_{args.suffix}.json'), "w"))
def compute_metrics(predictions: List[Dict[str, Any]], helper: PredictHelper, config: PredictionConfig) -> Dict[str, Any]: """ Computes metrics from a set of predictions. :param predictions: List of prediction JSON objects. :param helper: Instance of PredictHelper that wraps the nuScenes val set. :param config: Config file. :return: Metrics. Nested dictionary where keys are metric names and value is a dictionary mapping the Aggregator name to the results. """ n_preds = len(predictions) containers = { metric.name: np.zeros((n_preds, metric.shape)) for metric in config.metrics } for i, prediction_str in enumerate(predictions): prediction = Prediction.deserialize(prediction_str) ground_truth = helper.get_future_for_agent(prediction.instance, prediction.sample, config.seconds, in_agent_frame=False) for metric in config.metrics: containers[metric.name][i] = metric(ground_truth, prediction) aggregations: Dict[str, Dict[str, List[float]]] = defaultdict(dict) for metric in config.metrics: for agg in metric.aggregators: aggregations[metric.name][agg.name] = agg(containers[metric.name]) return aggregations
def __init__(self, DATAROOT='./data/sets/nuscenes', dataset_version='v1.0-mini'): self.DATAROOT = DATAROOT self.dataset_version = dataset_version self.nuscenes = NuScenes(dataset_version, dataroot=self.DATAROOT) self.helper = PredictHelper(self.nuscenes) # ['vehicle.car', 'vehicle.truck', 'vehicle.bus.rigid', 'vehicle.bus.bendy', 'vehicle.construction'] self.category_token_to_id = { "fd69059b62a3469fbaef25340c0eab7f": 1, # 'vehicle.car' "6021b5187b924d64be64a702e5570edf": 1, # 'vehicle.truck' "fedb11688db84088883945752e480c2c": 2, # 'vehicle.bus.rigid' "003edbfb9ca849ee8a7496e9af3025d4": 2, # 'vehicle.bus.bendy' "5b3cd6f2bca64b83aa3d0008df87d0e4": 3, # 'vehicle.construction' "7b2ff083a64e4d53809ae5d9be563504": 1 } # vehicle.emergency.police
def test_acceleration_nonzero(self): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 0.5e6 }, { 'token': '3', 'timestamp': 1e6 }] mock_annotations = copy.copy(self.mock_annotations) mock_annotations[2]['translation'] = [3, 3, 3] nusc = MockNuScenes(mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertAlmostEqual(helper.get_acceleration_for_agent('1', '3'), 2 * (np.sqrt(32) - np.sqrt(8)))
def main(args): print("Args:") print(vars(args)) print("Device:") print(device) # prepare output directories if not os.path.exists(args.experiment_dir): os.mkdir(args.experiment_dir) if not os.path.exists(os.path.join(args.experiment_dir, 'weights')): os.mkdir(os.path.join(args.experiment_dir, 'weights')) # store the arguments for reference config_fname = f'config_for_runtime_{RUN_TIME:%Y-%m-%d %Hh%Mm%Ss}.json' with open(os.path.join(args.experiment_dir, config_fname), 'w') as json_file: json.dump(vars(args), json_file) # load data nusc = NuScenes(version=args.version, dataroot=args.data_root) helper = PredictHelper(nusc) train_tokens = get_prediction_challenge_split(args.train_split_name, dataroot=args.data_root) val_tokens = get_prediction_challenge_split(args.val_split_name, dataroot=args.data_root) # apply downsampling train_tokens = np.random.choice( train_tokens, int(len(train_tokens) / args.train_downsample_factor), replace=False) val_tokens = np.random.choice( val_tokens, int(len(val_tokens) / args.val_downsample_factor), replace=False) # create data loaders train_dataset = get_dataset(train_tokens, helper, args) train_dataloader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers, shuffle=True) val_dataset = get_dataset(val_tokens, helper, args) val_dataloader = DataLoader(val_dataset, batch_size=args.batch_size, num_workers=args.num_workers, shuffle=False) # run training train_epochs(train_dataloader=train_dataloader, val_dataloader=val_dataloader, args=args)
def test_get_future_for_sample(self): mock_samples = [{ 'token': '1', 'timestamp': 0, 'anns': ['1', '1b'] }, { 'token': '2', 'timestamp': 1e6 }, { 'token': '3', 'timestamp': 2e6 }, { 'token': '4', 'timestamp': 3e6 }, { 'token': '5', 'timestamp': 4e6 }] nusc = MockNuScenes(self.multiagent_mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_sample("1", 3, False) answer = { '1': np.array([[1, 1], [2, 2], [3, 3]]), '2': np.array([[7, 7], [8, 8], [9, 9]]) } for k in answer: np.testing.assert_equal(answer[k], future[k]) future_in_sample = helper.get_future_for_sample("1", 3, True) answer_in_sample = { '1': np.array([[-1, 1], [-2, 2], [-3, 3]]), '2': np.array([[-1, 1], [-2, 2], [-3, 3]]) } for k in answer_in_sample: np.testing.assert_allclose(answer_in_sample[k], future_in_sample[k])
def test_get_past_for_agent_within_buffer(self, ): mock_samples = [{ 'token': '5', 'timestamp': 0 }, { 'token': '4', 'timestamp': -1e6 }, { 'token': '3', 'timestamp': -3.05e6 }, { 'token': '2', 'timestamp': -3.2e6 }] # Testing we get data if it is after future seconds but within buffer nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, False) np.testing.assert_equal(past, np.array([[3, 3], [2, 2]]))
def test_heading_change_rate_near_pi(self): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 0.5e6 }] mock_annotations = copy.copy(self.mock_annotations) mock_annotations[0]['rotation'] = [ np.cos((np.pi - 0.05) / 2), 0, 0, np.sin((np.pi - 0.05) / 2) ] mock_annotations[1]['rotation'] = [ np.cos((-np.pi + 0.05) / 2), 0, 0, np.sin((-np.pi + 0.05) / 2) ] nusc = MockNuScenes(mock_annotations, mock_samples) helper = PredictHelper(nusc) self.assertAlmostEqual( helper.get_heading_change_rate_for_agent('1', '2'), 0.2)
def test_get_future_for_agent_in_agent_frame(self): mock_samples = [{ 'token': '1', 'timestamp': 0 }, { 'token': '2', 'timestamp': 1e6 }, { 'token': '3', 'timestamp': 2e6 }, { 'token': '4', 'timestamp': 3e6 }, { 'token': '5', 'timestamp': 4e6 }] nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) future = helper.get_future_for_agent('1', '1', 3, True) np.testing.assert_allclose(future, np.array([[-1, 1], [-2, 2], [-3, 3]]))
def test_get_past_for_agent_less_amount(self, ): mock_samples = [{ 'token': '5', 'timestamp': 0 }, { 'token': '4', 'timestamp': -1e6 }, { 'token': '3', 'timestamp': -2.6e6 }, { 'token': '2', 'timestamp': -4e6 }, { 'token': '1', 'timestamp': -5.5e6 }] # Testing we do not include data after the past seconds nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, False) np.testing.assert_equal(past, np.array([[3, 3], [2, 2]]))
def test_get_past_for_agent_exact_amount(self, ): mock_samples = [{ 'token': '5', 'timestamp': 0 }, { 'token': '4', 'timestamp': -1e6 }, { 'token': '3', 'timestamp': -2e6 }, { 'token': '2', 'timestamp': -3e6 }, { 'token': '1', 'timestamp': -4e6 }] # Testing we can get the exact amount of past seconds available nusc = MockNuScenes(self.mock_annotations, mock_samples) helper = PredictHelper(nusc) past = helper.get_past_for_agent('1', '5', 3, False) np.testing.assert_equal(past, np.array([[3, 3], [2, 2], [1, 1]]))