def run(self, out_dir): if self.dataset_version.find("mini") != -1: train_agents = get_prediction_challenge_split( "mini_train", dataroot=self.DATAROOT) val_agents = get_prediction_challenge_split("mini_val", dataroot=self.DATAROOT) else: train_agents = get_prediction_challenge_split( "train", dataroot=self.DATAROOT) train_agents.extend( get_prediction_challenge_split("train_val", dataroot=self.DATAROOT)) val_agents = get_prediction_challenge_split("val", dataroot=self.DATAROOT) ## Statistics # mx =-1 # for in train_agents: # instance_token, sample_token = current_sample.split("_") # past_samples_local = self.helper.get_past_for_agent(instance_token, sample_token, 100, True, True)[::-1] # if len(past_samples_local) > mx: # mx = len(past_samples_local) # print("max length of the past sequences for trainval is:",mx) # for instance_token, sample_token in train_agents: # past_samples_local = self.helper.get_past_for_agent(instance_token, sample_token, 100, True, True)[::-1] # if len(past_samples_local) > mx: # mx = len(past_samples_local) # print("max length of the past sequence for val is:",mx) # return self.get_format_mha_jam( train_agents, os.path.join(out_dir, "states_train_" + self.dataset_version + ".txt")) self.get_format_mha_jam_context( os.path.join(out_dir, "states_train_" + self.dataset_version + ".txt"), os.path.join(out_dir, "context_train_" + self.dataset_version, "context_train_.txt")) self.get_format_mha_jam_maps( os.path.join(out_dir, "states_train_" + self.dataset_version + ".txt"), os.path.join(out_dir, "maps_train_" + self.dataset_version, "maps_train_.jpg")) # 25 self.get_format_mha_jam( val_agents, os.path.join(out_dir, "states_val_" + self.dataset_version + ".txt")) self.get_format_mha_jam_context( os.path.join(out_dir, "states_val_" + self.dataset_version + ".txt"), os.path.join(out_dir, "context_val_" + self.dataset_version, "context_val_.txt")) self.get_format_mha_jam_maps( os.path.join(out_dir, "states_val_" + self.dataset_version + ".txt"), os.path.join(out_dir, "maps_val_" + self.dataset_version, "maps_val_.jpg"))
def main(args): print("Args:") print(vars(args)) print("Device:") print(device) # prepare output directories if not os.path.exists(args.experiment_dir): os.mkdir(args.experiment_dir) if not os.path.exists(os.path.join(args.experiment_dir, 'weights')): os.mkdir(os.path.join(args.experiment_dir, 'weights')) # store the arguments for reference config_fname = f'config_for_runtime_{RUN_TIME:%Y-%m-%d %Hh%Mm%Ss}.json' with open(os.path.join(args.experiment_dir, config_fname), 'w') as json_file: json.dump(vars(args), json_file) # load data nusc = NuScenes(version=args.version, dataroot=args.data_root) helper = PredictHelper(nusc) train_tokens = get_prediction_challenge_split(args.train_split_name, dataroot=args.data_root) val_tokens = get_prediction_challenge_split(args.val_split_name, dataroot=args.data_root) # apply downsampling train_tokens = np.random.choice( train_tokens, int(len(train_tokens) / args.train_downsample_factor), replace=False) val_tokens = np.random.choice( val_tokens, int(len(val_tokens) / args.val_downsample_factor), replace=False) # create data loaders train_dataset = get_dataset(train_tokens, helper, args) train_dataloader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers, shuffle=True) val_dataset = get_dataset(val_tokens, helper, args) val_dataloader = DataLoader(val_dataset, batch_size=args.batch_size, num_workers=args.num_workers, shuffle=False) # run training train_epochs(train_dataloader=train_dataloader, val_dataloader=val_dataloader, args=args)
def main(version: str, data_root: str, split_name: str, output_dir: str, config_name: str = 'predict_2020_icra.json') -> None: """ Performs inference for all of the baseline models defined in the physics model module. :param version: nuScenes data set version. :param data_root: Directory where the NuScenes data is stored. :param split_name: nuScenes data split name, e.g. train, val, mini_train, etc. :param output_dir: Directory where predictions should be stored. :param config_name: Name of config file. """ nusc = NuScenes(version=version, dataroot=data_root) helper = PredictHelper(nusc) dataset = get_prediction_challenge_split(split_name) config = load_prediction_config(helper, config_name) oracle = PhysicsOracle(config.seconds, helper) cv_heading = ConstantVelocityHeading(config.seconds, helper) cv_preds = [] oracle_preds = [] for token in dataset: cv_preds.append(cv_heading(token).serialize()) oracle_preds.append(oracle(token).serialize()) json.dump(cv_preds, open(os.path.join(output_dir, "cv_preds.json"), "w")) json.dump(oracle_preds, open(os.path.join(output_dir, "oracle_preds.json"), "w"))
def __init__(self): DATAROOT = '/home/patrick/datasets/nuscenes' # This is the path where you stored your copy of the nuScenes dataset. self.nuscenes = NuScenes('v1.0-mini', dataroot=DATAROOT) self.mini_train = get_prediction_challenge_split("mini_train", dataroot=DATAROOT) self.helper = PredictHelper(self.nuscenes) self.physics_oracle = PhysicsOracle(sec_from_now=6, helper=self.helper) self.map_rasterizer = StaticLayerRasterizer(self.helper, meters_ahead=60, meters_behind=10, meters_left=35, meters_right=35) self.agent_rasterizer = FutureAgentBoxesWithFadedHistory( self.helper, meters_ahead=60, meters_behind=10, meters_left=35, meters_right=35) self.json_path = 'manual_results.json' self.annotations = [] if os.path.exists(self.json_path): with open(self.json_path) as json_file: self.annotations = json.load( json_file) # Load existing JSON file
def main(args): print("Running with args:") print(vars(args)) print("Device:") print(device) # load data nusc = NuScenes(version=args.version, dataroot=args.data_root) helper = PredictHelper(nusc) data_tokens = get_prediction_challenge_split(args.split_name, dataroot=args.data_root) if args.key == "covernet": dataset = CoverNetDataset(data_tokens, helper) elif args.key == "mtp": dataset = MTPDataset(data_tokens, helper) dataloader = DataLoader(dataset, batch_size=16, num_workers=0, shuffle=False) print(f"Loaded split {args.split_name}, length {len(dataset)}, in {len(dataloader)} batches.") # prepare model model = get_model(args) model.load_state_dict( torch.load(os.path.join(args.experiment_dir, 'weights', args.weights))) model.eval() predictions = get_predictions(args, dataloader, model) json.dump(predictions, open(os.path.join(args.experiment_dir, f'{args.key}_preds_{datetime.datetime.now():%Y-%m-%d %Hh%Mm%Ss}_{args.suffix}.json'), "w"))
def what_are_the_objects_types_used_in_prediction_tasks( DATAROOT='./data/sets/nuscenes', dataset_version='v1.0-mini'): nuscenes = NuScenes(dataset_version, DATAROOT) train_agents = get_prediction_challenge_split("train", dataroot=DATAROOT) val_agents = get_prediction_challenge_split("val", dataroot=DATAROOT) # # agents = get_prediction_challenge_split("train", dataroot=DATAROOT) agents = np.concatenate((train_agents, val_agents), axis=0) categories = {} token_to_name = {} for current_sample in agents: instance_token, sample_token = current_sample.split("_") category_token = nuscenes.get("instance", instance_token)["category_token"] category_name = nuscenes.get("category", category_token)["name"] categories[category_name] = category_token token_to_name[category_token] = category_name print(categories.items()) print(token_to_name.items())
def run(self, format_for_model): if self.dataset_version.find("mini") != -1: train_agents = get_prediction_challenge_split( "mini_train", dataroot=self.DATAROOT) val_agents = get_prediction_challenge_split("mini_val", dataroot=self.DATAROOT) else: train_agents = get_prediction_challenge_split( "train", dataroot=self.DATAROOT) train_agents.extend( get_prediction_challenge_split("train_val", dataroot=self.DATAROOT)) val_agents = get_prediction_challenge_split("val", dataroot=self.DATAROOT) self.get_new_format( train_agents, format_for_model, "/home/bassel/PycharmProjects/Trajectory-Transformer/datasets/nuscenes/bkup/transformer_train_" + self.dataset_version + ".txt") self.get_new_format( val_agents, format_for_model, "/home/bassel/PycharmProjects/Trajectory-Transformer/datasets/nuscenes/bkup/transformer_val_" + self.dataset_version + ".txt")
def save_cmu_dataset(self, save_dir, partition='all'): from nuscenes.eval.prediction.splits import get_prediction_challenge_split split_types = ['mini_train', 'mini_val', 'train', 'train_val', 'val'] if partition == 'mini': split_types = ['mini_train', 'mini_val'] if not os.path.isdir(save_dir): os.mkdir(save_dir) for split in tqdm(split_types, desc='split dataset'): partition_tokens = get_prediction_challenge_split( split, dataroot=self.root) tokens_dict = {} for token in partition_tokens: instance_token, sample_token = token.split('_') try: tokens_dict[sample_token].append(instance_token) except KeyError: tokens_dict[sample_token] = [instance_token] with open('{}/{}.tokens'.format(save_dir, split), 'wb') as f: pickle.dump(tokens_dict, f, pickle.HIGHEST_PROTOCOL) for sample_tk, instance_tks in tqdm(tokens_dict.items(), desc=split, total=len(tokens_dict)): sample_dir = os.path.join(save_dir, sample_tk) if not os.path.isdir(sample_dir): os.mkdir(sample_dir) scene_data = self.get_cmu_annotation( instance_tks, sample_tk) with open('{}/map.bin'.format(sample_dir), 'wb') as f: pickle.dump(scene_data['episode_img'], f, pickle.HIGHEST_PROTOCOL) with open('{}/viz.bin'.format(sample_dir), 'wb') as f: pickle.dump(scene_data['img_show'], f, pickle.HIGHEST_PROTOCOL) with open('{}/episode.bin'.format(sample_dir), 'wb') as f: pickle.dump(scene_data['episode'], f, pickle.HIGHEST_PROTOCOL) with open('{}/instance_tks.bin'.format(sample_dir), 'wb') as f: pickle.dump(instance_tks, f, pickle.HIGHEST_PROTOCOL) with open('{}/scene_txt.bin'.format(sample_dir), 'wb') as f: pickle.dump(scene_data['scene_txt'], f, pickle.HIGHEST_PROTOCOL)
def __init__(self, nusc, helper, maps_dir, save_maps_dataset=False, config_name='predict_2020_icra.json', history=1, num_examples=None, in_agent_frame=True): self.nusc = nusc self.helper = helper #initialize the data set if maps_dir == 'maps_train': dataset_version = "train" elif maps_dir == 'maps': dataset_version = "train_val" elif maps_dir == 'maps_val': dataset_version = "val" #initialize maps directory where everything will be saved self.maps_dir = os.path.join(os.getcwd(), maps_dir) self.data_set = get_prediction_challenge_split( dataset_version, dataroot=self.nusc.dataroot) if num_examples: self.data_set = self.data_set[:num_examples] #initialize rasterizers for map generation self.static_layer_rasterizer = StaticLayerRasterizer(self.helper) self.agent_rasterizer = AgentBoxesWithFadedHistory( self.helper, seconds_of_history=history) self.mtp_input_representation = InputRepresentation( self.static_layer_rasterizer, self.agent_rasterizer, Rasterizer()) self.in_agent_frame = in_agent_frame self.config = load_prediction_config(self.helper, config_name) self.save_maps_dataset = save_maps_dataset if self.save_maps_dataset: self.save_maps() self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu")
def main(version: str, data_root: str, split_name: str, output_dir: str, submission_name: str, config_name: str) \ -> None: """ Makes predictions for a submission to the nuScenes prediction challenge. :param version: NuScenes version. :param data_root: Directory storing NuScenes data. :param split_name: Data split to run inference on. :param output_dir: Directory to store the output file. :param submission_name: Name of the submission to use for the results file. :param config_name: Name of config file to use. """ nusc = NuScenes(version=version, dataroot=data_root) helper = PredictHelper(nusc) dataset = get_prediction_challenge_split(split_name) config = load_prediction_config(helper, config_name) predictions = do_inference_for_submission(helper, config, dataset) predictions = [prediction.serialize() for prediction in predictions] json.dump(predictions, open(os.path.join(output_dir, f"{submission_name}_inference.json"), "w"))
def main(version: str, data_root: str, split_name: str, output_dir: str, config_name: str = 'predict_2020_icra.json') -> None: """ Performs inference for all of the baseline models defined in the physics model module. :param version: nuScenes data set version. :param data_root: Directory where the NuScenes data is stored. :param split_name: nuScenes data split name, e.g. train, val, mini_train, etc. :param output_dir: Directory where predictions should be stored. :param config_name: Name of config file. """ print('timing point A') nusc = NuScenes(version=version, dataroot=data_root) print('timing point B') helper = PredictHelper(nusc) print('timing point C') dataset = get_prediction_challenge_split(split_name, dataroot=data_root) print('timing point D') config = load_prediction_config(helper, config_name) print('timing point E') # rasterization static_layer_rasterizer = StaticLayerRasterizer(helper) agent_rasterizer = AgentBoxesWithFadedHistory(helper, seconds_of_history=3) mtp_input_representation = InputRepresentation(static_layer_rasterizer, agent_rasterizer, Rasterizer()) # loop through training tasks for token in dataset[40:60:2]: fig, axes = plt.subplots(1, 3, figsize=(18, 9)) print(token) instance_token, sample_token = token.split('_') plot_cam_view(axes[1], nusc, token) plot_cam_view(axes[2], nusc, token, cam_name='CAM_FRONT_RIGHT') axes[0].imshow(mtp_input_representation.make_input_representation(instance_token, sample_token)) plt.show()
def process_data(data_path, version, output_path, half_dt, dynamic): nusc = NuScenes(version=version, dataroot=data_path, verbose=True) helper = PredictHelper(nusc) for data_class in [ 'val', 'train_val', 'train' ]: # Change this to 'mini_train', 'mini_val' if you want to do the mini set. env = Environment(node_type_list=['VEHICLE', 'PEDESTRIAN'], standardization=standardization) attention_radius = dict() attention_radius[(env.NodeType.PEDESTRIAN, env.NodeType.PEDESTRIAN)] = 10.0 attention_radius[(env.NodeType.PEDESTRIAN, env.NodeType.VEHICLE)] = 20.0 attention_radius[(env.NodeType.VEHICLE, env.NodeType.PEDESTRIAN)] = 20.0 attention_radius[(env.NodeType.VEHICLE, env.NodeType.VEHICLE)] = 30.0 env.attention_radius = attention_radius env.robot_type = env.NodeType.VEHICLE scenes = [] instance_sample_tokens = get_prediction_challenge_split( data_class, dataroot=data_path) processed_sample_tokens = set() for instance_sample_token in tqdm(instance_sample_tokens): _, sample_token = instance_sample_token.split("_") if sample_token in processed_sample_tokens: # and data_class != 'val': continue scene = process_scene(sample_token, processed_sample_tokens, env, nusc, helper, data_path, data_class, half_dt, dynamic) if scene is not None: if 'train' == data_class: # Change this to "in" if you want to do the mini set. scene.augmented = list() angles = np.arange(0, 360, 15) for angle in angles: scene.augmented.append(augment_scene(scene, angle)) scenes.append(scene) print(f'Processed {len(scenes)} scenes') env.scenes = scenes doubled_str = '_doubled' if half_dt else '' dynamic_str = '_dynamic' if dynamic else '' if len(scenes) > 0: data_dict_path = os.path.join( output_path, f'nuScenes_{data_class}_full{doubled_str}{dynamic_str}.pkl') with open(data_dict_path, 'wb') as f: dill.dump(env, f, protocol=dill.HIGHEST_PROTOCOL) print('Saved Environment!') global total global curv_0_2 global curv_0_1 print(f"Total Nodes: {total}") print(f"Curvature > 0.1 Nodes: {curv_0_1}") print(f"Curvature > 0.2 Nodes: {curv_0_2}") total = 0 curv_0_1 = 0 curv_0_2 = 0
def __init__(self, dataroot: str, split: str, t_h: float = 2, t_f: float = 6, grid_dim: int = 25, img_size: int = 200, horizon: int = 40, grid_extent: Tuple[int, int, int, int] = (-25, 25, -10, 40), num_actions: int = 4, image_extraction_mode: bool = False): """ Initializes dataset class for nuScenes prediction :param dataroot: Path to tables and data :param split: Dataset split for prediction benchmark ('train'/'train_val'/'val') :param t_h: Track history in seconds :param t_f: Prediction horizon in seconds :param grid_dim: Size of grid, default: 25x25 :param img_size: Size of raster map image in pixels, default: 200x200 :param horizon: MDP horizon :param grid_extent: Map extents in meters, (-left, right, -behind, front) :param num_actions: Number of actions for each state (4: [D,R,U,L] or 8: [D, R, U, L, DR, UR, DL, UL]) :param image_extraction_mode: Whether dataset class is being used for image extraction """ # Nuscenes dataset and predict helper self.dataroot = dataroot self.ns = NuScenes('v1.0-trainval', dataroot=dataroot) self.helper = PredictHelper(self.ns) self.token_list = get_prediction_challenge_split(split, dataroot=dataroot) # Useful parameters self.grid_dim = grid_dim self.grid_extent = grid_extent self.img_size = img_size self.t_f = t_f self.t_h = t_h self.horizon = horizon self.num_actions = num_actions # Map row, column and velocity states to actual values grid_size_m = self.grid_extent[1] - self.grid_extent[0] self.row_centers = np.linspace( self.grid_extent[3] - grid_size_m / (self.grid_dim * 2), self.grid_extent[2] + grid_size_m / (self.grid_dim * 2), self.grid_dim) self.col_centers = np.linspace( self.grid_extent[0] + grid_size_m / (self.grid_dim * 2), self.grid_extent[1] - grid_size_m / (self.grid_dim * 2), self.grid_dim) # Surrounding agent input representation: populate grid with velocity, acc, yaw-rate self.agent_ip = AgentMotionStatesOnGrid(self.helper, resolution=grid_size_m / img_size, meters_ahead=grid_extent[3], meters_behind=-grid_extent[2], meters_left=-grid_extent[0], meters_right=grid_extent[1]) # Image extraction mode is used for extracting map images offline prior to training self.image_extraction_mode = image_extraction_mode if self.image_extraction_mode: # Raster map representation self.map_ip = StaticLayerRasterizer(self.helper, resolution=grid_size_m / img_size, meters_ahead=grid_extent[3], meters_behind=-grid_extent[2], meters_left=-grid_extent[0], meters_right=grid_extent[1]) # Raster map with agent boxes. Only used for visualization static_layer_rasterizer = StaticLayerRasterizer( self.helper, resolution=grid_size_m / img_size, meters_ahead=grid_extent[3], meters_behind=-grid_extent[2], meters_left=-grid_extent[0], meters_right=grid_extent[1]) agent_rasterizer = AgentBoxesWithFadedHistory( self.helper, seconds_of_history=1, resolution=grid_size_m / img_size, meters_ahead=grid_extent[3], meters_behind=-grid_extent[2], meters_left=-grid_extent[0], meters_right=grid_extent[1]) self.map_ip_agents = InputRepresentation(static_layer_rasterizer, agent_rasterizer, Rasterizer())
parser.add_argument('--dataroot', required=True, type=str) parser.add_argument('--split', required=True, type=str) args = parser.parse_args('\ --dataroot /home/vilab/data/Nuscenes/v1.0-trainval_meta \ --split train_val'.split()) # args = parser.parse_args() DATAPATH = os.path.join('./dataset', args.split) dpathlist = ['image', 'state', 'traj'] nuscenes = NuScenes('v1.0-trainval', dataroot=args.dataroot) FUTURE_SEC = 6 inst_samp_pair_list = get_prediction_challenge_split(args.split, dataroot=args.dataroot) print(f"size of data : {len(inst_samp_pair_list):d}") helper = PredictHelper(nuscenes) dpathlist_ = [] for dlist in dpathlist: dpath = os.path.join(DATAPATH, dlist) dpathlist_.append(dpath) if not os.path.exists(dpath): os.makedirs(dpath) static_layer_rasterizer = StaticLayerRasterizer(helper) agent_rasterizer = AgentBoxesWithFadedHistory(helper, seconds_of_history=1) mtp_input_representation = InputRepresentation(static_layer_rasterizer, agent_rasterizer, Rasterizer())