def val_dataloader(self): # created chopped dataset rasterizer = build_rasterizer(cfg, dm) eval_cfg = cfg["valid_data_loader"] num_frames_to_chop = 100 eval_base_path = create_chopped_dataset( dm.require(eval_cfg["key"]), cfg["raster_params"]["filter_agents_threshold"], num_frames_to_chop, cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS) eval_zarr_path = str( Path(eval_base_path) / Path(dm.require(eval_cfg["key"])).name) eval_mask_path = str(Path(eval_base_path) / "mask.npz") eval_gt_path = str(Path(eval_base_path) / "gt.csv") self.eval_gt_path = eval_gt_path eval_zarr = ChunkedDataset(eval_zarr_path).open(cache_size_bytes=10e9) eval_mask = np.load(eval_mask_path)["arr_0"] eval_dataset = AgentDataset(cfg, eval_zarr, rasterizer, agents_mask=eval_mask) eval_dataloader = DataLoader(eval_dataset, shuffle=False, batch_size=eval_cfg["batch_size"], num_workers=8) return eval_dataloader
def build_chopped_dataset(): dm = get_dm() eval_cfg = cfg["val_data_loader"] MIN_FUTURE_STEPS = 10 num_frames_to_chop = [80, 130, 180] for chop_frame in num_frames_to_chop: eval_base_path = create_chopped_dataset( dm.require(eval_cfg["key"]), cfg["raster_params"]["filter_agents_threshold"], chop_frame, cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS) print(eval_base_path)
def main(): dm = LocalDataManager(None) rasterizer = build_rasterizer(cfg, dm) num_frames_to_chop = 10 eval_base_path = create_chopped_dataset( dm.require("scenes/validate.zarr"), cfg["raster_params"]["filter_agents_threshold"], num_frames_to_chop, cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS, ) print("Path:", eval_base_path)
def generate_eval_dataset(cfg, dm, rasterizer): eval_cfg = cfg["test_data_loader"] eval_dir = shutil.copytree(dm.require(eval_cfg["key"]), '/tmp/lyft/test.zarr') eval_cfg = cfg["test_data_loader"] num_frames_to_chop = 50 eval_base_path = create_chopped_dataset(eval_dir, cfg["raster_params"]["filter_agents_threshold"], num_frames_to_chop, cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS) eval_zarr_path = str(Path(eval_base_path) / "test.zarr") eval_mask_path = str(Path(eval_base_path) / "mask.npz") eval_gt_path = str(Path(eval_base_path) / "gt.csv") eval_zarr = ChunkedDataset(eval_zarr_path).open() eval_mask = np.load(eval_mask_path)["arr_0"] # ===== INIT DATASET AND LOAD MASK eval_dataset = AgentDataset(cfg, eval_zarr, rasterizer, agents_mask=eval_mask) eval_dataloader = DataLoader(eval_dataset, shuffle=eval_cfg["shuffle"], batch_size=eval_cfg["batch_size"], num_workers=eval_cfg["num_workers"]) eval_dataset_ego = EgoDataset(cfg, eval_zarr, rasterizer) return eval_dataset, eval_dataloader, eval_dataset_ego, eval_gt_path
train_dataset, np.arange(cfg["train_data_loader"]["batch_size"] * 40)) train_loader = DataLoader(train_dataset, shuffle=train_cfg["shuffle"], batch_size=train_cfg["batch_size"], num_workers=train_cfg["num_workers"]) print(train_agent_dataset) # GENERATE AND LOAD CHOPPED DATASET print('=' * 10 + 'Loading Validation' + '=' * 10) valid_cfg = cfg["valid_data_loader"] valid_path = "scenes/sample.zarr" if debug else valid_cfg["key"] num_frames_to_chop = 100 MIN_FUTURE_STEPS = 10 valid_base_path = create_chopped_dataset( dm.require(valid_path), cfg["raster_params"]["filter_agents_threshold"], num_frames_to_chop, cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS) valid_zarr_path = str( Path(valid_base_path) / Path(dm.require(valid_path)).name) valid_mask_path = str(Path(valid_base_path) / "mask.npz") valid_gt_path = str(Path(valid_base_path) / "gt.csv") valid_zarr = ChunkedDataset(valid_zarr_path).open() valid_mask = np.load(valid_mask_path)["arr_0"] # ===== INIT DATASET AND LOAD MASK valid_agent_dataset = AgentDataset(cfg, valid_zarr, rasterizer, agents_mask=valid_mask) valid_dataset = TransformDataset(valid_agent_dataset, transform) valid_loader = DataLoader(valid_dataset, shuffle=valid_cfg["shuffle"],
# Load data and model lyft_data_module = models.LyftDataModule(data_path, config_path) model = models.resnet_baseline.load_from_checkpoint( "/home/elias.nehme1/Documents/lyft-motion-prediction/epoch=0-v1.ckpt") # Generating a chopped dataset generate_chopped_dataset = False if generate_chopped_dataset: num_frames_to_chop = 100 eval_base_path = create_chopped_dataset( lyft_data_module.dm.require(lyft_data_module.val_cfg["key"]), lyft_data_module.cfg["raster_params"]["filter_agents_threshold"], num_frames_to_chop, lyft_data_module.cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS, ) eval_base_path = "/home/elias.nehme1/Documents/lyft-motion-prediction/data/scenes/validate_chopped_100" val_zarr_path = str( Path(eval_base_path) / Path(lyft_data_module.dm.require(lyft_data_module.val_cfg["key"])).name) val_mask_path = str(Path(eval_base_path) / "mask.npz") val_gt_path = str(Path(eval_base_path) / "gt.csv") val_zarr = ChunkedDataset(val_zarr_path).open() val_mask = np.load(val_mask_path)["arr_0"]
data = next(tr_it) except StopIteration: tr_it = iter(evaluation_dataloader) data = next(tr_it) model.eval() torch.set_grad_enabled(False) validate_loss, _ = forward(data, model, device, criterion) losses_evaluation.append(validate_loss.item()) progress_bar.set_description(f"loss: {validate_loss.item()} loss(avg): {np.mean(losses_evaluation)}") # ===== GENERATE AND LOAD CHOPPED DATASET num_frames_to_chop = 50 eval_cfg = training_cfg["val_data_loader"] eval_base_path = create_chopped_dataset(dm.require(eval_cfg["key"]), training_cfg["raster_params"]["filter_agents_threshold"], num_frames_to_chop, training_cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS) eval_zarr_path = str(Path(eval_base_path) / Path(dm.require(eval_cfg["key"])).name) eval_mask_path = str(Path(eval_base_path) / "mask.npz") eval_gt_path = str(Path(eval_base_path) / "gt.csv") eval_zarr = ChunkedDataset(eval_zarr_path).open() eval_mask = np.load(eval_mask_path)["arr_0"] # ===== INIT DATASET AND LOAD MASK eval_dataset = AgentDataset(training_cfg, eval_zarr, rasterizer, agents_mask=eval_mask) eval_dataloader = DataLoader(eval_dataset, shuffle=eval_cfg["shuffle"], batch_size=eval_cfg["batch_size"], num_workers=eval_cfg["num_workers"]) model.eval() torch.set_grad_enabled(False)
def evaluate(cfg, model, dm, rasterizer, first_time, iters, eval_dataloader, eval_gt_path): if first_time: num_frames_to_chop = 100 print("min_future_steps: ",MIN_FUTURE_STEPS) eval_cfg = cfg["val_data_loader"] eval_base_path = create_chopped_dataset(dm.require(eval_cfg["key"]), cfg["raster_params"]["filter_agents_threshold"], num_frames_to_chop, cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS) eval_zarr_path = str(Path(eval_base_path) / Path(dm.require(eval_cfg["key"])).name) eval_mask_path = str(Path(eval_base_path) / "mask.npz") eval_gt_path = str(Path(eval_base_path) / "gt.csv") eval_zarr = ChunkedDataset(eval_zarr_path).open() eval_mask = np.load(eval_mask_path)["arr_0"] eval_dataset = AgentDataset(cfg, eval_zarr, rasterizer, agents_mask=eval_mask) eval_dataloader = DataLoader(eval_dataset, shuffle=eval_cfg["shuffle"], batch_size=eval_cfg["batch_size"], num_workers=eval_cfg["num_workers"]) print(eval_dataset) first_time = False model.eval() torch.set_grad_enabled(False) future_coords_offsets_pd = [] timestamps = [] confidences_list = [] agent_ids = [] progress_bar = tqdm(eval_dataloader) for data in progress_bar: _, preds, confidences = forward(data, model) # convert agent coordinates into world offsets preds = preds.cpu().numpy() world_from_agents = data["world_from_agent"].numpy() centroids = data["centroid"].numpy() coords_offset = [] for idx in range(len(preds)): for mode in range(3): preds[idx, mode, :, :] = transform_points(preds[idx, mode, :, :], world_from_agents[idx]) - centroids[idx][:2] future_coords_offsets_pd.append(preds.copy()) confidences_list.append(confidences.cpu().numpy().copy()) timestamps.append(data["timestamp"].numpy().copy()) agent_ids.append(data["track_id"].numpy().copy()) model.train() torch.set_grad_enabled(True) pred_path = os.path.join(cfg["save_path"],f"pred_{iters}.csv") write_pred_csv(pred_path, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd), confs = np.concatenate(confidences_list) ) metrics = compute_metrics_csv(eval_gt_path, pred_path, [neg_multi_log_likelihood, time_displace]) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean) return first_time, eval_dataloader, eval_gt_path
def evaluate(model, device, data_path): # set env variable for data os.environ["L5KIT_DATA_FOLDER"] = data_path dm = LocalDataManager(None) cfg = model.cfg # ===== INIT DATASET test_cfg = cfg["test_data_loader"] # Rasterizer rasterizer = build_rasterizer(cfg, dm) # Test dataset/dataloader test_zarr = ChunkedDataset(dm.require(test_cfg["key"])).open() test_mask = np.load(f"{data_path}/scenes/mask.npz")["arr_0"] test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask) test_dataloader = DataLoader(test_dataset, shuffle=test_cfg["shuffle"], batch_size=test_cfg["batch_size"], num_workers=test_cfg["num_workers"]) test_dataloader = test_dataloader print(test_dataloader) # ==== EVAL LOOP model.eval() torch.set_grad_enabled(False) criterion = nn.MSELoss(reduction="none") # store information for evaluation future_coords_offsets_pd = [] timestamps = [] agent_ids = [] progress_bar = tqdm(test_dataloader) for data in progress_bar: _, outputs, _ = model.forward(data, device, criterion) future_coords_offsets_pd.append(outputs.cpu().numpy().copy()) timestamps.append(data["timestamp"].numpy().copy()) agent_ids.append(data["track_id"].numpy().copy()) # ==== Save Results pred_path = "./submission.csv" write_pred_csv(pred_path, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd)) # ===== GENERATE AND LOAD CHOPPED DATASET num_frames_to_chop = 56 test_cfg = cfg["test_data_loader"] test_base_path = create_chopped_dataset( zarr_path=dm.require(test_cfg["key"]), th_agent_prob=cfg["raster_params"]["filter_agents_threshold"], num_frames_to_copy=num_frames_to_chop, num_frames_gt=cfg["model_params"]["future_num_frames"], min_frame_future=MIN_FUTURE_STEPS) eval_zarr_path = str( Path(test_base_path) / Path(dm.require(test_cfg["key"])).name) print(eval_zarr_path) test_mask_path = str(Path(test_base_path) / "mask.npz") test_gt_path = str(Path(test_base_path) / "gt.csv") test_zarr = ChunkedDataset(eval_zarr_path).open() test_mask = np.load(test_mask_path)["arr_0"] # ===== INIT DATASET AND LOAD MASK test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask) test_dataloader = DataLoader(test_dataset, shuffle=test_cfg["shuffle"], batch_size=test_cfg["batch_size"], num_workers=test_cfg["num_workers"]) print(test_dataset) # ==== Perform Evaluation print(test_gt_path) metrics = compute_metrics_csv(test_gt_path, pred_path, [neg_multi_log_likelihood, time_displace]) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean)