def test_compute_mse_error(tmp_path: Path, zarr_dataset: ChunkedDataset, cfg: dict) -> None: render_context = RenderContext( np.asarray((10, 10)), np.asarray((0.25, 0.25)), np.asarray((0.5, 0.5)), set_origin_to_bottom=cfg["raster_params"]["set_origin_to_bottom"], ) rast = StubRasterizer(render_context) dataset = AgentDataset(cfg, zarr_dataset, rast) gt_coords = [] gt_avails = [] timestamps = [] track_ids = [] for idx, el in enumerate(dataset): # type: ignore gt_coords.append(el["target_positions"]) gt_avails.append(el["target_availabilities"]) timestamps.append(el["timestamp"]) track_ids.append(el["track_id"]) if idx == 100: break # speed up test gt_coords = np.asarray(gt_coords) gt_avails = np.asarray(gt_avails) timestamps = np.asarray(timestamps) track_ids = np.asarray(track_ids) # test same values error write_gt_csv(str(tmp_path / "gt1.csv"), timestamps, track_ids, gt_coords, gt_avails) write_pred_csv(str(tmp_path / "pred1.csv"), timestamps, track_ids, gt_coords, confs=None) metrics = compute_metrics_csv(str(tmp_path / "gt1.csv"), str(tmp_path / "pred1.csv"), [neg_multi_log_likelihood]) for metric_value in metrics.values(): assert np.all(metric_value == 0.0) # test different values error pred_coords = gt_coords.copy() pred_coords += np.random.randn(*pred_coords.shape) write_pred_csv(str(tmp_path / "pred3.csv"), timestamps, track_ids, pred_coords, confs=None) metrics = compute_metrics_csv(str(tmp_path / "gt1.csv"), str(tmp_path / "pred3.csv"), [neg_multi_log_likelihood]) for metric_value in metrics.values(): assert np.any(metric_value > 0.0) # test invalid conf by removing lines in gt1 with open(str(tmp_path / "pred4.csv"), "w") as fp: lines = open(str(tmp_path / "pred1.csv")).readlines() fp.writelines(lines[:-10]) with pytest.raises(ValueError): compute_metrics_csv(str(tmp_path / "gt1.csv"), str(tmp_path / "pred4.csv"), [neg_multi_log_likelihood])
def validation_epoch_end(self, outputs): timestamps = np.concatenate(self.timestamps_list) track_ids = np.concatenate(self.track_id_list) coords = np.concatenate(self.pred_coords_list) confs = np.concatenate(self.confidences_list) self.timestamps_list = [] self.track_id_list = [] self.pred_coords_list = [] self.confidences_list = [] csv_path = "val.csv" write_pred_csv(csv_path, timestamps=timestamps, track_ids=track_ids, coords=coords, confs=confs) try: metrics = compute_metrics_csv(eval_gt_path, csv_path, [neg_multi_log_likelihood]) target_metric = 0 for metric_name, metric_mean in metrics.items(): target_metric = metric_mean break except: target_metric = 1000 print('got target metric', target_metric) tensorboard_logs = {'val/_metric': target_metric} return {'avg_val_loss': target_metric, 'log': tensorboard_logs}
def validation(model, device): model.eval() torch.set_grad_enabled(False) # store information for evaluation future_coords_offsets_pd = [] timestamps = [] agent_ids = [] confidences_list = [] val_dataloader = load_val_data() # num_iter = iter(val_dataloader) # progress_bar = tqdm(range(cfg["train_params"]["val_num_steps"])) progress_bar = tqdm(val_dataloader) for data in progress_bar: # data = next(num_iter) preds, confs = forward(data, model, device) # convert agent coordinates into world offsets preds = preds.cpu().numpy() confs = confs.cpu().numpy() world_from_agents = data["world_from_agent"].numpy() centroids = data["centroid"].numpy() coords_offset = [] for pred, world_from_agent, centroid in zip(preds, world_from_agents, centroids): for mode in range(3): pred[mode] = transform_points(pred[mode], world_from_agent) - centroid[:2] coords_offset.append(pred) confidences_list.append(confs) future_coords_offsets_pd.append(np.stack(coords_offset)) timestamps.append(data["timestamp"].numpy().copy()) agent_ids.append(data["track_id"].numpy().copy()) pred_path = f"{gettempdir()}/pred.csv" write_pred_csv(pred_path, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd), confs=np.concatenate(confidences_list)) eval_base_path = '/home/axot/lyft/data/scenes/validate_chopped_31' eval_gt_path = str(Path(eval_base_path) / "gt.csv") metrics = compute_metrics_csv(eval_gt_path, pred_path, [neg_multi_log_likelihood, time_displace]) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean) return metrics['neg_multi_log_likelihood']
def model_validation_score(model, pred_path): # ==== EVAL LOOP model.eval() torch.set_grad_enabled(False) # store information for evaluation future_coords_offsets_pd = [] timestamps = [] confidences_list = [] agent_ids = [] progress_bar = tqdm(eval_dataloader) for data in progress_bar: _, preds, confidences = forward(data, model, device) #fix for the new environment preds = preds.cpu().numpy() world_from_agents = data["world_from_agent"].numpy() centroids = data["centroid"].numpy() coords_offset = [] # convert into world coordinates and compute offsets for idx in range(len(preds)): for mode in range(3): preds[idx, mode, :, :] = transform_points( preds[idx, mode, :, :], world_from_agents[idx]) - centroids[idx][:2] future_coords_offsets_pd.append(preds.copy()) confidences_list.append(confidences.cpu().numpy().copy()) timestamps.append(data["timestamp"].numpy().copy()) agent_ids.append(data["track_id"].numpy().copy()) write_pred_csv( pred_path, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd), confs=np.concatenate(confidences_list), ) metrics = compute_metrics_csv(eval_gt_path, pred_path, [neg_multi_log_likelihood, time_displace]) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean)
def eval(model, eval_dataloader, eval_dataset_ego, eval_dataset, criterion, device, eval_gt_path, cfg): # ===== EVAL Loop model.eval() torch.set_grad_enabled(False) # store information for evaluation future_coords_offsets_pd = [] timestamps = [] confidences_list = [] agent_ids = [] progress_bar = tqdm(eval_dataloader) for data, indices in progress_bar: loss, preds, confidences = mode.run(data, indices, model, eval_dataset_ego, eval_dataset, criterion, device = device) #fix for the new environment preds = preds.cpu().numpy() world_from_agents = data["world_from_agent"].numpy() centroids = data["centroid"].numpy() coords_offset = [] # convert into world coordinates and compute offsets for idx in range(len(preds)): for mode in range(cfg["model_params"]["num_trajectories"]): preds[idx,mode,:, :] = transform_points(preds[idx,mode,:, :], world_from_agents[idx]) - centroids[idx][:2] future_coords_offsets_pd.append(preds.copy()) confidences_list.append(confidences.cpu().numpy().copy()) timestamps.append(data["timestamp"].numpy().copy()) agent_ids.append(data["track_id"].numpy().copy()) pred_path = f"{gettempdir()}/pred.csv" write_pred_csv(pred_path, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd), confs = np.concatenate(confidences_list) ) metrics = compute_metrics_csv(eval_gt_path, pred_path, [neg_multi_log_likelihood, time_displace]) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean)
def valid_fn(model, loader, device, ground_truth_file, logdir, verbose=True): """Validation step. Args: model (nn.Module): model to train loader (DataLoader): loader with data device (str or torch.device): device to use for placing batches loss_fn (nn.Module): loss function, should be callable verbose (bool, optional): verbosity mode. Default is True. Returns: dict with metics computed during the validation on loader """ model.eval() future_coords_offsets_pd = [] timestamps = [] confidences_list = [] agent_ids = [] with torch.no_grad(), tqdm( total=len(loader), desc="valid", disable=not verbose ) as progress: for idx, batch in enumerate(loader): images, acceleration = t2d( [batch["image"], batch["xy_acceleration"]], device ) predictions, confidences = model(images, acceleration) _gt = batch["target_positions"].cpu().numpy().copy() predictions = predictions.cpu().numpy().copy() world_from_agents = batch["world_from_agent"].numpy() centroids = batch["centroid"].numpy() for idx in range(len(predictions)): for mode in range(3): # FIX predictions[idx, mode, :, :] = ( transform_points( predictions[idx, mode, :, :], world_from_agents[idx] ) - centroids[idx][:2] ) _gt[idx, :, :] = ( transform_points(_gt[idx, :, :], world_from_agents[idx]) - centroids[idx][:2] ) future_coords_offsets_pd.append(predictions.copy()) confidences_list.append(confidences.cpu().numpy().copy()) timestamps.append(batch["timestamp"].numpy().copy()) agent_ids.append(batch["track_id"].numpy().copy()) progress.update(1) if idx == DEBUG: break predictions_file = str(logdir / "preds_validate_chopped.csv") write_pred_csv( predictions_file, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd), confs=np.concatenate(confidences_list), ) metrics = compute_metrics_csv( ground_truth_file, predictions_file, [neg_multi_log_likelihood], ) return {"score": metrics["neg_multi_log_likelihood"]}
criterion = nn.MSELoss(reduction="none") torch.set_grad_enabled(False) future_coords_offsets_pd = [] timestamps = [] agent_ids = [] progress_bar = tqdm(eval_dataloader) for data in progress_bar: _, ouputs = forward(data, model, device, criterion) future_coords_offsets_pd.append(ouputs.cpu().numpy().copy()) timestamps.append(data["timestamp"].numpy().copy()) agent_ids.append(data["track_id"].numpy().copy()) pred_path = "pred.csv" write_pred_csv( pred_path, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd), ) metrics = compute_metrics_csv(eval_gt_path, pred_path, [neg_multi_log_likelihood, time_displace]) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean)
future_coords_offsets_pd.append(preds.copy()) confidences_list.append(confidences.cpu().numpy().copy()) timestamps.append(batch["timestamp"].numpy().copy()) agent_ids.append(batch["track_id"].numpy().copy()) progress.update(1) write_pred_csv(val_predictions_file, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd), confs=np.concatenate(confidences_list)) metrics = compute_metrics_csv( f"{DATA_DIR}/scenes/validate_chopped_100/gt.csv", val_predictions_file, [neg_multi_log_likelihood, time_displace], ) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean) #====== INIT TEST DATASET============================================================= rasterizer = build_rasterizer(cfg, dm) test_zarr = ChunkedDataset(dm.require("scenes/test.zarr")).open() test_mask = np.load(f"{DATA_DIR}/scenes/mask.npz")["arr_0"] test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask) test_dataloader = DataLoader(test_dataset, shuffle=False, batch_size=32, num_workers=30)
def evaluate(cfg, model, dm, rasterizer, first_time, iters, eval_dataloader, eval_gt_path): if first_time: num_frames_to_chop = 100 print("min_future_steps: ",MIN_FUTURE_STEPS) eval_cfg = cfg["val_data_loader"] eval_base_path = create_chopped_dataset(dm.require(eval_cfg["key"]), cfg["raster_params"]["filter_agents_threshold"], num_frames_to_chop, cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS) eval_zarr_path = str(Path(eval_base_path) / Path(dm.require(eval_cfg["key"])).name) eval_mask_path = str(Path(eval_base_path) / "mask.npz") eval_gt_path = str(Path(eval_base_path) / "gt.csv") eval_zarr = ChunkedDataset(eval_zarr_path).open() eval_mask = np.load(eval_mask_path)["arr_0"] eval_dataset = AgentDataset(cfg, eval_zarr, rasterizer, agents_mask=eval_mask) eval_dataloader = DataLoader(eval_dataset, shuffle=eval_cfg["shuffle"], batch_size=eval_cfg["batch_size"], num_workers=eval_cfg["num_workers"]) print(eval_dataset) first_time = False model.eval() torch.set_grad_enabled(False) future_coords_offsets_pd = [] timestamps = [] confidences_list = [] agent_ids = [] progress_bar = tqdm(eval_dataloader) for data in progress_bar: _, preds, confidences = forward(data, model) # convert agent coordinates into world offsets preds = preds.cpu().numpy() world_from_agents = data["world_from_agent"].numpy() centroids = data["centroid"].numpy() coords_offset = [] for idx in range(len(preds)): for mode in range(3): preds[idx, mode, :, :] = transform_points(preds[idx, mode, :, :], world_from_agents[idx]) - centroids[idx][:2] future_coords_offsets_pd.append(preds.copy()) confidences_list.append(confidences.cpu().numpy().copy()) timestamps.append(data["timestamp"].numpy().copy()) agent_ids.append(data["track_id"].numpy().copy()) model.train() torch.set_grad_enabled(True) pred_path = os.path.join(cfg["save_path"],f"pred_{iters}.csv") write_pred_csv(pred_path, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd), confs = np.concatenate(confidences_list) ) metrics = compute_metrics_csv(eval_gt_path, pred_path, [neg_multi_log_likelihood, time_displace]) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean) return first_time, eval_dataloader, eval_gt_path
def main(cfg: dict, args: argparse.Namespace) -> None: # set random seeds SEED = 42 os.environ["PYTHONHASHSEED"] = str(SEED) # set Python random seed random.seed(SEED) # set NumPy random seed np.random.seed(SEED) os.environ["CUDA_VISIBLE_DEVICES"] = args.visible_gpus # ===== Configure LYFT dataset # mypy error due to pl.DataModule.transfer_batch_to_device mpred_dm = LyftMpredDatamodule( # type: ignore[abstract] args.l5kit_data_folder, cfg, batch_size=args.batch_size, num_workers=args.num_workers, downsample_train=args.downsample_train, is_test=args.is_test, is_debug=args.is_debug, ) mpred_dm.prepare_data() mpred_dm.setup() if args.is_test: print("\t\t ==== TEST MODE ====") print("load from: ", args.ckpt_path) model = LitModel.load_from_checkpoint(args.ckpt_path, cfg=cfg) trainer = pl.Trainer(gpus=len(args.visible_gpus.split(","))) trainer.test(model, datamodule=mpred_dm) test_gt_path = os.path.join(os.path.dirname(mpred_dm.test_path), "gt.csv") if os.path.exists(test_gt_path): print( "test mode with validation chopped dataset, and check the metrics" ) print("validation ground truth path: ", test_gt_path) metrics = compute_metrics_csv( test_gt_path, CSV_PATH, [neg_multi_log_likelihood, time_displace]) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean) else: print("\t\t ==== TRAIN MODE ====") print("training samples: {}, valid samples: {}".format( len(mpred_dm.train_dataset), len(mpred_dm.val_dataset))) total_steps = args.epochs * len( mpred_dm.train_dataset) // args.batch_size val_check_interval = VAL_INTERVAL_SAMPLES // args.batch_size model = LitModel( cfg, lr=args.lr, backbone_name=args.backbone_name, num_modes=args.num_modes, optim_name=args.optim_name, ba_size=args.batch_size, epochs=args.epochs, data_size=len(mpred_dm.train_dataset), total_steps=total_steps, ) checkpoint_callback = pl.callbacks.ModelCheckpoint( monitor="val_loss", save_last=True, mode="min", verbose=True, ) pl.trainer.seed_everything(seed=SEED) trainer = pl.Trainer( gpus=len(args.visible_gpus.split(",")), max_steps=total_steps, val_check_interval=val_check_interval, precision=args.precision, benchmark=True, deterministic=False, checkpoint_callback=checkpoint_callback, ) # Run lr finder if args.find_lr: lr_finder = trainer.tuner.lr_find(model, datamodule=mpred_dm, num_training=500) lr_finder.plot(suggest=True) plt.show() sys.exit() # Run Training trainer.fit(model, datamodule=mpred_dm)
def evaluate(model, device, data_path): # set env variable for data os.environ["L5KIT_DATA_FOLDER"] = data_path dm = LocalDataManager(None) cfg = model.cfg # ===== INIT DATASET test_cfg = cfg["test_data_loader"] # Rasterizer rasterizer = build_rasterizer(cfg, dm) # Test dataset/dataloader test_zarr = ChunkedDataset(dm.require(test_cfg["key"])).open() test_mask = np.load(f"{data_path}/scenes/mask.npz")["arr_0"] test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask) test_dataloader = DataLoader(test_dataset, shuffle=test_cfg["shuffle"], batch_size=test_cfg["batch_size"], num_workers=test_cfg["num_workers"]) test_dataloader = test_dataloader print(test_dataloader) # ==== EVAL LOOP model.eval() torch.set_grad_enabled(False) criterion = nn.MSELoss(reduction="none") # store information for evaluation future_coords_offsets_pd = [] timestamps = [] agent_ids = [] progress_bar = tqdm(test_dataloader) for data in progress_bar: _, outputs, _ = model.forward(data, device, criterion) future_coords_offsets_pd.append(outputs.cpu().numpy().copy()) timestamps.append(data["timestamp"].numpy().copy()) agent_ids.append(data["track_id"].numpy().copy()) # ==== Save Results pred_path = "./submission.csv" write_pred_csv(pred_path, timestamps=np.concatenate(timestamps), track_ids=np.concatenate(agent_ids), coords=np.concatenate(future_coords_offsets_pd)) # ===== GENERATE AND LOAD CHOPPED DATASET num_frames_to_chop = 56 test_cfg = cfg["test_data_loader"] test_base_path = create_chopped_dataset( zarr_path=dm.require(test_cfg["key"]), th_agent_prob=cfg["raster_params"]["filter_agents_threshold"], num_frames_to_copy=num_frames_to_chop, num_frames_gt=cfg["model_params"]["future_num_frames"], min_frame_future=MIN_FUTURE_STEPS) eval_zarr_path = str( Path(test_base_path) / Path(dm.require(test_cfg["key"])).name) print(eval_zarr_path) test_mask_path = str(Path(test_base_path) / "mask.npz") test_gt_path = str(Path(test_base_path) / "gt.csv") test_zarr = ChunkedDataset(eval_zarr_path).open() test_mask = np.load(test_mask_path)["arr_0"] # ===== INIT DATASET AND LOAD MASK test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask) test_dataloader = DataLoader(test_dataset, shuffle=test_cfg["shuffle"], batch_size=test_cfg["batch_size"], num_workers=test_cfg["num_workers"]) print(test_dataset) # ==== Perform Evaluation print(test_gt_path) metrics = compute_metrics_csv(test_gt_path, pred_path, [neg_multi_log_likelihood, time_displace]) for metric_name, metric_mean in metrics.items(): print(metric_name, metric_mean)