def test_compute_mse_error(tmp_path: Path, zarr_dataset: ChunkedDataset, cfg: dict) -> None:
render_context = RenderContext(
np.asarray((10, 10)),
np.asarray((0.25, 0.25)),
np.asarray((0.5, 0.5)),
set_origin_to_bottom=cfg["raster_params"]["set_origin_to_bottom"],
)
rast = StubRasterizer(render_context)
dataset = AgentDataset(cfg, zarr_dataset, rast)
gt_coords = []
gt_avails = []
timestamps = []
track_ids = []
for idx, el in enumerate(dataset): # type: ignore
gt_coords.append(el["target_positions"])
gt_avails.append(el["target_availabilities"])
timestamps.append(el["timestamp"])
track_ids.append(el["track_id"])
if idx == 100:
break # speed up test
gt_coords = np.asarray(gt_coords)
gt_avails = np.asarray(gt_avails)
timestamps = np.asarray(timestamps)
track_ids = np.asarray(track_ids)
# test same values error
write_gt_csv(str(tmp_path / "gt1.csv"), timestamps, track_ids, gt_coords, gt_avails)
write_pred_csv(str(tmp_path / "pred1.csv"), timestamps, track_ids, gt_coords, confs=None)
metrics = compute_metrics_csv(str(tmp_path / "gt1.csv"), str(tmp_path / "pred1.csv"), [neg_multi_log_likelihood])
for metric_value in metrics.values():
assert np.all(metric_value == 0.0)
# test different values error
pred_coords = gt_coords.copy()
pred_coords += np.random.randn(*pred_coords.shape)
write_pred_csv(str(tmp_path / "pred3.csv"), timestamps, track_ids, pred_coords, confs=None)
metrics = compute_metrics_csv(str(tmp_path / "gt1.csv"), str(tmp_path / "pred3.csv"), [neg_multi_log_likelihood])
for metric_value in metrics.values():
assert np.any(metric_value > 0.0)
# test invalid conf by removing lines in gt1
with open(str(tmp_path / "pred4.csv"), "w") as fp:
lines = open(str(tmp_path / "pred1.csv")).readlines()
fp.writelines(lines[:-10])
with pytest.raises(ValueError):
compute_metrics_csv(str(tmp_path / "gt1.csv"), str(tmp_path / "pred4.csv"), [neg_multi_log_likelihood])
def validation_epoch_end(self, outputs):
timestamps = np.concatenate(self.timestamps_list)
track_ids = np.concatenate(self.track_id_list)
coords = np.concatenate(self.pred_coords_list)
confs = np.concatenate(self.confidences_list)
self.timestamps_list = []
self.track_id_list = []
self.pred_coords_list = []
self.confidences_list = []
csv_path = "val.csv"
write_pred_csv(csv_path,
timestamps=timestamps,
track_ids=track_ids,
coords=coords,
confs=confs)
try:
metrics = compute_metrics_csv(eval_gt_path, csv_path,
[neg_multi_log_likelihood])
target_metric = 0
for metric_name, metric_mean in metrics.items():
target_metric = metric_mean
break
except:
target_metric = 1000
print('got target metric', target_metric)
tensorboard_logs = {'val/_metric': target_metric}
return {'avg_val_loss': target_metric, 'log': tensorboard_logs}
def validation(model, device):
model.eval()
torch.set_grad_enabled(False)
# store information for evaluation
future_coords_offsets_pd = []
timestamps = []
agent_ids = []
confidences_list = []
val_dataloader = load_val_data()
# num_iter = iter(val_dataloader)
# progress_bar = tqdm(range(cfg["train_params"]["val_num_steps"]))
progress_bar = tqdm(val_dataloader)
for data in progress_bar:
# data = next(num_iter)
preds, confs = forward(data, model, device)
# convert agent coordinates into world offsets
preds = preds.cpu().numpy()
confs = confs.cpu().numpy()
world_from_agents = data["world_from_agent"].numpy()
centroids = data["centroid"].numpy()
coords_offset = []
for pred, world_from_agent, centroid in zip(preds, world_from_agents,
centroids):
for mode in range(3):
pred[mode] = transform_points(pred[mode],
world_from_agent) - centroid[:2]
coords_offset.append(pred)
confidences_list.append(confs)
future_coords_offsets_pd.append(np.stack(coords_offset))
timestamps.append(data["timestamp"].numpy().copy())
agent_ids.append(data["track_id"].numpy().copy())
pred_path = f"{gettempdir()}/pred.csv"
write_pred_csv(pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs=np.concatenate(confidences_list))
eval_base_path = '/home/axot/lyft/data/scenes/validate_chopped_31'
eval_gt_path = str(Path(eval_base_path) / "gt.csv")
metrics = compute_metrics_csv(eval_gt_path, pred_path,
[neg_multi_log_likelihood, time_displace])
for metric_name, metric_mean in metrics.items():
print(metric_name, metric_mean)
return metrics['neg_multi_log_likelihood']
def model_validation_score(model, pred_path):
# ==== EVAL LOOP
model.eval()
torch.set_grad_enabled(False)
# store information for evaluation
future_coords_offsets_pd = []
timestamps = []
confidences_list = []
agent_ids = []
progress_bar = tqdm(eval_dataloader)
for data in progress_bar:
_, preds, confidences = forward(data, model, device)
#fix for the new environment
preds = preds.cpu().numpy()
world_from_agents = data["world_from_agent"].numpy()
centroids = data["centroid"].numpy()
coords_offset = []
# convert into world coordinates and compute offsets
for idx in range(len(preds)):
for mode in range(3):
preds[idx, mode, :, :] = transform_points(
preds[idx, mode, :, :],
world_from_agents[idx]) - centroids[idx][:2]
future_coords_offsets_pd.append(preds.copy())
confidences_list.append(confidences.cpu().numpy().copy())
timestamps.append(data["timestamp"].numpy().copy())
agent_ids.append(data["track_id"].numpy().copy())
write_pred_csv(
pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs=np.concatenate(confidences_list),
)
metrics = compute_metrics_csv(eval_gt_path, pred_path,
[neg_multi_log_likelihood, time_displace])
for metric_name, metric_mean in metrics.items():
print(metric_name, metric_mean)
def eval(model, eval_dataloader, eval_dataset_ego, eval_dataset, criterion, device, eval_gt_path, cfg):
# ===== EVAL Loop
model.eval()
torch.set_grad_enabled(False)
# store information for evaluation
future_coords_offsets_pd = []
timestamps = []
confidences_list = []
agent_ids = []
progress_bar = tqdm(eval_dataloader)
for data, indices in progress_bar:
loss, preds, confidences = mode.run(data, indices, model, eval_dataset_ego, eval_dataset, criterion, device = device)
#fix for the new environment
preds = preds.cpu().numpy()
world_from_agents = data["world_from_agent"].numpy()
centroids = data["centroid"].numpy()
coords_offset = []
# convert into world coordinates and compute offsets
for idx in range(len(preds)):
for mode in range(cfg["model_params"]["num_trajectories"]):
preds[idx,mode,:, :] = transform_points(preds[idx,mode,:, :], world_from_agents[idx]) - centroids[idx][:2]
future_coords_offsets_pd.append(preds.copy())
confidences_list.append(confidences.cpu().numpy().copy())
timestamps.append(data["timestamp"].numpy().copy())
agent_ids.append(data["track_id"].numpy().copy())
pred_path = f"{gettempdir()}/pred.csv"
write_pred_csv(pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs = np.concatenate(confidences_list)
)
metrics = compute_metrics_csv(eval_gt_path, pred_path, [neg_multi_log_likelihood, time_displace])
for metric_name, metric_mean in metrics.items():
print(metric_name, metric_mean)
def valid_fn(model, loader, device, ground_truth_file, logdir, verbose=True):
"""Validation step.
Args:
model (nn.Module): model to train
loader (DataLoader): loader with data
device (str or torch.device): device to use for placing batches
loss_fn (nn.Module): loss function, should be callable
verbose (bool, optional): verbosity mode.
Default is True.
Returns:
dict with metics computed during the validation on loader
"""
model.eval()
future_coords_offsets_pd = []
timestamps = []
confidences_list = []
agent_ids = []
with torch.no_grad(), tqdm(
total=len(loader), desc="valid", disable=not verbose
) as progress:
for idx, batch in enumerate(loader):
images, acceleration = t2d(
[batch["image"], batch["xy_acceleration"]], device
)
predictions, confidences = model(images, acceleration)
_gt = batch["target_positions"].cpu().numpy().copy()
predictions = predictions.cpu().numpy().copy()
world_from_agents = batch["world_from_agent"].numpy()
centroids = batch["centroid"].numpy()
for idx in range(len(predictions)):
for mode in range(3):
# FIX
predictions[idx, mode, :, :] = (
transform_points(
predictions[idx, mode, :, :], world_from_agents[idx]
)
- centroids[idx][:2]
)
_gt[idx, :, :] = (
transform_points(_gt[idx, :, :], world_from_agents[idx])
- centroids[idx][:2]
)
future_coords_offsets_pd.append(predictions.copy())
confidences_list.append(confidences.cpu().numpy().copy())
timestamps.append(batch["timestamp"].numpy().copy())
agent_ids.append(batch["track_id"].numpy().copy())
progress.update(1)
if idx == DEBUG:
break
predictions_file = str(logdir / "preds_validate_chopped.csv")
write_pred_csv(
predictions_file,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs=np.concatenate(confidences_list),
)
metrics = compute_metrics_csv(
ground_truth_file,
predictions_file,
[neg_multi_log_likelihood],
)
return {"score": metrics["neg_multi_log_likelihood"]}
criterion = nn.MSELoss(reduction="none")
torch.set_grad_enabled(False)
future_coords_offsets_pd = []
timestamps = []
agent_ids = []
progress_bar = tqdm(eval_dataloader)
for data in progress_bar:
_, ouputs = forward(data, model, device, criterion)
future_coords_offsets_pd.append(ouputs.cpu().numpy().copy())
timestamps.append(data["timestamp"].numpy().copy())
agent_ids.append(data["track_id"].numpy().copy())
pred_path = "pred.csv"
write_pred_csv(
pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
)
metrics = compute_metrics_csv(eval_gt_path, pred_path,
[neg_multi_log_likelihood, time_displace])
for metric_name, metric_mean in metrics.items():
print(metric_name, metric_mean)
future_coords_offsets_pd.append(preds.copy())
confidences_list.append(confidences.cpu().numpy().copy())
timestamps.append(batch["timestamp"].numpy().copy())
agent_ids.append(batch["track_id"].numpy().copy())
progress.update(1)
write_pred_csv(val_predictions_file,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs=np.concatenate(confidences_list))
metrics = compute_metrics_csv(
f"{DATA_DIR}/scenes/validate_chopped_100/gt.csv",
val_predictions_file,
[neg_multi_log_likelihood, time_displace],
)
for metric_name, metric_mean in metrics.items():
print(metric_name, metric_mean)
#====== INIT TEST DATASET=============================================================
rasterizer = build_rasterizer(cfg, dm)
test_zarr = ChunkedDataset(dm.require("scenes/test.zarr")).open()
test_mask = np.load(f"{DATA_DIR}/scenes/mask.npz")["arr_0"]
test_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)
test_dataloader = DataLoader(test_dataset,
shuffle=False,
batch_size=32,
num_workers=30)
def evaluate(cfg, model, dm, rasterizer, first_time, iters, eval_dataloader, eval_gt_path):
if first_time:
num_frames_to_chop = 100
print("min_future_steps: ",MIN_FUTURE_STEPS)
eval_cfg = cfg["val_data_loader"]
eval_base_path = create_chopped_dataset(dm.require(eval_cfg["key"]), cfg["raster_params"]["filter_agents_threshold"],
num_frames_to_chop, cfg["model_params"]["future_num_frames"], MIN_FUTURE_STEPS)
eval_zarr_path = str(Path(eval_base_path) / Path(dm.require(eval_cfg["key"])).name)
eval_mask_path = str(Path(eval_base_path) / "mask.npz")
eval_gt_path = str(Path(eval_base_path) / "gt.csv")
eval_zarr = ChunkedDataset(eval_zarr_path).open()
eval_mask = np.load(eval_mask_path)["arr_0"]
eval_dataset = AgentDataset(cfg, eval_zarr, rasterizer, agents_mask=eval_mask)
eval_dataloader = DataLoader(eval_dataset, shuffle=eval_cfg["shuffle"], batch_size=eval_cfg["batch_size"],
num_workers=eval_cfg["num_workers"])
print(eval_dataset)
first_time = False
model.eval()
torch.set_grad_enabled(False)
future_coords_offsets_pd = []
timestamps = []
confidences_list = []
agent_ids = []
progress_bar = tqdm(eval_dataloader)
for data in progress_bar:
_, preds, confidences = forward(data, model)
# convert agent coordinates into world offsets
preds = preds.cpu().numpy()
world_from_agents = data["world_from_agent"].numpy()
centroids = data["centroid"].numpy()
coords_offset = []
for idx in range(len(preds)):
for mode in range(3):
preds[idx, mode, :, :] = transform_points(preds[idx, mode, :, :], world_from_agents[idx]) - centroids[idx][:2]
future_coords_offsets_pd.append(preds.copy())
confidences_list.append(confidences.cpu().numpy().copy())
timestamps.append(data["timestamp"].numpy().copy())
agent_ids.append(data["track_id"].numpy().copy())
model.train()
torch.set_grad_enabled(True)
pred_path = os.path.join(cfg["save_path"],f"pred_{iters}.csv")
write_pred_csv(pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd),
confs = np.concatenate(confidences_list)
)
metrics = compute_metrics_csv(eval_gt_path, pred_path, [neg_multi_log_likelihood, time_displace])
for metric_name, metric_mean in metrics.items():
print(metric_name, metric_mean)
return first_time, eval_dataloader, eval_gt_path
def main(cfg: dict, args: argparse.Namespace) -> None:
# set random seeds
SEED = 42
os.environ["PYTHONHASHSEED"] = str(SEED)
# set Python random seed
random.seed(SEED)
# set NumPy random seed
np.random.seed(SEED)
os.environ["CUDA_VISIBLE_DEVICES"] = args.visible_gpus
# ===== Configure LYFT dataset
# mypy error due to pl.DataModule.transfer_batch_to_device
mpred_dm = LyftMpredDatamodule( # type: ignore[abstract]
args.l5kit_data_folder,
cfg,
batch_size=args.batch_size,
num_workers=args.num_workers,
downsample_train=args.downsample_train,
is_test=args.is_test,
is_debug=args.is_debug,
)
mpred_dm.prepare_data()
mpred_dm.setup()
if args.is_test:
print("\t\t ==== TEST MODE ====")
print("load from: ", args.ckpt_path)
model = LitModel.load_from_checkpoint(args.ckpt_path, cfg=cfg)
trainer = pl.Trainer(gpus=len(args.visible_gpus.split(",")))
trainer.test(model, datamodule=mpred_dm)
test_gt_path = os.path.join(os.path.dirname(mpred_dm.test_path),
"gt.csv")
if os.path.exists(test_gt_path):
print(
"test mode with validation chopped dataset, and check the metrics"
)
print("validation ground truth path: ", test_gt_path)
metrics = compute_metrics_csv(
test_gt_path, CSV_PATH,
[neg_multi_log_likelihood, time_displace])
for metric_name, metric_mean in metrics.items():
print(metric_name, metric_mean)
else:
print("\t\t ==== TRAIN MODE ====")
print("training samples: {}, valid samples: {}".format(
len(mpred_dm.train_dataset), len(mpred_dm.val_dataset)))
total_steps = args.epochs * len(
mpred_dm.train_dataset) // args.batch_size
val_check_interval = VAL_INTERVAL_SAMPLES // args.batch_size
model = LitModel(
cfg,
lr=args.lr,
backbone_name=args.backbone_name,
num_modes=args.num_modes,
optim_name=args.optim_name,
ba_size=args.batch_size,
epochs=args.epochs,
data_size=len(mpred_dm.train_dataset),
total_steps=total_steps,
)
checkpoint_callback = pl.callbacks.ModelCheckpoint(
monitor="val_loss",
save_last=True,
mode="min",
verbose=True,
)
pl.trainer.seed_everything(seed=SEED)
trainer = pl.Trainer(
gpus=len(args.visible_gpus.split(",")),
max_steps=total_steps,
val_check_interval=val_check_interval,
precision=args.precision,
benchmark=True,
deterministic=False,
checkpoint_callback=checkpoint_callback,
)
# Run lr finder
if args.find_lr:
lr_finder = trainer.tuner.lr_find(model,
datamodule=mpred_dm,
num_training=500)
lr_finder.plot(suggest=True)
plt.show()
sys.exit()
# Run Training
trainer.fit(model, datamodule=mpred_dm)
def evaluate(model, device, data_path):
# set env variable for data
os.environ["L5KIT_DATA_FOLDER"] = data_path
dm = LocalDataManager(None)
cfg = model.cfg
# ===== INIT DATASET
test_cfg = cfg["test_data_loader"]
# Rasterizer
rasterizer = build_rasterizer(cfg, dm)
# Test dataset/dataloader
test_zarr = ChunkedDataset(dm.require(test_cfg["key"])).open()
test_mask = np.load(f"{data_path}/scenes/mask.npz")["arr_0"]
test_dataset = AgentDataset(cfg,
test_zarr,
rasterizer,
agents_mask=test_mask)
test_dataloader = DataLoader(test_dataset,
shuffle=test_cfg["shuffle"],
batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"])
test_dataloader = test_dataloader
print(test_dataloader)
# ==== EVAL LOOP
model.eval()
torch.set_grad_enabled(False)
criterion = nn.MSELoss(reduction="none")
# store information for evaluation
future_coords_offsets_pd = []
timestamps = []
agent_ids = []
progress_bar = tqdm(test_dataloader)
for data in progress_bar:
_, outputs, _ = model.forward(data, device, criterion)
future_coords_offsets_pd.append(outputs.cpu().numpy().copy())
timestamps.append(data["timestamp"].numpy().copy())
agent_ids.append(data["track_id"].numpy().copy())
# ==== Save Results
pred_path = "./submission.csv"
write_pred_csv(pred_path,
timestamps=np.concatenate(timestamps),
track_ids=np.concatenate(agent_ids),
coords=np.concatenate(future_coords_offsets_pd))
# ===== GENERATE AND LOAD CHOPPED DATASET
num_frames_to_chop = 56
test_cfg = cfg["test_data_loader"]
test_base_path = create_chopped_dataset(
zarr_path=dm.require(test_cfg["key"]),
th_agent_prob=cfg["raster_params"]["filter_agents_threshold"],
num_frames_to_copy=num_frames_to_chop,
num_frames_gt=cfg["model_params"]["future_num_frames"],
min_frame_future=MIN_FUTURE_STEPS)
eval_zarr_path = str(
Path(test_base_path) / Path(dm.require(test_cfg["key"])).name)
print(eval_zarr_path)
test_mask_path = str(Path(test_base_path) / "mask.npz")
test_gt_path = str(Path(test_base_path) / "gt.csv")
test_zarr = ChunkedDataset(eval_zarr_path).open()
test_mask = np.load(test_mask_path)["arr_0"]
# ===== INIT DATASET AND LOAD MASK
test_dataset = AgentDataset(cfg,
test_zarr,
rasterizer,
agents_mask=test_mask)
test_dataloader = DataLoader(test_dataset,
shuffle=test_cfg["shuffle"],
batch_size=test_cfg["batch_size"],
num_workers=test_cfg["num_workers"])
print(test_dataset)
# ==== Perform Evaluation
print(test_gt_path)
metrics = compute_metrics_csv(test_gt_path, pred_path,
[neg_multi_log_likelihood, time_displace])
for metric_name, metric_mean in metrics.items():
print(metric_name, metric_mean)
