for i, data in enumerate(tsDataloader):
st_time = time.time()
hist, nbrs, mask, lat_enc, lon_enc, fut, op_mask, graph_list, _, _, nbrs_idx = data
# Initialize Variables
if args['use_cuda']:
hist = hist.to(args['device'])
nbrs = nbrs.to(args['device'])
mask = mask.to(args['device'])
fut = fut.to(args['device'])
op_mask = op_mask.to(args['device'])
nbrs_idx = nbrs_idx.to(args['device'])
if metric == 'nll':
# Forward pass
fut_pred = net(hist, nbrs, mask, graph_list, nbrs_idx, mode='path')
l, c = maskedNLLTest(fut_pred, 0, 0, fut, op_mask, use_maneuvers=False)
else:
# Forward pass
fut_pred = net(hist, nbrs, mask, graph_list, nbrs_idx, mode='path')
l, c = maskedMSETest(fut_pred, fut, op_mask)
lossVals += l.detach()
counts += c.detach()
if metric == 'nll':
print(lossVals / counts)
else:
print(torch.pow(lossVals / counts, 0.5) *
0.3048) # Calculate RMSE and convert from feet to meters
def model_evaluate():
args = parser.parse_args()
## Initialize network
PiP = pipNet(args)
PiP.load_state_dict(
torch.load('./trained_models/{}/{}.tar'.format(
(args.name).split('-')[0], args.name)))
if args.use_cuda:
PiP = PiP.cuda()
## Evaluation Mode
PiP.eval()
PiP.train_output_flag = False
initLogging(log_file='./trained_models/{}/evaluation.log'.format((
args.name).split('-')[0]))
## Intialize dataset
logging.info("Loading test data from {}...".format(args.test_set))
tsSet = highwayTrajDataset(path=args.test_set,
targ_enc_size=args.social_context_size +
args.dynamics_encoding_size,
grid_size=args.grid_size,
fit_plan_traj=True,
fit_plan_further_ds=args.plan_info_ds)
logging.info("TOTAL :: {} test data.".format(len(tsSet)))
tsDataloader = DataLoader(tsSet,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=tsSet.collate_fn)
## Loss statistic
logging.info(
"<{}> evaluated by {}-based NLL & RMSE, with planning input of {}s step."
.format(args.name, args.metric, args.plan_info_ds * 0.2))
if args.metric == 'agent':
nll_loss_stat = np.zeros(
(np.max(tsSet.Data[:, 0]).astype(int) + 1,
np.max(tsSet.Data[:, 13:(13 + tsSet.grid_cells)]).astype(int) + 1,
args.out_length))
rmse_loss_stat = np.zeros(
(np.max(tsSet.Data[:, 0]).astype(int) + 1,
np.max(tsSet.Data[:, 13:(13 + tsSet.grid_cells)]).astype(int) + 1,
args.out_length))
both_count_stat = np.zeros(
(np.max(tsSet.Data[:, 0]).astype(int) + 1,
np.max(tsSet.Data[:, 13:(13 + tsSet.grid_cells)]).astype(int) + 1,
args.out_length))
elif args.metric == 'sample':
rmse_loss = torch.zeros(25).cuda()
rmse_counts = torch.zeros(25).cuda()
nll_loss = torch.zeros(25).cuda()
nll_counts = torch.zeros(25).cuda()
else:
raise RuntimeError("Wrong type of evaluation metric is specified")
avg_eva_time = 0
## Evaluation process
with torch.no_grad():
for i, data in enumerate(tsDataloader):
st_time = time.time()
nbsHist, nbsMask, planFut, planMask, targsHist, targsEncMask, targsFut, targsFutMask, lat_enc, lon_enc, idxs = data
# Initialize Variables
if args.use_cuda:
nbsHist = nbsHist.cuda()
nbsMask = nbsMask.cuda()
planFut = planFut.cuda()
planMask = planMask.cuda()
targsHist = targsHist.cuda()
targsEncMask = targsEncMask.cuda()
lat_enc = lat_enc.cuda()
lon_enc = lon_enc.cuda()
targsFut = targsFut.cuda()
targsFutMask = targsFutMask.cuda()
# Inference
fut_pred, lat_pred, lon_pred = PiP(nbsHist, nbsMask, planFut,
planMask, targsHist,
targsEncMask, lat_enc, lon_enc)
# Performance metric
if args.metric == 'agent':
dsIDs, targsIDs = tsSet.batchTargetVehsInfo(idxs)
l, c = maskedNLLTest(fut_pred,
lat_pred,
lon_pred,
targsFut,
targsFutMask,
separately=True)
# Select the trajectory with the largest probability of maneuver label when evaluating by RMSE
fut_pred_max = torch.zeros_like(fut_pred[0])
for k in range(lat_pred.shape[0]):
lat_man = torch.argmax(lat_pred[k, :]).detach()
lon_man = torch.argmax(lon_pred[k, :]).detach()
indx = lon_man * 3 + lat_man
fut_pred_max[:, k, :] = fut_pred[indx][:, k, :]
# Using the most probable trajectory
ll, cc = maskedMSETest(fut_pred_max,
targsFut,
targsFutMask,
separately=True)
l = l.detach().cpu().numpy()
ll = ll.detach().cpu().numpy()
c = c.detach().cpu().numpy()
cc = cc.detach().cpu().numpy()
for j, targ in enumerate(targsIDs):
dsID = dsIDs[j]
nll_loss_stat[dsID, targ, :] += l[:, j]
rmse_loss_stat[dsID, targ, :] += ll[:, j]
both_count_stat[dsID, targ, :] += c[:, j]
elif args.metric == 'sample':
l, c = maskedNLLTest(fut_pred, lat_pred, lon_pred, targsFut,
targsFutMask)
nll_loss += l.detach()
nll_counts += c.detach()
fut_pred_max = torch.zeros_like(fut_pred[0])
for k in range(lat_pred.shape[0]):
lat_man = torch.argmax(lat_pred[k, :]).detach()
lon_man = torch.argmax(lon_pred[k, :]).detach()
indx = lon_man * 3 + lat_man
fut_pred_max[:, k, :] = fut_pred[indx][:, k, :]
l, c = maskedMSETest(fut_pred_max, targsFut, targsFutMask)
rmse_loss += l.detach()
rmse_counts += c.detach()
# Time estimate
batch_time = time.time() - st_time
avg_eva_time += batch_time
if i % 100 == 99:
eta = avg_eva_time / 100 * (len(tsSet) / args.batch_size - i)
logging.info("Evaluation progress(%):{:.2f}".format(
i / (len(tsSet) / args.batch_size) * 100, ) +
" | ETA(s):{}".format(int(eta)))
avg_eva_time = 0
# Result Summary
if args.metric == 'agent':
# Loss averaged from all predicted vehicles.
ds_ids, veh_ids = both_count_stat[:, :, 0].nonzero()
num_vehs = len(veh_ids)
rmse_loss_averaged = np.zeros((args.out_length, num_vehs))
nll_loss_averaged = np.zeros((args.out_length, num_vehs))
count_averaged = np.zeros((args.out_length, num_vehs))
for i in range(num_vehs):
count_averaged[:, i] = \
both_count_stat[ds_ids[i], veh_ids[i], :].astype(bool)
rmse_loss_averaged[:,i] = rmse_loss_stat[ds_ids[i], veh_ids[i], :] \
* count_averaged[:, i] / (both_count_stat[ds_ids[i], veh_ids[i], :] + 1e-9)
nll_loss_averaged[:,i] = nll_loss_stat[ds_ids[i], veh_ids[i], :] \
* count_averaged[:, i] / (both_count_stat[ds_ids[i], veh_ids[i], :] + 1e-9)
rmse_loss_sum = np.sum(rmse_loss_averaged, axis=1)
nll_loss_sum = np.sum(nll_loss_averaged, axis=1)
count_sum = np.sum(count_averaged, axis=1)
rmseOverall = np.power(
rmse_loss_sum / count_sum,
0.5) * 0.3048 # Unit converted from feet to meter.
nllOverall = nll_loss_sum / count_sum
elif args.metric == 'sample':
rmseOverall = (torch.pow(rmse_loss / rmse_counts, 0.5) * 0.3048).cpu()
nllOverall = (nll_loss / nll_counts).cpu()
# Print the metrics every 5 time frame (1s)
logging.info("RMSE (m)\t=> {}, Mean={:.3f}".format(
rmseOverall[4::5], rmseOverall[4::5].mean()))
logging.info("NLL (nats)\t=> {}, Mean={:.3f}".format(
nllOverall[4::5], nllOverall[4::5].mean()))