def eval(gt, input_file, args):
# Ground Truth
reader_gt = trajnetplusplustools.Reader(gt, scene_type='paths')
scenes_gt = [s for _, s in reader_gt.scenes()]
scenes_id_gt = [s_id for s_id, _ in reader_gt.scenes()]
# Scene Predictions
reader_sub = trajnetplusplustools.Reader(input_file, scene_type='paths')
scenes_sub = [s for _, s in reader_sub.scenes()]
## indexes is dictionary deciding which scenes are in which type
indexes = {}
for i in range(1, 5):
indexes[i] = []
## sub-indexes
sub_indexes = {}
for i in range(1, 5):
sub_indexes[i] = []
for scene in reader_gt.scenes_by_id:
tags = reader_gt.scenes_by_id[scene].tag
main_tag = tags[0:1]
sub_tags = tags[1]
for ii in range(1, 5):
if ii in main_tag:
indexes[ii].append(scene)
if ii in sub_tags:
sub_indexes[ii].append(scene)
# Evaluate
evaluator = TrajnetEvaluator(reader_gt, scenes_gt, scenes_id_gt,
scenes_sub, indexes, sub_indexes, args)
evaluator.aggregate('kf', args.disable_collision)
return evaluator.result()
def prepare_data(path, subset='/train/', sample=1.0, goals=True):
""" Prepares the train/val scenes and corresponding goals
Parameters
----------
subset: String ['/train/', '/val/']
Determines the subset of data to be processed
sample: Float (0.0, 1.0]
Determines the ratio of data to be sampled
goals: Bool
If true, the goals of each track are extracted
The corresponding goal file must be present in the 'goal_files' folder
The name of the goal file must be the same as the name of the training file
Returns
-------
all_scenes: List
List of all processed scenes
all_goals: Dictionary
Dictionary of goals corresponding to each dataset file.
None if 'goals' argument is False.
Flag: Bool
True if the corresponding folder exists else False.
"""
## Check if folder exists
if not os.path.isdir(path + subset):
if 'train' in subset:
print(path + subset)
print("Train folder does NOT exist")
exit()
if 'val' in subset:
print("Validation folder does NOT exist")
return None, None, False
## read goal files
all_goals = {}
all_scenes = []
## List file names
files = [f.split('.')[-2] for f in os.listdir(path + subset) if f.endswith('.ndjson')]
## Iterate over file names
for file in files:
reader = trajnetplusplustools.Reader(path + subset + file + '.ndjson', scene_type='paths')
## Necessary modification of train scene to add filename
scene = [(file, s_id, s) for s_id, s in reader.scenes(sample=sample)]
if goals:
goal_dict = pickle.load(open('goal_files/' + subset + file +'.pkl', "rb"))
## Get goals corresponding to train scene
all_goals[file] = {s_id: [goal_dict[path[0].pedestrian] for path in s] for _, s_id, s in scene}
all_scenes += scene
if goals:
return all_scenes, all_goals, True
return all_scenes, None, True
def collision_test(list_sub, name, args):
""" Simple Collision Test """
submit_datasets = [args.path + name + '/' + f for f in list_sub if 'collision_test.ndjson' in f]
if len(submit_datasets):
# Scene Prediction
reader_sub = trajnetplusplustools.Reader(submit_datasets[0], scene_type='paths')
scenes_sub = [s for _, s in reader_sub.scenes()]
if trajnetplusplustools.metrics.collision(scenes_sub[0][0], scenes_sub[0][1], n_predictions=args.pred_length):
return "Fail"
return "Pass"
return "NA"
def main():
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
scenes = list(
trajnetplusplustools.Reader('data/train/biwi_hotel.ndjson').scenes(
limit=1))
pool = trajnetbaselines.lstm.Pooling(type_='social')
model = trajnetbaselines.lstm.LSTM(pool=pool)
trainer = trajnetbaselines.lstm.trainer.Trainer(model, device=device)
with torch.autograd.profiler.profile(
use_cuda=torch.cuda.is_available()) as prof:
trainer.train(scenes, epoch=0)
prof.export_chrome_trace('profile_trace.json')
def eval(input_file, dest_file, simulator, params, type_ids, args):
print('dataset', input_file)
reader = trajnetplusplustools.Reader(input_file, scene_type='paths')
scenes = [s for _, s in reader.scenes()]
## Filter scenes according to category type
# if type_ids is None:
# trajectory_type = 3
# interaction_type = 2
# type_ids = [scene_id for scene_id in reader.scenes_by_id \
# if interaction_type in reader.scenes_by_id[scene_id].tag[1]]
# scenes = [scenes[type_id] for type_id in type_ids]
## If final destination of pedestrian provided
dest_dict = None
if dest_file is not None:
dest_dict = pickle.load(open(dest_file, "rb"))
evaluator = Evaluator(scenes, dest_dict, params, args)
## Evaluate all
if simulator == 'all':
for dest_type in ['interp']:
evaluator.aggregate('orca' + dest_type, orca.predict, dest_type)
evaluator.aggregate('sf' + dest_type, socialforce.predict,
dest_type)
evaluator.aggregate('kf', kalman.predict)
# ORCA only
elif simulator == 'orca':
for dest_type in ['interp']:
evaluator.aggregate('orca' + dest_type, orca.predict, dest_type)
# Social Force only
elif simulator == 'sf':
for dest_type in ['interp']:
evaluator.aggregate('sf' + dest_type, socialforce.predict,
dest_type)
# Kalman only
elif simulator == 'kf':
evaluator.aggregate('kf', kalman.predict)
return evaluator.result()
def main(args=None):
## List of .json file inside the args.path (waiting to be predicted by the testing model)
datasets = sorted([f.split('.')[-2] for f in os.listdir(args.path.replace('_pred', '')) if not f.startswith('.') and f.endswith('.ndjson')])
all_goals = {}
seq_length = args.obs_length + args.pred_length
## Handcrafted Baselines (if included)
if args.kf:
args.output.append('/kf.pkl')
if args.sf:
args.output.append('/sf.pkl')
args.output.append('/sf_opt.pkl')
if args.orca:
args.output.append('/orca.pkl')
args.output.append('/orca_opt.pkl')
## Extract Model names from arguments and create its own folder in 'test_pred' for storing predictions
## WARNING: If Model predictions already exist from previous run, this process SKIPS WRITING
for model in args.output:
model_name = model.split('/')[-1].replace('.pkl', '')
model_name = model_name + '_modes' + str(args.modes)
## Check if model predictions already exist
if not os.path.exists(args.path):
os.makedirs(args.path)
if not os.path.exists(args.path + model_name):
os.makedirs(args.path + model_name)
else:
print('Predictions corresponding to {} already exist.'.format(model_name))
print('Loading the saved predictions')
continue
## Start writing predictions in dataset/test_pred
for dataset in datasets:
# Model's name
name = dataset.replace(args.path.replace('_pred', '') + 'test/', '') + '.ndjson'
print('NAME: ', name)
# Read Scenes from 'test' folder
reader = trajnetplusplustools.Reader(args.path.replace('_pred', '') + dataset + '.ndjson', scene_type='paths')
## Necessary modification of train scene to add filename (for goals)
scenes = [(dataset, s_id, s) for s_id, s in reader.scenes()]
## Consider goals
## Goal file must be present in 'goal_files/test_private' folder
## Goal file must have the same name as corresponding test file
if args.goals:
goal_dict = pickle.load(open('goal_files/test_private/' + dataset +'.pkl', "rb"))
all_goals[dataset] = {s_id: [goal_dict[path[0].pedestrian] for path in s] for _, s_id, s in scenes}
# Loading the APPROPRIATE model
## Keep Adding Different Model Architectures to this List
print("Model Name: ", model_name)
if model_name == 'kf':
print("Kalman")
predictor = trajnetbaselines.classical.kalman.predict
elif model_name in {'sf', 'sf_opt'}:
print("Social Force")
predictor = trajnetbaselines.classical.socialforce.predict
elif model_name in {'orca', 'orca_opt'}:
print("ORCA")
predictor = trajnetbaselines.classical.orca.predict
elif 'sgan' in model_name:
print("SGAN")
predictor = trajnetbaselines.sgan.SGANPredictor.load(model)
device = torch.device('cpu')
predictor.model.to(device)
elif 'lstm' in model_name:
print("LSTM")
predictor = trajnetbaselines.lstm.LSTMPredictor.load(model)
device = torch.device('cpu')
predictor.model.to(device)
else:
print("Model Architecture not recognized")
raise ValueError
# Get the model prediction and write them in corresponding test_pred file
# VERY IMPORTANT: Prediction Format
# The predictor function should output a dictionary. The keys of the dictionary should correspond to the prediction modes.
# ie. predictions[0] corresponds to the first mode. predictions[m] corresponds to the m^th mode.... Multimodal predictions!
# Each modal prediction comprises of primary prediction and neighbour (surrrounding) predictions i.e. predictions[m] = [primary_prediction, neigh_predictions]
# Note: Return [primary_prediction, []] if model does not provide neighbour predictions
# Shape of primary_prediction: Tensor of Shape (Prediction length, 2)
# Shape of Neighbour_prediction: Tensor of Shape (Prediction length, n_tracks - 1, 2).
# (See LSTMPredictor.py for more details)
with open(args.path + '{}/{}'.format(model_name, name), "a") as myfile:
for filename, scene_id, paths in scenes:
## Extract 1) first_frame, 2) frame_diff 3) ped_ids for writing predictions
observed_path = paths[0]
frame_diff = observed_path[1].frame - observed_path[0].frame
first_frame = observed_path[args.obs_length-1].frame + frame_diff
ped_id = observed_path[0].pedestrian
ped_id_ = []
for j, _ in enumerate(paths[1:]): ## Only need neighbour ids
ped_id_.append(paths[j+1][0].pedestrian)
## For each scene, get predictions
if model_name == 'sf_opt':
predictions = predictor(paths, sf_params=[0.5, 5.0, 0.3], n_predict=args.pred_length, obs_length=args.obs_length) ## optimal sf_params (no collision constraint) [0.5, 1.0, 0.1],
elif model_name == 'orca_opt':
predictions = predictor(paths, orca_params=[0.4, 1.0, 0.3], n_predict=args.pred_length, obs_length=args.obs_length) ## optimal orca_params (no collision constraint) [0.25, 1.0, 0.3]
elif model_name in {'sf', 'orca', 'kf'}:
predictions = predictor(paths, n_predict=args.pred_length, obs_length=args.obs_length, args=args)
else:
goals = get_goals(paths, all_goals, filename, scene_id) ## Zeros if no goals utilized
predictions = predictor(paths, goals, n_predict=args.pred_length, obs_length=args.obs_length, modes=args.modes, args=args)
## Write SceneRow
scenerow = trajnetplusplustools.SceneRow(scene_id, ped_id, observed_path[0].frame,
observed_path[0].frame + seq_length - 1, 2.5, 0)
myfile.write(trajnetplusplustools.writers.trajnet(scenerow))
myfile.write('\n')
for m in range(len(predictions)):
prediction, neigh_predictions = predictions[m]
## Write Primary
for i in range(len(prediction)):
track = trajnetplusplustools.TrackRow(first_frame + i * frame_diff, ped_id,
prediction[i, 0].item(), prediction[i, 1].item(), m, scene_id)
myfile.write(trajnetplusplustools.writers.trajnet(track))
myfile.write('\n')
## Write Neighbours (if non-empty)
if len(neigh_predictions):
for n in range(neigh_predictions.shape[1]):
neigh = neigh_predictions[:, n]
for j in range(len(neigh)):
track = trajnetplusplustools.TrackRow(first_frame + j * frame_diff, ped_id_[n],
neigh[j, 0].item(), neigh[j, 1].item(), m, scene_id)
myfile.write(trajnetplusplustools.writers.trajnet(track))
myfile.write('\n')
print('')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--path',
default='trajdata',
help='directory of data to test')
parser.add_argument('--output',
required=True,
nargs='+',
help='relative path to saved model')
parser.add_argument('--obs_length',
default=9,
type=int,
help='observation length')
parser.add_argument('--pred_length',
default=12,
type=int,
help='prediction length')
parser.add_argument('--disable-write',
action='store_true',
help='disable writing new files')
parser.add_argument('--disable-collision',
action='store_true',
help='disable collision metrics')
parser.add_argument('--labels',
required=False,
nargs='+',
help='labels of models')
parser.add_argument('--normalize_scene',
action='store_true',
help='augment scenes')
parser.add_argument('--unimodal',
action='store_true',
help='provide unimodal evaluation')
parser.add_argument('--topk',
action='store_true',
help='provide topk evaluation')
parser.add_argument('--multimodal',
action='store_true',
help='provide multimodal nll evaluation')
parser.add_argument('--modes',
default=1,
type=int,
help='number of modes to predict')
parser.add_argument('--scene_type',
default=0,
type=int,
choices=(0, 1, 2, 3, 4),
help='type of scene to evaluate')
parser.add_argument('--thresh',
default=0.0,
type=float,
help='noise thresh')
parser.add_argument('--ped_type',
default='primary',
help='type of ped to add noise to')
args = parser.parse_args()
scipy.seterr('ignore')
## Path to the data folder name to predict
args.path = 'DATA_BLOCK/' + args.path + '/'
## Test_pred: Folders for saving model predictions
args.path = args.path + 'test_pred/'
if (not args.unimodal) and (not args.topk) and (not args.multimodal):
args.unimodal = True # Compute unimodal metrics by default
if args.topk:
args.modes = 3
if args.multimodal:
args.modes = 20
enable_col1 = True
## drop pedestrians that appear post observation
def drop_post_obs(ground_truth, obs_length):
obs_end_frame = ground_truth[0][obs_length].frame
ground_truth = [
track for track in ground_truth if track[0].frame < obs_end_frame
]
return ground_truth
## Writes to Test_pred
## Does this overwrite existing predictions? No. ###
datasets = sorted([
f.split('.')[-2] for f in os.listdir(args.path.replace('_pred', ''))
if not f.startswith('.') and f.endswith('.ndjson')
])
## Model names are passed as arguments
for model in args.output:
model_name = model.split('/')[-1].replace('.pkl', '')
# Loading the appropriate model (functionality only for SGAN and LSTM)
print("Model Name: ", model_name)
if 'sgan' in model_name:
predictor = trajnetbaselines.sgan.SGANPredictor.load(model)
goal_flag = predictor.model.generator.goal_flag
elif 'vae' in model_name:
predictor = trajnetbaselines.vae.VAEPredictor.load(model)
goal_flag = predictor.model.goal_flag
else:
predictor = trajnetbaselines.lstm.LSTMPredictor.load(model)
goal_flag = predictor.model.goal_flag
# On CPU
device = torch.device('cpu')
predictor.model.to(device)
total_scenes = 0
average = 0
final = 0
gt_col = 0.
pred_col = 0.
neigh_scenes = 0
topk_average = 0
topk_final = 0
all_goals = {}
average_nll = 0
## Start writing in dataset/test_pred
for dataset in datasets:
# Model's name
name = dataset.replace(
args.path.replace('_pred', '') + 'test/', '')
# Copy file from test into test/train_pred folder
print('processing ' + name)
if 'collision_test' in name:
continue
## Filter for Scene Type
reader_tag = trajnetplusplustools.Reader(
args.path.replace('_pred', '_private') + dataset + '.ndjson',
scene_type='tags')
if args.scene_type != 0:
filtered_scene_ids = [
s_id for s_id, tag, s in reader_tag.scenes()
if tag[0] == args.scene_type
]
else:
filtered_scene_ids = [
s_id for s_id, _, _ in reader_tag.scenes()
]
# Read file from 'test'
reader = trajnetplusplustools.Reader(
args.path.replace('_pred', '') + dataset + '.ndjson',
scene_type='paths')
## Necessary modification of train scene to add filename (for goals)
scenes = [(dataset, s_id, s) for s_id, s in reader.scenes()
if s_id in filtered_scene_ids]
## Consider goals
## Goal file must be present in 'goal_files/test_private' folder
## Goal file must have the same name as corresponding test file
if goal_flag:
goal_dict = pickle.load(
open('goal_files/test_private/' + dataset + '.pkl', "rb"))
all_goals[dataset] = {
s_id: [goal_dict[path[0].pedestrian] for path in s]
for _, s_id, s in scenes
}
## Get Goals
if goal_flag:
scene_goals = [
np.array(all_goals[filename][scene_id])
for filename, scene_id, _ in scenes
]
else:
scene_goals = [
np.zeros((len(paths), 2)) for _, scene_id, paths in scenes
]
print("Getting Predictions")
scenes = tqdm(scenes)
## Get all predictions in parallel. Faster!
pred_list = Parallel(n_jobs=12)(
delayed(process_scene)(predictor, model_name, paths,
scene_goal, args)
for (_, _, paths), scene_goal in zip(scenes, scene_goals))
## GT Scenes
reader_gt = trajnetplusplustools.Reader(
args.path.replace('_pred', '_private') + dataset + '.ndjson',
scene_type='paths')
scenes_gt = [
s for s_id, s in reader_gt.scenes()
if s_id in filtered_scene_ids
]
total_scenes += len(scenes_gt)
print("Evaluating Predictions")
scenes = tqdm(scenes)
for (predictions, (_, scene_id, paths),
ground_truth) in zip(pred_list, scenes, scenes_gt):
## Extract 1) first_frame, 2) frame_diff 3) ped_ids for writing predictions
observed_path = paths[0]
frame_diff = observed_path[1].frame - observed_path[0].frame
first_frame = observed_path[args.obs_length -
1].frame + frame_diff
ped_id = observed_path[0].pedestrian
ped_id_ = []
for j, _ in enumerate(paths[1:]): ## Only need neighbour ids
ped_id_.append(paths[j + 1][0].pedestrian)
if args.unimodal: ## Unimodal
## ADE / FDE
prediction, neigh_predictions = predictions[0]
prediction = np.round(prediction, 2)
## make Track Rows
# primary
prediction = [
trajnetplusplustools.TrackRow(
first_frame + i * frame_diff, ped_id,
prediction[i, 0], prediction[i, 1], 0)
for i in range(len(prediction))
]
primary_tracks = [
t for t in prediction if t.prediction_number == 0
]
frame_gt = [
t.frame for t in ground_truth[0]
][args.obs_length:args.obs_length + args.pred_length]
frame_pred = [t.frame for t in primary_tracks]
## To verify if same scene
if frame_gt != frame_pred:
raise Exception('frame numbers are not consistent')
average_l2 = trajnetplusplustools.metrics.average_l2(
ground_truth[0][args.obs_length:args.obs_length +
args.pred_length],
primary_tracks,
n_predictions=args.pred_length)
final_l2 = trajnetplusplustools.metrics.final_l2(
ground_truth[0][args.obs_length:args.obs_length +
args.pred_length], primary_tracks)
# aggregate FDE and ADE
average += average_l2
final += final_l2
ground_truth = drop_post_obs(ground_truth, args.obs_length)
## Collision Metrics
for j in range(1, len(ground_truth)):
if trajnetplusplustools.metrics.collision(
primary_tracks,
ground_truth[j],
n_predictions=args.pred_length):
gt_col += 1
break
num_gt_neigh = len(ground_truth) - 1
num_predicted_neigh = neigh_predictions.shape[1]
if num_gt_neigh != num_predicted_neigh:
enable_col1 = False
# [Col-I] only if neighs in gt = neighs in prediction
if enable_col1:
neigh_scenes += 1
for n in range(neigh_predictions.shape[1]):
neigh = neigh_predictions[:, n]
neigh = np.round(neigh, 2)
neigh_track = [
trajnetplusplustools.TrackRow(
first_frame + j * frame_diff, n,
neigh[j, 0], neigh[j, 1], 0)
for j in range(len(neigh))
]
if trajnetplusplustools.metrics.collision(
primary_tracks,
neigh_track,
n_predictions=args.pred_length):
pred_col += 1
break
primary_tracks_all = [
trajnetplusplustools.TrackRow(first_frame + i * frame_diff,
ped_id, x, y, m)
for m, (prim, neighs) in predictions.items()
for i, (x, y) in enumerate(prim)
]
if args.topk:
topk_ade, topk_fde = trajnetplusplustools.metrics.topk(
primary_tracks_all,
ground_truth[0][args.obs_length:args.obs_length +
args.pred_length],
n_predictions=args.pred_length)
topk_average += topk_ade
topk_final += topk_fde
if args.multimodal:
nll_val = trajnetplusplustools.metrics.nll(
primary_tracks_all,
ground_truth[0],
n_predictions=args.pred_length,
n_samples=20)
average_nll += nll_val
if args.unimodal:
## Average ADE and FDE
average /= total_scenes
final /= total_scenes
gt_col /= (total_scenes * 0.01)
if not enable_col1:
pred_col = -1
else:
pred_col /= (neigh_scenes * 0.01)
print('ADE: ', np.round(average, 3))
print('FDE: ', np.round(final, 3))
print("Col-I: ", np.round(pred_col, 2))
print("Col-II: ", np.round(gt_col, 2))
if args.topk:
topk_average /= total_scenes
topk_final /= total_scenes
print('Topk_ADE: ', topk_average)
print('Topk_FDE: ', topk_final)
if args.multimodal:
average_nll /= total_scenes
print('Average NLL: ', average_nll)
def trajectory_type(rows, path, fps, track_id=0, args=None):
""" Categorization of all scenes """
## Read
reader = trajnetplusplustools.Reader(path, scene_type='paths')
scenes = [s for _, s in reader.scenes()]
## Filtered Frames and Scenes
new_frames = set()
new_scenes = []
start_frames = set()
###########################################################################
# scenes_test helps to handle both test and test_private simultaneously
# scenes_test correspond to Test
###########################################################################
test = 'test' in path
if test:
path_test = path.replace('test_private', 'test')
reader_test = trajnetplusplustools.Reader(path_test,
scene_type='paths')
scenes_test = [s for _, s in reader_test.scenes()]
## Filtered Test Frames and Test Scenes
new_frames_test = set()
new_scenes_test = []
## For ORCA (Sensitivity)
orca_sensitivity = False
if args.goal_file is not None:
goal_dict = pickle.load(open(args.goal_file, "rb"))
orca_sensitivity = True
print("Checking sensitivity to initial conditions")
## Initialize Tag Stats to be collected
tags = {1: [], 2: [], 3: [], 4: []}
mult_tags = {1: [], 2: [], 3: [], 4: []}
sub_tags = {1: [], 2: [], 3: [], 4: []}
col_count = 0
if not scenes:
raise Exception('No scenes found')
for index, scene in enumerate(scenes):
if (index + 1) % 50 == 0:
print(index)
## Primary Path
ped_interest = scene[0]
# if ped_interest[0].frame in start_frames:
# # print("Got common start")
# continue
# Assert Test Scene length
if test:
assert len(scenes_test[index][0]) >= args.obs_len, \
'Scene Test not adequate length'
## Check Collision
## Used in CFF Datasets to account for imperfect tracking
# if check_collision(scene, args.pred_len):
# col_count += 1
# continue
# ## Consider only those scenes where all pedestrians are present
# # Note: Different from removing incomplete trajectories
if args.all_present and (not all_ped_present(scene)):
continue
## Get Tag
tag, mult_tag, sub_tag = get_type(scene, args)
if np.random.uniform() < args.acceptance[tag - 1]:
## Check Validity
## Used in ORCA Datasets to account for rounding sensitivity
# 28-02-2021 14:00 - block uses orca_helper file. Since we do not have rov2 available, we will not use it
"""
if orca_sensitivity:
goals = [goal_dict[path[0].pedestrian] for path in scene]
# print('Type III')
if orca_validity(scene, goals, args.pred_len, args.obs_len, args.mode):
col_count += 1
continue
"""
## Update Tags
tags[tag].append(track_id)
for tt in mult_tag:
mult_tags[tt].append(track_id)
for st in sub_tag:
sub_tags[st].append(track_id)
## Define Scene_Tag
scene_tag = []
scene_tag.append(tag)
scene_tag.append(sub_tag)
## Filtered scenes and Frames
# start_frames |= set(ped_interest[i].frame for i in range(len(ped_interest[0:1])))
# print(start_frames)
new_frames |= set(ped_interest[i].frame
for i in range(len(ped_interest)))
new_scenes.append(
trajnetplusplustools.data.SceneRow(track_id,
ped_interest[0].pedestrian,
ped_interest[0].frame,
ped_interest[-1].frame, fps,
scene_tag))
## Append to list of scenes_test as well if Test Set
if test:
new_frames_test |= set(ped_interest[i].frame
for i in range(args.obs_len))
new_scenes_test.append(
trajnetplusplustools.data.SceneRow(
track_id, ped_interest[0].pedestrian,
ped_interest[0].frame, ped_interest[-1].frame, fps, 0))
track_id += 1
# Writes the Final Scenes and Frames
write(rows, path, new_scenes, new_frames)
if test:
write(rows, path_test, new_scenes_test, new_frames_test)
## Stats
# Number of collisions found
print("Col Count: ", col_count)
if scenes:
print("Total Scenes: ", index)
# Types:
print("Main Tags")
print("Type 1: ", len(tags[1]), "Type 2: ", len(tags[2]), "Type 3: ",
len(tags[3]), "Type 4: ", len(tags[4]))
print("Sub Tags")
print("LF: ", len(sub_tags[1]), "CA: ", len(sub_tags[2]), "Group: ",
len(sub_tags[3]), "Others: ", len(sub_tags[4]))
return track_id
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--path',
default='trajdata',
help='directory of data to test')
parser.add_argument('--output',
required=True,
nargs='+',
help='relative path to saved model')
parser.add_argument('--obs_length',
default=9,
type=int,
help='observation length')
parser.add_argument('--pred_length',
default=12,
type=int,
help='prediction length')
parser.add_argument('--disable-write',
action='store_true',
help='disable writing new files')
parser.add_argument('--disable-collision',
action='store_true',
help='disable collision metrics')
parser.add_argument('--labels',
required=False,
nargs='+',
help='labels of models')
parser.add_argument('--normalize_scene',
action='store_true',
help='augment scenes')
parser.add_argument('--goals',
action='store_true',
help='Considers goals during prediction')
parser.add_argument('--unimodal',
action='store_true',
help='provide unimodal evaluation')
parser.add_argument('--topk',
action='store_true',
help='provide topk evaluation')
parser.add_argument('--multimodal',
action='store_true',
help='provide multimodal nll evaluation')
parser.add_argument('--modes',
default=1,
type=int,
help='number of modes to predict')
args = parser.parse_args()
scipy.seterr('ignore')
## Path to the data folder name to predict
args.path = 'DATA_BLOCK/' + args.path + '/'
## Test_pred: Folders for saving model predictions
args.path = args.path + 'test_pred/'
if (not args.unimodal) and (not args.topk) and (not args.multimodal):
args.unimodal = True # Compute unimodal metrics by default
if args.multimodal:
args.modes = 20
## Writes to Test_pred
## Does this overwrite existing predictions? No. ###
datasets = sorted([
f.split('.')[-2] for f in os.listdir(args.path.replace('_pred', ''))
if not f.startswith('.') and f.endswith('.ndjson')
])
## Model names are passed as arguments
for model in args.output:
model_name = model.split('/')[-1].replace('.pkl', '')
# Loading the appropriate model (functionality only for SGAN and LSTM)
print("Model Name: ", model_name)
if 'sgan' in model_name:
predictor = trajnetbaselines.sgan.SGANPredictor.load(model)
else:
predictor = trajnetbaselines.lstm.LSTMPredictor.load(model)
# On CPU
device = torch.device('cpu')
predictor.model.to(device)
total_scenes = 0
average = 0
final = 0
gt_col = 0.
pred_col = 0.
neigh_scenes = 0
topk_average = 0
topk_final = 0
all_goals = {}
average_nll = 0
## Start writing in dataset/test_pred
for dataset in datasets:
# Model's name
name = dataset.replace(
args.path.replace('_pred', '') + 'test/', '')
# Copy file from test into test/train_pred folder
print('processing ' + name)
if 'collision_test' in name:
continue
# Read file from 'test'
reader = trajnetplusplustools.Reader(
args.path.replace('_pred', '') + dataset + '.ndjson',
scene_type='paths')
## Necessary modification of train scene to add filename (for goals)
scenes = [(dataset, s_id, s) for s_id, s in reader.scenes()]
## Consider goals
## Goal file must be present in 'goal_files/test_private' folder
## Goal file must have the same name as corresponding test file
if args.goals:
goal_dict = pickle.load(
open('goal_files/test_private/' + dataset + '.pkl', "rb"))
all_goals[dataset] = {
s_id: [goal_dict[path[0].pedestrian] for path in s]
for _, s_id, s in scenes
}
reader_gt = trajnetplusplustools.Reader(
args.path.replace('_pred', '_private') + dataset + '.ndjson',
scene_type='paths')
scenes_gt = [s for _, s in reader_gt.scenes()]
total_scenes += len(scenes_gt)
for i, (filename, scene_id, paths) in enumerate(scenes):
if i % 100 == 0:
print("Scenes evaluated: ",
'{}/{}'.format(i, len(scenes_gt)))
ground_truth = scenes_gt[i]
## Convert numpy array to Track Rows ##
## Extract 1) first_frame, 2) frame_diff 3) ped_ids for writing predictions
observed_path = paths[0]
frame_diff = observed_path[1].frame - observed_path[0].frame
first_frame = observed_path[args.obs_length -
1].frame + frame_diff
ped_id = observed_path[0].pedestrian
goals = get_goals(paths, all_goals, filename,
scene_id) ## Zeros if no goals utilized
predictions = predictor(paths,
goals,
n_predict=args.pred_length,
obs_length=args.obs_length,
modes=args.modes,
args=args)
if args.unimodal: ## Unimodal
## ADE / FDE
prediction, neigh_predictions = predictions[0]
prediction = np.round(prediction, 2)
## make Track Rows
# primary
prediction = [
trajnetplusplustools.TrackRow(
first_frame + i * frame_diff, ped_id,
prediction[i, 0], prediction[i, 1], 0)
for i in range(len(prediction))
]
primary_tracks = [
t for t in prediction if t.prediction_number == 0
]
frame_gt = [
t.frame for t in ground_truth[0]
][args.obs_length:args.obs_length + args.pred_length]
frame_pred = [t.frame for t in primary_tracks]
## To verify if same scene
if frame_gt != frame_pred:
raise Exception('frame numbers are not consistent')
average_l2 = trajnetplusplustools.metrics.average_l2(
ground_truth[0][args.obs_length:args.obs_length +
args.pred_length],
primary_tracks,
n_predictions=args.pred_length)
final_l2 = trajnetplusplustools.metrics.final_l2(
ground_truth[0][args.obs_length:args.obs_length +
args.pred_length], primary_tracks)
# aggregate FDE and ADE
average += average_l2
final += final_l2
## Collision Metrics
for j in range(1, len(ground_truth)):
if trajnetplusplustools.metrics.collision(
primary_tracks,
ground_truth[j],
n_predictions=args.pred_length):
gt_col += 1
break
# ## neighbours (if not empty) [Col-I]
if neigh_predictions.shape[1]:
neigh_scenes += 1
for n in range(neigh_predictions.shape[1]):
neigh = neigh_predictions[:, n]
neigh = np.round(neigh, 2)
neigh_track = [
trajnetplusplustools.TrackRow(
first_frame + j * frame_diff, n,
neigh[j, 0], neigh[j, 1], 0)
for j in range(len(neigh))
]
if trajnetplusplustools.metrics.collision(
primary_tracks,
neigh_track,
n_predictions=args.pred_length):
pred_col += 1
break
primary_tracks_all = [
trajnetplusplustools.TrackRow(first_frame + i * frame_diff,
ped_id, x, y, m)
for m, (prim, neighs) in predictions.items()
for i, (x, y) in enumerate(prim)
]
if args.topk:
topk_ade, topk_fde = trajnetplusplustools.metrics.topk(
primary_tracks_all,
ground_truth[0][args.obs_length:args.obs_length +
args.pred_length],
n_predictions=args.pred_length)
topk_average += topk_ade
topk_final += topk_fde
if args.multimodal:
nll_val = trajnetplusplustools.metrics.nll(
primary_tracks_all,
ground_truth[0],
n_predictions=args.pred_length,
n_samples=20)
average_nll += nll_val
if args.unimodal:
## Average ADE and FDE
average /= total_scenes
final /= total_scenes
gt_col /= (total_scenes * 0.01)
if neigh_scenes == 0:
pred_col = -1
else:
pred_col /= (neigh_scenes * 0.01)
print('ADE: ', average)
print('FDE: ', final)
print("Col-I: ", pred_col)
print("Col-II: ", gt_col)
if args.topk:
topk_average /= total_scenes
topk_final /= total_scenes
print('Topk_ADE: ', topk_average)
print('Topk_FDE: ', topk_final)
if args.multimodal:
average_nll /= total_scenes
print('Average NLL: ', average_nll)
