elif type(args[setting]) == type(None):
x = None
else:
x = args[setting]
config[setting] = x
eval_config = {
k: v
for k, v in config.items() if k in default_eval_config.keys()
}
dataset_config = {
k: v
for k, v in config.items() if k in default_dataset_config.keys()
}
metrics_config = {
k: v
for k, v in config.items() if k in default_metrics_config.keys()
}
# Run code
evaluator = trackeval.Evaluator(eval_config)
dataset_list = [trackeval.datasets.BDD100K2DBox(dataset_config)]
metrics_list = []
for metric in [
trackeval.metrics.HOTA, trackeval.metrics.CLEAR,
trackeval.metrics.Identity
]:
if metric.get_name() in metrics_config['METRICS']:
metrics_list.append(metric())
if len(metrics_list) == 0:
raise Exception('No metrics selected for evaluation')
evaluator.evaluate(dataset_list, metrics_list)
def evaluate(self,
results,
metric='track',
logger=None,
resfile_path=None,
bbox_iou_thr=0.5,
track_iou_thr=0.5):
"""Evaluation in MOT Challenge.
Args:
results (list[list | tuple]): Testing results of the dataset.
metric (str | list[str]): Metrics to be evaluated. Options are
'bbox', 'track'. Defaults to 'track'.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
resfile_path (str, optional): Path to save the formatted results.
Defaults to None.
bbox_iou_thr (float, optional): IoU threshold for detection
evaluation. Defaults to 0.5.
track_iou_thr (float, optional): IoU threshold for tracking
evaluation.. Defaults to 0.5.
Returns:
dict[str, float]: MOTChallenge style evaluation metric.
"""
eval_results = dict()
if isinstance(metric, list):
metrics = metric
elif isinstance(metric, str):
metrics = [metric]
else:
raise TypeError('metric must be a list or a str.')
allowed_metrics = ['bbox', 'track']
for metric in metrics:
if metric not in allowed_metrics:
raise KeyError(f'metric {metric} is not supported.')
if 'track' in metrics:
resfile_path, resfiles, names, tmp_dir = self.format_results(
results, resfile_path, metrics)
print_log('Evaluate CLEAR MOT results.', logger=logger)
distth = 1 - track_iou_thr
accs = []
for name in names:
if 'half-train' in self.ann_file:
gt_file = osp.join(self.img_prefix,
f'{name}/gt/gt_half-train.txt')
elif 'half-val' in self.ann_file:
gt_file = osp.join(self.img_prefix,
f'{name}/gt/gt_half-val.txt')
else:
gt_file = osp.join(self.img_prefix, f'{name}/gt/gt.txt')
res_file = osp.join(resfiles['track'], f'{name}.txt')
gt = mm.io.loadtxt(gt_file)
res = mm.io.loadtxt(res_file)
ini_file = osp.join(self.img_prefix, f'{name}/seqinfo.ini')
if osp.exists(ini_file) and 'MOT15' not in self.img_prefix:
acc, ana = mm.utils.CLEAR_MOT_M(gt,
res,
ini_file,
distth=distth)
else:
acc = mm.utils.compare_to_groundtruth(gt,
res,
distth=distth)
accs.append(acc)
mh = mm.metrics.create()
summary = mh.compute_many(accs,
names=names,
metrics=mm.metrics.motchallenge_metrics,
generate_overall=True)
if trackeval is None:
raise ImportError(
'Please run'
'pip install git+https://github.com/JonathonLuiten/TrackEval.git' # noqa
'to manually install trackeval')
seqmap = osp.join(resfile_path, 'videoseq.txt')
with open(seqmap, 'w') as f:
f.write('name\n')
for name in names:
f.write(name + '\n')
f.close
eval_config = trackeval.Evaluator.get_default_eval_config()
# tracker's name is set to 'track',
# so this word needs to be splited out
output_folder = resfiles['track'].rsplit(os.sep, 1)[0]
dataset_config = self.get_dataset_cfg_for_hota(
output_folder, seqmap)
evaluator = trackeval.Evaluator(eval_config)
dataset = [trackeval.datasets.MotChallenge2DBox(dataset_config)]
hota_metrics = [
trackeval.metrics.HOTA(dict(METRICS=['HOTA'], THRESHOLD=0.5))
]
output_res, _ = evaluator.evaluate(dataset, hota_metrics)
# modify HOTA results sequence according to summary list,
# indexes of summary are sequence names and 'OVERALL'
# while for hota they are sequence names and 'COMBINED_SEQ'
seq_list = list(summary.index)
seq_list.append('COMBINED_SEQ')
hota = [
np.average(output_res['MotChallenge2DBox']['track'][seq]
['pedestrian']['HOTA']['HOTA']) for seq in seq_list
if 'OVERALL' not in seq
]
eval_results.update({
mm.io.motchallenge_metric_names[k]: v['OVERALL']
for k, v in summary.to_dict().items()
})
eval_results['HOTA'] = hota[-1]
summary['HOTA'] = hota
str_summary = mm.io.render_summary(
summary,
formatters=mh.formatters,
namemap=mm.io.motchallenge_metric_names)
print(str_summary)
if tmp_dir is not None:
tmp_dir.cleanup()
if 'bbox' in metrics:
if isinstance(results, dict):
bbox_results = results['det_bboxes']
elif isinstance(results, list):
bbox_results = results
else:
raise TypeError('results must be a dict or a list.')
annotations = [self.get_ann_info(info) for info in self.data_infos]
mean_ap, _ = eval_map(bbox_results,
annotations,
iou_thr=bbox_iou_thr,
dataset=self.CLASSES,
logger=logger)
eval_results['mAP'] = mean_ap
for k, v in eval_results.items():
if isinstance(v, float):
eval_results[k] = float(f'{(v):.3f}')
return eval_results
