def evaluate_experiment(results_dir):
accumulators = []
experiment_names = []
for exp_info, trackingsOverTime, ground_truth in read_trackings(
results_dir):
acc = motmetrics.MOTAccumulator(auto_id=True)
accumulators.append(acc)
experiment_names.append(exp_info['datasetName'])
for tracks, gt in zip(trackingsOverTime, ground_truth):
if tracks:
x1, y1, x2, y2 = np.array([t['bbox'] for t in tracks]).T
else:
x1, y1, x2, y2 = [np.array([])] * 4
pred_bboxes = np.array([x1, y1, x2 - x1, y2 - y1]).T
iou = motmetrics.distances.iou_matrix(
gt[:, 1:5] if len(gt) else np.empty((0, 0)),
pred_bboxes,
max_iou=1)
acc.update(gt[:, 0] if len(gt) else np.empty((0)),
[t['id'] for t in tracks], iou)
mh = motmetrics.metrics.create()
summary = mh.compute_many(accumulators,
metrics=motmetrics.metrics.motchallenge_metrics,
names=experiment_names,
generate_overall=True)
strsummary = motmetrics.io.render_summary(
summary,
formatters=mh.formatters,
namemap=motmetrics.io.motchallenge_metric_names)
strsummary = "MOTA and MOTP are wrong. See py-motmetrics README. Don't use these.\n" + strsummary
return strsummary
def test_assignment_metrics_with_empty_groundtruth():
acc = mm.MOTAccumulator(auto_id=True)
# Empty groundtruth.
acc.update([], [1, 2, 3, 4], [])
acc.update([], [1, 2, 3, 4], [])
acc.update([], [1, 2, 3, 4], [])
acc.update([], [1, 2, 3, 4], [])
mh = mm.metrics.create()
metr = mh.compute(acc,
return_dataframe=False,
metrics=[
'num_matches',
'num_false_positives',
'num_misses',
'idtp',
'idfp',
'idfn',
])
assert metr['num_matches'] == 0
assert metr['num_false_positives'] == 16
assert metr['num_misses'] == 0
assert metr['idtp'] == 0
assert metr['idfp'] == 16
assert metr['idfn'] == 0
def test_events():
acc = mm.MOTAccumulator()
# All FP
acc.update([], ['a', 'b'], [], frameid=0)
# All miss
acc.update([1, 2], [], [], frameid=1)
# Match
acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=2)
# Switch
acc.update([1, 2], ['a', 'b'], [[0.2, np.nan], [np.nan, 0.1]], frameid=3)
# Match. Better new match is available but should prefer history
acc.update([1, 2], ['a', 'b'], [[5, 1], [1, 5]], frameid=4)
# No data
acc.update([], [], [], frameid=5)
expect = mm.MOTAccumulator.new_event_dataframe()
expect.loc[(0, 0), :] = ['FP', np.nan, 'a', np.nan]
expect.loc[(0, 1), :] = ['FP', np.nan, 'b', np.nan]
expect.loc[(1, 0), :] = ['MISS', 1, np.nan, np.nan]
expect.loc[(1, 1), :] = ['MISS', 2, np.nan, np.nan]
expect.loc[(2, 0), :] = ['MATCH', 1, 'b', 0.5]
expect.loc[(2, 1), :] = ['MATCH', 2, 'a', 0.3]
expect.loc[(3, 0), :] = ['SWITCH', 1, 'a', 0.2]
expect.loc[(3, 1), :] = ['SWITCH', 2, 'b', 0.1]
expect.loc[(4, 0), :] = ['MATCH', 1, 'a', 5.]
expect.loc[(4, 1), :] = ['MATCH', 2, 'b', 5.]
# frame 5 generates no events
assert pd.DataFrame.equals(acc.events, expect)
mh = mm.metrics.create()
metr = mh.compute(acc, metrics=['motp', 'mota'], return_dataframe=False)
assert metr['motp'] == approx(11.1 / 6)
assert metr['mota'] == approx(1. - (2 + 2 + 2) / 8)
def test_assignment_metrics_with_both_empty():
"""Tests metrics when there are no ground-truth objects or predictions."""
acc = mm.MOTAccumulator(auto_id=True)
# Empty groundtruth and empty predictions.
acc.update([], [], [])
acc.update([], [], [])
acc.update([], [], [])
acc.update([], [], [])
mh = mm.metrics.create()
metr = mh.compute(acc,
return_dataframe=False,
metrics=[
'num_matches',
'num_false_positives',
'num_misses',
'idtp',
'idfp',
'idfn',
'num_frames',
])
assert metr['num_matches'] == 0
assert metr['num_false_positives'] == 0
assert metr['num_misses'] == 0
assert metr['idtp'] == 0
assert metr['idfp'] == 0
assert metr['idfn'] == 0
assert metr['num_frames'] == 4
def evaluate(objects, hypothesis, verbose=True):
obj_json = JSONTracks(objects, from_xml=(objects.suffix == ".xml"))
hyp_json = JSONTracks(hypothesis, from_xml=(hypothesis.suffix == ".xml"))
assert obj_json.nframes == hyp_json.nframes, \
f"Objects (nframes={obj_json.nframes}) and Hypothesis (nframes={hyp_json.nframes}) " \
f"must have equal number of frames."
acc = mm.MOTAccumulator(auto_id=True)
for i in range(obj_json.nframes):
if verbose:
print(f"\rAccumulatating annotations {i + 1}/{obj_json.nframes}.",
end="")
gt_coords, gt_ids = obj_json.get_frame_coordinates(
i), obj_json.get_frame_ids(i)
hy_coords, hy_ids = hyp_json.get_frame_coordinates(
i), hyp_json.get_frame_ids(i)
dis = mm.distances.norm2squared_matrix(gt_coords, hy_coords, 15)
acc.update(gt_ids, hy_ids, dis)
if verbose:
print("\nComputing metrics...")
mh = mm.metrics.create()
summary = mh.compute_many([acc],
metrics=mm.metrics.motchallenge_metrics,
names=['value'])
str_summary = mm.io.render_summary(summary,
formatters=mh.formatters,
namemap=mm.io.motchallenge_metric_names)
if verbose:
print(50 * "#" + "\n" + str_summary + "\n" + 50 * "#")
return summary
def test_extract_counts():
"""Tests events_to_df_map() and extract_counts_from_df_map()."""
acc = mm.MOTAccumulator()
# All FP
acc.update([], [1, 2], [], frameid=0)
# All miss
acc.update([1, 2], [], [], frameid=1)
# Match
acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=2)
# Switch
acc.update([1, 2], [1, 2], [[0.2, np.nan], [np.nan, 0.1]], frameid=3)
# Match. Better new match is available but should prefer history
acc.update([1, 2], [1, 2], [[5, 1], [1, 5]], frameid=4)
# No data
acc.update([], [], [], frameid=5)
ocs, hcs, tps = _extract_counts(acc)
assert ocs == {1: 4, 2: 4}
assert hcs == {1: 4, 2: 4}
expected_tps = {
(1, 1): 3,
(1, 2): 2,
(2, 1): 2,
(2, 2): 3,
}
assert tps == expected_tps
def __init__(
self,
folder,
draw_,
min_iou_,
max_miss_,
conf_v,
eval_=False,
):
self.dataset = Dataset(folder, eval_)
self.Tracks = OrderedDict()
self.num_frames = self.dataset.getNumberFrames()
self.next_id = 0
self.draw_objects, self.show_objects, self.draw_video = draw_
self.eval = eval_
self.min_iou = min_iou_
self.max_miss = max_miss_
self.conf_value = conf_v
self._create_new_tracks(self.dataset.getDetections(1))
if self.draw_objects:
self.list_frames = [self._drawObjects(1)]
if self.eval:
self.acc = mm.MOTAccumulator(auto_id=True)
self.mh = mm.metrics.create()
self._eval(1)
def print_result(gt, ht):
interested_frame = set(gt.keys()).union(set(ht.keys()))
acc = mm.MOTAccumulator(auto_id=True)
for frame_id in list(interested_frame):
gt_ids = []
gt_bbs = []
ht_ids = []
ht_bbs = []
if frame_id in gt:
for tracker in gt[frame_id]:
gt_ids.append(tracker[0])
gt_bbs.append(tracker[1])
if frame_id in ht:
for tracker in ht[frame_id]:
ht_ids.append(tracker[0])
ht_bbs.append(tracker[1])
distance_matrix = mm.distances.iou_matrix(gt_bbs, ht_bbs, max_iou=0.8)
acc.update(gt_ids, ht_ids, distance_matrix)
mh = mm.metrics.create()
summary = mh.compute_many([acc],
metrics=mm.metrics.motchallenge_metrics,
names=['Value'])
strsummary = mm.io.render_summary(summary,
formatters=mh.formatters,
namemap=mm.io.motchallenge_metric_names)
print(strsummary)
def test_mota_motp():
acc = mm.MOTAccumulator()
# All FP
acc.update([], ['a', 'b'], [], frameid=0)
# All miss
acc.update([1, 2], [], [], frameid=1)
# Match
acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=2)
# Switch
acc.update([1, 2], ['a', 'b'], [[0.2, np.nan], [np.nan, 0.1]], frameid=3)
# Match. Better new match is available but should prefer history
acc.update([1, 2], ['a', 'b'], [[5, 1], [1, 5]], frameid=4)
# No data
acc.update([], [], [], frameid=5)
mh = mm.metrics.create()
metr = mh.compute(acc,
metrics=['motp', 'mota', 'num_predictions'],
return_dataframe=False,
return_cached=True)
assert metr['num_matches'] == 4
assert metr['num_false_positives'] == 2
assert metr['num_misses'] == 2
assert metr['num_switches'] == 2
assert metr['num_detections'] == 6
assert metr['num_objects'] == 8
assert metr['num_predictions'] == 8
assert metr['mota'] == approx(1. - (2 + 2 + 2) / 8)
assert metr['motp'] == approx(11.1 / 6)
def compute_map(self, epoch):
global GT
if GT is None:
self.collect_gt()
root = self.args.res_dir
unique_fps = set()
for x in os.listdir(root):
name = x.split('.')[0]
unique_fps.add(name)
dr = {}
for x in unique_fps:
if int(x.split('_')[-1]) % 10 != 0:
continue
if x not in dr:
dr[x] = {'bboxes': [], 'scores': [], 'ids': []}
with open(os.path.join(root, x + '.txt')) as f:
tracks = f.read().split('\n')
tracks = [list(map(float, x.split())) for x in tracks if x]
for track in tracks:
score, _id, *box = track
dr[x]['scores'].append(score)
dr[x]['bboxes'].append(self.args.down_ratio * np.array(box))
dr[x]['ids'].append(int(_id))
inters = set(GT.keys()).intersection(dr.keys())
unique_fids = set()
for x in inters:
unique_fids.add(x.split('.')[0].split('_frame')[0])
unique_fids = list(unique_fids)
accs = []
for k in unique_fids:
acc = mm.MOTAccumulator(auto_id=True)
fids = sorted([x.split('.')[0] for x in inters if k in x],
key=lambda x: int(x.split('.')[0].split('_')[-1]))
for fid in fids:
gt_id, dr_id, dist = load(fid,
GT,
dr,
use_ids=True,
max_iou=0.55)
acc.update(gt_id, dr_id, dist)
accs.append(acc)
mh = mm.metrics.create()
summary = mh.compute_many(
accs,
metrics=[
'num_frames', 'num_objects', 'num_predictions', 'num_matches',
'num_misses', 'num_false_positives', 'num_switches', 'mota',
'motp', 'recall', 'idp', 'idr', 'idf1'
],
generate_overall=True,
names=unique_fids,
)
summary.to_csv(
os.path.join(self.args.weights_dir,
"map_result_{}.csv".format(epoch)))
def add(self, tracks, gt_frames, name='test', frame_range=None):
assert frame_range is not None
if not tracks:
return
acc = motmetrics.MOTAccumulator()
self.accumulators.append(acc)
self.names.append(name.split('.')[0][-40:])
self.update(tracks, gt_frames, frame_range)
def test_events():
"""Tests that expected events are created by MOTAccumulator.update()."""
acc = mm.MOTAccumulator()
# All FP
acc.update([], [1, 2], [], frameid=0)
# All miss
acc.update([1, 2], [], [], frameid=1)
# Match
acc.update([1, 2], [1, 2], [[1, 0.5], [0.3, 1]], frameid=2)
# Switch
acc.update([1, 2], [1, 2], [[0.2, np.nan], [np.nan, 0.1]], frameid=3)
# Match. Better new match is available but should prefer history
acc.update([1, 2], [1, 2], [[5, 1], [1, 5]], frameid=4)
# No data
acc.update([], [], [], frameid=5)
expect = mm.MOTAccumulator.new_event_dataframe()
expect.loc[(0, 0), :] = ['RAW', np.nan, np.nan, np.nan]
expect.loc[(0, 1), :] = ['RAW', np.nan, 1, np.nan]
expect.loc[(0, 2), :] = ['RAW', np.nan, 2, np.nan]
expect.loc[(0, 3), :] = ['FP', np.nan, 1, np.nan]
expect.loc[(0, 4), :] = ['FP', np.nan, 2, np.nan]
expect.loc[(1, 0), :] = ['RAW', np.nan, np.nan, np.nan]
expect.loc[(1, 1), :] = ['RAW', 1, np.nan, np.nan]
expect.loc[(1, 2), :] = ['RAW', 2, np.nan, np.nan]
expect.loc[(1, 3), :] = ['MISS', 1, np.nan, np.nan]
expect.loc[(1, 4), :] = ['MISS', 2, np.nan, np.nan]
expect.loc[(2, 0), :] = ['RAW', np.nan, np.nan, np.nan]
expect.loc[(2, 1), :] = ['RAW', 1, 1, 1.0]
expect.loc[(2, 2), :] = ['RAW', 1, 2, 0.5]
expect.loc[(2, 3), :] = ['RAW', 2, 1, 0.3]
expect.loc[(2, 4), :] = ['RAW', 2, 2, 1.0]
expect.loc[(2, 5), :] = ['MATCH', 1, 2, 0.5]
expect.loc[(2, 6), :] = ['MATCH', 2, 1, 0.3]
expect.loc[(3, 0), :] = ['RAW', np.nan, np.nan, np.nan]
expect.loc[(3, 1), :] = ['RAW', 1, 1, 0.2]
expect.loc[(3, 2), :] = ['RAW', 2, 2, 0.1]
expect.loc[(3, 3), :] = ['TRANSFER', 1, 1, 0.2]
expect.loc[(3, 4), :] = ['SWITCH', 1, 1, 0.2]
expect.loc[(3, 5), :] = ['TRANSFER', 2, 2, 0.1]
expect.loc[(3, 6), :] = ['SWITCH', 2, 2, 0.1]
expect.loc[(4, 0), :] = ['RAW', np.nan, np.nan, np.nan]
expect.loc[(4, 1), :] = ['RAW', 1, 1, 5.]
expect.loc[(4, 2), :] = ['RAW', 1, 2, 1.]
expect.loc[(4, 3), :] = ['RAW', 2, 1, 1.]
expect.loc[(4, 4), :] = ['RAW', 2, 2, 5.]
expect.loc[(4, 5), :] = ['MATCH', 1, 1, 5.]
expect.loc[(4, 6), :] = ['MATCH', 2, 2, 5.]
expect.loc[(5, 0), :] = ['RAW', np.nan, np.nan, np.nan]
pd.util.testing.assert_frame_equal(acc.events, expect)
def test_max_switch_time():
acc = mm.MOTAccumulator(max_switch_time=1)
acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]],
frameid=1) # 1->a, 2->b
frameid = acc.update([1, 2], ['a', 'b'], [[0.5, np.nan], [np.nan, 0.5]],
frameid=2) # 1->b, 2->a
df = acc.events.loc[frameid]
assert ((df.Type == 'SWITCH') | (df.Type == 'RAW')).all()
acc = mm.MOTAccumulator(max_switch_time=1)
acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]],
frameid=1) # 1->a, 2->b
frameid = acc.update([1, 2], ['a', 'b'], [[0.5, np.nan], [np.nan, 0.5]],
frameid=5) # Later frame 1->b, 2->a
df = acc.events.loc[frameid]
assert ((df.Type == 'MATCH') | (df.Type == 'RAW')).all()
def acc_single_video_mots(
gts: List[str],
results: List[str],
iou_thr: float = 0.5,
ignore_iof_thr: float = 0.5,
) -> List[mm.MOTAccumulator]:
"""Accumulate results for one video."""
assert len(gts) == len(results)
num_classes = len(CLASSES)
accs = [mm.MOTAccumulator(auto_id=True) for _ in range(num_classes)]
for gt, result in zip(gts, results):
assert os.path.isfile(gt)
assert os.path.isfile(result)
gt_masks = np.asarray(Image.open(gt))
res_masks = np.asarray(Image.open(result))
gt_masks, gt_ids, gt_attrs, gt_cats = parse_bitmasks(gt_masks)
pred_masks, pred_ids, pred_attrs, pred_cats = parse_bitmasks(res_masks)
ious, iofs = mask_intersection_rate(gt_masks, pred_masks)
gt_valids = np.logical_not((gt_attrs & 3).astype(np.bool8))
pred_valids = np.logical_not((pred_attrs & 3).astype(np.bool8))
for i in range(num_classes):
# cats starts from 1 and i starts from 0
gt_inds = (gt_cats == i + 1) * gt_valids
pred_inds = (pred_cats == i + 1) * pred_valids
gt_ids_c, pred_ids_c = gt_ids[gt_inds], pred_ids[pred_inds]
if gt_ids_c.shape[0] == 0 and pred_ids_c.shape[0] != 0:
distances = np.full((0, pred_ids_c.shape[0]), np.nan)
elif gt_ids_c.shape[0] != 0 and pred_ids_c.shape[0] == 0:
distances = np.full((gt_ids_c.shape[0], 0), np.nan)
else:
ious_c = ious[gt_inds, :][:, pred_inds]
distances = 1 - ious_c
distances = np.where(distances > 1 - iou_thr, np.nan,
distances)
gt_invalid = np.logical_not(gt_valids)
if (gt_invalid).any():
# 1. assign gt and preds
fps = np.ones(pred_ids_c.shape[0]).astype(np.bool8)
le, ri = mm.lap.linear_sum_assignment(distances)
for m, n in zip(le, ri):
if np.isfinite(distances[m, n]):
fps[n] = False
# 2. ignore by iof
iofs_c = iofs[gt_invalid, :][:, pred_inds]
ignores = (iofs_c > ignore_iof_thr).any(axis=0)
# 3. filter preds
valid_inds = ~(fps & ignores)
pred_ids_c = pred_ids_c[valid_inds]
distances = distances[:, valid_inds]
if distances.shape != (0, 0):
accs[i].update(gt_ids_c, pred_ids_c, distances)
return accs
def acc_single_video_mot(
gts: List[Frame],
results: List[Frame],
classes: List[str],
iou_thr: float = 0.5,
ignore_iof_thr: float = 0.5,
) -> List[mm.MOTAccumulator]:
"""Accumulate results for one video."""
assert len(gts) == len(results)
get_frame_index = (
lambda x: x.frame_index if x.frame_index is not None else 0
)
num_classes = len(classes)
gts = sorted(gts, key=get_frame_index)
results = sorted(results, key=get_frame_index)
accs = [mm.MOTAccumulator(auto_id=True) for _ in range(num_classes)]
for gt, result in zip(gts, results):
assert gt.frame_index == result.frame_index
gt_bboxes, gt_labels, gt_ids, gt_ignores = parse_objects(
gt.labels if gt.labels is not None else [], classes
)
pred_bboxes, pred_labels, pred_ids, _ = parse_objects(
result.labels if result.labels is not None else [], classes
)
for i in range(num_classes):
gt_inds, pred_inds = gt_labels == i, pred_labels == i
gt_bboxes_c, gt_ids_c = gt_bboxes[gt_inds], gt_ids[gt_inds]
pred_bboxes_c, pred_ids_c = (
pred_bboxes[pred_inds],
pred_ids[pred_inds],
)
if gt_bboxes_c.shape[0] == 0 and pred_bboxes_c.shape[0] != 0:
distances = np.full((0, pred_bboxes_c.shape[0]), np.nan)
elif gt_bboxes_c.shape[0] != 0 and pred_bboxes_c.shape[0] == 0:
distances = np.full((gt_bboxes_c.shape[0], 0), np.nan)
else:
distances = mm.distances.iou_matrix(
gt_bboxes_c, pred_bboxes_c, max_iou=1 - iou_thr
)
if gt_ignores.shape[0] > 0:
# 1. assign gt and preds
fps = np.ones(pred_bboxes_c.shape[0]).astype(np.bool8)
le, ri = mm.lap.linear_sum_assignment(distances)
for m, n in zip(le, ri):
if np.isfinite(distances[m, n]):
fps[n] = False
# 2. ignore by iof
iofs = intersection_over_area(pred_bboxes_c, gt_ignores)
ignores = (iofs > ignore_iof_thr).any(axis=1)
# 3. filter preds
valid_inds = ~(fps & ignores)
pred_ids_c = pred_ids_c[valid_inds]
distances = distances[:, valid_inds]
if distances.shape != (0, 0):
accs[i].update(gt_ids_c, pred_ids_c, distances)
return accs
def getMetrics(tracks, params, gt_file):
num_frames = params["frames"]
ground_truth, list_of_trajectories = parseXML(gt_file, num_frames)
list_of_tracks, hypothesis = build_tracks_at_frame(tracks, num_frames)
acc = mm.MOTAccumulator(auto_id=True)
acc = update_accumulator(acc, ground_truth, hypothesis, list_of_trajectories, list_of_tracks, num_frames)
results = generate_results(acc)
return getResultsAsString(results)
def calculate_ap(det_bb, gt_bb, ini_frame, last_frame, mode):
lst_gt = [item[0] for item in gt_bb]
lst_det = [item[0] for item in det_bb]
acc = mm.MOTAccumulator(auto_id=True)
AP = 0
for f_val in range(ini_frame, last_frame):
frame_gt_bb = [
gt_bb[i] for i, num in enumerate(lst_gt) if num == f_val
]
n_gt = len(frame_gt_bb)
frame_det_bb = [
det_bb[i] for i, num in enumerate(lst_det) if num == f_val
]
if mode == 'sort':
frame_det_bb = sorted(frame_det_bb,
key=lambda x: x[-1],
reverse=True)
f_det_bb = [item[:-1] for item in frame_det_bb]
AP = AP + frame_AP(n_gt, f_det_bb, copy.deepcopy(frame_gt_bb))
elif mode == 'random':
#Random shuffle
f_ap = 0
for i in range(0, 10):
shuffle(frame_det_bb)
a = frame_AP(n_gt, copy.deepcopy(frame_det_bb),
copy.deepcopy(frame_gt_bb))
f_ap = f_ap + a
AP = AP + f_ap / 10
else:
#Sorted by area
frame_det_bb = sorted(frame_det_bb,
key=lambda x: (x[5] - x[3]) * (x[5] - x[3]),
reverse=True)
AP = AP + frame_AP(n_gt, copy.deepcopy(frame_det_bb),
copy.deepcopy(frame_gt_bb))
y_pred = {det[2]: det[3:7] for det in frame_det_bb}
y_true = {gt[2]: gt[3:] for gt in frame_gt_bb}
distances = [[
np.sqrt(np.sum((np.array(i) - np.array(j))**2))
for j in y_pred.values()
] for i in y_true.values()]
acc.update([*y_true.keys()], [*y_pred.keys()], distances)
summary = mm.metrics.create().compute(acc,
metrics=['idf1'],
name='idf1_score')
print(summary)
AP = AP / last_frame
return AP
def _accum_random_uniform(rand, seq_len, num_objs, num_hyps, objs_per_frame,
hyps_per_frame):
acc = mm.MOTAccumulator(auto_id=True)
for _ in range(seq_len):
# Choose subset of objects present in this frame.
objs = rand.choice(num_objs, objs_per_frame, replace=False)
# Choose subset of hypotheses present in this frame.
hyps = rand.choice(num_hyps, hyps_per_frame, replace=False)
dist = rand.uniform(size=(objs_per_frame, hyps_per_frame))
acc.update(objs, hyps, dist)
return acc
def test_events():
acc = mm.MOTAccumulator()
# All FP
acc.update([], ['a', 'b'], [], frameid=0)
# All miss
acc.update([1, 2], [], [], frameid=1)
# Match
acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=2)
# Switch
acc.update([1, 2], ['a', 'b'], [[0.2, np.nan], [np.nan, 0.1]], frameid=3)
# Match. Better new match is available but should prefer history
acc.update([1, 2], ['a', 'b'], [[5, 1], [1, 5]], frameid=4)
# No data
acc.update([], [], [], frameid=5)
expect = mm.MOTAccumulator.new_event_dataframe()
expect.loc[(0, 0), :] = ['RAW', np.nan, 'a', np.nan]
expect.loc[(0, 1), :] = ['RAW', np.nan, 'b', np.nan]
expect.loc[(0, 2), :] = ['FP', np.nan, 'a', np.nan]
expect.loc[(0, 3), :] = ['FP', np.nan, 'b', np.nan]
expect.loc[(1, 0), :] = ['RAW', 1, np.nan, np.nan]
expect.loc[(1, 1), :] = ['RAW', 2, np.nan, np.nan]
expect.loc[(1, 2), :] = ['MISS', 1, np.nan, np.nan]
expect.loc[(1, 3), :] = ['MISS', 2, np.nan, np.nan]
expect.loc[(2, 0), :] = ['RAW', 1, 'a', 1.0]
expect.loc[(2, 1), :] = ['RAW', 1, 'b', 0.5]
expect.loc[(2, 2), :] = ['RAW', 2, 'a', 0.3]
expect.loc[(2, 3), :] = ['RAW', 2, 'b', 1.0]
expect.loc[(2, 4), :] = ['MATCH', 1, 'b', 0.5]
expect.loc[(2, 5), :] = ['MATCH', 2, 'a', 0.3]
expect.loc[(3, 0), :] = ['RAW', 1, 'a', 0.2]
expect.loc[(3, 1), :] = ['RAW', 1, 'b', np.nan]
expect.loc[(3, 2), :] = ['RAW', 2, 'a', np.nan]
expect.loc[(3, 3), :] = ['RAW', 2, 'b', 0.1]
expect.loc[(3, 4), :] = ['TRANSFER', 1, 'a', 0.2]
expect.loc[(3, 5), :] = ['SWITCH', 1, 'a', 0.2]
expect.loc[(3, 6), :] = ['TRANSFER', 2, 'b', 0.1]
expect.loc[(3, 7), :] = ['SWITCH', 2, 'b', 0.1]
expect.loc[(4, 0), :] = ['RAW', 1, 'a', 5.]
expect.loc[(4, 1), :] = ['RAW', 1, 'b', 1.]
expect.loc[(4, 2), :] = ['RAW', 2, 'a', 1.]
expect.loc[(4, 3), :] = ['RAW', 2, 'b', 5.]
expect.loc[(4, 4), :] = ['MATCH', 1, 'a', 5.]
expect.loc[(4, 5), :] = ['MATCH', 2, 'b', 5.]
# frame 5 generates no events
from pandas.util.testing import assert_frame_equal
assert_frame_equal(acc.events, expect)
def accumulate_events(gt_dict, pred_dict, args):
acc = mm.MOTAccumulator()
count = 0
for i in tqdm(range(len(gt_dict))):
for t in range(args.start_step, args.stop_step):
gt_dict_frame = gt_dict[i][t]
pred_dict_frame = pred_dict[i][t]
dist, gt_ids, pred_ids = compute_dists_per_frame(
gt_dict_frame, pred_dict_frame, args)
acc.update(gt_ids, pred_ids, dist, frameid=count)
count += 1
return acc
def get_mot_accum(results, seq):
mot_accum = mm.MOTAccumulator(auto_id=True)
for i, data in enumerate(seq):
gt = data["gt"]
gt_ids = []
if gt:
gt_boxes = []
for gt_id, box in gt.items():
gt_ids.append(gt_id)
gt_boxes.append(box)
gt_boxes = np.stack(gt_boxes, axis=0)
# x1, y1, x2, y2 --> x1, y1, width, height
gt_boxes = np.stack(
(
gt_boxes[:, 0],
gt_boxes[:, 1],
gt_boxes[:, 2] - gt_boxes[:, 0],
gt_boxes[:, 3] - gt_boxes[:, 1],
),
axis=1,
)
else:
gt_boxes = np.array([])
track_ids = []
track_boxes = []
for track_id, frames in results.items():
if i in frames:
track_ids.append(track_id)
# frames = x1, y1, x2, y2, score
track_boxes.append(frames[i][:4])
if track_ids:
track_boxes = np.stack(track_boxes, axis=0)
# x1, y1, x2, y2 --> x1, y1, width, height
track_boxes = np.stack(
(
track_boxes[:, 0],
track_boxes[:, 1],
track_boxes[:, 2] - track_boxes[:, 0],
track_boxes[:, 3] - track_boxes[:, 1],
),
axis=1,
)
else:
track_boxes = np.array([])
distance = mm.distances.iou_matrix(gt_boxes, track_boxes, max_iou=0.5)
mot_accum.update(gt_ids, track_ids, distance)
return mot_accum
def build_mot_accumulator(truth_framedata: {},
model_framedata: {}) -> mm.MOTAccumulator:
""" Parses the provided frame data and builds the motmetrics accumulator with
it.
Params
------
truth_framedata: The data for the ground-truth output, parsed as a dictionary
where the keys are frame numbers mapping to lists of detections for that
frame. (as given by parse_XML_by_frame)
model_framedata: The data for the model's output, in the same format as the
truth_framedata.
Returns
-------
A motmetrics.MOTAccumulator that stores the matches and MOTEvents for all the
frames in the input framedata.
"""
# Get the maximum number of frames.
max_frame = -1
if truth_framedata:
truth_frames = list(map(lambda x: int(x), truth_framedata.keys()))
max_frame = max(truth_frames)
if model_framedata:
model_frames = list(map(lambda x: int(x), model_framedata.keys()))
max_model_frame = max(model_frames)
max_frame = max(max_frame, max_model_frame)
#print("max_frame is {}".format(max_frame))
acc = mm.MOTAccumulator()
# Loop through each frame and build the distance matrix.
for i in range(max_frame + 1):
# Get the array of detections for the current frame
truth_curr_frame = truth_framedata.get(str(i), [])
model_curr_frame = model_framedata.get(str(i), [])
# Build the distance/cost matrix.
dist_matrix = build_cost_matrix(truth_curr_frame,
model_curr_frame,
iou=True)
# Get the object indices of detections for the truth and model as arrays.
truth_indices = list(map(lambda det: int(det['id']), truth_curr_frame))
model_indices = list(map(lambda det: int(det['id']), model_curr_frame))
# Pass into the motmetrics accumulator, which will automatically match
# the indices.
acc.update(truth_indices, model_indices, dist_matrix, frameid=i)
return acc
def initialize_base(self, ground_truth, working_dir_path):
# Create an accumulator that will be updated during each frame
self.acc = mm.MOTAccumulator(auto_id=True)
self.ground_truth = ground_truth
if isinstance(ground_truth, str):
self.ground_truth = pandas_loader.load_csv(working_dir_path,
ground_truth)
self.gt_frame_numbers_cam = set(
self.ground_truth["frame_no_cam"].tolist())
self.gt_frame_numbers_cam = list(map(int, self.gt_frame_numbers_cam))
def test_metrics_with_no_events():
acc = mm.MOTAccumulator()
mh = mm.metrics.create()
metr = mh.compute(acc,
metrics=['motp', 'mota', 'num_predictions'],
return_dataframe=False,
return_cached=True)
assert np.isnan(metr['mota'])
assert np.isnan(metr['motp'])
assert metr['num_predictions'] == 0
assert metr['num_objects'] == 0
assert metr['num_detections'] == 0
def test_ids():
"""Test metrics with frame IDs specified manually."""
acc = mm.MOTAccumulator()
# No data
acc.update([], [], [], frameid=0)
# Match
acc.update([1, 2], [1, 2], [[1, 0], [0, 1]], frameid=1)
# Switch also Transfer
acc.update([1, 2], [1, 2], [[0.4, np.nan], [np.nan, 0.4]], frameid=2)
# Match
acc.update([1, 2], [1, 2], [[0, 1], [1, 0]], frameid=3)
# Ascend (switch)
acc.update([1, 2], [2, 3], [[1, 0], [0.4, 0.7]], frameid=4)
# Migrate (transfer)
acc.update([1, 3], [2, 3], [[1, 0], [0.4, 0.7]], frameid=5)
# No data
acc.update([], [], [], frameid=6)
mh = mm.metrics.create()
metr = mh.compute(acc,
return_dataframe=False,
return_cached=True,
metrics=[
'num_matches',
'num_false_positives',
'num_misses',
'num_switches',
'num_transfer',
'num_ascend',
'num_migrate',
'num_detections',
'num_objects',
'num_predictions',
'mota',
'motp',
'num_frames',
])
assert metr['num_matches'] == 7
assert metr['num_false_positives'] == 0
assert metr['num_misses'] == 0
assert metr['num_switches'] == 3
assert metr['num_transfer'] == 3
assert metr['num_ascend'] == 1
assert metr['num_migrate'] == 1
assert metr['num_detections'] == 10
assert metr['num_objects'] == 10
assert metr['num_predictions'] == 10
assert metr['mota'] == approx(1. - (0 + 0 + 3) / 10)
assert metr['motp'] == approx(1.6 / 10)
assert metr['num_frames'] == 7
def idf1_score(list_gt_bboxes, tracked_detections):
"""The lists passed are of the form [label, tly, tlx, width, height, confidence, id]"""
for frames in list_gt_bboxes:
for element in frames:
frame = getattr(element, 'frame')
xtl = int(float(getattr(element, 'xtl')))
ytl = int(float(getattr(element, 'ytl')))
xbr = int(float(getattr(element, 'xbr')))
ybr = int(float(getattr(element, 'ybr')))
height = ybr - ytl
width = xbr - xtl
id_det = int(getattr(element, 'id'))
if getattr(element, 'score') is None:
confidence = np.random
else: confidence = getattr(element, 'score')
'''if frame == getattr(element, 'frame'):
element.append(['car', xtl, ytl, height, width, random(), id_det])
else:'''
element[frame] = ['car', xtl, ytl, height, width, confidence, id_det]
acc = mm.MOTAccumulator(auto_id=True)
for gt_elements_frame, det_elements_frame in zip(bboxes, tracked_detections):
det_ids = []
gt_ids = []
mm_det_bboxes = []
mm_gt_bboxes = []
for gt_bbox in gt_elements_frame:
# mm_gt_bboxes.append([(gt_bbox[1]+gt_bbox[3])/2, (gt_bbox[2]+gt_bbox[4])/2, gt_bbox[3]-gt_bbox[1],
# gt_bbox[4]-gt_bbox[2]])
mm_gt_bboxes.append([gt_bbox[2], gt_bbox[1], gt_bbox[3], gt_bbox[4]])
gt_ids.append(gt_bbox[-1])
for det_bbox in det_elements_frame:
# mm_det_bboxes.append([(det_bbox[1]+det_bbox[3])/2, (det_bbox[2]+det_bbox[4])/2, det_bbox[3]-det_bbox[1],
# det_bbox[4]-det_bbox[2]])
mm_det_bboxes.append([det_bbox[2], det_bbox[1], det_bbox[3], det_bbox[4]])
det_ids.append(det_bbox[-1])
distances_gt_det = mm.distances.iou_matrix(mm_gt_bboxes, mm_det_bboxes, max_iou=1.)
acc.update(gt_ids, det_ids, distances_gt_det)
mh = mm.metrics.create()
summary = mh.compute(acc, metrics=['idf1'], name='acc')
print(summary)
return summary
def get_MOTA_accumulator(seq_id, size_limit, tolerance, tracker_dir):
data = get_data(seq_id, size_limit, tolerance, tracker_dir)
# Create an accumulator that will be updated during each frame
acc = mm.MOTAccumulator(auto_id=True)
for frame in data:
acc.update(
frame[0], # Ground truth objects in this frame
frame[1], # Detector hypotheses in this frame
frame[2] # Distances from objects to hypotheses
)
return acc
def __init__(self):
self.position_errors = []
self.velocity_errors = []
self.position_gt = []
self.velocity_gt = []
self.position_det = []
self.velocity_det = []
self.velocity_success = []
self.tf_buffer = tf2_ros.Buffer()
self.tf_listener = tf2_ros.TransformListener(self.tf_buffer)
self.acc = mot.MOTAccumulator(auto_id=True)
self.rc_car = OptiTrackData('rc_car', self.tf_buffer)
self.oncoming_car = OptiTrackData('oncoming_car', self.tf_buffer)
def initialize_base(self):
self.load_evaluation_files()
# Create an accumulator that will be updated during each frame
self.acc = mm.MOTAccumulator(auto_id=True)
self.overall_frame_count = 0
self.gt_cams_df_frame_numbers = []
for gt_cam_dataframe in self.ground_truth_dataframes:
gt_cam_df_grouped = gt_cam_dataframe.groupby("frame_no_cam", as_index=False).mean()
gt_cam_frame_numbers = gt_cam_df_grouped["frame_no_cam"].tolist()
gt_cam_frame_numbers = list(map(int, gt_cam_frame_numbers))
self.overall_frame_count += len(gt_cam_frame_numbers)
self.gt_cams_df_frame_numbers.extend(gt_cam_frame_numbers)
self.max_all_frame_numbers = max(self.gt_cams_df_frame_numbers)
self.min_all_frame_numbers = min(self.gt_cams_df_frame_numbers)
def test_issue19():
acc = mm.MOTAccumulator()
g0 = [0, 1]
p0 = [0, 1]
d0 = [[0.2, np.nan], [np.nan, 0.2]]
g1 = [2, 3]
p1 = [2, 3, 4, 5]
d1 = [[0.28571429, 0.5, 0.0, np.nan], [ np.nan, 0.44444444, np.nan, 0.0 ]]
acc.update(g0, p0, d0, 0)
acc.update(g1, p1, d1, 1)
mh = mm.metrics.create()
result = mh.compute(acc)
