def compute_3rd_party_metrics(self):
"""
Computes the metrics defined in
- Stiefelhagen 2008: Evaluating Multiple Object Tracking Performance: The CLEAR MOT Metrics
MOTA, MOTAL, MOTP
- Nevatia 2008: Global Data Association for Multi-Object Tracking Using Network Flows
mostly_tracked/partialy_tracked/mostly_lost
"""
# construct Munkres object for Hungarian Method association
hm = Munkres()
max_cost = 1e9
# go through all frames and associate ground truth and tracker results
# groundtruth and tracker contain lists for every single frame containing lists of KITTI format detections
seq_gt = self.groundtruth
seq_dc = self.dcareas # don't care areas
seq_result_data = self.result_data
seq_trajectories = defaultdict(list)
seq_ignored = defaultdict(list)
last_frame_ids = [[], []]
for i_frame in tqdm(range(len(seq_gt))):
frame_gts = seq_gt[i_frame]
frame_dcs = seq_dc[i_frame]
frame_results = seq_result_data[i_frame]
# counting total number of ground truth and tracker objects
self.n_gt += len(frame_gts)
self.n_tr += len(frame_results)
# use hungarian method to associate, using boxoverlap 0..1 as cost
# build cost matrix
cost_matrix = []
frame_ids = [[], []]
# loop over ground truth objects in one frame
for gt in frame_gts:
# save current ids
frame_ids[0].append(gt.track_id)
frame_ids[1].append(-1)
gt.tracker = -1
gt.id_switch = 0
gt.fragmentation = 0
cost_row = []
# loop over tracked objects in one frame
for result in frame_results:
# overlap == 1 means cost == 0
# Rect(cx, cy, l, w, angle)
r1 = Rect(gt.cx, gt.cy, gt.l, gt.w, gt.yaw)
r2 = Rect(result.cx, result.cy, result.l, result.w,
result.yaw)
iou = r1.intersection_over_union(r2)
cost = 1 - iou
# gating for boxoverlap
if cost <= self.min_overlap:
cost_row.append(cost)
else:
cost_row.append(max_cost) # = 1e9
# return
cost_matrix.append(cost_row)
# all ground truth trajectories are initially not associated
# extend groundtruth trajectories lists (merge lists)
seq_trajectories[gt.track_id].append(-1)
seq_ignored[gt.track_id].append(False)
if len(frame_gts) is 0:
cost_matrix = [[]]
# associate
association_matrix = hm.compute(cost_matrix)
# tmp variables for sanity checks
tmptp = 0
tmpfp = 0
tmpfn = 0
# mapping for tracker ids and ground truth ids
for row, col in association_matrix:
# apply gating on boxoverlap
c = cost_matrix[row][col]
if c < max_cost:
frame_gts[row].tracker = frame_results[col].track_id
frame_ids[1][row] = frame_results[col].track_id
frame_results[col].valid = True
frame_gts[row].distance = c
seq_trajectories[frame_gts[row].track_id][
-1] = frame_results[col].track_id
# true positives are only valid associations
self.tp += 1
tmptp += 1
else:
# wrong data association
frame_gts[row].tracker = -1
self.fn += 1
tmpfn += 1
# associate tracker and DontCare areas
# ignore tracker in neighboring classes
nignoredtracker = 0 # number of ignored tracker detections
ignoredtrackers = dict() # will associate the track_id with -1
# if it is not ignored and 1 if it is
# ignored;
# this is used to avoid double counting ignored
# cases, see the next loop
# check for ignored FN/TP (truncation or neighboring object class)
nignoredfn = 0 # the number of ignored false negatives
nignoredtp = 0 # the number of ignored true positives
gi = 0
for gt in frame_gts:
if gt.tracker < 0:
if gt.occlusion > self.max_occlusion or gt.truncation > self.max_truncation:
seq_ignored[gt.track_id][-1] = True
gt.ignored = True
nignoredfn += 1
elif gt.tracker >= 0:
if gt.occlusion > self.max_occlusion or gt.truncation > self.max_truncation:
seq_ignored[gt.track_id][-1] = True
gt.ignored = True
nignoredtp += 1
gi += 1
# the below might be confusion, check the comments in __init__
# to see what the individual statistics represent
# correct TP by number of ignored TP due to truncation
# ignored TP are shown as tracked in visualization
tmptp -= nignoredtp
# count the number of ignored true positives
self.itp += nignoredtp
# adjust the number of ground truth objects considered
self.n_gt -= (nignoredfn + nignoredtp)
# count the number of ignored ground truth objects
self.n_igt += nignoredfn + nignoredtp
# count the number of ignored tracker objects
self.n_itr += nignoredtracker
# false negatives = associated gt bboxes exceding association threshold + non-associated gt bboxes
tmpfn += len(frame_gts) - len(association_matrix) - nignoredfn
self.fn += len(frame_gts) - len(association_matrix) - nignoredfn
self.ifn += nignoredfn
# false positives = tracker bboxes - associated tracker bboxes
tmpfp += len(frame_results) - tmptp - nignoredtracker - nignoredtp
self.fp += len(
frame_results) - tmptp - nignoredtracker - nignoredtp
# sanity checks
# - the number of true positives minues ignored true positives
# should be greater or equal to 0
# - the number of false negatives should be greater or equal to 0
# - the number of false positives needs to be greater or equal to 0
# otherwise ignored detections might be counted double
# - the number of counted true positives (plus ignored ones)
# and the number of counted false negatives (plus ignored ones)
# should match the total number of ground truth objects
# - the number of counted true positives (plus ignored ones)
# and the number of counted false positives
# plus the number of ignored tracker detections should
# match the total number of tracker detections; note that
# nignoredpairs is subtracted here to avoid double counting
# of ignored detection sin nignoredtp and nignoredtracker
if tmptp < 0:
print(tmptp, nignoredtp)
raise NameError("Something went wrong! TP is negative")
if tmpfn < 0:
print(tmpfn, len(frame_gts), len(association_matrix),
nignoredfn, nignoredpairs)
raise NameError("Something went wrong! FN is negative")
if tmpfp < 0:
print(tmpfp, len(frame_results), tmptp, nignoredtracker,
nignoredtp, nignoredpairs)
raise NameError("Something went wrong! FP is negative")
if tmptp + tmpfn is not len(frame_gts) - nignoredfn - nignoredtp:
print("seqidx", seq_idx)
print("frame ", f)
print("TP ", tmptp)
print("FN ", tmpfn)
print("FP ", tmpfp)
print("nGT ", len(frame_gts))
print("nAss ", len(association_matrix))
print("ign GT", nignoredfn)
print("ign TP", nignoredtp)
raise NameError(
"Something went wrong! nGroundtruth is not TP+FN")
if tmptp + tmpfp + nignoredtp + nignoredtracker is not len(
frame_results):
print(seq_idx, f, len(frame_results), tmptp, tmpfp)
print(len(association_matrix), association_matrix)
raise NameError("Something went wrong! nTracker is not TP+FP")
# loop over ground truth track_id
# check for id switches or fragmentations
for i, gt_id in enumerate(frame_ids[0]):
if gt_id in last_frame_ids[0]:
idx = last_frame_ids[0].index(gt_id)
tid = frame_ids[1][i]
lid = last_frame_ids[1][idx]
if tid != lid and lid != -1 and tid != -1:
if frame_gts[i].truncation < self.max_truncation:
frame_gts[i].id_switch = 1
if tid != lid and lid != -1:
if frame_gts[i].truncation < self.max_truncation:
frame_gts[i].fragmentation = 1
# save current index
last_frame_ids = frame_ids
# compute mostly_tracked/partialy_tracked/mostly_lost, fragments, idswitches for all groundtruth trajectories
n_ignored_tr_total = 0
if len(seq_trajectories) == 0:
print("Error: There is no trajectories data")
return
n_ignored_tr = 0
for g, ign_g in zip(seq_trajectories.values(), seq_ignored.values()):
# all frames of this gt trajectory are ignored
if all(ign_g):
n_ignored_tr += 1
n_ignored_tr_total += 1
continue
# all frames of this gt trajectory are not assigned to any detections
if all([this == -1 for this in g]):
self.mostly_lost += 1
continue
# compute tracked frames in trajectory
last_id = g[0]
# first detection (necessary to be in gt_trajectories) is always tracked
tracked = 1 if g[0] >= 0 else 0
lgt = 0 if ign_g[0] else 1
for f in range(1, len(g)):
if ign_g[f]:
last_id = -1
continue
lgt += 1
if last_id != g[f] and last_id != -1 and g[f] != -1 and g[
f - 1] != -1:
self.id_switches += 1
if f < len(g) - 1 and g[f - 1] != g[f] and last_id != -1 and g[
f] != -1 and g[f + 1] != -1:
self.fragments += 1
if g[f] != -1:
tracked += 1
last_id = g[f]
# handle last frame; tracked state is handled in for loop (g[f]!=-1)
if len(g) > 1 and g[f - 1] != g[f] and last_id != -1 and g[
f] != -1 and not ign_g[f]:
self.fragments += 1
# compute mostly_tracked/partialy_tracked/mostly_lost
tracking_ratio = tracked / float(len(g) - sum(ign_g))
if tracking_ratio > 0.8:
self.mostly_tracked += 1
elif tracking_ratio < 0.2:
self.mostly_lost += 1
else: # 0.2 <= tracking_ratio <= 0.8
self.partialy_tracked += 1
if (self.n_gt_trajectories - n_ignored_tr_total) == 0:
self.mostly_tracked = 0.
self.partialy_tracked = 0.
self.mostly_lost = 0.
else:
self.mostly_tracked /= float(self.n_gt_trajectories -
n_ignored_tr_total)
self.partialy_tracked /= float(self.n_gt_trajectories -
n_ignored_tr_total)
self.mostly_lost /= float(self.n_gt_trajectories -
n_ignored_tr_total)
# precision/recall
if (self.fp + self.tp) == 0 or (self.tp + self.fn) == 0:
self.recall = 0.
self.precision = 0.
else:
self.recall = self.tp / float(self.tp + self.fn)
self.precision = self.tp / float(self.fp + self.tp)
return True
