def compute_3rd_party_metrics(self): """ Computes the metrics defined in - Stiefelhagen 2008: Evaluating Multiple Object Tracking Performance: The CLEAR MOT Metrics MOTA, MOTAL, MOTP - Nevatia 2008: Global Data Association for Multi-Object Tracking Using Network Flows mostly_tracked/partialy_tracked/mostly_lost """ # construct Munkres object for Hungarian Method association hm = Munkres() max_cost = 1e9 # go through all frames and associate ground truth and tracker results # groundtruth and tracker contain lists for every single frame containing lists of KITTI format detections seq_gt = self.groundtruth seq_dc = self.dcareas # don't care areas seq_result_data = self.result_data seq_trajectories = defaultdict(list) seq_ignored = defaultdict(list) last_frame_ids = [[], []] for i_frame in tqdm(range(len(seq_gt))): frame_gts = seq_gt[i_frame] frame_dcs = seq_dc[i_frame] frame_results = seq_result_data[i_frame] # counting total number of ground truth and tracker objects self.n_gt += len(frame_gts) self.n_tr += len(frame_results) # use hungarian method to associate, using boxoverlap 0..1 as cost # build cost matrix cost_matrix = [] frame_ids = [[], []] # loop over ground truth objects in one frame for gt in frame_gts: # save current ids frame_ids[0].append(gt.track_id) frame_ids[1].append(-1) gt.tracker = -1 gt.id_switch = 0 gt.fragmentation = 0 cost_row = [] # loop over tracked objects in one frame for result in frame_results: # overlap == 1 means cost == 0 # Rect(cx, cy, l, w, angle) r1 = Rect(gt.cx, gt.cy, gt.l, gt.w, gt.yaw) r2 = Rect(result.cx, result.cy, result.l, result.w, result.yaw) iou = r1.intersection_over_union(r2) cost = 1 - iou # gating for boxoverlap if cost <= self.min_overlap: cost_row.append(cost) else: cost_row.append(max_cost) # = 1e9 # return cost_matrix.append(cost_row) # all ground truth trajectories are initially not associated # extend groundtruth trajectories lists (merge lists) seq_trajectories[gt.track_id].append(-1) seq_ignored[gt.track_id].append(False) if len(frame_gts) is 0: cost_matrix = [[]] # associate association_matrix = hm.compute(cost_matrix) # tmp variables for sanity checks tmptp = 0 tmpfp = 0 tmpfn = 0 # mapping for tracker ids and ground truth ids for row, col in association_matrix: # apply gating on boxoverlap c = cost_matrix[row][col] if c < max_cost: frame_gts[row].tracker = frame_results[col].track_id frame_ids[1][row] = frame_results[col].track_id frame_results[col].valid = True frame_gts[row].distance = c seq_trajectories[frame_gts[row].track_id][ -1] = frame_results[col].track_id # true positives are only valid associations self.tp += 1 tmptp += 1 else: # wrong data association frame_gts[row].tracker = -1 self.fn += 1 tmpfn += 1 # associate tracker and DontCare areas # ignore tracker in neighboring classes nignoredtracker = 0 # number of ignored tracker detections ignoredtrackers = dict() # will associate the track_id with -1 # if it is not ignored and 1 if it is # ignored; # this is used to avoid double counting ignored # cases, see the next loop # check for ignored FN/TP (truncation or neighboring object class) nignoredfn = 0 # the number of ignored false negatives nignoredtp = 0 # the number of ignored true positives gi = 0 for gt in frame_gts: if gt.tracker < 0: if gt.occlusion > self.max_occlusion or gt.truncation > self.max_truncation: seq_ignored[gt.track_id][-1] = True gt.ignored = True nignoredfn += 1 elif gt.tracker >= 0: if gt.occlusion > self.max_occlusion or gt.truncation > self.max_truncation: seq_ignored[gt.track_id][-1] = True gt.ignored = True nignoredtp += 1 gi += 1 # the below might be confusion, check the comments in __init__ # to see what the individual statistics represent # correct TP by number of ignored TP due to truncation # ignored TP are shown as tracked in visualization tmptp -= nignoredtp # count the number of ignored true positives self.itp += nignoredtp # adjust the number of ground truth objects considered self.n_gt -= (nignoredfn + nignoredtp) # count the number of ignored ground truth objects self.n_igt += nignoredfn + nignoredtp # count the number of ignored tracker objects self.n_itr += nignoredtracker # false negatives = associated gt bboxes exceding association threshold + non-associated gt bboxes tmpfn += len(frame_gts) - len(association_matrix) - nignoredfn self.fn += len(frame_gts) - len(association_matrix) - nignoredfn self.ifn += nignoredfn # false positives = tracker bboxes - associated tracker bboxes tmpfp += len(frame_results) - tmptp - nignoredtracker - nignoredtp self.fp += len( frame_results) - tmptp - nignoredtracker - nignoredtp # sanity checks # - the number of true positives minues ignored true positives # should be greater or equal to 0 # - the number of false negatives should be greater or equal to 0 # - the number of false positives needs to be greater or equal to 0 # otherwise ignored detections might be counted double # - the number of counted true positives (plus ignored ones) # and the number of counted false negatives (plus ignored ones) # should match the total number of ground truth objects # - the number of counted true positives (plus ignored ones) # and the number of counted false positives # plus the number of ignored tracker detections should # match the total number of tracker detections; note that # nignoredpairs is subtracted here to avoid double counting # of ignored detection sin nignoredtp and nignoredtracker if tmptp < 0: print(tmptp, nignoredtp) raise NameError("Something went wrong! TP is negative") if tmpfn < 0: print(tmpfn, len(frame_gts), len(association_matrix), nignoredfn, nignoredpairs) raise NameError("Something went wrong! FN is negative") if tmpfp < 0: print(tmpfp, len(frame_results), tmptp, nignoredtracker, nignoredtp, nignoredpairs) raise NameError("Something went wrong! FP is negative") if tmptp + tmpfn is not len(frame_gts) - nignoredfn - nignoredtp: print("seqidx", seq_idx) print("frame ", f) print("TP ", tmptp) print("FN ", tmpfn) print("FP ", tmpfp) print("nGT ", len(frame_gts)) print("nAss ", len(association_matrix)) print("ign GT", nignoredfn) print("ign TP", nignoredtp) raise NameError( "Something went wrong! nGroundtruth is not TP+FN") if tmptp + tmpfp + nignoredtp + nignoredtracker is not len( frame_results): print(seq_idx, f, len(frame_results), tmptp, tmpfp) print(len(association_matrix), association_matrix) raise NameError("Something went wrong! nTracker is not TP+FP") # loop over ground truth track_id # check for id switches or fragmentations for i, gt_id in enumerate(frame_ids[0]): if gt_id in last_frame_ids[0]: idx = last_frame_ids[0].index(gt_id) tid = frame_ids[1][i] lid = last_frame_ids[1][idx] if tid != lid and lid != -1 and tid != -1: if frame_gts[i].truncation < self.max_truncation: frame_gts[i].id_switch = 1 if tid != lid and lid != -1: if frame_gts[i].truncation < self.max_truncation: frame_gts[i].fragmentation = 1 # save current index last_frame_ids = frame_ids # compute mostly_tracked/partialy_tracked/mostly_lost, fragments, idswitches for all groundtruth trajectories n_ignored_tr_total = 0 if len(seq_trajectories) == 0: print("Error: There is no trajectories data") return n_ignored_tr = 0 for g, ign_g in zip(seq_trajectories.values(), seq_ignored.values()): # all frames of this gt trajectory are ignored if all(ign_g): n_ignored_tr += 1 n_ignored_tr_total += 1 continue # all frames of this gt trajectory are not assigned to any detections if all([this == -1 for this in g]): self.mostly_lost += 1 continue # compute tracked frames in trajectory last_id = g[0] # first detection (necessary to be in gt_trajectories) is always tracked tracked = 1 if g[0] >= 0 else 0 lgt = 0 if ign_g[0] else 1 for f in range(1, len(g)): if ign_g[f]: last_id = -1 continue lgt += 1 if last_id != g[f] and last_id != -1 and g[f] != -1 and g[ f - 1] != -1: self.id_switches += 1 if f < len(g) - 1 and g[f - 1] != g[f] and last_id != -1 and g[ f] != -1 and g[f + 1] != -1: self.fragments += 1 if g[f] != -1: tracked += 1 last_id = g[f] # handle last frame; tracked state is handled in for loop (g[f]!=-1) if len(g) > 1 and g[f - 1] != g[f] and last_id != -1 and g[ f] != -1 and not ign_g[f]: self.fragments += 1 # compute mostly_tracked/partialy_tracked/mostly_lost tracking_ratio = tracked / float(len(g) - sum(ign_g)) if tracking_ratio > 0.8: self.mostly_tracked += 1 elif tracking_ratio < 0.2: self.mostly_lost += 1 else: # 0.2 <= tracking_ratio <= 0.8 self.partialy_tracked += 1 if (self.n_gt_trajectories - n_ignored_tr_total) == 0: self.mostly_tracked = 0. self.partialy_tracked = 0. self.mostly_lost = 0. else: self.mostly_tracked /= float(self.n_gt_trajectories - n_ignored_tr_total) self.partialy_tracked /= float(self.n_gt_trajectories - n_ignored_tr_total) self.mostly_lost /= float(self.n_gt_trajectories - n_ignored_tr_total) # precision/recall if (self.fp + self.tp) == 0 or (self.tp + self.fn) == 0: self.recall = 0. self.precision = 0. else: self.recall = self.tp / float(self.tp + self.fn) self.precision = self.tp / float(self.fp + self.tp) return True