def test_change_solver(): def mysolver(x): mysolver.called += 1 return np.array([]), np.array([]) mysolver.called = 0 costs = np.array([[6, 9, 1], [10, 3, 2], [8, 7, 4.]]) with lap.set_default_solver(mysolver): rids, cids = lap.linear_sum_assignment(costs) assert mysolver.called == 1 rids, cids = lap.linear_sum_assignment(costs) assert mysolver.called == 1
def test_change_solver(): """Tests effect of lap.set_default_solver.""" def mysolver(_): mysolver.called += 1 return np.array([]), np.array([]) mysolver.called = 0 costs = np.asfarray([[6, 9, 1], [10, 3, 2], [8, 7, 4]]) with lap.set_default_solver(mysolver): lap.linear_sum_assignment(costs) assert mysolver.called == 1 lap.linear_sum_assignment(costs) assert mysolver.called == 1
def test_assign_empty(solver): costs = np.asfarray([[]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) np.testing.assert_equal(np.size(result), 0) np.testing.assert_equal(costs, costs_copy)
def test_unbalanced_disallowed_tall(solver): costs = np.asfarray([[np.nan, 9], [11, np.nan], [8, 7]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) expected = np.array([[0, 2], [1, 0]]) np.testing.assert_equal(result, expected) np.testing.assert_equal(costs, costs_copy)
def test_unbalanced_tall(solver): costs = np.asfarray([[6, 10], [4, 8], [1, 2]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) expected = np.array([[1, 2], [0, 1]]) np.testing.assert_equal(result, expected) np.testing.assert_equal(costs, costs_copy)
def test_assign_disallowed(solver): costs = np.asfarray([[5, 9, np.nan], [10, np.nan, 2], [8, 7, 4]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) expected = np.array([[0, 1, 2], [0, 2, 1]]) np.testing.assert_equal(result, expected) np.testing.assert_equal(costs, costs_copy)
def test_assign_full_negative(solver): costs = -7 + np.asfarray([[5, 5, 6], [1, 2, 5], [2, 4, 5]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) # Optimal matching is (0, 2), (1, 1), (2, 0) for 5 + 1 + 1. expected = np.array([[0, 1, 2], [2, 1, 0]]) np.testing.assert_equal(result, expected) np.testing.assert_equal(costs, costs_copy)
def test_assign_easy(solver): """Problem that could be solved by a greedy algorithm.""" costs = np.asfarray([[6, 9, 1], [10, 3, 2], [8, 7, 4]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) expected = np.array([[0, 1, 2], [2, 1, 0]]) np.testing.assert_equal(result, expected) np.testing.assert_equal(costs, costs_copy)
def preprocessResult(res, gt, inifile): """Preprocesses data for utils.CLEAR_MOT_M. Returns a subset of the predictions. """ # pylint: disable=too-many-locals st = time.time() labels = [ 'ped', # 1 'person_on_vhcl', # 2 'car', # 3 'bicycle', # 4 'mbike', # 5 'non_mot_vhcl', # 6 'static_person', # 7 'distractor', # 8 'occluder', # 9 'occluder_on_grnd', # 10 'occluder_full', # 11 'reflection', # 12 'crowd', # 13 ] distractors = [ 'person_on_vhcl', 'static_person', 'distractor', 'reflection' ] is_distractor = {i + 1: x in distractors for i, x in enumerate(labels)} for i in distractors: is_distractor[i] = 1 seqIni = ConfigParser() seqIni.read(inifile, encoding='utf8') F = int(seqIni['Sequence']['seqLength']) todrop = [] for t in range(1, F + 1): if t not in res.index or t not in gt.index: continue resInFrame = res.loc[t] GTInFrame = gt.loc[t] A = GTInFrame[['X', 'Y', 'Width', 'Height']].values B = resInFrame[['X', 'Y', 'Width', 'Height']].values disM = mmd.iou_matrix(A, B, max_iou=0.5) le, ri = linear_sum_assignment(disM) flags = [ 1 if is_distractor[it['ClassId']] or it['Visibility'] < 0. else 0 for i, (k, it) in enumerate(GTInFrame.iterrows()) ] hid = [k for k, it in resInFrame.iterrows()] for i, j in zip(le, ri): if not np.isfinite(disM[i, j]): continue if flags[i]: todrop.append((t, hid[j])) ret = res.drop(labels=todrop) logging.info('Preprocess take %.3f seconds and remove %d boxes.', time.time() - st, len(todrop)) return ret
def test_assign_full(solver): """Problem that would be incorrect using a greedy algorithm.""" costs = np.asfarray([[5, 5, 6], [1, 2, 5], [2, 4, 5]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) # Optimal matching is (0, 2), (1, 1), (2, 0) for 6 + 2 + 2. expected = np.asfarray([[0, 1, 2], [2, 1, 0]]) np.testing.assert_equal(result, expected) np.testing.assert_equal(costs, costs_copy)
def test_assign_attractive_broken_ring(solver): """Graph contains cheap broken ring and expensive unbroken ring.""" costs = np.asfarray([[np.nan, 1000, np.nan], [np.nan, 1, 1000], [1000, np.nan, 1]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) # Optimal solution is (0, 1), (1, 2), (2, 0) with cost 1000 + 1000 + 1000. # Solver might choose (0, 0), (1, 1), (2, 2) with cost inf + 1 + 1. expected = np.array([[0, 1, 2], [1, 2, 0]]) np.testing.assert_equal(result, expected) np.testing.assert_equal(costs, costs_copy)
def test_assign_attractive_disallowed(solver): """Graph contains an attractive edge that cannot be used.""" costs = np.asfarray([[-10000, -1], [-1, np.nan]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) # The optimal solution is (0, 1), (1, 0) for a cost of -2. # Ensure that the algorithm does not choose the (0, 0) edge. # This would not be a perfect matching. expected = np.array([[0, 1], [1, 0]]) np.testing.assert_equal(result, expected) np.testing.assert_equal(costs, costs_copy)
def id_global_assignment(df): """ID measures: Global min-cost assignment for ID measures.""" oids = df.full['OId'].dropna().unique() hids = df.full['HId'].dropna().unique() hids_idx = dict((h, i) for i, h in enumerate(hids)) hcs = [len(df.raw[(df.raw.HId == h)].groupby(level=0)) for h in hids] ocs = [len(df.raw[(df.raw.OId == o)].groupby(level=0)) for o in oids] no = oids.shape[0] nh = hids.shape[0] df = df.raw.reset_index() df = df.set_index(['OId', 'HId']) df = df.sort_index(level=[0, 1]) fpmatrix = np.full((no + nh, no + nh), 0.) fnmatrix = np.full((no + nh, no + nh), 0.) fpmatrix[no:, :nh] = np.nan fnmatrix[:no, nh:] = np.nan for r, oc in enumerate(ocs): fnmatrix[r, :nh] = oc fnmatrix[r, nh + r] = oc for c, hc in enumerate(hcs): fpmatrix[:no, c] = hc fpmatrix[c + no, c] = hc for r, o in enumerate(oids): try: df_o = df.loc[o, 'D'].dropna() except IndexError: continue for h, ex in df_o.groupby(level=0).count().iteritems(): c = hids_idx[h] fpmatrix[r, c] -= ex fnmatrix[r, c] -= ex costs = fpmatrix + fnmatrix rids, cids = linear_sum_assignment(costs) return { 'fpmatrix': fpmatrix, 'fnmatrix': fnmatrix, 'rids': rids, 'cids': cids, 'costs': costs, 'min_cost': costs[rids, cids].sum() }
def test_assign_infeasible(solver): """Tests that minimum-cost solution with most edges is found.""" costs = np.asfarray([[np.nan, np.nan, 2], [np.nan, np.nan, 1], [8, 7, 4]]) costs_copy = costs.copy() result = lap.linear_sum_assignment(costs, solver=solver) # Optimal matching is (1, 2), (2, 1). expected = np.array([[1, 2], [2, 1]]) np.testing.assert_equal(result, expected) np.testing.assert_equal(costs, costs_copy)
def test_lap_solvers(): assert len(lap.available_solvers) > 0 print(lap.available_solvers) costs = np.array([[6, 9, 1], [10, 3, 2], [8, 7, 4.]]) costs_copy = costs.copy() results = [ lap.linear_sum_assignment(costs, solver=s) for s in lap.available_solvers ] expected = np.array([[0, 1, 2], [2, 1, 0]]) [np.testing.assert_allclose(r, expected) for r in results] np.testing.assert_allclose(costs, costs_copy) costs = np.array([[5, 9, np.nan], [10, np.nan, 2], [8, 7, 4.]]) costs_copy = costs.copy() results = [ lap.linear_sum_assignment(costs, solver=s) for s in lap.available_solvers ] expected = np.array([[0, 1, 2], [0, 2, 1]]) [np.testing.assert_allclose(r, expected) for r in results] np.testing.assert_allclose(costs, costs_copy)
def preprocessResult(res, gt, inifile): st = time.time() labels = ['ped', # 1 'person_on_vhcl', # 2 'car', # 3 'bicycle', # 4 'mbike', # 5 'non_mot_vhcl', # 6 'static_person', # 7 'distractor', # 8 'occluder', # 9 'occluder_on_grnd', #10 'occluder_full', # 11 'reflection', # 12 'crowd' # 13 ] distractors_ = ['person_on_vhcl','static_person','distractor','reflection'] distractors = {i+1 : x in distractors_ for i,x in enumerate(labels)} for i in distractors_: distractors[i] = 1 seqIni = ConfigParser() seqIni.read(inifile, encoding='utf8') F = int(seqIni['Sequence']['seqLength']) todrop = [] for t in range(1,F+1): if t not in res.index or t not in gt.index: continue #st = time.time() resInFrame = res.loc[t] N = len(resInFrame) GTInFrame = gt.loc[t] Ngt = len(GTInFrame) A = GTInFrame[['X','Y','Width','Height']].values B = resInFrame[['X','Y','Width','Height']].values disM = mmd.iou_matrix(A, B, max_iou = 0.5) #en = time.time() #print('----', 'disM', en - st) le, ri = linear_sum_assignment(disM) flags = [1 if distractors[it['ClassId']] or it['Visibility']<0. else 0 for i,(k,it) in enumerate(GTInFrame.iterrows())] hid = [k for k,it in resInFrame.iterrows()] for i, j in zip(le, ri): if not np.isfinite(disM[i, j]): continue if flags[i]: todrop.append((t, hid[j])) #en = time.time() #print('Frame %d: '%t, en - st) ret = res.drop(labels=todrop) logging.info('Preprocess take %.3f seconds and remove %d boxes.'%(time.time() - st, len(todrop))) return ret
def acc_single_video(results, gts, iou_thr=0.5, ignore_iof_thr=0.5, ignore_by_classes=False): """Accumulate results in a single video.""" num_classes = len(results[0]) accumulators = [ mm.MOTAccumulator(auto_id=True) for i in range(num_classes) ] for result, gt in zip(results, gts): if ignore_by_classes: gt_ignore = outs2results(bboxes=gt['bboxes_ignore'], labels=gt['labels_ignore'], num_classes=num_classes)['bbox_results'] else: gt_ignore = [gt['bboxes_ignore'] for i in range(num_classes)] gt = outs2results(bboxes=gt['bboxes'], labels=gt['labels'], ids=gt['instance_ids'], num_classes=num_classes)['bbox_results'] for i in range(num_classes): gt_ids, gt_bboxes = gt[i][:, 0].astype(np.int), gt[i][:, 1:] pred_ids, pred_bboxes = result[i][:, 0].astype( np.int), result[i][:, 1:-1] dist = bbox_distances(gt_bboxes, pred_bboxes, iou_thr) if gt_ignore[i].shape[0] > 0: # 1. assign gt and preds fps = np.ones(pred_bboxes.shape[0]).astype(np.bool) row, col = linear_sum_assignment(dist) for m, n in zip(row, col): if not np.isfinite(dist[m, n]): continue fps[n] = False # 2. ignore by iof iofs = bbox_overlaps(pred_bboxes, gt_ignore[i], mode='iof') ignores = (iofs > ignore_iof_thr).any(axis=1) # 3. filter preds valid_inds = ~(fps & ignores) pred_ids = pred_ids[valid_inds] dist = dist[:, valid_inds] if dist.shape != (0, 0): accumulators[i].update(gt_ids, pred_ids, dist) return accumulators
def id_global_assignment(df, ana=None): """ID measures: Global min-cost assignment for ID measures.""" # pylint: disable=too-many-locals del ana # unused ocs, hcs, tps = extract_counts_from_df_map(df) oids = sorted(ocs.keys()) hids = sorted(hcs.keys()) oids_idx = dict((o, i) for i, o in enumerate(oids)) hids_idx = dict((h, i) for i, h in enumerate(hids)) no = len(ocs) nh = len(hcs) fpmatrix = np.full((no + nh, no + nh), 0.) fnmatrix = np.full((no + nh, no + nh), 0.) fpmatrix[no:, :nh] = np.nan fnmatrix[:no, nh:] = np.nan for oid, oc in ocs.items(): r = oids_idx[oid] fnmatrix[r, :nh] = oc fnmatrix[r, nh + r] = oc for hid, hc in hcs.items(): c = hids_idx[hid] fpmatrix[:no, c] = hc fpmatrix[c + no, c] = hc for (oid, hid), ex in tps.items(): r = oids_idx[oid] c = hids_idx[hid] fpmatrix[r, c] -= ex fnmatrix[r, c] -= ex costs = fpmatrix + fnmatrix rids, cids = linear_sum_assignment(costs) return { 'fpmatrix': fpmatrix, 'fnmatrix': fnmatrix, 'rids': rids, 'cids': cids, 'costs': costs, 'min_cost': costs[rids, cids].sum() }
def track(self, img, img_metas, model, bboxes, labels, frame_id, rescale=False, **kwargs): """Tracking forward function. Args: img (Tensor): of shape (N, C, H, W) encoding input images. Typically these should be mean centered and std scaled. img_metas (list[dict]): list of image info dict where each dict has: 'img_shape', 'scale_factor', 'flip', and may also contain 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. model (nn.Module): MOT model. bboxes (Tensor): of shape (N, 5). labels (Tensor): of shape (N, ). frame_id (int): The id of current frame, 0-index. rescale (bool, optional): If True, the bounding boxes should be rescaled to fit the original scale of the image. Defaults to False. Returns: tuple: Tracking results. """ if not hasattr(self, 'kf'): self.kf = model.motion if self.with_reid: if self.reid.get('img_norm_cfg', False): reid_img = imrenormalize(img, img_metas[0]['img_norm_cfg'], self.reid['img_norm_cfg']) else: reid_img = img.clone() valid_inds = bboxes[:, -1] > self.obj_score_thr bboxes = bboxes[valid_inds] labels = labels[valid_inds] if self.empty or bboxes.size(0) == 0: num_new_tracks = bboxes.size(0) ids = torch.arange(self.num_tracks, self.num_tracks + num_new_tracks, dtype=torch.long) self.num_tracks += num_new_tracks if self.with_reid: embeds = model.reid.simple_test( self.crop_imgs(reid_img, img_metas, bboxes[:, :4].clone(), rescale)) else: ids = torch.full((bboxes.size(0), ), -1, dtype=torch.long) # motion if model.with_motion: self.tracks, costs = model.motion.track( self.tracks, bbox_xyxy_to_cxcyah(bboxes)) active_ids = self.confirmed_ids if self.with_reid: embeds = model.reid.simple_test( self.crop_imgs(reid_img, img_metas, bboxes[:, :4].clone(), rescale)) # reid if len(active_ids) > 0: track_embeds = self.get('embeds', active_ids, self.reid.get('num_samples', None), behavior='mean') reid_dists = torch.cdist(track_embeds, embeds).cpu().numpy() valid_inds = [list(self.ids).index(_) for _ in active_ids] reid_dists[~np.isfinite(costs[valid_inds, :])] = np.nan row, col = linear_sum_assignment(reid_dists) for r, c in zip(row, col): dist = reid_dists[r, c] if not np.isfinite(dist): continue if dist <= self.reid['match_score_thr']: ids[c] = active_ids[r] active_ids = [ id for id in self.ids if id not in ids and self.tracks[id].frame_ids[-1] == frame_id - 1 ] if len(active_ids) > 0: active_dets = torch.nonzero(ids == -1).squeeze(1) track_bboxes = self.get('bboxes', active_ids) ious = bbox_overlaps( track_bboxes, bboxes[active_dets][:, :-1]).cpu().numpy() dists = 1 - ious row, col = linear_sum_assignment(dists) for r, c in zip(row, col): dist = dists[r, c] if dist < 1 - self.match_iou_thr: ids[active_dets[c]] = active_ids[r] new_track_inds = ids == -1 ids[new_track_inds] = torch.arange(self.num_tracks, self.num_tracks + new_track_inds.sum(), dtype=torch.long) self.num_tracks += new_track_inds.sum() self.update(ids=ids, bboxes=bboxes[:, :4], scores=bboxes[:, -1], labels=labels, embeds=embeds if self.with_reid else None, frame_ids=frame_id) return bboxes, labels, ids
def update(self, oids, hids, dists, frameid=None, vf=''): """Updates the accumulator with frame specific objects/detections. This method generates events based on the following algorithm [1]: 1. Try to carry forward already established tracks. If any paired object / hypothesis from previous timestamps are still visible in the current frame, create a 'MATCH' event between them. 2. For the remaining constellations minimize the total object / hypothesis distance error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous match create a 'SWITCH' else a 'MATCH' event. 3. Create 'MISS' events for all remaining unassigned objects. 4. Create 'FP' events for all remaining unassigned hypotheses. Params ------ oids : N array Array of object ids. hids : M array Array of hypothesis ids. dists: NxM array Distance matrix. np.nan values to signal do-not-pair constellations. See `distances` module for support methods. Kwargs ------ frameId : id Unique frame id. Optional when MOTAccumulator.auto_id is specified during construction. vf: file to log details Returns ------- frame_events : pd.DataFrame Dataframe containing generated events References ---------- 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10. """ # pylint: disable=too-many-locals, too-many-statements self.dirty_events = True oids = np.asarray(oids) oids_masked = np.zeros_like(oids, dtype=np.bool) hids = np.asarray(hids) hids_masked = np.zeros_like(hids, dtype=np.bool) dists = np.atleast_2d(dists).astype(float).reshape( oids.shape[0], hids.shape[0]).copy() if frameid is None: assert self.auto_id, 'auto-id is not enabled' if len(self._indices['FrameId']) > 0: frameid = self._indices['FrameId'][-1] + 1 else: frameid = 0 else: assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled' eid = itertools.count() # 0. Record raw events no = len(oids) nh = len(hids) # Add a RAW event simply to ensure the frame is counted. self._append_to_indices(frameid, next(eid)) self._append_to_events('RAW', np.nan, np.nan, np.nan) # There must be at least one RAW event per object and hypothesis. # Record all finite distances as RAW events. valid_i, valid_j = np.where(np.isfinite(dists)) valid_dists = dists[valid_i, valid_j] for i, j, dist_ij in zip(valid_i, valid_j, valid_dists): self._append_to_indices(frameid, next(eid)) self._append_to_events('RAW', oids[i], hids[j], dist_ij) # Add a RAW event for objects and hypotheses that were present but did # not overlap with anything. used_i = np.unique(valid_i) used_j = np.unique(valid_j) unused_i = np.setdiff1d(np.arange(no), used_i) unused_j = np.setdiff1d(np.arange(nh), used_j) for oid in oids[unused_i]: self._append_to_indices(frameid, next(eid)) self._append_to_events('RAW', oid, np.nan, np.nan) for hid in hids[unused_j]: self._append_to_indices(frameid, next(eid)) self._append_to_events('RAW', np.nan, hid, np.nan) if oids.size * hids.size > 0: # 1. Try to re-establish tracks from previous correspondences for i in range(oids.shape[0]): # No need to check oids_masked[i] here. if oids[i] not in self.m: continue hprev = self.m[oids[i]] j, = np.where(~hids_masked & (hids == hprev)) if j.shape[0] == 0: continue j = j[0] if np.isfinite(dists[i, j]): o = oids[i] h = hids[j] oids_masked[i] = True hids_masked[j] = True self.m[oids[i]] = hids[j] self._append_to_indices(frameid, next(eid)) self._append_to_events('MATCH', oids[i], hids[j], dists[i, j]) self.last_match[o] = frameid self.hypHistory[h] = frameid # 2. Try to remaining objects/hypotheses dists[oids_masked, :] = np.nan dists[:, hids_masked] = np.nan rids, cids = linear_sum_assignment(dists) for i, j in zip(rids, cids): if not np.isfinite(dists[i, j]): continue o = oids[i] h = hids[j] is_switch = (o in self.m and self.m[o] != h and abs(frameid - self.last_occurrence[o]) <= self.max_switch_time) cat1 = 'SWITCH' if is_switch else 'MATCH' if cat1 == 'SWITCH': if h not in self.hypHistory: subcat = 'ASCEND' self._append_to_indices(frameid, next(eid)) self._append_to_events(subcat, oids[i], hids[j], dists[i, j]) # ignore the last condition temporarily is_transfer = (h in self.res_m and self.res_m[h] != o) # is_transfer = (h in self.res_m and # self.res_m[h] != o and # abs(frameid - self.last_occurrence[o]) <= self.max_switch_time) cat2 = 'TRANSFER' if is_transfer else 'MATCH' if cat2 == 'TRANSFER': if o not in self.last_match: subcat = 'MIGRATE' self._append_to_indices(frameid, next(eid)) self._append_to_events(subcat, oids[i], hids[j], dists[i, j]) self._append_to_indices(frameid, next(eid)) self._append_to_events(cat2, oids[i], hids[j], dists[i, j]) if vf != '' and (cat1 != 'MATCH' or cat2 != 'MATCH'): if cat1 == 'SWITCH': vf.write('%s %d %d %d %d %d\n' % (subcat[:2], o, self.last_match[o], self.m[o], frameid, h)) if cat2 == 'TRANSFER': vf.write('%s %d %d %d %d %d\n' % (subcat[:2], h, self.hypHistory[h], self.res_m[h], frameid, o)) self.hypHistory[h] = frameid self.last_match[o] = frameid self._append_to_indices(frameid, next(eid)) self._append_to_events(cat1, oids[i], hids[j], dists[i, j]) oids_masked[i] = True hids_masked[j] = True self.m[o] = h self.res_m[h] = o # 3. All remaining objects are missed for o in oids[~oids_masked]: self._append_to_indices(frameid, next(eid)) self._append_to_events('MISS', o, np.nan, np.nan) if vf != '': vf.write('FN %d %d\n' % (frameid, o)) # 4. All remaining hypotheses are false alarms for h in hids[~hids_masked]: self._append_to_indices(frameid, next(eid)) self._append_to_events('FP', np.nan, h, np.nan) if vf != '': vf.write('FP %d %d\n' % (frameid, h)) # 5. Update occurance state for o in oids: self.last_occurrence[o] = frameid if self.return_fn and self.return_fp: return frameid, oids_masked, hids_masked return frameid
def update(self, oids, hids, dists, oignores=None, frameid=None): """Updates the accumulator with frame specific objects/detections. This method generates events based on the following algorithm [1]: 1. Try to carry forward already established tracks. If any paired object / hypothesis from previous timestamps are still visible in the current frame, create a 'MATCH' event between them. 2. For the remaining constellations minimize the total object / hypothesis distance error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous match create a 'SWITCH' else a 'MATCH' event. 3. Create 'MISS' events for all remaining unassigned objects. 4. Create 'FP' events for all remaining unassigned hypotheses. Params ------ oids : N array Array of object ids. hids : M array Array of hypothesis ids. dists: NxM array Distance matrix. np.nan values to signal do-not-pair constellations. See `distances` module for support methods. oignores : N array or None Boolean array matching the size of ``oids``, that indicates if any object needs to be ignored. This might be useful when an object is tagged as occluded, and the multiple object tracking algorithm can be given the benefit of MATCHing if it has the capability, or disregard SWITCHes or MISSes, if it's not designed to track occluded objects. Kwargs ------ frameId : id Unique frame id. Optional when MOTAccumulator.auto_id is specified during construction. Returns ------- frame_events : pd.DataFrame Dataframe containing generated events References ---------- 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10. """ self.dirty_events = True oids = ma.array(oids, mask=np.zeros(len(oids))) hids = ma.array(hids, mask=np.zeros(len(hids))) dists = np.atleast_2d(dists).astype(float).reshape(oids.shape[0], hids.shape[0]).copy() if oignores is None: oignores = np.array([False] * oids.shape[0]) oignores = np.array(oignores) assert oignores.dtype == bool, 'Ignored objects must be boolean' if frameid is None: assert self.auto_id, 'auto-id is not enabled' if len(self._indices) > 0: frameid = self._indices[-1][0] + 1 else: frameid = 0 else: assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled' eid = count() # 0. Record raw events no = len(oids) nh = len(hids) if no * nh > 0: for i in range(no): for j in range(nh): self._indices.append((frameid, next(eid))) self._events.append(['RAW', oids[i], hids[j], dists[i,j]]) elif no == 0: for i in range(nh): self._indices.append((frameid, next(eid))) self._events.append(['RAW', np.nan, hids[i], np.nan]) elif nh == 0: for i in range(no): self._indices.append((frameid, next(eid))) self._events.append(['RAW', oids[i], np.nan, np.nan]) if oids.size * hids.size > 0: # 1. Try to re-establish tracks from previous correspondences for i in range(oids.shape[0]): if not oids[i] in self.m: continue hprev = self.m[oids[i]] j, = np.where(hids==hprev) if j.shape[0] == 0: continue j = j[0] if np.isfinite(dists[i,j]): oids[i] = ma.masked hids[j] = ma.masked self.m[oids.data[i]] = hids.data[j] self._indices.append((frameid, next(eid))) self._events.append(['MATCH', oids.data[i], hids.data[j], dists[i, j]]) # If matched, we don't care about ignoring. We'll give # the tracker the benefit of counting a MATCH # 2. Try to remaining objects/hypotheses dists[oids.mask, :] = np.nan dists[:, hids.mask] = np.nan rids, cids = linear_sum_assignment(dists) for i, j in zip(rids, cids): if not np.isfinite(dists[i,j]): continue o = oids[i] h = hids.data[j] is_switch = o in self.m and \ self.m[o] != h and \ abs(frameid - self.last_occurrence[o]) <= self.max_switch_time # If not ignoring object, update normally if not oignores[i]: cat = 'SWITCH' if is_switch else 'MATCH' self._indices.append((frameid, next(eid))) self._events.append([cat, oids.data[i], hids.data[j], dists[i, j]]) self.m[o] = h # On the other hand, if ignoring object, and # we have a MATCH, we'll let it through, but # not count a SWITCH elif not is_switch: self._indices.append((frameid, next(eid))) self._events.append(['MATCH', oids.data[i], hids.data[j], dists[i, j]]) self.m[o] = h # Ignored or not, account for the object # and hypothesis in this round oids[i] = ma.masked hids[j] = ma.masked # 3. All remaining objects are missed for o, oi in zip(oids[~oids.mask], oignores[~oids.mask]): # If object marked to be ignored, don't account a MISS if not oi: self._indices.append((frameid, next(eid))) self._events.append(['MISS', o, np.nan, np.nan]) # 4. All remaining hypotheses are false alarms for h in hids[~hids.mask]: self._indices.append((frameid, next(eid))) self._events.append(['FP', np.nan, h, np.nan]) # 5. Update occurance state for o in oids.data: self.last_occurrence[o] = frameid return frameid
def id_global_assignment(df, ana = None): """ID measures: Global min-cost assignment for ID measures.""" #st1 = time.time() oids = df.full['OId'].dropna().unique() hids = df.full['HId'].dropna().unique() hids_idx = dict((h,i) for i,h in enumerate(hids)) #print('----'*2, '1', time.time()-st1) if ana is None: hcs = [len(df.raw[(df.raw.HId==h)].groupby(level=0)) for h in hids] ocs = [len(df.raw[(df.raw.OId==o)].groupby(level=0)) for o in oids] else: hcs = [ana['hyp'][int(h)] for h in hids if h!='nan' and np.isfinite(float(h))] ocs = [ana['obj'][int(o)] for o in oids if o!='nan' and np.isfinite(float(o))] #print('----'*2, '2', time.time()-st1) no = oids.shape[0] nh = hids.shape[0] df = df.raw.reset_index() df = df.set_index(['OId','HId']) df = df.sort_index(level=[0,1]) #print('----'*2, '3', time.time()-st1) fpmatrix = np.full((no+nh, no+nh), 0.) fnmatrix = np.full((no+nh, no+nh), 0.) fpmatrix[no:, :nh] = np.nan fnmatrix[:no, nh:] = np.nan #print('----'*2, '4', time.time()-st1) for r, oc in enumerate(ocs): fnmatrix[r, :nh] = oc fnmatrix[r,nh+r] = oc for c, hc in enumerate(hcs): fpmatrix[:no, c] = hc fpmatrix[c+no,c] = hc #print('----'*2, '5', time.time()-st1) for r, o in enumerate(oids): df_o = df.loc[o, 'D'].dropna() for h, ex in df_o.groupby(level=0).count().iteritems(): c = hids_idx[h] fpmatrix[r,c] -= ex fnmatrix[r,c] -= ex #print('----'*2, '6', time.time()-st1) #print(fpmatrix.shape, fnmatrix.shape) costs = fpmatrix + fnmatrix #print(costs.shape) rids, cids = linear_sum_assignment(costs) #print('----'*2, '7', time.time()-st1) return { 'fpmatrix' : fpmatrix, 'fnmatrix' : fnmatrix, 'rids' : rids, 'cids' : cids, 'costs' : costs, 'min_cost' : costs[rids, cids].sum() }
def preprocessResult(res, anns, cats_mapping, crowd_ioa_thr=0.5): """Preprocesses data for utils.CLEAR_MOT_M. Returns a subset of the predictions. """ # pylint: disable=too-many-locals # fast indexing annsByAttr = defaultdict(lambda: defaultdict(list)) for i, bbox in enumerate(anns['annotations']): annsByAttr[bbox['image_id']][cats_mapping[bbox['category_id']]].append( i) dropped_gt_ids = set() dropped_gts = [] drops = 0 print('Results before drop:', sum([len(i) for i in res])) # match for (r, img) in zip(res, anns['images']): anns_in_frame = [ anns['annotations'][i] for v in annsByAttr[img['id']].values() for i in v ] gt_bboxes = [a['bbox'] for a in anns_in_frame if not a['iscrowd']] res_bboxes = [xyxy2xywh(v['bbox'][:-1]) for v in r.values()] res_ids = list(r.keys()) dropped_pred = [] # drop preds that match with ignored labels dist = mm.distances.iou_matrix(gt_bboxes, res_bboxes, max_iou=0.5) le, ri = linear_sum_assignment(dist) ignore_gt = [a['ignore'] for a in anns_in_frame if not a['iscrowd']] fp_ids = set(res_ids) for i, j in zip(le, ri): if not np.isfinite(dist[i, j]): continue fp_ids.remove(res_ids[j]) if ignore_gt[i]: # remove from results dropped_gt_ids.add(anns_in_frame[i]['id']) dropped_pred.append(res_ids[j]) dropped_gts.append(i) # drop fps that fall in crowd regions crowd_gt_labels = [a['bbox'] for a in anns_in_frame if a['iscrowd']] if len(crowd_gt_labels) > 0 and len(fp_ids) > 0: ioas = np.max( intersection_over_area( [xyxy2xywh(r[k]['bbox'][:-1]) for k in fp_ids], crowd_gt_labels), axis=1) for i, ioa in zip(fp_ids, ioas): if ioa > crowd_ioa_thr: dropped_pred.append(i) for p in dropped_pred: del r[p] print('Results after drop:', sum([len(i) for i in res]))
def id_global_assignment(df): """ID measures: Global min-cost assignment for ID measures.""" import multiprocessing as mp oids = df.full['OId'].dropna().unique() hids = df.full['HId'].dropna().unique() hids_idx = dict((h, i) for i, h in enumerate(hids)) oids_idx = dict((o, i) for i, o in enumerate(oids)) idx_hids = dict((i, h) for i, h in enumerate(hids)) idx_oids = dict((i, o) for i, o in enumerate(oids)) #These two rows take a huge amout of time print("Calculating hcs. hids len: {}".format(len(hids))) hcs = calc_hcs(df, hids) print("Calculating ocs. oids len: {}".format(len(oids))) ocs = calc_ocs(df, oids) no = oids.shape[0] nh = hids.shape[0] df = df.raw.reset_index() df = df.set_index(['OId', 'HId']) df = df.sort_index(level=[0, 1]) fpmatrix = np.full((no + nh, no + nh), 0.) fnmatrix = np.full((no + nh, no + nh), 0.) fpmatrix[no:, :nh] = np.nan fnmatrix[:no, nh:] = np.nan for r, oc in enumerate(ocs): fnmatrix[r, :nh] = oc fnmatrix[r, nh + r] = oc for c, hc in enumerate(hcs): fpmatrix[:no, c] = hc fpmatrix[c + no, c] = hc for r, o in enumerate(oids): df_o = df.loc[o, 'D'].dropna() for h, ex in df_o.groupby(level=0).count().iteritems(): c = hids_idx[h] fpmatrix[r, c] -= ex fnmatrix[r, c] -= ex costs = fpmatrix + fnmatrix import time start = time.time() print("Starting linear assignment solving") print("Global assignment matrix costs shape: {}".format(str(costs.shape))) rids, cids = linear_sum_assignment(costs, solver="lapsolver") end = time.time() print("Assignment calculation time: {}".format(end - start)) return { 'fpmatrix': fpmatrix, 'fnmatrix': fnmatrix, 'rids': rids, 'cids': cids, 'costs': costs, 'oids_idx': oids_idx, 'idx_oids': idx_oids, 'idx_hids': idx_hids, 'min_cost': costs[rids, cids].sum() }
def update(self, oids, hids, dists, frameid=None, vf=''): """Updates the accumulator with frame specific objects/detections. This method generates events based on the following algorithm [1]: 1. Try to carry forward already established tracks. If any paired object / hypothesis from previous timestamps are still visible in the current frame, create a 'MATCH' event between them. 2. For the remaining constellations minimize the total object / hypothesis distance error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous match create a 'SWITCH' else a 'MATCH' event. 3. Create 'MISS' events for all remaining unassigned objects. 4. Create 'FP' events for all remaining unassigned hypotheses. Params ------ oids : N array Array of object ids. hids : M array Array of hypothesis ids. dists: NxM array Distance matrix. np.nan values to signal do-not-pair constellations. See `distances` module for support methods. Kwargs ------ frameId : id Unique frame id. Optional when MOTAccumulator.auto_id is specified during construction. vf: file to log details Returns ------- frame_events : pd.DataFrame Dataframe containing generated events References ---------- 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10. """ self.dirty_events = True oids = ma.array(oids, mask=np.zeros(len(oids))) hids = ma.array(hids, mask=np.zeros(len(hids))) dists = np.atleast_2d(dists).astype(float).reshape( oids.shape[0], hids.shape[0]).copy() if frameid is None: assert self.auto_id, 'auto-id is not enabled' if len(self._indices) > 0: frameid = self._indices[-1][0] + 1 else: frameid = 0 else: assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled' eid = count() # 0. Record raw events no = len(oids) nh = len(hids) if no * nh > 0: for i in range(no): for j in range(nh): self._indices.append((frameid, next(eid))) self._events.append(['RAW', oids[i], hids[j], dists[i, j]]) elif no == 0: for i in range(nh): self._indices.append((frameid, next(eid))) self._events.append(['RAW', np.nan, hids[i], np.nan]) elif nh == 0: for i in range(no): self._indices.append((frameid, next(eid))) self._events.append(['RAW', oids[i], np.nan, np.nan]) if oids.size * hids.size > 0: # 1. Try to re-establish tracks from previous correspondences for i in range(oids.shape[0]): if not oids[i] in self.m: continue hprev = self.m[oids[i]] j, = np.where(hids == hprev) if j.shape[0] == 0: continue j = j[0] if np.isfinite(dists[i, j]): o = oids[i] h = hids[j] oids[i] = ma.masked hids[j] = ma.masked self.m[oids.data[i]] = hids.data[j] self._indices.append((frameid, next(eid))) self._events.append( ['MATCH', oids.data[i], hids.data[j], dists[i, j]]) self.last_match[o] = frameid self.hypHistory[h] = frameid # 2. Try to remaining objects/hypotheses dists[oids.mask, :] = np.nan dists[:, hids.mask] = np.nan rids, cids = linear_sum_assignment(dists) for i, j in zip(rids, cids): if not np.isfinite(dists[i, j]): continue o = oids[i] h = hids.data[j] is_switch = o in self.m and \ self.m[o] != h and \ abs(frameid - self.last_occurrence[o]) <= self.max_switch_time cat1 = 'SWITCH' if is_switch else 'MATCH' if cat1 == 'SWITCH': if h not in self.hypHistory: subcat = 'ASCEND' self._indices.append((frameid, next(eid))) self._events.append( [subcat, oids.data[i], hids.data[j], dists[i, j]]) is_transfer = h in self.res_m and \ self.res_m[h] != o #and \ # abs(frameid - self.last_occurrence[o]) <= self.max_switch_time # ignore this condition temporarily cat2 = 'TRANSFER' if is_transfer else 'MATCH' if cat2 == 'TRANSFER': if o not in self.last_match: subcat = 'MIGRATE' self._indices.append((frameid, next(eid))) self._events.append( [subcat, oids.data[i], hids.data[j], dists[i, j]]) self._indices.append((frameid, next(eid))) self._events.append( [cat2, oids.data[i], hids.data[j], dists[i, j]]) if vf != '' and (cat1 != 'MATCH' or cat2 != 'MATCH'): if cat1 == 'SWITCH': vf.write('%s %d %d %d %d %d\n' % (subcat[:2], o, self.last_match[o], self.m[o], frameid, h)) if cat2 == 'TRANSFER': vf.write('%s %d %d %d %d %d\n' % (subcat[:2], h, self.hypHistory[h], self.res_m[h], frameid, o)) self.hypHistory[h] = frameid self.last_match[o] = frameid self._indices.append((frameid, next(eid))) self._events.append( [cat1, oids.data[i], hids.data[j], dists[i, j]]) oids[i] = ma.masked hids[j] = ma.masked self.m[o] = h self.res_m[h] = o # 3. All remaining objects are missed for o in oids[~oids.mask]: self._indices.append((frameid, next(eid))) self._events.append(['MISS', o, np.nan, np.nan]) if vf != '': vf.write('FN %d %d\n' % (frameid, o)) # 4. All remaining hypotheses are false alarms for h in hids[~hids.mask]: self._indices.append((frameid, next(eid))) self._events.append(['FP', np.nan, h, np.nan]) if vf != '': vf.write('FP %d %d\n' % (frameid, h)) # 5. Update occurance state for o in oids.data: self.last_occurrence[o] = frameid return frameid
def update(self, oids, hids, dists, frameid=None): """Updates the accumulator with frame specific objects/detections. This method generates events based on the following algorithm [1]: 1. Try to carry forward already established tracks. If any paired object / hypothesis from previous timestamps are still visible in the current frame, create a 'MATCH' event between them. 2. For the remaining constellations minimize the total object / hypothesis distance error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous match create a 'SWITCH' else a 'MATCH' event. 3. Create 'MISS' events for all remaining unassigned objects. 4. Create 'FP' events for all remaining unassigned hypotheses. Params ------ oids : N array Array of object ids. hids : M array Array of hypothesis ids. dists: NxM array Distance matrix. np.nan values to signal do-not-pair constellations. See `distances` module for support methods. Kwargs ------ frameId : id Unique frame id. Optional when MOTAccumulator.auto_id is specified during construction. Returns ------- frame_events : pd.DataFrame Dataframe containing generated events References ---------- 1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics." EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10. """ self.dirty_events = True oids = ma.array(oids, mask=np.zeros(len(oids))) hids = ma.array(hids, mask=np.zeros(len(hids))) dists = np.atleast_2d(dists).astype(float).reshape(oids.shape[0], hids.shape[0]).copy() if frameid is None: assert self.auto_id, 'auto-id is not enabled' if len(self._indices) > 0: frameid = self._indices[-1][0] + 1 #frameid = 1 at the next call of update else: frameid = 0 #frameid = 0 at first call of update else: assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled' eid = count() # 0. Record raw events no = len(oids) nh = len(hids) if no * nh > 0: for i in range(no): for j in range(nh): self._indices.append((frameid, next(eid))) self._events.append(['RAW', oids[i], hids[j], dists[i,j]]) elif no == 0: for i in range(nh): self._indices.append((frameid, next(eid))) self._events.append(['RAW', np.nan, hids[i], np.nan]) elif nh == 0: for i in range(no): self._indices.append((frameid, next(eid))) self._events.append(['RAW', oids[i], np.nan, np.nan]) if oids.size * hids.size > 0: # 1. Try to re-establish tracks from previous correspondences for i in range(oids.shape[0]): if not oids[i] in self.m: #if idx of object correspond to some hypothesis in previous frame continue #if index of object does not correspond to any hypothesis in previous frame( new object or false negative in previous frame), then continue hprev = self.m[oids[i]]#then get index of that previous hypothesis. j, = np.where(hids==hprev) #position of that index in the new hypothesis list if j.shape[0] == 0:# if index of that hypothesis in previous frame does not exist in current hypothesis list (false negative in current frame) continue j = j[0] if np.isfinite(dists[i,j]): oids[i] = ma.masked #masked the object that already MATCH, so that we can set it to nan in 2nd step hids[j] = ma.masked self.m[oids.data[i]] = hids.data[j] self._indices.append((frameid, next(eid))) self._events.append(['MATCH', oids.data[i], hids.data[j], dists[i, j]]) #re-establish the matched pairs in previous frame # 2. Try to correspond remaining objects/hypotheses after re-establish ( objects already match above are ignore by setting dist to np.nan) dists[oids.mask, :] = np.nan dists[:, hids.mask] = np.nan rids, cids = linear_sum_assignment(dists) #only return rows and columns indices that not np.nan for i, j in zip(rids, cids): if not np.isfinite(dists[i,j]): continue o = oids[i] h = hids.data[j] is_switch = o in self.m and \ #if object id used to correspond to another id not the current one self.m[o] != h and \# abs(frameid - self.last_occurrence[o]) <= self.max_switch_time# Then that is a switch in id of hypothesis cat = 'SWITCH' if is_switch else 'MATCH' self._indices.append((frameid, next(eid))) self._events.append([cat, oids.data[i], hids.data[j], dists[i, j]]) oids[i] = ma.masked hids[j] = ma.masked self.m[o] = h