def test_change_solver():
def mysolver(x):
mysolver.called += 1
return np.array([]), np.array([])
mysolver.called = 0
costs = np.array([[6, 9, 1], [10, 3, 2], [8, 7, 4.]])
with lap.set_default_solver(mysolver):
rids, cids = lap.linear_sum_assignment(costs)
assert mysolver.called == 1
rids, cids = lap.linear_sum_assignment(costs)
assert mysolver.called == 1
def test_change_solver():
"""Tests effect of lap.set_default_solver."""
def mysolver(_):
mysolver.called += 1
return np.array([]), np.array([])
mysolver.called = 0
costs = np.asfarray([[6, 9, 1], [10, 3, 2], [8, 7, 4]])
with lap.set_default_solver(mysolver):
lap.linear_sum_assignment(costs)
assert mysolver.called == 1
lap.linear_sum_assignment(costs)
assert mysolver.called == 1
def test_assign_empty(solver):
costs = np.asfarray([[]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
np.testing.assert_equal(np.size(result), 0)
np.testing.assert_equal(costs, costs_copy)
def test_unbalanced_disallowed_tall(solver):
costs = np.asfarray([[np.nan, 9], [11, np.nan], [8, 7]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
expected = np.array([[0, 2], [1, 0]])
np.testing.assert_equal(result, expected)
np.testing.assert_equal(costs, costs_copy)
def test_unbalanced_tall(solver):
costs = np.asfarray([[6, 10], [4, 8], [1, 2]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
expected = np.array([[1, 2], [0, 1]])
np.testing.assert_equal(result, expected)
np.testing.assert_equal(costs, costs_copy)
def test_assign_disallowed(solver):
costs = np.asfarray([[5, 9, np.nan], [10, np.nan, 2], [8, 7, 4]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
expected = np.array([[0, 1, 2], [0, 2, 1]])
np.testing.assert_equal(result, expected)
np.testing.assert_equal(costs, costs_copy)
def test_assign_full_negative(solver):
costs = -7 + np.asfarray([[5, 5, 6], [1, 2, 5], [2, 4, 5]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
# Optimal matching is (0, 2), (1, 1), (2, 0) for 5 + 1 + 1.
expected = np.array([[0, 1, 2], [2, 1, 0]])
np.testing.assert_equal(result, expected)
np.testing.assert_equal(costs, costs_copy)
def test_assign_easy(solver):
"""Problem that could be solved by a greedy algorithm."""
costs = np.asfarray([[6, 9, 1], [10, 3, 2], [8, 7, 4]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
expected = np.array([[0, 1, 2], [2, 1, 0]])
np.testing.assert_equal(result, expected)
np.testing.assert_equal(costs, costs_copy)
def preprocessResult(res, gt, inifile):
"""Preprocesses data for utils.CLEAR_MOT_M.
Returns a subset of the predictions.
"""
# pylint: disable=too-many-locals
st = time.time()
labels = [
'ped', # 1
'person_on_vhcl', # 2
'car', # 3
'bicycle', # 4
'mbike', # 5
'non_mot_vhcl', # 6
'static_person', # 7
'distractor', # 8
'occluder', # 9
'occluder_on_grnd', # 10
'occluder_full', # 11
'reflection', # 12
'crowd', # 13
]
distractors = [
'person_on_vhcl', 'static_person', 'distractor', 'reflection'
]
is_distractor = {i + 1: x in distractors for i, x in enumerate(labels)}
for i in distractors:
is_distractor[i] = 1
seqIni = ConfigParser()
seqIni.read(inifile, encoding='utf8')
F = int(seqIni['Sequence']['seqLength'])
todrop = []
for t in range(1, F + 1):
if t not in res.index or t not in gt.index:
continue
resInFrame = res.loc[t]
GTInFrame = gt.loc[t]
A = GTInFrame[['X', 'Y', 'Width', 'Height']].values
B = resInFrame[['X', 'Y', 'Width', 'Height']].values
disM = mmd.iou_matrix(A, B, max_iou=0.5)
le, ri = linear_sum_assignment(disM)
flags = [
1 if is_distractor[it['ClassId']] or it['Visibility'] < 0. else 0
for i, (k, it) in enumerate(GTInFrame.iterrows())
]
hid = [k for k, it in resInFrame.iterrows()]
for i, j in zip(le, ri):
if not np.isfinite(disM[i, j]):
continue
if flags[i]:
todrop.append((t, hid[j]))
ret = res.drop(labels=todrop)
logging.info('Preprocess take %.3f seconds and remove %d boxes.',
time.time() - st, len(todrop))
return ret
def test_assign_full(solver):
"""Problem that would be incorrect using a greedy algorithm."""
costs = np.asfarray([[5, 5, 6], [1, 2, 5], [2, 4, 5]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
# Optimal matching is (0, 2), (1, 1), (2, 0) for 6 + 2 + 2.
expected = np.asfarray([[0, 1, 2], [2, 1, 0]])
np.testing.assert_equal(result, expected)
np.testing.assert_equal(costs, costs_copy)
def test_assign_attractive_broken_ring(solver):
"""Graph contains cheap broken ring and expensive unbroken ring."""
costs = np.asfarray([[np.nan, 1000, np.nan], [np.nan, 1, 1000], [1000, np.nan, 1]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
# Optimal solution is (0, 1), (1, 2), (2, 0) with cost 1000 + 1000 + 1000.
# Solver might choose (0, 0), (1, 1), (2, 2) with cost inf + 1 + 1.
expected = np.array([[0, 1, 2], [1, 2, 0]])
np.testing.assert_equal(result, expected)
np.testing.assert_equal(costs, costs_copy)
def test_assign_attractive_disallowed(solver):
"""Graph contains an attractive edge that cannot be used."""
costs = np.asfarray([[-10000, -1], [-1, np.nan]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
# The optimal solution is (0, 1), (1, 0) for a cost of -2.
# Ensure that the algorithm does not choose the (0, 0) edge.
# This would not be a perfect matching.
expected = np.array([[0, 1], [1, 0]])
np.testing.assert_equal(result, expected)
np.testing.assert_equal(costs, costs_copy)
def id_global_assignment(df):
"""ID measures: Global min-cost assignment for ID measures."""
oids = df.full['OId'].dropna().unique()
hids = df.full['HId'].dropna().unique()
hids_idx = dict((h, i) for i, h in enumerate(hids))
hcs = [len(df.raw[(df.raw.HId == h)].groupby(level=0)) for h in hids]
ocs = [len(df.raw[(df.raw.OId == o)].groupby(level=0)) for o in oids]
no = oids.shape[0]
nh = hids.shape[0]
df = df.raw.reset_index()
df = df.set_index(['OId', 'HId'])
df = df.sort_index(level=[0, 1])
fpmatrix = np.full((no + nh, no + nh), 0.)
fnmatrix = np.full((no + nh, no + nh), 0.)
fpmatrix[no:, :nh] = np.nan
fnmatrix[:no, nh:] = np.nan
for r, oc in enumerate(ocs):
fnmatrix[r, :nh] = oc
fnmatrix[r, nh + r] = oc
for c, hc in enumerate(hcs):
fpmatrix[:no, c] = hc
fpmatrix[c + no, c] = hc
for r, o in enumerate(oids):
try:
df_o = df.loc[o, 'D'].dropna()
except IndexError:
continue
for h, ex in df_o.groupby(level=0).count().iteritems():
c = hids_idx[h]
fpmatrix[r, c] -= ex
fnmatrix[r, c] -= ex
costs = fpmatrix + fnmatrix
rids, cids = linear_sum_assignment(costs)
return {
'fpmatrix': fpmatrix,
'fnmatrix': fnmatrix,
'rids': rids,
'cids': cids,
'costs': costs,
'min_cost': costs[rids, cids].sum()
}
def test_assign_infeasible(solver):
"""Tests that minimum-cost solution with most edges is found."""
costs = np.asfarray([[np.nan, np.nan, 2],
[np.nan, np.nan, 1],
[8, 7, 4]])
costs_copy = costs.copy()
result = lap.linear_sum_assignment(costs, solver=solver)
# Optimal matching is (1, 2), (2, 1).
expected = np.array([[1, 2], [2, 1]])
np.testing.assert_equal(result, expected)
np.testing.assert_equal(costs, costs_copy)
def test_lap_solvers():
assert len(lap.available_solvers) > 0
print(lap.available_solvers)
costs = np.array([[6, 9, 1], [10, 3, 2], [8, 7, 4.]])
costs_copy = costs.copy()
results = [
lap.linear_sum_assignment(costs, solver=s)
for s in lap.available_solvers
]
expected = np.array([[0, 1, 2], [2, 1, 0]])
[np.testing.assert_allclose(r, expected) for r in results]
np.testing.assert_allclose(costs, costs_copy)
costs = np.array([[5, 9, np.nan], [10, np.nan, 2], [8, 7, 4.]])
costs_copy = costs.copy()
results = [
lap.linear_sum_assignment(costs, solver=s)
for s in lap.available_solvers
]
expected = np.array([[0, 1, 2], [0, 2, 1]])
[np.testing.assert_allclose(r, expected) for r in results]
np.testing.assert_allclose(costs, costs_copy)
def preprocessResult(res, gt, inifile):
st = time.time()
labels = ['ped', # 1
'person_on_vhcl', # 2
'car', # 3
'bicycle', # 4
'mbike', # 5
'non_mot_vhcl', # 6
'static_person', # 7
'distractor', # 8
'occluder', # 9
'occluder_on_grnd', #10
'occluder_full', # 11
'reflection', # 12
'crowd' # 13
]
distractors_ = ['person_on_vhcl','static_person','distractor','reflection']
distractors = {i+1 : x in distractors_ for i,x in enumerate(labels)}
for i in distractors_:
distractors[i] = 1
seqIni = ConfigParser()
seqIni.read(inifile, encoding='utf8')
F = int(seqIni['Sequence']['seqLength'])
todrop = []
for t in range(1,F+1):
if t not in res.index or t not in gt.index: continue
#st = time.time()
resInFrame = res.loc[t]
N = len(resInFrame)
GTInFrame = gt.loc[t]
Ngt = len(GTInFrame)
A = GTInFrame[['X','Y','Width','Height']].values
B = resInFrame[['X','Y','Width','Height']].values
disM = mmd.iou_matrix(A, B, max_iou = 0.5)
#en = time.time()
#print('----', 'disM', en - st)
le, ri = linear_sum_assignment(disM)
flags = [1 if distractors[it['ClassId']] or it['Visibility']<0. else 0 for i,(k,it) in enumerate(GTInFrame.iterrows())]
hid = [k for k,it in resInFrame.iterrows()]
for i, j in zip(le, ri):
if not np.isfinite(disM[i, j]):
continue
if flags[i]:
todrop.append((t, hid[j]))
#en = time.time()
#print('Frame %d: '%t, en - st)
ret = res.drop(labels=todrop)
logging.info('Preprocess take %.3f seconds and remove %d boxes.'%(time.time() - st, len(todrop)))
return ret
def acc_single_video(results,
gts,
iou_thr=0.5,
ignore_iof_thr=0.5,
ignore_by_classes=False):
"""Accumulate results in a single video."""
num_classes = len(results[0])
accumulators = [
mm.MOTAccumulator(auto_id=True) for i in range(num_classes)
]
for result, gt in zip(results, gts):
if ignore_by_classes:
gt_ignore = outs2results(bboxes=gt['bboxes_ignore'],
labels=gt['labels_ignore'],
num_classes=num_classes)['bbox_results']
else:
gt_ignore = [gt['bboxes_ignore'] for i in range(num_classes)]
gt = outs2results(bboxes=gt['bboxes'],
labels=gt['labels'],
ids=gt['instance_ids'],
num_classes=num_classes)['bbox_results']
for i in range(num_classes):
gt_ids, gt_bboxes = gt[i][:, 0].astype(np.int), gt[i][:, 1:]
pred_ids, pred_bboxes = result[i][:, 0].astype(
np.int), result[i][:, 1:-1]
dist = bbox_distances(gt_bboxes, pred_bboxes, iou_thr)
if gt_ignore[i].shape[0] > 0:
# 1. assign gt and preds
fps = np.ones(pred_bboxes.shape[0]).astype(np.bool)
row, col = linear_sum_assignment(dist)
for m, n in zip(row, col):
if not np.isfinite(dist[m, n]):
continue
fps[n] = False
# 2. ignore by iof
iofs = bbox_overlaps(pred_bboxes, gt_ignore[i], mode='iof')
ignores = (iofs > ignore_iof_thr).any(axis=1)
# 3. filter preds
valid_inds = ~(fps & ignores)
pred_ids = pred_ids[valid_inds]
dist = dist[:, valid_inds]
if dist.shape != (0, 0):
accumulators[i].update(gt_ids, pred_ids, dist)
return accumulators
def id_global_assignment(df, ana=None):
"""ID measures: Global min-cost assignment for ID measures."""
# pylint: disable=too-many-locals
del ana # unused
ocs, hcs, tps = extract_counts_from_df_map(df)
oids = sorted(ocs.keys())
hids = sorted(hcs.keys())
oids_idx = dict((o, i) for i, o in enumerate(oids))
hids_idx = dict((h, i) for i, h in enumerate(hids))
no = len(ocs)
nh = len(hcs)
fpmatrix = np.full((no + nh, no + nh), 0.)
fnmatrix = np.full((no + nh, no + nh), 0.)
fpmatrix[no:, :nh] = np.nan
fnmatrix[:no, nh:] = np.nan
for oid, oc in ocs.items():
r = oids_idx[oid]
fnmatrix[r, :nh] = oc
fnmatrix[r, nh + r] = oc
for hid, hc in hcs.items():
c = hids_idx[hid]
fpmatrix[:no, c] = hc
fpmatrix[c + no, c] = hc
for (oid, hid), ex in tps.items():
r = oids_idx[oid]
c = hids_idx[hid]
fpmatrix[r, c] -= ex
fnmatrix[r, c] -= ex
costs = fpmatrix + fnmatrix
rids, cids = linear_sum_assignment(costs)
return {
'fpmatrix': fpmatrix,
'fnmatrix': fnmatrix,
'rids': rids,
'cids': cids,
'costs': costs,
'min_cost': costs[rids, cids].sum()
}
def track(self,
img,
img_metas,
model,
bboxes,
labels,
frame_id,
rescale=False,
**kwargs):
"""Tracking forward function.
Args:
img (Tensor): of shape (N, C, H, W) encoding input images.
Typically these should be mean centered and std scaled.
img_metas (list[dict]): list of image info dict where each dict
has: 'img_shape', 'scale_factor', 'flip', and may also contain
'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'.
model (nn.Module): MOT model.
bboxes (Tensor): of shape (N, 5).
labels (Tensor): of shape (N, ).
frame_id (int): The id of current frame, 0-index.
rescale (bool, optional): If True, the bounding boxes should be
rescaled to fit the original scale of the image. Defaults to
False.
Returns:
tuple: Tracking results.
"""
if not hasattr(self, 'kf'):
self.kf = model.motion
if self.with_reid:
if self.reid.get('img_norm_cfg', False):
reid_img = imrenormalize(img, img_metas[0]['img_norm_cfg'],
self.reid['img_norm_cfg'])
else:
reid_img = img.clone()
valid_inds = bboxes[:, -1] > self.obj_score_thr
bboxes = bboxes[valid_inds]
labels = labels[valid_inds]
if self.empty or bboxes.size(0) == 0:
num_new_tracks = bboxes.size(0)
ids = torch.arange(self.num_tracks,
self.num_tracks + num_new_tracks,
dtype=torch.long)
self.num_tracks += num_new_tracks
if self.with_reid:
embeds = model.reid.simple_test(
self.crop_imgs(reid_img, img_metas, bboxes[:, :4].clone(),
rescale))
else:
ids = torch.full((bboxes.size(0), ), -1, dtype=torch.long)
# motion
if model.with_motion:
self.tracks, costs = model.motion.track(
self.tracks, bbox_xyxy_to_cxcyah(bboxes))
active_ids = self.confirmed_ids
if self.with_reid:
embeds = model.reid.simple_test(
self.crop_imgs(reid_img, img_metas, bboxes[:, :4].clone(),
rescale))
# reid
if len(active_ids) > 0:
track_embeds = self.get('embeds',
active_ids,
self.reid.get('num_samples', None),
behavior='mean')
reid_dists = torch.cdist(track_embeds,
embeds).cpu().numpy()
valid_inds = [list(self.ids).index(_) for _ in active_ids]
reid_dists[~np.isfinite(costs[valid_inds, :])] = np.nan
row, col = linear_sum_assignment(reid_dists)
for r, c in zip(row, col):
dist = reid_dists[r, c]
if not np.isfinite(dist):
continue
if dist <= self.reid['match_score_thr']:
ids[c] = active_ids[r]
active_ids = [
id for id in self.ids if id not in ids
and self.tracks[id].frame_ids[-1] == frame_id - 1
]
if len(active_ids) > 0:
active_dets = torch.nonzero(ids == -1).squeeze(1)
track_bboxes = self.get('bboxes', active_ids)
ious = bbox_overlaps(
track_bboxes, bboxes[active_dets][:, :-1]).cpu().numpy()
dists = 1 - ious
row, col = linear_sum_assignment(dists)
for r, c in zip(row, col):
dist = dists[r, c]
if dist < 1 - self.match_iou_thr:
ids[active_dets[c]] = active_ids[r]
new_track_inds = ids == -1
ids[new_track_inds] = torch.arange(self.num_tracks,
self.num_tracks +
new_track_inds.sum(),
dtype=torch.long)
self.num_tracks += new_track_inds.sum()
self.update(ids=ids,
bboxes=bboxes[:, :4],
scores=bboxes[:, -1],
labels=labels,
embeds=embeds if self.with_reid else None,
frame_ids=frame_id)
return bboxes, labels, ids
def update(self, oids, hids, dists, frameid=None, vf=''):
"""Updates the accumulator with frame specific objects/detections.
This method generates events based on the following algorithm [1]:
1. Try to carry forward already established tracks. If any paired object / hypothesis
from previous timestamps are still visible in the current frame, create a 'MATCH'
event between them.
2. For the remaining constellations minimize the total object / hypothesis distance
error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous
match create a 'SWITCH' else a 'MATCH' event.
3. Create 'MISS' events for all remaining unassigned objects.
4. Create 'FP' events for all remaining unassigned hypotheses.
Params
------
oids : N array
Array of object ids.
hids : M array
Array of hypothesis ids.
dists: NxM array
Distance matrix. np.nan values to signal do-not-pair constellations.
See `distances` module for support methods.
Kwargs
------
frameId : id
Unique frame id. Optional when MOTAccumulator.auto_id is specified during
construction.
vf: file to log details
Returns
-------
frame_events : pd.DataFrame
Dataframe containing generated events
References
----------
1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics."
EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10.
"""
# pylint: disable=too-many-locals, too-many-statements
self.dirty_events = True
oids = np.asarray(oids)
oids_masked = np.zeros_like(oids, dtype=np.bool)
hids = np.asarray(hids)
hids_masked = np.zeros_like(hids, dtype=np.bool)
dists = np.atleast_2d(dists).astype(float).reshape(
oids.shape[0], hids.shape[0]).copy()
if frameid is None:
assert self.auto_id, 'auto-id is not enabled'
if len(self._indices['FrameId']) > 0:
frameid = self._indices['FrameId'][-1] + 1
else:
frameid = 0
else:
assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled'
eid = itertools.count()
# 0. Record raw events
no = len(oids)
nh = len(hids)
# Add a RAW event simply to ensure the frame is counted.
self._append_to_indices(frameid, next(eid))
self._append_to_events('RAW', np.nan, np.nan, np.nan)
# There must be at least one RAW event per object and hypothesis.
# Record all finite distances as RAW events.
valid_i, valid_j = np.where(np.isfinite(dists))
valid_dists = dists[valid_i, valid_j]
for i, j, dist_ij in zip(valid_i, valid_j, valid_dists):
self._append_to_indices(frameid, next(eid))
self._append_to_events('RAW', oids[i], hids[j], dist_ij)
# Add a RAW event for objects and hypotheses that were present but did
# not overlap with anything.
used_i = np.unique(valid_i)
used_j = np.unique(valid_j)
unused_i = np.setdiff1d(np.arange(no), used_i)
unused_j = np.setdiff1d(np.arange(nh), used_j)
for oid in oids[unused_i]:
self._append_to_indices(frameid, next(eid))
self._append_to_events('RAW', oid, np.nan, np.nan)
for hid in hids[unused_j]:
self._append_to_indices(frameid, next(eid))
self._append_to_events('RAW', np.nan, hid, np.nan)
if oids.size * hids.size > 0:
# 1. Try to re-establish tracks from previous correspondences
for i in range(oids.shape[0]):
# No need to check oids_masked[i] here.
if oids[i] not in self.m:
continue
hprev = self.m[oids[i]]
j, = np.where(~hids_masked & (hids == hprev))
if j.shape[0] == 0:
continue
j = j[0]
if np.isfinite(dists[i, j]):
o = oids[i]
h = hids[j]
oids_masked[i] = True
hids_masked[j] = True
self.m[oids[i]] = hids[j]
self._append_to_indices(frameid, next(eid))
self._append_to_events('MATCH', oids[i], hids[j], dists[i,
j])
self.last_match[o] = frameid
self.hypHistory[h] = frameid
# 2. Try to remaining objects/hypotheses
dists[oids_masked, :] = np.nan
dists[:, hids_masked] = np.nan
rids, cids = linear_sum_assignment(dists)
for i, j in zip(rids, cids):
if not np.isfinite(dists[i, j]):
continue
o = oids[i]
h = hids[j]
is_switch = (o in self.m and self.m[o] != h
and abs(frameid - self.last_occurrence[o]) <=
self.max_switch_time)
cat1 = 'SWITCH' if is_switch else 'MATCH'
if cat1 == 'SWITCH':
if h not in self.hypHistory:
subcat = 'ASCEND'
self._append_to_indices(frameid, next(eid))
self._append_to_events(subcat, oids[i], hids[j],
dists[i, j])
# ignore the last condition temporarily
is_transfer = (h in self.res_m and self.res_m[h] != o)
# is_transfer = (h in self.res_m and
# self.res_m[h] != o and
# abs(frameid - self.last_occurrence[o]) <= self.max_switch_time)
cat2 = 'TRANSFER' if is_transfer else 'MATCH'
if cat2 == 'TRANSFER':
if o not in self.last_match:
subcat = 'MIGRATE'
self._append_to_indices(frameid, next(eid))
self._append_to_events(subcat, oids[i], hids[j],
dists[i, j])
self._append_to_indices(frameid, next(eid))
self._append_to_events(cat2, oids[i], hids[j], dists[i, j])
if vf != '' and (cat1 != 'MATCH' or cat2 != 'MATCH'):
if cat1 == 'SWITCH':
vf.write('%s %d %d %d %d %d\n' %
(subcat[:2], o, self.last_match[o], self.m[o],
frameid, h))
if cat2 == 'TRANSFER':
vf.write('%s %d %d %d %d %d\n' %
(subcat[:2], h, self.hypHistory[h],
self.res_m[h], frameid, o))
self.hypHistory[h] = frameid
self.last_match[o] = frameid
self._append_to_indices(frameid, next(eid))
self._append_to_events(cat1, oids[i], hids[j], dists[i, j])
oids_masked[i] = True
hids_masked[j] = True
self.m[o] = h
self.res_m[h] = o
# 3. All remaining objects are missed
for o in oids[~oids_masked]:
self._append_to_indices(frameid, next(eid))
self._append_to_events('MISS', o, np.nan, np.nan)
if vf != '':
vf.write('FN %d %d\n' % (frameid, o))
# 4. All remaining hypotheses are false alarms
for h in hids[~hids_masked]:
self._append_to_indices(frameid, next(eid))
self._append_to_events('FP', np.nan, h, np.nan)
if vf != '':
vf.write('FP %d %d\n' % (frameid, h))
# 5. Update occurance state
for o in oids:
self.last_occurrence[o] = frameid
if self.return_fn and self.return_fp:
return frameid, oids_masked, hids_masked
return frameid
def update(self, oids, hids, dists, oignores=None, frameid=None):
"""Updates the accumulator with frame specific objects/detections.
This method generates events based on the following algorithm [1]:
1. Try to carry forward already established tracks. If any paired object / hypothesis
from previous timestamps are still visible in the current frame, create a 'MATCH'
event between them.
2. For the remaining constellations minimize the total object / hypothesis distance
error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous
match create a 'SWITCH' else a 'MATCH' event.
3. Create 'MISS' events for all remaining unassigned objects.
4. Create 'FP' events for all remaining unassigned hypotheses.
Params
------
oids : N array
Array of object ids.
hids : M array
Array of hypothesis ids.
dists: NxM array
Distance matrix. np.nan values to signal do-not-pair constellations.
See `distances` module for support methods.
oignores : N array or None
Boolean array matching the size of ``oids``, that indicates if any
object needs to be ignored. This might be useful when an object
is tagged as occluded, and the multiple object tracking algorithm
can be given the benefit of MATCHing if it has the capability, or
disregard SWITCHes or MISSes, if it's not designed to track
occluded objects.
Kwargs
------
frameId : id
Unique frame id. Optional when MOTAccumulator.auto_id is specified during
construction.
Returns
-------
frame_events : pd.DataFrame
Dataframe containing generated events
References
----------
1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics."
EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10.
"""
self.dirty_events = True
oids = ma.array(oids, mask=np.zeros(len(oids)))
hids = ma.array(hids, mask=np.zeros(len(hids)))
dists = np.atleast_2d(dists).astype(float).reshape(oids.shape[0], hids.shape[0]).copy()
if oignores is None:
oignores = np.array([False] * oids.shape[0])
oignores = np.array(oignores)
assert oignores.dtype == bool, 'Ignored objects must be boolean'
if frameid is None:
assert self.auto_id, 'auto-id is not enabled'
if len(self._indices) > 0:
frameid = self._indices[-1][0] + 1
else:
frameid = 0
else:
assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled'
eid = count()
# 0. Record raw events
no = len(oids)
nh = len(hids)
if no * nh > 0:
for i in range(no):
for j in range(nh):
self._indices.append((frameid, next(eid)))
self._events.append(['RAW', oids[i], hids[j], dists[i,j]])
elif no == 0:
for i in range(nh):
self._indices.append((frameid, next(eid)))
self._events.append(['RAW', np.nan, hids[i], np.nan])
elif nh == 0:
for i in range(no):
self._indices.append((frameid, next(eid)))
self._events.append(['RAW', oids[i], np.nan, np.nan])
if oids.size * hids.size > 0:
# 1. Try to re-establish tracks from previous correspondences
for i in range(oids.shape[0]):
if not oids[i] in self.m:
continue
hprev = self.m[oids[i]]
j, = np.where(hids==hprev)
if j.shape[0] == 0:
continue
j = j[0]
if np.isfinite(dists[i,j]):
oids[i] = ma.masked
hids[j] = ma.masked
self.m[oids.data[i]] = hids.data[j]
self._indices.append((frameid, next(eid)))
self._events.append(['MATCH', oids.data[i], hids.data[j], dists[i, j]])
# If matched, we don't care about ignoring. We'll give
# the tracker the benefit of counting a MATCH
# 2. Try to remaining objects/hypotheses
dists[oids.mask, :] = np.nan
dists[:, hids.mask] = np.nan
rids, cids = linear_sum_assignment(dists)
for i, j in zip(rids, cids):
if not np.isfinite(dists[i,j]):
continue
o = oids[i]
h = hids.data[j]
is_switch = o in self.m and \
self.m[o] != h and \
abs(frameid - self.last_occurrence[o]) <= self.max_switch_time
# If not ignoring object, update normally
if not oignores[i]:
cat = 'SWITCH' if is_switch else 'MATCH'
self._indices.append((frameid, next(eid)))
self._events.append([cat, oids.data[i], hids.data[j], dists[i, j]])
self.m[o] = h
# On the other hand, if ignoring object, and
# we have a MATCH, we'll let it through, but
# not count a SWITCH
elif not is_switch:
self._indices.append((frameid, next(eid)))
self._events.append(['MATCH', oids.data[i], hids.data[j], dists[i, j]])
self.m[o] = h
# Ignored or not, account for the object
# and hypothesis in this round
oids[i] = ma.masked
hids[j] = ma.masked
# 3. All remaining objects are missed
for o, oi in zip(oids[~oids.mask], oignores[~oids.mask]):
# If object marked to be ignored, don't account a MISS
if not oi:
self._indices.append((frameid, next(eid)))
self._events.append(['MISS', o, np.nan, np.nan])
# 4. All remaining hypotheses are false alarms
for h in hids[~hids.mask]:
self._indices.append((frameid, next(eid)))
self._events.append(['FP', np.nan, h, np.nan])
# 5. Update occurance state
for o in oids.data:
self.last_occurrence[o] = frameid
return frameid
def id_global_assignment(df, ana = None):
"""ID measures: Global min-cost assignment for ID measures."""
#st1 = time.time()
oids = df.full['OId'].dropna().unique()
hids = df.full['HId'].dropna().unique()
hids_idx = dict((h,i) for i,h in enumerate(hids))
#print('----'*2, '1', time.time()-st1)
if ana is None:
hcs = [len(df.raw[(df.raw.HId==h)].groupby(level=0)) for h in hids]
ocs = [len(df.raw[(df.raw.OId==o)].groupby(level=0)) for o in oids]
else:
hcs = [ana['hyp'][int(h)] for h in hids if h!='nan' and np.isfinite(float(h))]
ocs = [ana['obj'][int(o)] for o in oids if o!='nan' and np.isfinite(float(o))]
#print('----'*2, '2', time.time()-st1)
no = oids.shape[0]
nh = hids.shape[0]
df = df.raw.reset_index()
df = df.set_index(['OId','HId'])
df = df.sort_index(level=[0,1])
#print('----'*2, '3', time.time()-st1)
fpmatrix = np.full((no+nh, no+nh), 0.)
fnmatrix = np.full((no+nh, no+nh), 0.)
fpmatrix[no:, :nh] = np.nan
fnmatrix[:no, nh:] = np.nan
#print('----'*2, '4', time.time()-st1)
for r, oc in enumerate(ocs):
fnmatrix[r, :nh] = oc
fnmatrix[r,nh+r] = oc
for c, hc in enumerate(hcs):
fpmatrix[:no, c] = hc
fpmatrix[c+no,c] = hc
#print('----'*2, '5', time.time()-st1)
for r, o in enumerate(oids):
df_o = df.loc[o, 'D'].dropna()
for h, ex in df_o.groupby(level=0).count().iteritems():
c = hids_idx[h]
fpmatrix[r,c] -= ex
fnmatrix[r,c] -= ex
#print('----'*2, '6', time.time()-st1)
#print(fpmatrix.shape, fnmatrix.shape)
costs = fpmatrix + fnmatrix
#print(costs.shape)
rids, cids = linear_sum_assignment(costs)
#print('----'*2, '7', time.time()-st1)
return {
'fpmatrix' : fpmatrix,
'fnmatrix' : fnmatrix,
'rids' : rids,
'cids' : cids,
'costs' : costs,
'min_cost' : costs[rids, cids].sum()
}
def preprocessResult(res, anns, cats_mapping, crowd_ioa_thr=0.5):
"""Preprocesses data for utils.CLEAR_MOT_M.
Returns a subset of the predictions.
"""
# pylint: disable=too-many-locals
# fast indexing
annsByAttr = defaultdict(lambda: defaultdict(list))
for i, bbox in enumerate(anns['annotations']):
annsByAttr[bbox['image_id']][cats_mapping[bbox['category_id']]].append(
i)
dropped_gt_ids = set()
dropped_gts = []
drops = 0
print('Results before drop:', sum([len(i) for i in res]))
# match
for (r, img) in zip(res, anns['images']):
anns_in_frame = [
anns['annotations'][i] for v in annsByAttr[img['id']].values()
for i in v
]
gt_bboxes = [a['bbox'] for a in anns_in_frame if not a['iscrowd']]
res_bboxes = [xyxy2xywh(v['bbox'][:-1]) for v in r.values()]
res_ids = list(r.keys())
dropped_pred = []
# drop preds that match with ignored labels
dist = mm.distances.iou_matrix(gt_bboxes, res_bboxes, max_iou=0.5)
le, ri = linear_sum_assignment(dist)
ignore_gt = [a['ignore'] for a in anns_in_frame if not a['iscrowd']]
fp_ids = set(res_ids)
for i, j in zip(le, ri):
if not np.isfinite(dist[i, j]):
continue
fp_ids.remove(res_ids[j])
if ignore_gt[i]:
# remove from results
dropped_gt_ids.add(anns_in_frame[i]['id'])
dropped_pred.append(res_ids[j])
dropped_gts.append(i)
# drop fps that fall in crowd regions
crowd_gt_labels = [a['bbox'] for a in anns_in_frame if a['iscrowd']]
if len(crowd_gt_labels) > 0 and len(fp_ids) > 0:
ioas = np.max(
intersection_over_area(
[xyxy2xywh(r[k]['bbox'][:-1]) for k in fp_ids],
crowd_gt_labels),
axis=1)
for i, ioa in zip(fp_ids, ioas):
if ioa > crowd_ioa_thr:
dropped_pred.append(i)
for p in dropped_pred:
del r[p]
print('Results after drop:', sum([len(i) for i in res]))
def id_global_assignment(df):
"""ID measures: Global min-cost assignment for ID measures."""
import multiprocessing as mp
oids = df.full['OId'].dropna().unique()
hids = df.full['HId'].dropna().unique()
hids_idx = dict((h, i) for i, h in enumerate(hids))
oids_idx = dict((o, i) for i, o in enumerate(oids))
idx_hids = dict((i, h) for i, h in enumerate(hids))
idx_oids = dict((i, o) for i, o in enumerate(oids))
#These two rows take a huge amout of time
print("Calculating hcs. hids len: {}".format(len(hids)))
hcs = calc_hcs(df, hids)
print("Calculating ocs. oids len: {}".format(len(oids)))
ocs = calc_ocs(df, oids)
no = oids.shape[0]
nh = hids.shape[0]
df = df.raw.reset_index()
df = df.set_index(['OId', 'HId'])
df = df.sort_index(level=[0, 1])
fpmatrix = np.full((no + nh, no + nh), 0.)
fnmatrix = np.full((no + nh, no + nh), 0.)
fpmatrix[no:, :nh] = np.nan
fnmatrix[:no, nh:] = np.nan
for r, oc in enumerate(ocs):
fnmatrix[r, :nh] = oc
fnmatrix[r, nh + r] = oc
for c, hc in enumerate(hcs):
fpmatrix[:no, c] = hc
fpmatrix[c + no, c] = hc
for r, o in enumerate(oids):
df_o = df.loc[o, 'D'].dropna()
for h, ex in df_o.groupby(level=0).count().iteritems():
c = hids_idx[h]
fpmatrix[r, c] -= ex
fnmatrix[r, c] -= ex
costs = fpmatrix + fnmatrix
import time
start = time.time()
print("Starting linear assignment solving")
print("Global assignment matrix costs shape: {}".format(str(costs.shape)))
rids, cids = linear_sum_assignment(costs, solver="lapsolver")
end = time.time()
print("Assignment calculation time: {}".format(end - start))
return {
'fpmatrix': fpmatrix,
'fnmatrix': fnmatrix,
'rids': rids,
'cids': cids,
'costs': costs,
'oids_idx': oids_idx,
'idx_oids': idx_oids,
'idx_hids': idx_hids,
'min_cost': costs[rids, cids].sum()
}
def update(self, oids, hids, dists, frameid=None, vf=''):
"""Updates the accumulator with frame specific objects/detections.
This method generates events based on the following algorithm [1]:
1. Try to carry forward already established tracks. If any paired object / hypothesis
from previous timestamps are still visible in the current frame, create a 'MATCH'
event between them.
2. For the remaining constellations minimize the total object / hypothesis distance
error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous
match create a 'SWITCH' else a 'MATCH' event.
3. Create 'MISS' events for all remaining unassigned objects.
4. Create 'FP' events for all remaining unassigned hypotheses.
Params
------
oids : N array
Array of object ids.
hids : M array
Array of hypothesis ids.
dists: NxM array
Distance matrix. np.nan values to signal do-not-pair constellations.
See `distances` module for support methods.
Kwargs
------
frameId : id
Unique frame id. Optional when MOTAccumulator.auto_id is specified during
construction.
vf: file to log details
Returns
-------
frame_events : pd.DataFrame
Dataframe containing generated events
References
----------
1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics."
EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10.
"""
self.dirty_events = True
oids = ma.array(oids, mask=np.zeros(len(oids)))
hids = ma.array(hids, mask=np.zeros(len(hids)))
dists = np.atleast_2d(dists).astype(float).reshape(
oids.shape[0], hids.shape[0]).copy()
if frameid is None:
assert self.auto_id, 'auto-id is not enabled'
if len(self._indices) > 0:
frameid = self._indices[-1][0] + 1
else:
frameid = 0
else:
assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled'
eid = count()
# 0. Record raw events
no = len(oids)
nh = len(hids)
if no * nh > 0:
for i in range(no):
for j in range(nh):
self._indices.append((frameid, next(eid)))
self._events.append(['RAW', oids[i], hids[j], dists[i, j]])
elif no == 0:
for i in range(nh):
self._indices.append((frameid, next(eid)))
self._events.append(['RAW', np.nan, hids[i], np.nan])
elif nh == 0:
for i in range(no):
self._indices.append((frameid, next(eid)))
self._events.append(['RAW', oids[i], np.nan, np.nan])
if oids.size * hids.size > 0:
# 1. Try to re-establish tracks from previous correspondences
for i in range(oids.shape[0]):
if not oids[i] in self.m:
continue
hprev = self.m[oids[i]]
j, = np.where(hids == hprev)
if j.shape[0] == 0:
continue
j = j[0]
if np.isfinite(dists[i, j]):
o = oids[i]
h = hids[j]
oids[i] = ma.masked
hids[j] = ma.masked
self.m[oids.data[i]] = hids.data[j]
self._indices.append((frameid, next(eid)))
self._events.append(
['MATCH', oids.data[i], hids.data[j], dists[i, j]])
self.last_match[o] = frameid
self.hypHistory[h] = frameid
# 2. Try to remaining objects/hypotheses
dists[oids.mask, :] = np.nan
dists[:, hids.mask] = np.nan
rids, cids = linear_sum_assignment(dists)
for i, j in zip(rids, cids):
if not np.isfinite(dists[i, j]):
continue
o = oids[i]
h = hids.data[j]
is_switch = o in self.m and \
self.m[o] != h and \
abs(frameid - self.last_occurrence[o]) <= self.max_switch_time
cat1 = 'SWITCH' if is_switch else 'MATCH'
if cat1 == 'SWITCH':
if h not in self.hypHistory:
subcat = 'ASCEND'
self._indices.append((frameid, next(eid)))
self._events.append(
[subcat, oids.data[i], hids.data[j], dists[i, j]])
is_transfer = h in self.res_m and \
self.res_m[h] != o #and \
# abs(frameid - self.last_occurrence[o]) <= self.max_switch_time # ignore this condition temporarily
cat2 = 'TRANSFER' if is_transfer else 'MATCH'
if cat2 == 'TRANSFER':
if o not in self.last_match:
subcat = 'MIGRATE'
self._indices.append((frameid, next(eid)))
self._events.append(
[subcat, oids.data[i], hids.data[j], dists[i, j]])
self._indices.append((frameid, next(eid)))
self._events.append(
[cat2, oids.data[i], hids.data[j], dists[i, j]])
if vf != '' and (cat1 != 'MATCH' or cat2 != 'MATCH'):
if cat1 == 'SWITCH':
vf.write('%s %d %d %d %d %d\n' %
(subcat[:2], o, self.last_match[o], self.m[o],
frameid, h))
if cat2 == 'TRANSFER':
vf.write('%s %d %d %d %d %d\n' %
(subcat[:2], h, self.hypHistory[h],
self.res_m[h], frameid, o))
self.hypHistory[h] = frameid
self.last_match[o] = frameid
self._indices.append((frameid, next(eid)))
self._events.append(
[cat1, oids.data[i], hids.data[j], dists[i, j]])
oids[i] = ma.masked
hids[j] = ma.masked
self.m[o] = h
self.res_m[h] = o
# 3. All remaining objects are missed
for o in oids[~oids.mask]:
self._indices.append((frameid, next(eid)))
self._events.append(['MISS', o, np.nan, np.nan])
if vf != '':
vf.write('FN %d %d\n' % (frameid, o))
# 4. All remaining hypotheses are false alarms
for h in hids[~hids.mask]:
self._indices.append((frameid, next(eid)))
self._events.append(['FP', np.nan, h, np.nan])
if vf != '':
vf.write('FP %d %d\n' % (frameid, h))
# 5. Update occurance state
for o in oids.data:
self.last_occurrence[o] = frameid
return frameid
def update(self, oids, hids, dists, frameid=None):
"""Updates the accumulator with frame specific objects/detections.
This method generates events based on the following algorithm [1]:
1. Try to carry forward already established tracks. If any paired object / hypothesis
from previous timestamps are still visible in the current frame, create a 'MATCH'
event between them.
2. For the remaining constellations minimize the total object / hypothesis distance
error (Kuhn-Munkres algorithm). If a correspondence made contradicts a previous
match create a 'SWITCH' else a 'MATCH' event.
3. Create 'MISS' events for all remaining unassigned objects.
4. Create 'FP' events for all remaining unassigned hypotheses.
Params
------
oids : N array
Array of object ids.
hids : M array
Array of hypothesis ids.
dists: NxM array
Distance matrix. np.nan values to signal do-not-pair constellations.
See `distances` module for support methods.
Kwargs
------
frameId : id
Unique frame id. Optional when MOTAccumulator.auto_id is specified during
construction.
Returns
-------
frame_events : pd.DataFrame
Dataframe containing generated events
References
----------
1. Bernardin, Keni, and Rainer Stiefelhagen. "Evaluating multiple object tracking performance: the CLEAR MOT metrics."
EURASIP Journal on Image and Video Processing 2008.1 (2008): 1-10.
"""
self.dirty_events = True
oids = ma.array(oids, mask=np.zeros(len(oids)))
hids = ma.array(hids, mask=np.zeros(len(hids)))
dists = np.atleast_2d(dists).astype(float).reshape(oids.shape[0], hids.shape[0]).copy()
if frameid is None:
assert self.auto_id, 'auto-id is not enabled'
if len(self._indices) > 0:
frameid = self._indices[-1][0] + 1 #frameid = 1 at the next call of update
else:
frameid = 0 #frameid = 0 at first call of update
else:
assert not self.auto_id, 'Cannot provide frame id when auto-id is enabled'
eid = count()
# 0. Record raw events
no = len(oids)
nh = len(hids)
if no * nh > 0:
for i in range(no):
for j in range(nh):
self._indices.append((frameid, next(eid)))
self._events.append(['RAW', oids[i], hids[j], dists[i,j]])
elif no == 0:
for i in range(nh):
self._indices.append((frameid, next(eid)))
self._events.append(['RAW', np.nan, hids[i], np.nan])
elif nh == 0:
for i in range(no):
self._indices.append((frameid, next(eid)))
self._events.append(['RAW', oids[i], np.nan, np.nan])
if oids.size * hids.size > 0:
# 1. Try to re-establish tracks from previous correspondences
for i in range(oids.shape[0]):
if not oids[i] in self.m: #if idx of object correspond to some hypothesis in previous frame
continue #if index of object does not correspond to any hypothesis in previous frame( new object or false negative in previous frame), then continue
hprev = self.m[oids[i]]#then get index of that previous hypothesis.
j, = np.where(hids==hprev) #position of that index in the new hypothesis list
if j.shape[0] == 0:# if index of that hypothesis in previous frame does not exist in current hypothesis list (false negative in current frame)
continue
j = j[0]
if np.isfinite(dists[i,j]):
oids[i] = ma.masked #masked the object that already MATCH, so that we can set it to nan in 2nd step
hids[j] = ma.masked
self.m[oids.data[i]] = hids.data[j]
self._indices.append((frameid, next(eid)))
self._events.append(['MATCH', oids.data[i], hids.data[j], dists[i, j]]) #re-establish the matched pairs in previous frame
# 2. Try to correspond remaining objects/hypotheses after re-establish ( objects already match above are ignore by setting dist to np.nan)
dists[oids.mask, :] = np.nan
dists[:, hids.mask] = np.nan
rids, cids = linear_sum_assignment(dists) #only return rows and columns indices that not np.nan
for i, j in zip(rids, cids):
if not np.isfinite(dists[i,j]):
continue
o = oids[i]
h = hids.data[j]
is_switch = o in self.m and \ #if object id used to correspond to another id not the current one
self.m[o] != h and \#
abs(frameid - self.last_occurrence[o]) <= self.max_switch_time# Then that is a switch in id of hypothesis
cat = 'SWITCH' if is_switch else 'MATCH'
self._indices.append((frameid, next(eid)))
self._events.append([cat, oids.data[i], hids.data[j], dists[i, j]])
oids[i] = ma.masked
hids[j] = ma.masked
self.m[o] = h
