def test_all_inf():
cost = np.empty((5, 5), dtype=float)
cost[:] = np.inf
with raises(ValueError):
lapmod(*sparse_from_dense(cost))
with raises(ValueError):
lapmod(*sparse_from_masked(cost))
def test_all_inf():
cost = np.empty((5, 5), dtype=float)
cost[:] = np.inf
with raises(ValueError):
lapmod(*sparse_from_dense(cost))
with raises(ValueError):
lapmod(*sparse_from_masked(cost))
def test_inf_row():
cost = np.array(
[[0., 0., 0., 0., np.inf], [np.inf, np.inf, 0., 0., 0.],
[np.inf, np.inf, np.inf, np.inf, np.inf],
[np.inf, np.inf, np.inf, 0., 0.], [0., 0., 0., np.inf, np.inf]],
dtype=float)
with raises(ValueError):
ret = lapmod(*sparse_from_dense(cost))
ret = lapmod(*sparse_from_masked(cost))
assert len(ret) == 3
assert ret[0] == np.inf
def test_inf_row():
cost = np.array([[0., 0., 0., 0., np.inf],
[np.inf, np.inf, 0., 0., 0.],
[np.inf, np.inf, np.inf, np.inf, np.inf],
[np.inf, np.inf, np.inf, 0., 0.],
[0., 0., 0., np.inf, np.inf]], dtype=float)
with raises(ValueError):
ret = lapmod(*sparse_from_dense(cost))
ret = lapmod(*sparse_from_masked(cost))
assert len(ret) == 3
assert ret[0] == np.inf
def test_inf_unique():
cost = np.array([[1000, 4, 1], [1, 1000, 3], [5, 1, 1000]])
cost_ext = np.empty((4, 4))
cost_ext[:] = np.inf
cost_ext[:3, :3] = cost
cost_ext[3, 3] = 0
with raises(ValueError):
ret = lapmod(*sparse_from_dense(cost_ext))
ret = lapmod(*sparse_from_masked(cost_ext))
assert len(ret) == 3
assert ret[0] == 3.
assert np.all(ret[1] == [2, 0, 1, 3])
def test_inf_unique():
cost = np.array([[1000, 4, 1],
[1, 1000, 3],
[5, 1, 1000]])
cost_ext = np.empty((4, 4))
cost_ext[:] = np.inf
cost_ext[:3, :3] = cost
cost_ext[3, 3] = 0
with raises(ValueError):
ret = lapmod(*sparse_from_dense(cost_ext))
ret = lapmod(*sparse_from_masked(cost_ext))
assert len(ret) == 3
assert ret[0] == 3.
assert np.all(ret[1] == [2, 0, 1, 3])
def test_dense_100x100_int(dense_100x100_int):
cost, opt = dense_100x100_int
ret = lapmod(*sparse_from_dense(cost))
assert len(ret) == 3
assert ret[0] == opt
lapjv_ret = lapjv(cost)
assert ret[0] == lapjv_ret[0]
def test_dense_100x100_int(dense_100x100_int):
cost, opt = dense_100x100_int
ret = lapmod(*sparse_from_dense(cost))
assert len(ret) == 3
assert ret[0] == opt
lapjv_ret = lapjv(cost)
assert ret[0] == lapjv_ret[0]
def test_dense_1kx1k_int_hard(dense_1kx1k_int_hard):
cost, opt = dense_1kx1k_int_hard
ret = lapmod(*sparse_from_dense(cost))
assert len(ret) == 3
assert ret[0] == opt
lapjv_ret = lapjv(cost)
assert ret[0] == lapjv_ret[0]
def test_dense_1kx1k_int_hard(dense_1kx1k_int_hard):
cost, opt = dense_1kx1k_int_hard
ret = lapmod(*sparse_from_dense(cost))
assert len(ret) == 3
assert ret[0] == opt
lapjv_ret = lapjv(cost)
assert ret[0] == lapjv_ret[0]
def test_sparse_4kx4k_int(sparse_4kx4k_int):
cost, mask, opt = sparse_4kx4k_int
ret = lapmod(*sparse_from_masked(cost, mask))
assert len(ret) == 3
assert ret[0] == opt
cost[~mask] = get_platform_maxint()
lapjv_ret = lapjv(cost)
assert ret[0] == lapjv_ret[0]
def test_sparse_4kx4k_int(sparse_4kx4k_int):
cost, mask, opt = sparse_4kx4k_int
ret = lapmod(*sparse_from_masked(cost, mask))
assert len(ret) == 3
assert ret[0] == opt
cost[~mask] = get_platform_maxint()
lapjv_ret = lapjv(cost)
assert ret[0] == lapjv_ret[0]
def test_square(cost, expected):
ret = lapmod(*sparse_from_dense(cost))
assert len(ret) == len(expected)
assert cost[range(cost.shape[0]), ret[1]].sum() == ret[0]
assert cost[ret[2], range(cost.shape[1])].sum() == ret[0]
assert ret[0] == expected[0]
assert np.all(ret[1] == expected[1])
assert np.all(ret[2] == expected[2])
dense_ret = lapjv(cost)
assert ret[0] == dense_ret[0]
assert np.all(ret[1] == dense_ret[1])
assert np.all(ret[2] == dense_ret[2])
def linker(Nparticles, Cs):
"""Link a set of detections."""
assert all(Cs.A > 0)
assert all(isfinite(Cs.A))
assert all(isnan(Cs.A) == False)
vals, kk, offsets = Cs.convert()
_, start = array(
lapmod(2 * Nparticles, vals, offsets, kk, return_cost=False))
end = arange(start.size)
linkinds = (start < Nparticles) & (end < Nparticles)
S, E = start[linkinds], end[linkinds]
return S, E
def test_sparse_square(cost, expected):
ret = lapmod(*sparse_from_masked(cost))
assert len(ret) == len(expected)
assert cost[range(cost.shape[0]), ret[1]].sum() == ret[0]
assert cost[ret[2], range(cost.shape[1])].sum() == ret[0]
assert ret[0] == expected[0]
assert np.all(ret[1] == expected[1])
assert np.all(ret[2] == expected[2])
dense_ret = lapjv(cost)
assert ret[0] == dense_ret[0]
assert np.all(ret[1] == dense_ret[1])
assert np.all(ret[2] == dense_ret[2])
def test_infs_unsolvable():
cost = np.array([[0., 0., 0., np.inf, np.inf],
[np.inf, np.inf, np.inf, 0., 0.],
[np.inf, np.inf, np.inf, 0., 0.],
[np.inf, np.inf, np.inf, 0., 0.],
[0., 0., 0., np.inf, np.inf]], dtype=float)
lapjv_ret = lapjv(cost)
assert lapjv_ret[0] == np.inf
ret = lapmod(*sparse_from_masked(cost))
assert len(ret) == 3
assert ret[0] == np.inf
cost = np.array([[19., 22., 16., np.inf, np.inf],
[np.inf, np.inf, np.inf, 4., 13.],
[np.inf, np.inf, np.inf, 3., 14.],
[np.inf, np.inf, np.inf, 10., 12.],
[11., 14., 13., np.inf, np.inf]], dtype=float)
lapjv_ret = lapjv(cost)
assert lapjv_ret[0] == np.inf
ret = lapmod(*sparse_from_masked(cost))
assert len(ret) == 3
assert ret[0] == np.inf
def test_infs_unsolvable():
cost = np.array(
[[0., 0., 0., np.inf, np.inf], [np.inf, np.inf, np.inf, 0., 0.],
[np.inf, np.inf, np.inf, 0., 0.], [np.inf, np.inf, np.inf, 0., 0.],
[0., 0., 0., np.inf, np.inf]],
dtype=float)
lapjv_ret = lapjv(cost)
assert lapjv_ret[0] == np.inf
ret = lapmod(*sparse_from_masked(cost))
assert len(ret) == 3
assert ret[0] == np.inf
cost = np.array(
[[19., 22., 16., np.inf, np.inf], [np.inf, np.inf, np.inf, 4., 13.],
[np.inf, np.inf, np.inf, 3., 14.], [np.inf, np.inf, np.inf, 10., 12.],
[11., 14., 13., np.inf, np.inf]],
dtype=float)
lapjv_ret = lapjv(cost)
assert lapjv_ret[0] == np.inf
ret = lapmod(*sparse_from_masked(cost))
assert len(ret) == 3
assert ret[0] == np.inf
def test_lapmod_arr_loop():
shape = (7, 3)
cc = np.array([
2.593883482138951146e-01, 3.080381437461217620e-01,
1.976243020727339317e-01, 2.462740976049606068e-01,
4.203993396282833528e-01, 4.286184525458427985e-01,
1.706431415909629434e-01, 2.192929371231896185e-01,
2.117769622802734286e-01, 2.604267578125001315e-01])
ii = np.array([0, 0, 1, 1, 2, 2, 5, 5, 6, 6])
jj = np.array([0, 1, 0, 1, 1, 2, 0, 1, 0, 1])
cost_limit = 1e3
cc, ii, kk = prepare_sparse_cost(shape, cc, ii, jj, cost_limit)
opt, ind1, ind0 = lapmod(len(ii)-1, cc, ii, kk, return_cost=True)
ind1[ind1 >= shape[1]] = -1
ind0[ind0 >= shape[0]] = -1
ind1 = ind1[:shape[0]]
ind0 = ind0[:shape[1]]
assert opt == approx(4000.8455356917416, 1e-10)
assert np.all(ind0 == [5, 1, 2]) or np.all(ind0 == [1, 5, 2])
def _link(self, search_radius=15,params=[],**kwargs):
"""
Helper function : link adjecent frames using JV lap.
TODO: extend for a general cost function. make class of cost functions that returns shape, cc, ii, jj
Parameters
----------
params : [(channel,weight)] list of tuples
search_radius : [25]
"""
#ch = NucChannel
#if ch is None:
# ch = self.channels[0]
# print('No channel provided, using '+ch)
cents = self.centroid_um
nums = self.num
if params:
Cp = [p[0] for p in params if p[0] in self.channels]
Wp = [p[1] for p in params if p[0] in self.channels]
ints = {}
else:
Cp=[]
Wp=[]
for i in np.arange(nums.shape[0]-1):
sys.stdout.write("\r"+'linking frame '+ str(i))
sys.stdout.flush()
T = KDTree(cents[i+1])
#We calculate points in centroid(n+1) that are less than distance_upper_bound from points in centroid(n)
dists, idx = T.query(cents[i], k=12, distance_upper_bound=search_radius)
dists = [r[r<1E308] for r in dists]
idx = [r[r<cents[i+1].shape[0]] for r in idx]
#possible matches in n+1
jj = np.concatenate(idx)
#possible correspondence in n
j=0
ii=[]
for r in idx:
ii.append(j*np.ones_like(r))
j+=1
ii = np.concatenate(ii)
#ii jj cc are now sparse matrix in COO format
ampRatio=1
eps = 10**-72
for cp, wp in zip(Cp, Wp):
ints = self.ninetyint(cp)
ampRatio = ampRatio + wp*(np.array(eps+np.maximum(list(ints[i][ii]), list(ints[i+1][jj])))/np.array(eps+np.minimum(list(ints[i][ii]), list(ints[i+1][jj])))-1)
#costs of match
cc = np.concatenate(dists)*ampRatio
cc[cc>1000000]=999999
shape = (nums[i], nums[i+1])
cc, ii, kk = prepare_sparse_cost(shape, cc, ii, jj, cost_limit=300)
ind1, ind0 = lap.lapmod(len(ii)-1, cc, ii, kk, return_cost=False)
ind1[ind1 >= shape[1]] = -1
ind0[ind0 >= shape[0]] = -1
#inds in n+1 that match inds (1:N) in n
ind1 = ind1[:shape[0]]
#inds in n that match inds (1:N) in n+1
ind0 = ind0[:shape[1]]
self.framelabels[i].link1in2 = ind1
self.framelabels[nums.shape[0]-1].link1in2=np.array([])
def _split(self, search_radius=20, params=[],maxAmpRatio=5, **kwargs):
"""
Helper function : find splits using JV lap.
----------
params : [(channel,weight)] list of tuples
maxAmpRatio : [5] max allowd ratio of amplitudes for linking
search_radius : [40] search radius
"""
#ch=NucChannel
#if ch is None:
# ch = self.channels[0]
if params:
Cp = [p[0] for p in params if p[0] in self.channels]
Wp = [p[1] for p in params if p[0] in self.channels]
ints = {}
for cp in Cp:
ints[cp] = self.ninetyint(cp)
else:
Cp=[]
Wp=[]
trackbits = self.trackinds
cents = self.centroid_um
relatives = [[]]*len(trackbits)
notdoneflag=1
while notdoneflag:
trackstarts = np.array([np.where(~np.isnan(r.astype('float')))[0][0] for r in trackbits])
trackends = np.array([np.where(~np.isnan(r.astype('float')))[0] for r in trackbits])
# dtmat = np.expand_dims(trackstarts,1)-np.expand_dims(trackends,0)
# dt1array = np.zeros_like(dtmat)
# for (x, y), element in np.ndenumerate(dtmat):
# try:
# dt1array[x,y] = np.nonzero(element==1)[0][0]
# except:
# dt1array[x,y] = None
# possiblelinks = np.transpose(np.where(dt1array!=None))
possiblelinks = np.empty((0, 2), int)
for J in np.unique(trackstarts):
link_a = np.where(trackstarts==J)
link_b = np.where(list(map(lambda x: np.any(x==J-1), trackends)))
if np.any(link_a):
f = lambda x: np.pad(link_b,((1,0),(0,0)), constant_values=x)
d = np.transpose(np.hstack(list(map(f, link_a[0]))))
possiblelinks = np.concatenate((possiblelinks,d))
ii = []
jj = []
cc = []
for i in np.arange(len(possiblelinks)):
# frame1 - end of possible to link
frame2 = trackstarts[possiblelinks[i][0]]
# frame1 - end of possible to link
frame1 = frame2-1
# cell label in frame 1 to link
ind1 = trackbits[possiblelinks[i][1]][frame1]
# cell label in frame 2 to link
ind2 = trackbits[possiblelinks[i][0]][frame2]
dr = np.linalg.norm(cents[frame1][ind1]-cents[frame2][ind2])
if dr <= search_radius:
da=1
for cp, wp in zip(Cp, Wp):
da = da + wp*(np.maximum(ints[cp][frame1][ind1], ints[cp][frame2][ind2])/np.minimum(ints[cp][frame1][ind1], ints[cp][frame2][ind2])-1)
if da<=maxAmpRatio:
ii.append(possiblelinks[i][1])
jj.append(possiblelinks[i][0])
#maybe one day we'll change this somehow. Not sure how rn
cost = dr*da
cc.append(cost)
ii = np.array(ii)
jj = np.array(jj)
cc = np.array(cc)
#print(ii)
if len(ii)==0:
print('\nFinished finding splits')
notdoneflag=0
break
shape = (len(trackbits),len(trackbits))
cc, ii, kk = prepare_sparse_cost(shape,cc,ii,jj, 1000)
match1, _ = lap.lapmod(len(ii)-1, cc, ii, kk, return_cost=False)
match1[match1 >= shape[1]] = -1
#inds in n+1 that match inds (1:N) in n
match1 = np.array(match1[:shape[0]])
trackindstofill = np.nonzero(match1+1)[0]
trackindstoadd = match1[np.nonzero(match1+1)]
fa = {trackindstofill[i]: trackindstoadd[i] for i in range(len(trackindstoadd))}
for i in fa:
sf = trackstarts[i]
ef = trackstarts[fa[i]]
trackbits[fa[i]][np.arange(sf,ef)]=trackbits[i][np.arange(sf,ef)]
relatives[i]=relatives[i]+[fa[i]]
relatives[fa[i]]=relatives[fa[i]]+[i]
self.relatives = relatives
self.trackinds = trackbits
def lat_var(xp, sims, n_similar, n_repeats, batch_size, asym):
"""
Run the matching in the E-step of the latent-variable model.
:param xp: numpy or cupy, depending whether we run on CPU or GPU.
:param xw: the xw matrix
:param zw: the zw matrix
:param sims: an matrix of shape (src_size, trg_size) where the similarity values
between each source word and target words are stored
:param best_sim_forward: an array of shape (src_size), which stores the best similarity
scores for each
:param n_similar:
:param n_repeats:
:param batch_size:
:param asym:
:return:
"""
src_size = sims.shape[0]
cc = np.empty(
src_size * n_similar
) # 1D array of all finite elements of the assignement cost matrix
kk = np.empty(
src_size * n_similar
) # 1D array of the column indices. Must be sorted within one row.
ii = np.empty((src_size * n_repeats + 1, ),
dtype=int) # 1D array of indices of the row starts in cc.
ii[0] = 0
# if each src id should be matched to trg id, then we need to double the source indices
for i in range(1, src_size * n_repeats + 1):
ii[i] = ii[i - 1] + n_similar
for i in range(0, src_size, batch_size):
# j = min(x.shape[0], i + batch_size)
j = min(i + batch_size, src_size)
sim = sims[i:j]
trg_indices = xp.argpartition(
sim, -n_similar)[:,
-n_similar:] # get indices of n largest elements
if xp != np:
trg_indices = xp.asnumpy(trg_indices)
trg_indices.sort() # sort the target indices
trg_indices = trg_indices.flatten()
row_indices = np.array([[i] * n_similar
for i in range(j - i)]).flatten()
sim_scores = sim[row_indices, trg_indices]
costs = 1 - sim_scores
if xp != np:
costs = xp.asnumpy(costs)
cc[i * n_similar:j * n_similar] = costs
kk[i * n_similar:j * n_similar] = trg_indices
if n_repeats > 1:
# duplicate costs and target indices
new_cc = cc
new_kk = kk
for i in range(1, n_repeats):
new_cc = np.concatenate([new_cc, cc], axis=0)
if asym == '1:2':
# for 1:2, we don't duplicate the target indices
new_kk = np.concatenate([new_kk, kk], axis=0)
else:
# update target indices so that they refer to new columns
new_kk = np.concatenate([new_kk, kk + src_size * i], axis=0)
cc = new_cc
kk = new_kk
# trg indices are targets assigned to each row id from 0-(n_rows-1)
cost, trg_indices, _ = lapmod(src_size * n_repeats, cc, ii, kk)
src_indices = np.concatenate([np.arange(src_size)] * n_repeats, 0)
src_indices, trg_indices = xp.asarray(src_indices), xp.asarray(trg_indices)
# remove the pairs in which a source word was connected to a target
# which was not one of its k most similar words
wrong_inds = []
for i, trgind in enumerate(trg_indices):
krow = ii[i]
candidates = kk[krow:krow + n_similar]
if trgind not in candidates:
wrong_inds.append(i)
trg_indices = np.delete(trg_indices, wrong_inds)
src_indices = np.delete(src_indices, wrong_inds)
for i in range(len(src_indices)):
src_idx, trg_idx = src_indices[i], trg_indices[i]
# we do this if args.n_repeats > 0 to assign the target
# indices in the cost matrix to the correct idx
while trg_idx >= src_size:
# if we repeat, we have indices that are > n_rows
trg_idx -= src_size
trg_indices[i] = trg_idx
return src_indices, trg_indices
def test_sparse_100x100_int(sparse_100x100_int):
cost, mask, opt = sparse_100x100_int
ret = lapmod(*sparse_from_masked(cost, mask))
assert len(ret) == 3
assert ret[0] == opt
def test_eps(dense_eps):
cost, opt = dense_eps
ret = lapmod(*sparse_from_dense(cost))
assert len(ret) == 3
assert ret[0] == opt
def test_eps(dense_eps):
cost, opt = dense_eps
ret = lapmod(*sparse_from_dense(cost))
assert len(ret) == 3
assert ret[0] == opt
def test_sparse_100x100_int(sparse_100x100_int):
cost, mask, opt = sparse_100x100_int
ret = lapmod(*sparse_from_masked(cost, mask))
assert len(ret) == 3
assert ret[0] == opt
def _closegaps(self, maxStep=10, params=[],maxAmpRatio=5,maxTimeJump=4, mintracklength=30, **kwargs):
"""
Helper function : close gaps between open stubs using JV lap.
todo: split. When a stub that starts in the middle has a plausible link, make a compound track
----------
#NucChannel : ['DeepBlue']
params : [(channel,weight)] list of tuples
maxAmpRatio : [2] max allowd ratio of amplitudes for linking
mintracklength : [30] final minimum length of a track
"""
#ch=NucChannel
#if ch is None:
# ch = self.channels[0]
if params:
Cp = [p[0] for p in params if p[0] in self.channels]
Wp = [p[1] for p in params if p[0] in self.channels]
ints = {}
for cp in Cp:
ints[cp] = self.ninetyint(cp)
else:
Cp=[]
Wp=[]
trackbits = self._getAllContinuousTrackSegs(**kwargs)
cents = self.centroid_um
notdoneflag=1
while notdoneflag:
trackstarts = np.array([np.where(~np.isnan(r.astype('float')))[0][0] for r in trackbits])
trackends = np.array([np.where(~np.isnan(r.astype('float')))[0][-1] for r in trackbits])
dtmat = np.expand_dims(trackstarts,1)-np.expand_dims(trackends,0)
possiblelinks = np.transpose(np.nonzero((dtmat>0)*(dtmat<maxTimeJump)))
ii = []
jj = []
cc = []
for i in np.arange(len(possiblelinks)):
# frame1 - end of possible to link
frame1 = trackends[possiblelinks[i][1]]
# frame2 - beginning of possible link
frame2 = trackstarts[possiblelinks[i][0]]
# cell label in frame 1 to link
ind1 = trackbits[possiblelinks[i][1]][frame1]
# cell label in frame 2 to link
ind2 = trackbits[possiblelinks[i][0]][frame2]
dt = frame2-frame1
dr = np.linalg.norm(cents[frame1][ind1]-cents[frame2][ind2])
da=1
eps=10**-72
for cp, wp in zip(Cp, Wp):
da = da + wp*((eps+np.maximum(ints[cp][frame1][ind1], ints[cp][frame2][ind2]))/(eps+np.minimum(ints[cp][frame1][ind1], ints[cp][frame2][ind2]))-1)
if dr <= (np.sqrt(dt)*maxStep):
if da<=maxAmpRatio:
ii.append(possiblelinks[i][1])
jj.append(possiblelinks[i][0])
#maybe one day we'll change this somehow. Not sure how rn
cost = dr*da*dt
cc.append(cost)
ii = np.array(ii)
jj = np.array(jj)
cc = np.array(cc)
if len(ii)==0:
print('\nFinished connecting tracks')
notdoneflag=0
break
shape = (len(trackbits),len(trackbits))
cc, ii, kk = prepare_sparse_cost(shape,cc,ii,jj, 1000)
match1, _ = lap.lapmod(len(ii)-1, cc, ii, kk, return_cost=False)
match1[match1 >= shape[1]] = -1
#inds in n+1 that match inds (1:N) in n
match1 = np.array(match1[:shape[0]])
trackindstofill = np.nonzero(match1+1)[0]
trackindstoadd = match1[np.nonzero(match1+1)]
fa = {trackindstofill[i]: trackindstoadd[i] for i in range(len(trackindstoadd))}
for i in fa:
sf = trackstarts[fa[i]]
ef = trackends[fa[i]]+1
trackbits[i][np.arange(sf,ef)]=trackbits[fa[i]][np.arange(sf,ef)]
trackbits[fa[i]][np.arange(sf,ef)]=None
#add nans in gaps
#eef = trackends[i]+1
#trackbits[i][np.arange(eef,sf)]=np.nan
#remove lines that are all Nones
trackbits = trackbits[[any(~np.isnan(r.astype('float'))) for r in trackbits]]
trackbits = trackbits[np.array([sum(~np.isnan(r.astype('float'))) for r in trackbits])>=mintracklength]
sortind = np.lexsort((np.arange(len(trackbits)) ,[np.sum(np.isnan(r.astype('float'))) for r in trackbits] ,[np.sum(r==None) for r in trackbits]))
self.trackinds = trackbits[sortind]
