def test_all_inf(): cost = np.empty((5, 5), dtype=float) cost[:] = np.inf with raises(ValueError): lapmod(*sparse_from_dense(cost)) with raises(ValueError): lapmod(*sparse_from_masked(cost))
def test_all_inf(): cost = np.empty((5, 5), dtype=float) cost[:] = np.inf with raises(ValueError): lapmod(*sparse_from_dense(cost)) with raises(ValueError): lapmod(*sparse_from_masked(cost))
def test_inf_row(): cost = np.array( [[0., 0., 0., 0., np.inf], [np.inf, np.inf, 0., 0., 0.], [np.inf, np.inf, np.inf, np.inf, np.inf], [np.inf, np.inf, np.inf, 0., 0.], [0., 0., 0., np.inf, np.inf]], dtype=float) with raises(ValueError): ret = lapmod(*sparse_from_dense(cost)) ret = lapmod(*sparse_from_masked(cost)) assert len(ret) == 3 assert ret[0] == np.inf
def test_inf_row(): cost = np.array([[0., 0., 0., 0., np.inf], [np.inf, np.inf, 0., 0., 0.], [np.inf, np.inf, np.inf, np.inf, np.inf], [np.inf, np.inf, np.inf, 0., 0.], [0., 0., 0., np.inf, np.inf]], dtype=float) with raises(ValueError): ret = lapmod(*sparse_from_dense(cost)) ret = lapmod(*sparse_from_masked(cost)) assert len(ret) == 3 assert ret[0] == np.inf
def test_inf_unique(): cost = np.array([[1000, 4, 1], [1, 1000, 3], [5, 1, 1000]]) cost_ext = np.empty((4, 4)) cost_ext[:] = np.inf cost_ext[:3, :3] = cost cost_ext[3, 3] = 0 with raises(ValueError): ret = lapmod(*sparse_from_dense(cost_ext)) ret = lapmod(*sparse_from_masked(cost_ext)) assert len(ret) == 3 assert ret[0] == 3. assert np.all(ret[1] == [2, 0, 1, 3])
def test_inf_unique(): cost = np.array([[1000, 4, 1], [1, 1000, 3], [5, 1, 1000]]) cost_ext = np.empty((4, 4)) cost_ext[:] = np.inf cost_ext[:3, :3] = cost cost_ext[3, 3] = 0 with raises(ValueError): ret = lapmod(*sparse_from_dense(cost_ext)) ret = lapmod(*sparse_from_masked(cost_ext)) assert len(ret) == 3 assert ret[0] == 3. assert np.all(ret[1] == [2, 0, 1, 3])
def test_dense_100x100_int(dense_100x100_int): cost, opt = dense_100x100_int ret = lapmod(*sparse_from_dense(cost)) assert len(ret) == 3 assert ret[0] == opt lapjv_ret = lapjv(cost) assert ret[0] == lapjv_ret[0]
def test_dense_100x100_int(dense_100x100_int): cost, opt = dense_100x100_int ret = lapmod(*sparse_from_dense(cost)) assert len(ret) == 3 assert ret[0] == opt lapjv_ret = lapjv(cost) assert ret[0] == lapjv_ret[0]
def test_dense_1kx1k_int_hard(dense_1kx1k_int_hard): cost, opt = dense_1kx1k_int_hard ret = lapmod(*sparse_from_dense(cost)) assert len(ret) == 3 assert ret[0] == opt lapjv_ret = lapjv(cost) assert ret[0] == lapjv_ret[0]
def test_dense_1kx1k_int_hard(dense_1kx1k_int_hard): cost, opt = dense_1kx1k_int_hard ret = lapmod(*sparse_from_dense(cost)) assert len(ret) == 3 assert ret[0] == opt lapjv_ret = lapjv(cost) assert ret[0] == lapjv_ret[0]
def test_sparse_4kx4k_int(sparse_4kx4k_int): cost, mask, opt = sparse_4kx4k_int ret = lapmod(*sparse_from_masked(cost, mask)) assert len(ret) == 3 assert ret[0] == opt cost[~mask] = get_platform_maxint() lapjv_ret = lapjv(cost) assert ret[0] == lapjv_ret[0]
def test_sparse_4kx4k_int(sparse_4kx4k_int): cost, mask, opt = sparse_4kx4k_int ret = lapmod(*sparse_from_masked(cost, mask)) assert len(ret) == 3 assert ret[0] == opt cost[~mask] = get_platform_maxint() lapjv_ret = lapjv(cost) assert ret[0] == lapjv_ret[0]
def test_square(cost, expected): ret = lapmod(*sparse_from_dense(cost)) assert len(ret) == len(expected) assert cost[range(cost.shape[0]), ret[1]].sum() == ret[0] assert cost[ret[2], range(cost.shape[1])].sum() == ret[0] assert ret[0] == expected[0] assert np.all(ret[1] == expected[1]) assert np.all(ret[2] == expected[2]) dense_ret = lapjv(cost) assert ret[0] == dense_ret[0] assert np.all(ret[1] == dense_ret[1]) assert np.all(ret[2] == dense_ret[2])
def linker(Nparticles, Cs): """Link a set of detections.""" assert all(Cs.A > 0) assert all(isfinite(Cs.A)) assert all(isnan(Cs.A) == False) vals, kk, offsets = Cs.convert() _, start = array( lapmod(2 * Nparticles, vals, offsets, kk, return_cost=False)) end = arange(start.size) linkinds = (start < Nparticles) & (end < Nparticles) S, E = start[linkinds], end[linkinds] return S, E
def test_sparse_square(cost, expected): ret = lapmod(*sparse_from_masked(cost)) assert len(ret) == len(expected) assert cost[range(cost.shape[0]), ret[1]].sum() == ret[0] assert cost[ret[2], range(cost.shape[1])].sum() == ret[0] assert ret[0] == expected[0] assert np.all(ret[1] == expected[1]) assert np.all(ret[2] == expected[2]) dense_ret = lapjv(cost) assert ret[0] == dense_ret[0] assert np.all(ret[1] == dense_ret[1]) assert np.all(ret[2] == dense_ret[2])
def test_infs_unsolvable(): cost = np.array([[0., 0., 0., np.inf, np.inf], [np.inf, np.inf, np.inf, 0., 0.], [np.inf, np.inf, np.inf, 0., 0.], [np.inf, np.inf, np.inf, 0., 0.], [0., 0., 0., np.inf, np.inf]], dtype=float) lapjv_ret = lapjv(cost) assert lapjv_ret[0] == np.inf ret = lapmod(*sparse_from_masked(cost)) assert len(ret) == 3 assert ret[0] == np.inf cost = np.array([[19., 22., 16., np.inf, np.inf], [np.inf, np.inf, np.inf, 4., 13.], [np.inf, np.inf, np.inf, 3., 14.], [np.inf, np.inf, np.inf, 10., 12.], [11., 14., 13., np.inf, np.inf]], dtype=float) lapjv_ret = lapjv(cost) assert lapjv_ret[0] == np.inf ret = lapmod(*sparse_from_masked(cost)) assert len(ret) == 3 assert ret[0] == np.inf
def test_infs_unsolvable(): cost = np.array( [[0., 0., 0., np.inf, np.inf], [np.inf, np.inf, np.inf, 0., 0.], [np.inf, np.inf, np.inf, 0., 0.], [np.inf, np.inf, np.inf, 0., 0.], [0., 0., 0., np.inf, np.inf]], dtype=float) lapjv_ret = lapjv(cost) assert lapjv_ret[0] == np.inf ret = lapmod(*sparse_from_masked(cost)) assert len(ret) == 3 assert ret[0] == np.inf cost = np.array( [[19., 22., 16., np.inf, np.inf], [np.inf, np.inf, np.inf, 4., 13.], [np.inf, np.inf, np.inf, 3., 14.], [np.inf, np.inf, np.inf, 10., 12.], [11., 14., 13., np.inf, np.inf]], dtype=float) lapjv_ret = lapjv(cost) assert lapjv_ret[0] == np.inf ret = lapmod(*sparse_from_masked(cost)) assert len(ret) == 3 assert ret[0] == np.inf
def test_lapmod_arr_loop(): shape = (7, 3) cc = np.array([ 2.593883482138951146e-01, 3.080381437461217620e-01, 1.976243020727339317e-01, 2.462740976049606068e-01, 4.203993396282833528e-01, 4.286184525458427985e-01, 1.706431415909629434e-01, 2.192929371231896185e-01, 2.117769622802734286e-01, 2.604267578125001315e-01]) ii = np.array([0, 0, 1, 1, 2, 2, 5, 5, 6, 6]) jj = np.array([0, 1, 0, 1, 1, 2, 0, 1, 0, 1]) cost_limit = 1e3 cc, ii, kk = prepare_sparse_cost(shape, cc, ii, jj, cost_limit) opt, ind1, ind0 = lapmod(len(ii)-1, cc, ii, kk, return_cost=True) ind1[ind1 >= shape[1]] = -1 ind0[ind0 >= shape[0]] = -1 ind1 = ind1[:shape[0]] ind0 = ind0[:shape[1]] assert opt == approx(4000.8455356917416, 1e-10) assert np.all(ind0 == [5, 1, 2]) or np.all(ind0 == [1, 5, 2])
def _link(self, search_radius=15,params=[],**kwargs): """ Helper function : link adjecent frames using JV lap. TODO: extend for a general cost function. make class of cost functions that returns shape, cc, ii, jj Parameters ---------- params : [(channel,weight)] list of tuples search_radius : [25] """ #ch = NucChannel #if ch is None: # ch = self.channels[0] # print('No channel provided, using '+ch) cents = self.centroid_um nums = self.num if params: Cp = [p[0] for p in params if p[0] in self.channels] Wp = [p[1] for p in params if p[0] in self.channels] ints = {} else: Cp=[] Wp=[] for i in np.arange(nums.shape[0]-1): sys.stdout.write("\r"+'linking frame '+ str(i)) sys.stdout.flush() T = KDTree(cents[i+1]) #We calculate points in centroid(n+1) that are less than distance_upper_bound from points in centroid(n) dists, idx = T.query(cents[i], k=12, distance_upper_bound=search_radius) dists = [r[r<1E308] for r in dists] idx = [r[r<cents[i+1].shape[0]] for r in idx] #possible matches in n+1 jj = np.concatenate(idx) #possible correspondence in n j=0 ii=[] for r in idx: ii.append(j*np.ones_like(r)) j+=1 ii = np.concatenate(ii) #ii jj cc are now sparse matrix in COO format ampRatio=1 eps = 10**-72 for cp, wp in zip(Cp, Wp): ints = self.ninetyint(cp) ampRatio = ampRatio + wp*(np.array(eps+np.maximum(list(ints[i][ii]), list(ints[i+1][jj])))/np.array(eps+np.minimum(list(ints[i][ii]), list(ints[i+1][jj])))-1) #costs of match cc = np.concatenate(dists)*ampRatio cc[cc>1000000]=999999 shape = (nums[i], nums[i+1]) cc, ii, kk = prepare_sparse_cost(shape, cc, ii, jj, cost_limit=300) ind1, ind0 = lap.lapmod(len(ii)-1, cc, ii, kk, return_cost=False) ind1[ind1 >= shape[1]] = -1 ind0[ind0 >= shape[0]] = -1 #inds in n+1 that match inds (1:N) in n ind1 = ind1[:shape[0]] #inds in n that match inds (1:N) in n+1 ind0 = ind0[:shape[1]] self.framelabels[i].link1in2 = ind1 self.framelabels[nums.shape[0]-1].link1in2=np.array([])
def _split(self, search_radius=20, params=[],maxAmpRatio=5, **kwargs): """ Helper function : find splits using JV lap. ---------- params : [(channel,weight)] list of tuples maxAmpRatio : [5] max allowd ratio of amplitudes for linking search_radius : [40] search radius """ #ch=NucChannel #if ch is None: # ch = self.channels[0] if params: Cp = [p[0] for p in params if p[0] in self.channels] Wp = [p[1] for p in params if p[0] in self.channels] ints = {} for cp in Cp: ints[cp] = self.ninetyint(cp) else: Cp=[] Wp=[] trackbits = self.trackinds cents = self.centroid_um relatives = [[]]*len(trackbits) notdoneflag=1 while notdoneflag: trackstarts = np.array([np.where(~np.isnan(r.astype('float')))[0][0] for r in trackbits]) trackends = np.array([np.where(~np.isnan(r.astype('float')))[0] for r in trackbits]) # dtmat = np.expand_dims(trackstarts,1)-np.expand_dims(trackends,0) # dt1array = np.zeros_like(dtmat) # for (x, y), element in np.ndenumerate(dtmat): # try: # dt1array[x,y] = np.nonzero(element==1)[0][0] # except: # dt1array[x,y] = None # possiblelinks = np.transpose(np.where(dt1array!=None)) possiblelinks = np.empty((0, 2), int) for J in np.unique(trackstarts): link_a = np.where(trackstarts==J) link_b = np.where(list(map(lambda x: np.any(x==J-1), trackends))) if np.any(link_a): f = lambda x: np.pad(link_b,((1,0),(0,0)), constant_values=x) d = np.transpose(np.hstack(list(map(f, link_a[0])))) possiblelinks = np.concatenate((possiblelinks,d)) ii = [] jj = [] cc = [] for i in np.arange(len(possiblelinks)): # frame1 - end of possible to link frame2 = trackstarts[possiblelinks[i][0]] # frame1 - end of possible to link frame1 = frame2-1 # cell label in frame 1 to link ind1 = trackbits[possiblelinks[i][1]][frame1] # cell label in frame 2 to link ind2 = trackbits[possiblelinks[i][0]][frame2] dr = np.linalg.norm(cents[frame1][ind1]-cents[frame2][ind2]) if dr <= search_radius: da=1 for cp, wp in zip(Cp, Wp): da = da + wp*(np.maximum(ints[cp][frame1][ind1], ints[cp][frame2][ind2])/np.minimum(ints[cp][frame1][ind1], ints[cp][frame2][ind2])-1) if da<=maxAmpRatio: ii.append(possiblelinks[i][1]) jj.append(possiblelinks[i][0]) #maybe one day we'll change this somehow. Not sure how rn cost = dr*da cc.append(cost) ii = np.array(ii) jj = np.array(jj) cc = np.array(cc) #print(ii) if len(ii)==0: print('\nFinished finding splits') notdoneflag=0 break shape = (len(trackbits),len(trackbits)) cc, ii, kk = prepare_sparse_cost(shape,cc,ii,jj, 1000) match1, _ = lap.lapmod(len(ii)-1, cc, ii, kk, return_cost=False) match1[match1 >= shape[1]] = -1 #inds in n+1 that match inds (1:N) in n match1 = np.array(match1[:shape[0]]) trackindstofill = np.nonzero(match1+1)[0] trackindstoadd = match1[np.nonzero(match1+1)] fa = {trackindstofill[i]: trackindstoadd[i] for i in range(len(trackindstoadd))} for i in fa: sf = trackstarts[i] ef = trackstarts[fa[i]] trackbits[fa[i]][np.arange(sf,ef)]=trackbits[i][np.arange(sf,ef)] relatives[i]=relatives[i]+[fa[i]] relatives[fa[i]]=relatives[fa[i]]+[i] self.relatives = relatives self.trackinds = trackbits
def lat_var(xp, sims, n_similar, n_repeats, batch_size, asym): """ Run the matching in the E-step of the latent-variable model. :param xp: numpy or cupy, depending whether we run on CPU or GPU. :param xw: the xw matrix :param zw: the zw matrix :param sims: an matrix of shape (src_size, trg_size) where the similarity values between each source word and target words are stored :param best_sim_forward: an array of shape (src_size), which stores the best similarity scores for each :param n_similar: :param n_repeats: :param batch_size: :param asym: :return: """ src_size = sims.shape[0] cc = np.empty( src_size * n_similar ) # 1D array of all finite elements of the assignement cost matrix kk = np.empty( src_size * n_similar ) # 1D array of the column indices. Must be sorted within one row. ii = np.empty((src_size * n_repeats + 1, ), dtype=int) # 1D array of indices of the row starts in cc. ii[0] = 0 # if each src id should be matched to trg id, then we need to double the source indices for i in range(1, src_size * n_repeats + 1): ii[i] = ii[i - 1] + n_similar for i in range(0, src_size, batch_size): # j = min(x.shape[0], i + batch_size) j = min(i + batch_size, src_size) sim = sims[i:j] trg_indices = xp.argpartition( sim, -n_similar)[:, -n_similar:] # get indices of n largest elements if xp != np: trg_indices = xp.asnumpy(trg_indices) trg_indices.sort() # sort the target indices trg_indices = trg_indices.flatten() row_indices = np.array([[i] * n_similar for i in range(j - i)]).flatten() sim_scores = sim[row_indices, trg_indices] costs = 1 - sim_scores if xp != np: costs = xp.asnumpy(costs) cc[i * n_similar:j * n_similar] = costs kk[i * n_similar:j * n_similar] = trg_indices if n_repeats > 1: # duplicate costs and target indices new_cc = cc new_kk = kk for i in range(1, n_repeats): new_cc = np.concatenate([new_cc, cc], axis=0) if asym == '1:2': # for 1:2, we don't duplicate the target indices new_kk = np.concatenate([new_kk, kk], axis=0) else: # update target indices so that they refer to new columns new_kk = np.concatenate([new_kk, kk + src_size * i], axis=0) cc = new_cc kk = new_kk # trg indices are targets assigned to each row id from 0-(n_rows-1) cost, trg_indices, _ = lapmod(src_size * n_repeats, cc, ii, kk) src_indices = np.concatenate([np.arange(src_size)] * n_repeats, 0) src_indices, trg_indices = xp.asarray(src_indices), xp.asarray(trg_indices) # remove the pairs in which a source word was connected to a target # which was not one of its k most similar words wrong_inds = [] for i, trgind in enumerate(trg_indices): krow = ii[i] candidates = kk[krow:krow + n_similar] if trgind not in candidates: wrong_inds.append(i) trg_indices = np.delete(trg_indices, wrong_inds) src_indices = np.delete(src_indices, wrong_inds) for i in range(len(src_indices)): src_idx, trg_idx = src_indices[i], trg_indices[i] # we do this if args.n_repeats > 0 to assign the target # indices in the cost matrix to the correct idx while trg_idx >= src_size: # if we repeat, we have indices that are > n_rows trg_idx -= src_size trg_indices[i] = trg_idx return src_indices, trg_indices
def test_sparse_100x100_int(sparse_100x100_int): cost, mask, opt = sparse_100x100_int ret = lapmod(*sparse_from_masked(cost, mask)) assert len(ret) == 3 assert ret[0] == opt
def test_eps(dense_eps): cost, opt = dense_eps ret = lapmod(*sparse_from_dense(cost)) assert len(ret) == 3 assert ret[0] == opt
def test_eps(dense_eps): cost, opt = dense_eps ret = lapmod(*sparse_from_dense(cost)) assert len(ret) == 3 assert ret[0] == opt
def test_sparse_100x100_int(sparse_100x100_int): cost, mask, opt = sparse_100x100_int ret = lapmod(*sparse_from_masked(cost, mask)) assert len(ret) == 3 assert ret[0] == opt
def _closegaps(self, maxStep=10, params=[],maxAmpRatio=5,maxTimeJump=4, mintracklength=30, **kwargs): """ Helper function : close gaps between open stubs using JV lap. todo: split. When a stub that starts in the middle has a plausible link, make a compound track ---------- #NucChannel : ['DeepBlue'] params : [(channel,weight)] list of tuples maxAmpRatio : [2] max allowd ratio of amplitudes for linking mintracklength : [30] final minimum length of a track """ #ch=NucChannel #if ch is None: # ch = self.channels[0] if params: Cp = [p[0] for p in params if p[0] in self.channels] Wp = [p[1] for p in params if p[0] in self.channels] ints = {} for cp in Cp: ints[cp] = self.ninetyint(cp) else: Cp=[] Wp=[] trackbits = self._getAllContinuousTrackSegs(**kwargs) cents = self.centroid_um notdoneflag=1 while notdoneflag: trackstarts = np.array([np.where(~np.isnan(r.astype('float')))[0][0] for r in trackbits]) trackends = np.array([np.where(~np.isnan(r.astype('float')))[0][-1] for r in trackbits]) dtmat = np.expand_dims(trackstarts,1)-np.expand_dims(trackends,0) possiblelinks = np.transpose(np.nonzero((dtmat>0)*(dtmat<maxTimeJump))) ii = [] jj = [] cc = [] for i in np.arange(len(possiblelinks)): # frame1 - end of possible to link frame1 = trackends[possiblelinks[i][1]] # frame2 - beginning of possible link frame2 = trackstarts[possiblelinks[i][0]] # cell label in frame 1 to link ind1 = trackbits[possiblelinks[i][1]][frame1] # cell label in frame 2 to link ind2 = trackbits[possiblelinks[i][0]][frame2] dt = frame2-frame1 dr = np.linalg.norm(cents[frame1][ind1]-cents[frame2][ind2]) da=1 eps=10**-72 for cp, wp in zip(Cp, Wp): da = da + wp*((eps+np.maximum(ints[cp][frame1][ind1], ints[cp][frame2][ind2]))/(eps+np.minimum(ints[cp][frame1][ind1], ints[cp][frame2][ind2]))-1) if dr <= (np.sqrt(dt)*maxStep): if da<=maxAmpRatio: ii.append(possiblelinks[i][1]) jj.append(possiblelinks[i][0]) #maybe one day we'll change this somehow. Not sure how rn cost = dr*da*dt cc.append(cost) ii = np.array(ii) jj = np.array(jj) cc = np.array(cc) if len(ii)==0: print('\nFinished connecting tracks') notdoneflag=0 break shape = (len(trackbits),len(trackbits)) cc, ii, kk = prepare_sparse_cost(shape,cc,ii,jj, 1000) match1, _ = lap.lapmod(len(ii)-1, cc, ii, kk, return_cost=False) match1[match1 >= shape[1]] = -1 #inds in n+1 that match inds (1:N) in n match1 = np.array(match1[:shape[0]]) trackindstofill = np.nonzero(match1+1)[0] trackindstoadd = match1[np.nonzero(match1+1)] fa = {trackindstofill[i]: trackindstoadd[i] for i in range(len(trackindstoadd))} for i in fa: sf = trackstarts[fa[i]] ef = trackends[fa[i]]+1 trackbits[i][np.arange(sf,ef)]=trackbits[fa[i]][np.arange(sf,ef)] trackbits[fa[i]][np.arange(sf,ef)]=None #add nans in gaps #eef = trackends[i]+1 #trackbits[i][np.arange(eef,sf)]=np.nan #remove lines that are all Nones trackbits = trackbits[[any(~np.isnan(r.astype('float'))) for r in trackbits]] trackbits = trackbits[np.array([sum(~np.isnan(r.astype('float'))) for r in trackbits])>=mintracklength] sortind = np.lexsort((np.arange(len(trackbits)) ,[np.sum(np.isnan(r.astype('float'))) for r in trackbits] ,[np.sum(r==None) for r in trackbits])) self.trackinds = trackbits[sortind]