def update_spatial_components(Y, C=None, f=None, A_in=None, sn=None, dims=None, min_size=3, max_size=8, dist=3, normalize_yyt_one=True,
method='ellipse', expandCore=None, dview=None, n_pixels_per_process=128,
medw=(3, 3), thr_method='nrg', maxthr=0.1, nrgthr=0.9999, extract_cc=True,
se=np.ones((3, 3), dtype=np.int), ss=np.ones((3, 3), dtype=np.int), nb=1, method_ls='nnls_L0'):
"""update spatial footprints and background through Basis Pursuit Denoising
for each pixel i solve the problem
[A(i,:),b(i)] = argmin sum(A(i,:))
subject to
|| Y(i,:) - A(i,:)*C + b(i)*f || <= sn(i)*sqrt(T);
for each pixel the search is limited to a few spatial components
Parameters
----------
Y: np.ndarray (2D or 3D)
movie, raw data in 2D or 3D (pixels x time).
C: np.ndarray
calcium activity of each neuron.
f: np.ndarray
temporal profile of background activity.
A_in: np.ndarray
spatial profile of background activity. If A_in is boolean then it defines the spatial support of A.
Otherwise it is used to determine it through determine_search_location
dims: [optional] tuple
x, y[, z] movie dimensions
min_size: [optional] int
max_size: [optional] int
dist: [optional] int
sn: [optional] float
noise associated with each pixel if known
backend [optional] str
'ipyparallel', 'single_thread'
single_thread:no parallelization. It can be used with small datasets.
ipyparallel: uses ipython clusters and then send jobs to each of them
SLURM: use the slurm scheduler
n_pixels_per_process: [optional] int
number of pixels to be processed by each thread
method: [optional] string
method used to expand the search for pixels 'ellipse' or 'dilate'
expandCore: [optional] scipy.ndimage.morphology
if method is dilate this represents the kernel used for expansion
dview: view on ipyparallel client
you need to create an ipyparallel client and pass a view on the processors (client = Client(), dview=client[:])
medw, thr_method, maxthr, nrgthr, extract_cc, se, ss: [optional]
Parameters for components post-processing. Refer to spatial.threshold_components for more details
nb: [optional] int
Number of background components
method_ls:
method to perform the regression for the basis pursuit denoising.
'nnls_L0'. Nonnegative least square with L0 penalty
'lasso_lars' lasso lars function from scikit learn
'lasso_lars_old' lasso lars from old implementation, will be deprecated
normalize_yyt_one: bool
wheter to norrmalize the C and A matrices so that diag(C*C.T) are ones
Returns
--------
A: np.ndarray
new estimate of spatial footprints
b: np.ndarray
new estimate of spatial background
C: np.ndarray
temporal components (updated only when spatial components are completely removed)
f: np.ndarray
same as f_in except if empty component deleted.
"""
C = np.array(C)
if normalize_yyt_one:
# cct=np.diag(C.dot(C.T))
nr_C = np.shape(C)[0]
d = scipy.sparse.lil_matrix((nr_C, nr_C))
d.setdiag(np.sqrt(np.sum(C**2, 1)))
A_in = A_in * d
C = old_div(C, np.sqrt(np.sum(C**2, 1)[:, np.newaxis]))
if expandCore is None:
expandCore = iterate_structure(generate_binary_structure(2, 1), 2).astype(int)
if dims is None:
raise Exception('You need to define the input dimensions')
if Y.ndim < 2 and not isinstance(Y, basestring):
Y = np.atleast_2d(Y)
if Y.shape[1] == 1:
raise Exception('Dimension of Matrix Y must be pixels x time')
if C is not None:
C = np.atleast_2d(C)
if C.shape[1] == 1:
raise Exception('Dimension of Matrix C must be neurons x time')
if f is not None:
f = np.atleast_2d(f)
if f.shape[1] == 1:
raise Exception('Dimension of Matrix f must be background comps x time ')
if (A_in is None) and (C is None):
raise Exception('Either A or C need to be determined')
if A_in is not None:
if len(A_in.shape) == 1:
A_in = np.atleast_2d(A_in).T
if A_in.shape[0] == 1:
raise Exception('Dimension of Matrix A must be pixels x neurons ')
start_time = time.time()
[d, T] = np.shape(Y)
if A_in is None:
A_in = np.ones((d, np.shape(C)[1]), dtype=bool)
if n_pixels_per_process > d:
print(
'The number of pixels per process (n_pixels_per_process) is larger than the total number of pixels!! Decreasing suitably.')
n_pixels_per_process = d
if f is not None:
nb = f.shape[0]
else:
if b is not None:
nb = b.shape[1]
if A_in.dtype == bool:
IND = A_in.copy()
print("spatial support for each components given by the user")
if C is None:
INDav = old_div(IND.astype('float32'), np.sum(IND, axis=0))
px = (np.sum(IND, axis=1) > 0)
model = NMF(n_components=nb, init='random', random_state=0)
b = model.fit_transform(np.maximum(Y[~px, :], 0))
f = model.components_.squeeze()
#f = np.mean(Y[~px,:],axis=0)
Y_resf = np.dot(Y, f.T)
b = np.fmax(Y_resf.dot(np.linalg.inv(f.dot(f.T)), 0))
#b = np.fmax(Y_resf / scipy.linalg.norm(f)**2, 0)
C = np.fmax(csr_matrix(INDav.T).dot(Y) - np.outer(INDav.T.dot(b), f), 0)
f = np.atleast_2d(f)
else:
IND = determine_search_location(
A_in, dims, method=method, min_size=min_size, max_size=max_size, dist=dist, expandCore=expandCore, dview=dview)
print("found spatial support for each component")
if C is None:
raise Exception('You need to provide estimate of C and f')
print((np.shape(A_in)))
Cf = np.vstack((C, f)) # create matrix that include background components
nr, _ = np.shape(C) # number of neurons
ind2_ = [np.hstack((np.where(iid_)[0], nr + np.arange(f.shape[0])))
if np.size(np.where(iid_)[0]) > 0 else [] for iid_ in IND]
if os.environ.get('SLURM_SUBMIT_DIR') is not None:
tmpf = os.environ.get('SLURM_SUBMIT_DIR')
print(('cluster temporary folder:' + tmpf))
folder = tempfile.mkdtemp(dir=tmpf)
else:
folder = tempfile.mkdtemp()
if dview is None:
Y_name = Y
C_name = Cf
else:
C_name = os.path.join(folder, 'C_temp.npy')
np.save(C_name, Cf)
if type(Y) is np.core.memmap: # if input file is already memory mapped then find the filename
Y_name = Y.filename
# if not create a memory mapped version (necessary for parallelization)
elif isinstance(Y, basestring) or dview is None:
Y_name = Y
else:
raise Exception('Not implemented consistently')
Y_name = os.path.join(folder, 'Y_temp.npy')
np.save(Y_name, Y)
Y, _, _, _ = load_memmap(Y_name)
# create arguments to be passed to the function. Here we are grouping
# bunch of pixels to be processed by each thread
# pixel_groups = [(Y_name, C_name, sn, ind2_, range(i, i + n_pixels_per_process))
# for i in range(0, np.prod(dims) - n_pixels_per_process + 1,
# n_pixels_per_process)]
cct = np.diag(C.dot(C.T))
rank_f = nb
pixel_groups = []
for i in range(0, np.prod(dims) - n_pixels_per_process + 1, n_pixels_per_process):
pixel_groups.append([Y_name, C_name, sn, ind2_, list(
range(i, i + n_pixels_per_process)), method_ls, cct, rank_f])
if i < np.prod(dims):
pixel_groups.append([Y_name, C_name, sn, ind2_, list(
range(i, np.prod(dims))), method_ls, cct, rank_f])
A_ = np.zeros((d, nr + np.size(f, 0)))
print('Starting Update Spatial Components')
#serial_result = map(lars_regression_noise_ipyparallel, pixel_groups)
if dview is not None:
parallel_result = dview.map_sync(regression_ipyparallel, pixel_groups)
dview.results.clear()
for chunk in parallel_result:
for pars in chunk:
px, idxs_, a = pars
A_[px, idxs_] = a
else:
parallel_result = list(map(regression_ipyparallel, pixel_groups))
for chunk in parallel_result:
for pars in chunk:
px, idxs_, a = pars
A_[px, idxs_] = a
##
# Cf_ = [Cf[idx_, :] for idx_ in ind2_]
#
# #% LARS regression
# A_ = np.hstack((np.zeros((d, nr)), np.zeros((d, np.size(f, 0)))))
#
# for c, y, s, id2_, px in zip(Cf_, Y, sn, ind2_, range(d)):
# if px % 1000 == 0:
# print px
# if np.size(c) > 0:
# _, _, a, _, _ = lars_regression_noise_old(y, np.array(c.T), 1, sn[px]**2 * T)
# if np.isscalar(a):
# A_[px, id2_] = a
# else:
# A_[px, id2_] = a.T
##
#%
print('Updated Spatial Components')
A_ = threshold_components(A_, dims, dview=dview, medw=medw, thr_method=thr_method,
maxthr=maxthr, nrgthr=nrgthr, extract_cc=extract_cc, se=se, ss=ss)
print("threshold")
ff = np.where(np.sum(A_, axis=0) == 0) # remove empty components
if np.size(ff) > 0:
ff = ff[0]
print('eliminating {} empty components!!'.format(len(ff)))
A_ = np.delete(A_, list(ff), 1)
C = np.delete(C, list(ff), 0)
background_ff = list(filter(lambda i: i > 0, ff-nr))
f = np.delete(f, background_ff, 0)
nr = nr - (len(ff) - len(background_ff))
nb = nb - len(background_ff)
A_ = A_[:, :nr]
A_ = coo_matrix(A_)
#import pdb
# pdb.set_trace()
# Y_resf = np.dot(Y, f.T) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T))
print("Computing residuals")
if 'memmap' in str(type(Y)):
Y_resf = parallel_dot_product(Y, f.T, block_size=1000, dview=dview) - \
A_.dot(coo_matrix(C[:nr, :]).dot(f.T))
else:
Y_resf = np.dot(Y, f.T) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T))
print("Computing A_bas")
A_bas = np.fmax(Y_resf.dot(np.linalg.inv(f.dot(f.T))), 0) # update baseline based on residual
# A_bas = np.fmax(Y_resf / scipy.linalg.norm(f)**2, 0) # update baseline based on residual
# baseline based on residual
b = A_bas
print(("--- %s seconds ---" % (time.time() - start_time)))
try: # clean up
# remove temporary file created
print("Remove temporary file created")
shutil.rmtree(folder)
except:
raise Exception("Failed to delete: " + folder)
# if A_in.dtype == bool:
# return A_, b, C, f
# else:
# return A_, b, C
return A_, b, C, f
def update_temporal_components(Y,
A,
b,
Cin,
fin,
bl=None,
c1=None,
g=None,
sn=None,
nb=1,
ITER=2,
method_foopsi='constrained_foopsi',
block_size=20000,
memory_efficient=False,
debug=False,
dview=None,
**kwargs):
"""Update temporal components and background given spatial components using a block coordinate descent approach.
Parameters
-----------
Y: np.ndarray (2D)
input data with time in the last axis (d x T)
A: sparse matrix (crc format)
matrix of temporal components (d x K)
b: ndarray (dx1)
current estimate of background component
Cin: np.ndarray
current estimate of temporal components (K x T)
fin: np.ndarray
current estimate of temporal background (vector of length T)
g: np.ndarray
Global time constant (not used)
bl: np.ndarray
baseline for fluorescence trace for each column in A
c1: np.ndarray
initial concentration for each column in A
g: np.ndarray
discrete time constant for each column in A
sn: np.ndarray
noise level for each column in A
nb: [optional] int
Number of background components
ITER: positive integer
Maximum number of block coordinate descent loops.
method_foopsi: string
Method of deconvolution of neural activity. constrained_foopsi is the only method supported at the moment.
n_processes: int
number of processes to use for parallel computation. Should be less than the number of processes started with ipcluster.
backend: 'str'
single_thread no parallelization
ipyparallel, parallelization using the ipyparallel cluster. You should start the cluster (install ipyparallel and then type
ipcluster -n 6, where 6 is the number of processes).
SLURM: using SLURM scheduler
memory_efficient: Bool
whether or not to optimize for memory usage (longer running times). nevessary with very large datasets
**kwargs: dict
all parameters passed to constrained_foopsi except bl,c1,g,sn (see documentation). Some useful parameters are
p: int
order of the autoregression model
method: [optional] string
solution method for constrained foopsi. Choices are
'cvx': using cvxopt and picos (slow especially without the MOSEK solver)
'cvxpy': using cvxopt and cvxpy with the ECOS solver (faster, default)
solvers: list string
primary and secondary (if problem unfeasible for approx solution) solvers to be used with cvxpy, default is ['ECOS','SCS']
Note
--------
The temporal components are updated in parallel by default by forming of sequence of vertex covers.
Returns
--------
C: np.ndarray
matrix of temporal components (K x T)
f: np.array
vector of temporal background (length T)
S: np.ndarray
matrix of merged deconvolved activity (spikes) (K x T)
bl: float
same as input
c1: float
same as input
g: float
same as input
sn: float
same as input
YrA: np.ndarray
matrix of spatial component filtered raw data, after all contributions have been removed.
YrA corresponds to the residual trace for each component and is used for faster plotting (K x T)
"""
if 'p' not in kwargs or kwargs['p'] is None:
raise Exception("You have to provide a value for p")
d, T = np.shape(Y)
nr = np.shape(A)[-1]
if b is not None:
if b.shape[0] < b.shape[1]:
b = b.T
nb = b.shape[1]
if bl is None:
bl = np.repeat(None, nr)
if c1 is None:
c1 = np.repeat(None, nr)
if g is None:
g = np.repeat(None, nr)
if sn is None:
sn = np.repeat(None, nr)
A = scipy.sparse.hstack((A, coo_matrix(b)))
S = np.zeros(np.shape(Cin))
Cin = np.vstack((Cin, fin))
C = Cin
nA = np.squeeze(np.array(np.sum(np.square(A.todense()), axis=0)))
Cin = coo_matrix(Cin)
#YrA = ((A.T.dot(Y)).T-Cin.T.dot(A.T.dot(A)))
print('Generating residuals')
# YA = (A.T.dot(Y).T)*spdiags(1./nA,0,nr+nb,nr+nb)
if 'memmap' in str(type(Y)):
if block_size >= 500:
print('Forcing single thread for memory issues')
dview_res = None
else:
print(
'Using thread. If memory issues set block_size larger than 500'
)
dview_res = dview
YA = parallel_dot_product(
Y, A, dview=dview_res, block_size=block_size,
transpose=True) * spdiags(old_div(1., nA), 0, nr + nb, nr + nb)
else:
YA = (A.T.dot(Y).T) * spdiags(old_div(1., nA), 0, nr + nb, nr + nb)
print('Done')
# print np.allclose(YA,YA1)
AA = ((A.T.dot(A)) * spdiags(old_div(1., nA), 0, nr + nb, nr + nb)).tocsr()
YrA = YA - Cin.T.dot(AA)
#YrA = ((A.T.dot(Y)).T-Cin.T.dot(A.T.dot(A)))*spdiags(1./nA,0,nr+1,nr+1)
Cin = np.array(Cin.todense())
for iter in range(ITER):
O, lo = update_order(A.tocsc()[:, :nr])
P_ = []
for count, jo_ in enumerate(O):
jo = np.array(list(jo_))
#Ytemp = YrA[:,jo.flatten()] + (np.dot(np.diag(nA[jo]),Cin[jo,:])).T
Ytemp = YrA[:, jo.flatten()] + Cin[jo, :].T
Ctemp = np.zeros((np.size(jo), T))
Stemp = np.zeros((np.size(jo), T))
btemp = np.zeros((np.size(jo), 1))
sntemp = btemp.copy()
c1temp = btemp.copy()
gtemp = np.zeros((np.size(jo), kwargs['p']))
nT = nA[jo]
#args_in=[(np.squeeze(np.array(Ytemp[:,jj])), nT[jj], jj, bl[jo[jj]], c1[jo[jj]], g[jo[jj]], sn[jo[jj]], kwargs) for jj in range(len(jo))]
args_in = [(np.squeeze(np.array(Ytemp[:, jj])), nT[jj], jj, None,
None, None, None, kwargs) for jj in range(len(jo))]
# import pdb
# pdb.set_trace()
if dview is not None:
#
if debug:
results = dview.map_async(constrained_foopsi_parallel,
args_in)
results.get()
for outp in results.stdout:
print((outp[:-1]))
sys.stdout.flush()
for outp in results.stderr:
print((outp[:-1]))
sys.stderr.flush()
else:
results = dview.map_sync(constrained_foopsi_parallel,
args_in)
else:
results = list(map(constrained_foopsi_parallel, args_in))
for chunk in results:
pars = dict()
C_, Sp_, Ytemp_, cb_, c1_, sn_, gn_, jj_ = chunk
Ctemp[jj_, :] = C_[None, :]
Stemp[jj_, :] = Sp_
Ytemp[:, jj_] = Ytemp_[:, None]
btemp[jj_] = cb_
c1temp[jj_] = c1_
sntemp[jj_] = sn_
gtemp[jj_, :] = gn_.T
bl[jo[jj_]] = cb_
c1[jo[jj_]] = c1_
sn[jo[jj_]] = sn_
g[jo[jj_]] = gn_.T if kwargs['p'] > 0 else [] #gtemp[jj,:]
pars['b'] = cb_
pars['c1'] = c1_
pars['neuron_sn'] = sn_
pars['gn'] = gtemp[jj_, np.abs(gtemp[jj_, :]) > 0]
pars['neuron_id'] = jo[jj_]
P_.append(pars)
YrA -= (Ctemp - C[jo, :]).T * AA[jo, :]
#YrA[:,jo] = Ytemp
C[jo, :] = Ctemp.copy()
S[jo, :] = Stemp
# if (np.sum(lo[:jo])+1)%1 == 0:
print((str(np.sum(lo[:count + 1])) + ' out of total ' + str(nr) +
' temporal components updated'))
ii = nr
# Delete those who do not spike(?)
#YrA[:,ii] = YrA[:,ii] + np.atleast_2d(Cin[ii,:]).T
#cc = np.maximum(YrA[:,ii],0)
for ii in np.arange(nr, nr + nb):
cc = np.maximum(YrA[:, ii] + np.atleast_2d(Cin[ii, :]).T, 0)
YrA -= (cc - np.atleast_2d(Cin[ii, :]).T) * AA[ii, :]
C[ii, :] = cc.T
#YrA = YA - C.T.dot(AA)
#YrA[:,ii] = YrA[:,ii] - np.atleast_2d(C[ii,:]).T
if dview is not None:
dview.results.clear()
if old_div(scipy.linalg.norm(Cin - C, 'fro'),
scipy.linalg.norm(C, 'fro')) <= 1e-3:
# stop if the overall temporal component does not change by much
print(
"stopping: overall temporal component not changing significantly"
)
break
else:
Cin = C
f = C[nr:, :]
C = C[:nr, :]
YrA = np.array(YrA[:, :nr]).T
P_ = sorted(P_, key=lambda k: k['neuron_id'])
return C, f, S, bl, c1, sn, g, YrA #,P_
def update_spatial_components(Y, C=None, f=None, A_in=None, sn=None, dims=None, min_size=3, max_size=8, dist=3,normalize_yyt_one=True,
method='ellipse', expandCore=None, dview=None, n_pixels_per_process=128,
medw=(3, 3), thr_method='nrg', maxthr=0.1, nrgthr=0.9999, extract_cc=True,
se=np.ones((3, 3), dtype=np.int), ss=np.ones((3, 3), dtype=np.int), nb=1, method_ls='nnls_L0'):
"""update spatial footprints and background through Basis Pursuit Denoising
for each pixel i solve the problem
[A(i,:),b(i)] = argmin sum(A(i,:))
subject to
|| Y(i,:) - A(i,:)*C + b(i)*f || <= sn(i)*sqrt(T);
for each pixel the search is limited to a few spatial components
Parameters
----------
Y: np.ndarray (2D or 3D)
movie, raw data in 2D or 3D (pixels x time).
C: np.ndarray
calcium activity of each neuron.
f: np.ndarray
temporal profile of background activity.
A_in: np.ndarray
spatial profile of background activity. If A_in is boolean then it defines the spatial support of A.
Otherwise it is used to determine it through determine_search_location
dims: [optional] tuple
x, y[, z] movie dimensions
min_size: [optional] int
max_size: [optional] int
dist: [optional] int
sn: [optional] float
noise associated with each pixel if known
backend [optional] str
'ipyparallel', 'single_thread'
single_thread:no parallelization. It can be used with small datasets.
ipyparallel: uses ipython clusters and then send jobs to each of them
SLURM: use the slurm scheduler
n_pixels_per_process: [optional] int
number of pixels to be processed by each thread
method: [optional] string
method used to expand the search for pixels 'ellipse' or 'dilate'
expandCore: [optional] scipy.ndimage.morphology
if method is dilate this represents the kernel used for expansion
dview: view on ipyparallel client
you need to create an ipyparallel client and pass a view on the processors (client = Client(), dview=client[:])
medw, thr_method, maxthr, nrgthr, extract_cc, se, ss: [optional]
Parameters for components post-processing. Refer to spatial.threshold_components for more details
nb: [optional] int
Number of background components
method_ls:
method to perform the regression for the basis pursuit denoising.
'nnls_L0'. Nonnegative least square with L0 penalty
'lasso_lars' lasso lars function from scikit learn
'lasso_lars_old' lasso lars from old implementation, will be deprecated
normalize_yyt_one: bool
wheter to norrmalize the C and A matrices so that diag(C*C.T) are ones
Returns
--------
A: np.ndarray
new estimate of spatial footprints
b: np.ndarray
new estimate of spatial background
C: np.ndarray
temporal components (updated only when spatial components are completely removed)
"""
C=np.array(C)
if normalize_yyt_one:
# cct=np.diag(C.dot(C.T))
nr_C=np.shape(C)[0]
d = scipy.sparse.lil_matrix((nr_C,nr_C))
d.setdiag(np.sqrt(np.sum(C**2,1)))
A_in=A_in*d
C=old_div(C,np.sqrt(np.sum(C**2,1)[:,np.newaxis]))
if expandCore is None:
expandCore = iterate_structure(generate_binary_structure(2, 1), 2).astype(int)
if dims is None:
raise Exception('You need to define the input dimensions')
if Y.ndim < 2 and not isinstance(Y, basestring):
Y = np.atleast_2d(Y)
if Y.shape[1] == 1:
raise Exception('Dimension of Matrix Y must be pixels x time')
if C is not None:
C = np.atleast_2d(C)
if C.shape[1] == 1:
raise Exception('Dimension of Matrix C must be neurons x time')
if f is not None:
f = np.atleast_2d(f)
if f.shape[1] == 1:
raise Exception('Dimension of Matrix f must be background comps x time ')
if (A_in is None) and (C is None):
raise Exception('Either A or C need to be determined')
if A_in is not None:
if len(A_in.shape) == 1:
A_in = np.atleast_2d(A_in).T
if A_in.shape[0] == 1:
raise Exception('Dimension of Matrix A must be pixels x neurons ')
start_time = time.time()
[d, T] = np.shape(Y)
if A_in is None:
A_in = np.ones((d, np.shape(C)[1]), dtype=bool)
if n_pixels_per_process > d:
raise Exception(
'The number of pixels per process (n_pixels_per_process) is larger than the total number of pixels!! Decrease suitably.')
if f is not None:
nb = f.shape[0]
else:
if b is not None:
nb = b.shape[1]
if A_in.dtype == bool:
IND = A_in.copy()
print("spatial support for each components given by the user")
if C is None:
INDav = old_div(IND.astype('float32'), np.sum(IND, axis=0))
px = (np.sum(IND, axis=1) > 0)
model = NMF(n_components=nb, init='random', random_state=0)
b = model.fit_transform(np.maximum(Y[~px, :], 0))
f = model.components_.squeeze()
#f = np.mean(Y[~px,:],axis=0)
Y_resf = np.dot(Y, f.T)
b = np.fmax(Y_resf.dot(np.linalg.inv(f.dot(f.T)), 0))
#b = np.fmax(Y_resf / scipy.linalg.norm(f)**2, 0)
C = np.fmax(csr_matrix(INDav.T).dot(Y) - np.outer(INDav.T.dot(b), f), 0)
f = np.atleast_2d(f)
else:
IND = determine_search_location(
A_in, dims, method=method, min_size=min_size, max_size=max_size, dist=dist, expandCore=expandCore, dview=dview)
print("found spatial support for each component")
if C is None:
raise Exception('You need to provide estimate of C and f')
print((np.shape(A_in)))
Cf = np.vstack((C, f)) # create matrix that include background components
nr, _ = np.shape(C) # number of neurons
ind2_ = [np.hstack((np.where(iid_)[0], nr + np.arange(f.shape[0])))
if np.size(np.where(iid_)[0]) > 0 else [] for iid_ in IND]
if os.environ.get('SLURM_SUBMIT_DIR') is not None:
tmpf = os.environ.get('SLURM_SUBMIT_DIR')
print(('cluster temporary folder:' + tmpf))
folder = tempfile.mkdtemp(dir=tmpf)
else:
folder = tempfile.mkdtemp()
if dview is None:
Y_name = Y
C_name = Cf
else:
C_name = os.path.join(folder, 'C_temp.npy')
np.save(C_name, Cf)
if type(Y) is np.core.memmap: # if input file is already memory mapped then find the filename
Y_name = Y.filename
# if not create a memory mapped version (necessary for parallelization)
elif isinstance(Y, basestring) or dview is None:
Y_name = Y
else:
raise Exception('Not implemented consistently')
Y_name = os.path.join(folder, 'Y_temp.npy')
np.save(Y_name, Y)
Y, _, _, _ = load_memmap(Y_name)
# create arguments to be passed to the function. Here we are grouping
# bunch of pixels to be processed by each thread
# pixel_groups = [(Y_name, C_name, sn, ind2_, range(i, i + n_pixels_per_process))
# for i in range(0, np.prod(dims) - n_pixels_per_process + 1,
# n_pixels_per_process)]
cct=np.diag(C.dot(C.T))
rank_f=nb
pixel_groups = []
for i in range(0, np.prod(dims) - n_pixels_per_process + 1, n_pixels_per_process):
pixel_groups.append([Y_name, C_name, sn, ind2_, list(range(i, i + n_pixels_per_process)), method_ls, cct,rank_f])
if i < np.prod(dims):
pixel_groups.append([Y_name, C_name, sn, ind2_, list(range(i, np.prod(dims))), method_ls, cct,rank_f])
A_ = np.zeros((d, nr + np.size(f, 0)))
print('Starting Update Spatial Components')
#serial_result = map(lars_regression_noise_ipyparallel, pixel_groups)
if dview is not None:
parallel_result = dview.map_sync(regression_ipyparallel, pixel_groups)
dview.results.clear()
for chunk in parallel_result:
for pars in chunk:
px, idxs_, a = pars
A_[px, idxs_] = a
else:
parallel_result = list(map(regression_ipyparallel, pixel_groups))
for chunk in parallel_result:
for pars in chunk:
px, idxs_, a = pars
A_[px, idxs_] = a
##
# Cf_ = [Cf[idx_, :] for idx_ in ind2_]
#
# #% LARS regression
# A_ = np.hstack((np.zeros((d, nr)), np.zeros((d, np.size(f, 0)))))
#
# for c, y, s, id2_, px in zip(Cf_, Y, sn, ind2_, range(d)):
# if px % 1000 == 0:
# print px
# if np.size(c) > 0:
# _, _, a, _, _ = lars_regression_noise_old(y, np.array(c.T), 1, sn[px]**2 * T)
# if np.isscalar(a):
# A_[px, id2_] = a
# else:
# A_[px, id2_] = a.T
##
#%
print('Updated Spatial Components')
A_ = threshold_components(A_, dims, dview=dview, medw=(3, 3), thr_method=thr_method, maxthr=maxthr, nrgthr=nrgthr, extract_cc=extract_cc,
se=se, ss=ss)
print("threshold")
ff = np.where(np.sum(A_, axis=0) == 0) # remove empty components
if np.size(ff) > 0:
ff = ff[0]
print('eliminating empty components!!')
nr = nr - len(ff)
A_ = np.delete(A_, list(ff), 1)
C = np.delete(C, list(ff), 0)
A_ = A_[:, :nr]
A_ = coo_matrix(A_)
#import pdb
# pdb.set_trace()
# Y_resf = np.dot(Y, f.T) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T))
print("Computing residuals")
if 'memmap' in str(type(Y)):
Y_resf = parallel_dot_product(Y,f.T,block_size=5000,dview=dview) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T))
else:
Y_resf = np.dot(Y, f.T) - A_.dot(coo_matrix(C[:nr, :]).dot(f.T))
print("Computing A_bas")
A_bas = np.fmax(Y_resf.dot(np.linalg.inv(f.dot(f.T))), 0) # update baseline based on residual
# A_bas = np.fmax(Y_resf / scipy.linalg.norm(f)**2, 0) # update baseline based on residual
# baseline based on residual
b = A_bas
print(("--- %s seconds ---" % (time.time() - start_time)))
try: # clean up
# remove temporary file created
print("Remove temporary file created")
shutil.rmtree(folder)
except:
raise Exception("Failed to delete: " + folder)
if A_in.dtype == bool:
return A_, b, C, f
else:
return A_, b, C
def update_temporal_components(Y, A, b, Cin, fin, bl = None, c1 = None, g = None, sn = None, nb = 1, ITER=2, method_foopsi='constrained_foopsi', memory_efficient=False, debug=False, dview=None,**kwargs):
"""Update temporal components and background given spatial components using a block coordinate descent approach.
Parameters
-----------
Y: np.ndarray (2D)
input data with time in the last axis (d x T)
A: sparse matrix (crc format)
matrix of temporal components (d x K)
b: ndarray (dx1)
current estimate of background component
Cin: np.ndarray
current estimate of temporal components (K x T)
fin: np.ndarray
current estimate of temporal background (vector of length T)
g: np.ndarray
Global time constant (not used)
bl: np.ndarray
baseline for fluorescence trace for each column in A
c1: np.ndarray
initial concentration for each column in A
g: np.ndarray
discrete time constant for each column in A
sn: np.ndarray
noise level for each column in A
nb: [optional] int
Number of background components
ITER: positive integer
Maximum number of block coordinate descent loops.
method_foopsi: string
Method of deconvolution of neural activity. constrained_foopsi is the only method supported at the moment.
n_processes: int
number of processes to use for parallel computation. Should be less than the number of processes started with ipcluster.
backend: 'str'
single_thread no parallelization
ipyparallel, parallelization using the ipyparallel cluster. You should start the cluster (install ipyparallel and then type
ipcluster -n 6, where 6 is the number of processes).
SLURM: using SLURM scheduler
memory_efficient: Bool
whether or not to optimize for memory usage (longer running times). nevessary with very large datasets
**kwargs: dict
all parameters passed to constrained_foopsi except bl,c1,g,sn (see documentation). Some useful parameters are
p: int
order of the autoregression model
method: [optional] string
solution method for constrained foopsi. Choices are
'cvx': using cvxopt and picos (slow especially without the MOSEK solver)
'cvxpy': using cvxopt and cvxpy with the ECOS solver (faster, default)
solvers: list string
primary and secondary (if problem unfeasible for approx solution) solvers to be used with cvxpy, default is ['ECOS','SCS']
Note
--------
The temporal components are updated in parallel by default by forming of sequence of vertex covers.
Returns
--------
C: np.ndarray
matrix of temporal components (K x T)
f: np.array
vector of temporal background (length T)
S: np.ndarray
matrix of merged deconvolved activity (spikes) (K x T)
bl: float
same as input
c1: float
same as input
g: float
same as input
sn: float
same as input
YrA: np.ndarray
matrix of spatial component filtered raw data, after all contributions have been removed.
YrA corresponds to the residual trace for each component and is used for faster plotting (K x T)
"""
if 'p' not in kwargs or kwargs['p'] is None:
raise Exception("You have to provide a value for p")
d,T = np.shape(Y);
nr = np.shape(A)[-1]
if b is not None:
if b.shape[0]<b.shape[1]:
b = b.T
nb = b.shape[1]
if bl is None:
bl=np.repeat(None,nr)
if c1 is None:
c1=np.repeat(None,nr)
if g is None:
g=np.repeat(None,nr)
if sn is None:
sn=np.repeat(None,nr)
A = scipy.sparse.hstack((A,coo_matrix(b)))
S = np.zeros(np.shape(Cin));
Cin = np.vstack((Cin,fin));
C = Cin;
nA = np.squeeze(np.array(np.sum(np.square(A.todense()),axis=0)))
Cin=coo_matrix(Cin)
#YrA = ((A.T.dot(Y)).T-Cin.T.dot(A.T.dot(A)))
print ('Generating residuals')
# YA = (A.T.dot(Y).T)*spdiags(1./nA,0,nr+nb,nr+nb)
if 'memmap' in str(type(Y)):
YA = parallel_dot_product(Y,A,dview=None,block_size=20000,transpose=True)*spdiags(old_div(1.,nA),0,nr+nb,nr+nb)
else:
YA = (A.T.dot(Y).T)*spdiags(old_div(1.,nA),0,nr+nb,nr+nb)
print ('Done')
#
# print np.allclose(YA,YA1)
AA = ((A.T.dot(A))*spdiags(old_div(1.,nA),0,nr+nb,nr+nb)).tocsr()
YrA = YA - Cin.T.dot(AA)
#YrA = ((A.T.dot(Y)).T-Cin.T.dot(A.T.dot(A)))*spdiags(1./nA,0,nr+1,nr+1)
Cin=np.array(Cin.todense())
for iter in range(ITER):
O,lo = update_order(A.tocsc()[:,:nr])
P_=[];
for count,jo_ in enumerate(O):
jo=np.array(list(jo_))
#Ytemp = YrA[:,jo.flatten()] + (np.dot(np.diag(nA[jo]),Cin[jo,:])).T
Ytemp = YrA[:,jo.flatten()] + Cin[jo,:].T
Ctemp = np.zeros((np.size(jo),T))
Stemp = np.zeros((np.size(jo),T))
btemp = np.zeros((np.size(jo),1))
sntemp = btemp.copy()
c1temp = btemp.copy()
gtemp = np.zeros((np.size(jo),kwargs['p']));
nT = nA[jo]
# args_in=[(np.squeeze(np.array(Ytemp[:,jj])), nT[jj], jj, bl[jo[jj]], c1[jo[jj]], g[jo[jj]], sn[jo[jj]], kwargs) for jj in range(len(jo))]
args_in=[(np.squeeze(np.array(Ytemp[:,jj])), nT[jj], jj, None, None, None, None, kwargs) for jj in range(len(jo))]
# import pdb
# pdb.set_trace()
if dview is not None:
#
if debug:
results = dview.map_async(constrained_foopsi_parallel,args_in)
results.get()
for outp in results.stdout:
print((outp[:-1]))
sys.stdout.flush()
for outp in results.stderr:
print((outp[:-1]))
sys.stderr.flush()
else:
results = dview.map_sync(constrained_foopsi_parallel,args_in)
else:
results = list(map(constrained_foopsi_parallel,args_in))
for chunk in results:
pars=dict()
C_,Sp_,Ytemp_,cb_,c1_,sn_,gn_,jj_=chunk
Ctemp[jj_,:] = C_[None,:]
Stemp[jj_,:] = Sp_
Ytemp[:,jj_] = Ytemp_[:,None]
btemp[jj_] = cb_
c1temp[jj_] = c1_
sntemp[jj_] = sn_
gtemp[jj_,:] = gn_.T
bl[jo[jj_]] = cb_
c1[jo[jj_]] = c1_
sn[jo[jj_]] = sn_
g[jo[jj_]] = gn_.T if kwargs['p'] > 0 else [] #gtemp[jj,:]
pars['b'] = cb_
pars['c1'] = c1_
pars['neuron_sn'] = sn_
pars['gn'] = gtemp[jj_,np.abs(gtemp[jj_,:])>0]
pars['neuron_id'] = jo[jj_]
P_.append(pars)
YrA -= (Ctemp-C[jo,:]).T*AA[jo,:]
#YrA[:,jo] = Ytemp
C[jo,:] = Ctemp.copy()
S[jo,:] = Stemp
# if (np.sum(lo[:jo])+1)%1 == 0:
print((str(np.sum(lo[:count+1])) + ' out of total ' + str(nr) + ' temporal components updated'))
ii=nr
#YrA[:,ii] = YrA[:,ii] + np.atleast_2d(Cin[ii,:]).T
#cc = np.maximum(YrA[:,ii],0)
for ii in np.arange(nr,nr+nb):
cc = np.maximum(YrA[:,ii] + np.atleast_2d(Cin[ii,:]).T,0)
YrA -= (cc-np.atleast_2d(Cin[ii,:]).T)*AA[ii,:]
C[ii,:] = cc.T
#YrA = YA - C.T.dot(AA)
#YrA[:,ii] = YrA[:,ii] - np.atleast_2d(C[ii,:]).T
if dview is not None:
dview.results.clear()
if old_div(scipy.linalg.norm(Cin - C,'fro'),scipy.linalg.norm(C,'fro')) <= 1e-3:
# stop if the overall temporal component does not change by much
print("stopping: overall temporal component not changing significantly")
break
else:
Cin = C
f = C[nr:,:]
C = C[:nr,:]
YrA = np.array(YrA[:,:nr]).T
P_ = sorted(P_, key=lambda k: k['neuron_id'])
return C,f,S,bl,c1,sn,g,YrA #,P_
