def update_temporal(self):
self.C, self.f, self.Y_res, self.Pnew = update_temporal_components(
self.Yr,
self.A,
self.b,
self.Cin,
self.fin,
ITER=2,
deconv_method='spgl1')
def mergeROIS(Y_res,A,b,C,f,d1,d2,dz,nr,P_,thr=0.8,mx=50,sn=None,deconv_method='spgl1',min_size=3,max_size=8,dist=3,method_exp = 'ellipse', expandCore = iterate_structure(generate_binary_structure(2,1), 2).astype(int)):
"""
merging of spatially overlapping components that have highly correlated tmeporal activity
% The correlation threshold for merging overlapping components is user specified in P.merge_thr (default value 0.85)
% Inputs:
% Y_res: residual movie after subtracting all found components
% A: matrix of spatial components
% b: spatial background
% C: matrix of temporal components
% f: temporal background
% P: parameter struct
% Outputs:
% A: matrix of new spatial components
% C: matrix of new temporal components
% nr: new number of components
% merged_ROIs: list of old components that were merged
% Written by:
% Andrea Giovannucci from implementation of Eftychios A. Pnevmatikakis, Simons Foundation, 2015
"""
#%
N = len(nr)
[d,T] = np.shape(Y_res)
C_corr = np.corrcoef(C[:N,:],C[:N,:])[:N,:N];
FF1=C_corr>=thr; #find graph of strongly correlated temporal components
A_corr=A.T*A
A_corr.setdiag(0)
FF2=A_corr>0 # % find graph of overlapping spatial components
FF3=np.logical_and(FF1,FF2.todense())
FF3=coo_matrix(FF3)
c,l=csgraph.connected_components(FF3) # % extract connected components
p=len(P_[0]['gn'])
MC=[];
for i in range(c):
if np.sum(l==i)>1:
MC.append((l==i).T)
MC=np.asarray(MC).T
if MC.ndim>1:
cor = np.zeros((np.shape(MC)[1],1));
for i in range(np.size(cor)):
fm = np.where(MC[:,i])[0]
for j1 in range(np.size(fm)):
for j2 in range(j1+1,np.size(fm)):
print j1,j2
cor[i] = cor[i] +C_corr[fm[j1],fm[j2]]
Y_res = Y_res + np.dot(b,f);
if np.size(cor) > 1:
ind=np.argsort(np.squeeze(cor))[::-1]
else:
ind = [0]
nm = min((np.size(ind),mx)) # number of merging operations
A_merged = coo_matrix((d,nm)).tocsr();
C_merged = np.zeros((nm,T));
nr_merged = [0]*nm
import pdb; pdb.set_trace()
P_merged=[];
merged_ROIs = []
#%
for i in range(nm):
P_cycle=dict()
merged_ROI=np.where(MC[:,ind[i]])[0]
merged_ROIs.append(merged_ROI)
nC = np.sqrt(np.sum(np.array(C[merged_ROI,:])**2,axis=1)) #SVR need to cast to array otherwise assumes matrix power
# A_merged[:,i] = np.squeeze((A[:,merged_ROI]*spdiags(nC,0,len(nC),len(nC))).sum(axis=1))
A = A.tocsr() #SVR A comes as coo_matrix which has no __get_item__
A_merged[:,i] = csr_matrix((A[:,merged_ROI]*spdiags(nC,0,len(nC),len(nC))).sum(axis=1))
Y_res = Y_res + A[:,merged_ROI]*C[merged_ROI,:]
nr_merged[i] = nr[merged_ROI[0]]
aa_1=scipy.sparse.linalg.spsolve(spdiags(nC,0,len(nC),len(nC)),C[merged_ROI,:])
aa_2=(aa_1).mean(axis=0)
ff = np.nonzero(A_merged[:,i])[0]
cc,_,_,Ptemp = update_temporal_components(np.asarray(Y_res[ff,:]),A_merged[ff,i],b[ff],aa_2,f,p=p,deconv_method=deconv_method)
aa,bb,cc,_ = update_spatial_components(np.asarray(Y_res),cc,f,A_merged[:,i],d1=d1,d2=d2,dz=dz,nr=[nr_merged[i]],sn=sn,min_size=min_size,max_size=max_size,dist=dist,method = method_exp, expandCore =expandCore)
A_merged[:,i] = aa.tocsr();
cc,_,_,Ptemp = update_temporal_components(Y_res[ff,:],A_merged[ff,i],bb[ff],cc,f,p=p,deconv_method=deconv_method)
P_cycle=P_[merged_ROI[0]].copy()
P_cycle['gn']=Ptemp[0]['gn']
P_cycle['b']=Ptemp[0]['b']
P_cycle['c1']=Ptemp[0]['c1']
P_cycle['neuron_sn']=Ptemp[0]['neuron_sn']
P_merged.append(P_cycle)
C_merged[i,:] = cc
if i+1 < nm:
Y_res[ff,:] = Y_res[ff,:] - A_merged[ff,i]*cc
#%
neur_id = np.unique(np.hstack(merged_ROIs))
good_neurons=np.setdiff1d(range(N),neur_id)
A= scipy.sparse.hstack((A[:,good_neurons],A_merged.tocsc()))
C = np.vstack((C[good_neurons,:],C_merged))
nr = [nrv for nri,nrv in enumerate(nr) if nri in good_neurons] + nr_merged
# P_new=list(P_[good_neurons].copy())
P_new=[P_[pp] for pp in good_neurons]
for p in P_merged:
P_new.append(p)
#SVR TODO: update nr appropriately after merge
#nr = nr - len(neur_id) + nm
else:
warnings.warn('No neurons merged!')
merged_ROIs=[];
P_new=P_
return A,C,nr_merged,merged_ROIs,P_new
#demo_=np.load('demo_post_spatial.npz')
#Y=demo_['Y']
##A=demo_['A_out']
#b=demo_['b_out']
#fin=demo_['f_in']
#Cin=demo_['C_in'];
#g=demo_['g']
#sn=demo_['sn']
#d1=demo_['d1']
#d2=demo_['d2']
#P=demo_['P']
#%%
#TODO: test with restimate_g=True
#TODO: test reordering of list
start=time.time()
C_out,f_out,Y_res_out,P_=update_temporal_components(Y,A,b,C_in,f_in,ITER=2,method='constrained_foopsi',g=None,bas_nonneg=False,p=2,fudge_factor=1);
print time.time()-start
kkk
#%%
#np.savez('after_temporal.npz',P_=P_)
##%%
#P_=np.load('after_temporal.npz')['arr_3']
#thr=0.85
#mx=50
#d1=P.d1
#d2=P.d2
#sn=P.sn
#from scipy.sparse import spdiags,coo_matrix,csgraph
#import scipy
#import numpy as np
#%% nmf Y_res = Yr - np.dot(Ain,Cin) model = ProjectedGradientNMF(n_components=1, init='random', random_state=0) model.fit(np.maximum(Y_res,0)) fin = model.components_.squeeze() #%% update spatial components t1 = time() A,b = update_spatial_components(Yr, Cin, fin, Ain, d1=d1, d2=d2, sn = P['sn']) t_elSPATIAL = time() - t1 #%% t1 = time() C,f,Y_res,Pnew = update_temporal_components(Yr,A,b,Cin,fin,ITER=2) t_elTEMPORAL1 = time() - t1 #%% solving using spgl1 for deconvolution t1 = time() C2,f2,Y_res2,Pnew2 = update_temporal_components(Yr,A,b,Cin,fin,ITER=2,deconv_method = 'spgl1') t_elTEMPORAL2 = time() - t1 #%% t1 = time() A_m,C_m,nr_m,merged_ROIs,P_m=mergeROIS(Y_res,A.tocsc(),b,np.array(C2),f2,d1,d2,Pnew2,sn=P['sn'],deconv_method = 'spgl1') t_elMERGE = time() - t1
def update_temporal(self):
self.C,self.f,self.Y_res,self.Pnew = update_temporal_components(self.Yr,self.A,self.b,self.Cin,self.fin,ITER=2,deconv_method = 'spgl1')
p = 2; P = arpfit(Yr,p=1,pixels = active_pixels) Y_res = Yr - np.dot(Ain,Cin) model = ProjectedGradientNMF(n_components=1, init='random', random_state=0) model.fit(np.maximum(Y_res,0)) fin = model.components_.squeeze() #%% t1 = time() A,b,Cin = update_spatial_components(Yr, Cin, fin, Ain, d1=d1, d2=d2, sn = P['sn'],dist=2,max_size=8,min_size=3) t_elSPATIAL = time() - t1 #%% crd = plot_contours(A,Cn2,thr=0.9,cmap=pl.cm.gray) #%% t1 = time() C,f,Y_res,Pnew = update_temporal_components(Yr,A,b,Cin,fin,ITER=2,deconv_method = 'spgl1') t_elTEMPORAL2 = time() - t1 #%% t1 = time() A_sp=A.tocsc(); A_m,C_m,nr_m,merged_ROIs,P_m=mergeROIS(Y_res,A_sp,b,np.array(C),f,d1,d2,Pnew,sn=P['sn'],thr=.7,deconv_method='spgl1',min_size=3,max_size=8,dist=2) t_elMERGE = time() - t1 #%% crd = plot_contours(A_m,Cn2,thr=0.9) #%% A2,b2,C_m_ = update_spatial_components(Yr, C_m, f, A_m, d1=d1, d2=d2, sn = P['sn'],dist=3,max_size=8,min_size=3) C2,f2,Y_res2,Pnew2 = update_temporal_components(Yr,A2,b2,C_m_,f,ITER=2,deconv_method = 'spgl1') #%% crd = plot_contours(A2,Cn2,thr=0.9,cmap=pl.cm.gray) #%%
#Cin=demo_['C_in'];
#g=demo_['g']
#sn=demo_['sn']
#d1=demo_['d1']
#d2=demo_['d2']
#P=demo_['P']
#%%
#TODO: test with restimate_g=True
#TODO: test reordering of list
start = time.time()
C_out, f_out, Y_res_out, P_ = update_temporal_components(
Y,
A,
b,
C_in,
f_in,
ITER=2,
method='constrained_foopsi',
g=None,
bas_nonneg=False,
p=2,
fudge_factor=1)
print time.time() - start
kkk
#%%
#np.savez('after_temporal.npz',P_=P_)
##%%
#P_=np.load('after_temporal.npz')['arr_3']
#thr=0.85
#mx=50
#d1=P.d1
def mergeROIS(Y_res,
A,
b,
C,
f,
d1,
d2,
P_,
thr=0.8,
mx=50,
sn=None,
deconv_method='spgl1',
min_size=3,
max_size=8,
dist=3,
method_exp='ellipse',
expandCore=iterate_structure(generate_binary_structure(2, 1),
2).astype(int)):
"""
merging of spatially overlapping components that have highly correlated tmeporal activity
% The correlation threshold for merging overlapping components is user specified in P.merge_thr (default value 0.85)
% Inputs:
% Y_res: residual movie after subtracting all found components
% A: matrix of spatial components
% b: spatial background
% C: matrix of temporal components
% f: temporal background
% P: parameter struct
% Outputs:
% A: matrix of new spatial components
% C: matrix of new temporal components
% nr: new number of components
% merged_ROIs: list of old components that were merged
% Written by:
% Andrea Giovannucci from implementation of Eftychios A. Pnevmatikakis, Simons Foundation, 2015
"""
#%
nr = A.shape[1]
[d, T] = np.shape(Y_res)
C_corr = np.corrcoef(C[:nr, :], C[:nr, :])[:nr, :nr]
FF1 = C_corr >= thr
#find graph of strongly correlated temporal components
A_corr = A.T * A
A_corr.setdiag(0)
FF2 = A_corr > 0 # % find graph of overlapping spatial components
FF3 = np.logical_and(FF1, FF2.todense())
FF3 = coo_matrix(FF3)
c, l = csgraph.connected_components(FF3) # % extract connected components
p = len(P_[0]['gn'])
MC = []
for i in range(c):
if np.sum(l == i) > 1:
MC.append((l == i).T)
MC = np.asarray(MC).T
if MC.ndim > 1:
cor = np.zeros((np.shape(MC)[1], 1))
for i in range(np.size(cor)):
fm = np.where(MC[:, i])[0]
for j1 in range(np.size(fm)):
for j2 in range(j1 + 1, np.size(fm)):
print j1, j2
cor[i] = cor[i] + C_corr[fm[j1], fm[j2]]
Y_res = Y_res + np.dot(b, f)
if np.size(cor) > 1:
ind = np.argsort(np.squeeze(cor))[::-1]
else:
ind = [0]
nm = min((np.size(ind), mx)) # number of merging operations
A_merged = coo_matrix((d, nm)).tocsr()
C_merged = np.zeros((nm, T))
P_merged = []
merged_ROIs = []
#%
for i in range(nm):
P_cycle = dict()
merged_ROI = np.where(MC[:, ind[i]])[0]
merged_ROIs.append(merged_ROI)
nC = np.sqrt(np.sum(C[merged_ROI, :]**2, axis=1))
# A_merged[:,i] = np.squeeze((A[:,merged_ROI]*spdiags(nC,0,len(nC),len(nC))).sum(axis=1))
A_merged[:, i] = csr_matrix(
(A[:, merged_ROI] *
spdiags(nC, 0, len(nC), len(nC))).sum(axis=1))
Y_res = Y_res + A[:, merged_ROI] * C[merged_ROI, :]
aa_1 = scipy.sparse.linalg.spsolve(
spdiags(nC, 0, len(nC), len(nC)), C[merged_ROI, :])
aa_2 = (aa_1).mean(axis=0)
ff = np.nonzero(A_merged[:, i])[0]
cc, _, _, Ptemp = update_temporal_components(
np.asarray(Y_res[ff, :]),
A_merged[ff, i],
b[ff],
aa_2,
f,
p=p,
deconv_method=deconv_method)
aa, bb, cc = update_spatial_components(np.asarray(Y_res),
cc,
f,
A_merged[:, i],
d1=d1,
d2=d2,
sn=sn,
min_size=min_size,
max_size=max_size,
dist=dist,
method=method_exp,
expandCore=expandCore)
A_merged[:, i] = aa.tocsr()
cc, _, _, Ptemp = update_temporal_components(
Y_res[ff, :],
A_merged[ff, i],
bb[ff],
cc,
f,
p=p,
deconv_method=deconv_method)
P_cycle = P_[merged_ROI[0]].copy()
P_cycle['gn'] = Ptemp[0]['gn']
P_cycle['b'] = Ptemp[0]['b']
P_cycle['c1'] = Ptemp[0]['c1']
P_cycle['neuron_sn'] = Ptemp[0]['neuron_sn']
P_merged.append(P_cycle)
C_merged[i, :] = cc
if i + 1 < nm:
Y_res[ff, :] = Y_res[ff, :] - A_merged[ff, i] * cc
#%
neur_id = np.unique(np.hstack(merged_ROIs))
good_neurons = np.setdiff1d(range(nr), neur_id)
A = scipy.sparse.hstack((A[:, good_neurons], A_merged.tocsc()))
C = np.vstack((C[good_neurons, :], C_merged))
# P_new=list(P_[good_neurons].copy())
P_new = [P_[pp] for pp in good_neurons]
for p in P_merged:
P_new.append(p)
nr = nr - len(neur_id) + nm
else:
warnings.warn('No neurons merged!')
merged_ROIs = []
P_new = P_
return A, C, nr, merged_ROIs, P_new
t1 = time()
A,b = update_spatial_components(Yr, Cin, fin, Ain, d1=d1, d2=d2, min_size=3, max_size=8, dist=3,sn = P['sn'], method = 'ellipse')
t_elSPATIAL = time() - t1
#%% check with matlab
demo_results= sio.loadmat('demo_results.mat', struct_as_record=False, squeeze_me=True)
A_m=demo_results['A']*1.0
P_m=demo_results['P']
b_m =np.expand_dims(demo_results['b']*1.0,axis=1)
print np.sum(np.abs(A-A_m).todense())/np.sum(np.abs(A).todense()) # should give 0.0035824510737
print np.sum(np.abs(b-b_m))/np.sum(np.abs(b_m)) # should give 0.0032486662048
#%%
t1 = time()
C,f,Y_res_temp,P_temp = update_temporal_components(Yr,A,b,Cin,fin,ITER=2,method='constrained_foopsi',deconv_method = 'cvx', g='None')
t_elTEMPORAL1 = time() - t1
print t_elTEMPORAL1
#%%
t1 = time()
C2,f2,Y_res_temp2,P_temp2 = update_temporal_components(Yr,A,b,Cin,fin,ITER=2,method='constrained_foopsi',deconv_method = 'spgl1', g='None')
t_elTEMPORAL2 = time() - t1
#%% compare with matlab
demo_results= sio.loadmat('demo_results.mat', struct_as_record=False, squeeze_me=True)
C_m=demo_results['C']*1.0
f_m=demo_results['f']*1.0
P_temp_m=demo_results['P_temp']
Y_res_temp_m=demo_results['Y_res_temp']*1.0
C_cor=np.squeeze(np.array([np.array(ca-pt['b']) for pt,ca in zip(P_temp,C)]))
C_cor_m=np.squeeze(np.array([(ca-P_temp_m.b[idx]) if np.size(P_temp_m.b[idx])>0 else ca for idx,ca in enumerate(C)]))
