def process_data(haussio_data, mask=None, p=2, nrois_init=200):
fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat'
tiffs_to_cnmf(haussio_data, mask)
sys.stdout.write('Loading from {0}... '.format(
haussio_data.dirname_comp + '_Y*.npy'))
Y = np.load(haussio_data.dirname_comp + '_Y.npy', mmap_mode='r')
d1, d2, T = Y.shape
if not os.path.exists(fn_cnmf):
cse.utilities.stop_server()
sys.stdout.flush()
t0 = time.time()
Yr = np.load(haussio_data.dirname_comp + '_Yr.npy', mmap_mode='r')
sys.stdout.write('took {0:.2f} s\n'.format(time.time()-t0))
# how to subdivide the work among processes
n_pixels_per_process = d1*d2/NCPUS
options = cse.utilities.CNMFSetParms(Y, K=nrois_init, p=p, gSig=[9, 9])
options['preprocess_params']['n_processes'] = NCPUS
options['preprocess_params'][
'n_pixels_per_process'] = n_pixels_per_process
options['init_params']['nIter'] = 10
options['init_params']['maxIter'] = 10
options['init_params']['use_hals'] = False
options['spatial_params']['n_processes'] = NCPUS
options['spatial_params'][
'n_pixels_per_process'] = n_pixels_per_process
options['temporal_params']['n_processes'] = NCPUS
options['temporal_params'][
'n_pixels_per_process'] = n_pixels_per_process
cse.utilities.start_server(NCPUS)
t0 = time.time()
sys.stdout.write("Preprocessing... ")
sys.stdout.flush()
Yr, sn, g = cse.preprocess_data(Yr, **options['preprocess_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 224.94s
t0 = time.time()
sys.stdout.write("Initializing components... ")
sys.stdout.flush()
Ain, Cin, b_in, f_in, center = cse.initialize_components(
Y, **options['init_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 2281.37s
t0 = time.time()
sys.stdout.write("Updating spatial components... ")
sys.stdout.flush()
A, b, Cin = cse.update_spatial_components(
Yr, Cin, f_in, Ain, sn=sn, **options['spatial_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 252.57s
t0 = time.time()
sys.stdout.write("Updating temporal components... ")
sys.stdout.flush()
C, f, S, bl, c1, neurons_sn, g, YrA = \
cse.update_temporal_components(
Yr, A, b, Cin, f_in, bl=None, c1=None, sn=None, g=None,
**options['temporal_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 455.14s
t0 = time.time()
sys.stdout.write("Merging ROIs... ")
sys.stdout.flush()
A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = \
cse.merge_components(
Yr, A, b, C, f, S, sn, options['temporal_params'],
options['spatial_params'], bl=bl, c1=c1, sn=neurons_sn, g=g,
thr=0.7, mx=100, fast_merge=True)
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 702.55s
t0 = time.time()
sys.stdout.write("Updating spatial components... ")
sys.stdout.flush()
A2, b2, C2 = cse.update_spatial_components(
Yr, C_m, f, A_m, sn=sn, **options['spatial_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 77.16s
t0 = time.time()
sys.stdout.write("Updating temporal components... ")
sys.stdout.flush()
C2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = \
cse.update_temporal_components(
Yr, A2, b2, C2, f, bl=None, c1=None, sn=None, g=None,
**options['temporal_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 483.41s
# A: spatial components (ROIs)
# C: denoised [Ca2+]
# YrA: residuals ("noise")
# S: Spikes
savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2})
else:
resdict = loadmat(fn_cnmf)
A2 = resdict["A"]
C2 = resdict["C"]
YrA = resdict["YrA"]
S2 = resdict["S"]
bl2 = resdict["bl"]
proj_fn = haussio_data.dirname_comp + "_proj.npy"
if not os.path.exists(proj_fn):
zproj = utils.zproject(np.transpose(Y, (2, 0, 1)))
np.save(proj_fn, zproj)
else:
zproj = np.load(proj_fn)
# DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2)
t0 = time.time()
sys.stdout.write("Ordering components... ")
sys.stdout.flush()
A_or, C_or, srt = cse.order_components(A2, C2)
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
cse.utilities.stop_server()
polygons = contour(A2, d1, d2, thr=0.9)
rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons])
return rois, C2, haussio_data, zproj, S2, Y, YrA
A,b,Cin = cse.update_spatial_components_parallel(Yr, Cin, f_in, Ain, sn=sn, **spatial_params) t_elSPATIAL = time() - t1 print t_elSPATIAL crd = cse.plot_contours(A,Cn,thr=0.9) #%% update_temporal_components t1 = time() C,f,Y_res,S,bl,c1,neurons_sn,g = cse.update_temporal_components_parallel(Yr,A,b,Cin,f_in,bl=None,c1=None,sn=None,g=None,**temporal_params) t_elTEMPORAL2 = time() - t1 print t_elTEMPORAL2 # took 98 sec #%% merge components corresponding to the same neuron t1 = time() A_m,C_m,nr_m,merged_ROIs,S_m,bl_m,c1_m,sn_m,g_m=cse.mergeROIS_parallel(Y_res,A,b,C,f,S,sn,temporal_params, spatial_params, bl=bl, c1=c1, sn=neurons_sn, g=g, thr=0.8, mx=50) t_elMERGE = time() - t1 print t_elMERGE #%% view results. Notice that in order to scroll components you need to click on the plot A_or, C_or, srt = cse.order_components(A_m,C_m) cse.view_patches(Yr,coo_matrix(A_or),C_or,b,f, d1,d2,secs=0) #%% crd = cse.plot_contours(A_m,Cn,thr=0.9) #%% refine spatial and temporal t1 = time() A2,b2,C2 = cse.update_spatial_components_parallel(Yr, C_m, f, A_m, sn=sn, **spatial_params) #C2,f2,Y_res2,S2,bl2,c12,neurons_sn2,g21 = cse.update_temporal_components_parallel(Yr,A2,b2,C2,f,bl=bl_m,c1=c1_m,sn=sn_m,g=g_m,**temporal_params) C2,f2,Y_res2,S2,bl2,c12,neurons_sn2,g21 = cse.update_temporal_components_parallel(Yr,A2,b2,C2,f,bl=None,c1=None,sn=None,g=None,**temporal_params) print time() - t1 # 100 seconds #%% A_or, C_or, srt = cse.order_components(A2,C2) cse.view_patches(Yr,coo_matrix(A_or),C_or,b2,f2, d1,d2,secs=0) #%% STOP CLUSTER print "Stopping Cluster...." sys.stdout.flush()
def process_data(haussio_data, mask=None, p=2, nrois_init=400):
fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat'
tiffs_to_cnmf(haussio_data, mask)
sys.stdout.write('Loading from {0}... '.format(
haussio_data.dirname_comp + '_Y*.npy'))
Y = np.load(haussio_data.dirname_comp + '_Y.npy', mmap_mode='r')
d1, d2, T = Y.shape
if not os.path.exists(fn_cnmf):
cse.utilities.stop_server()
sys.stdout.flush()
t0 = time.time()
Yr = np.load(haussio_data.dirname_comp + '_Yr.npy', mmap_mode='r')
sys.stdout.write('took {0:.2f} s\n'.format(time.time()-t0))
# how to subdivide the work among processes
n_pixels_per_process = d1*d2/NCPUS
options = cse.utilities.CNMFSetParms(
Y, NCPUS, K=nrois_init, p=p, gSig=[9, 9], ssub=2, tsub=2)
options['preprocess_params']['n_processes'] = NCPUS
options['preprocess_params'][
'n_pixels_per_process'] = n_pixels_per_process
options['init_params']['nIter'] = 10
options['init_params']['maxIter'] = 10
options['init_params']['use_hals'] = True
options['spatial_params']['n_processes'] = NCPUS
options['spatial_params'][
'n_pixels_per_process'] = n_pixels_per_process
options['temporal_params']['n_processes'] = NCPUS
options['temporal_params'][
'n_pixels_per_process'] = n_pixels_per_process
cse.utilities.start_server()
t0 = time.time()
sys.stdout.write("Preprocessing... ")
sys.stdout.flush()
Yr, sn, g, psx = cse.preprocess_data(Yr, **options['preprocess_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 224.94s
# 2016-05-24: 146.30s
t0 = time.time()
sys.stdout.write("Initializing components... ")
sys.stdout.flush()
Ain, Cin, b_in, f_in, center = cse.initialize_components(
Y, **options['init_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 2281.37s
# 2016-05-24: 1054.72s
t0 = time.time()
sys.stdout.write("Updating spatial components... ")
sys.stdout.flush()
A, b, Cin = cse.update_spatial_components(
Yr, Cin, f_in, Ain, sn=sn, **options['spatial_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 252.57s
# 2016-05-24: 445.95s
t0 = time.time()
sys.stdout.write("Updating temporal components... ")
sys.stdout.flush()
C, f, S, bl, c1, neurons_sn, g, YrA = \
cse.update_temporal_components(
Yr, A, b, Cin, f_in, bl=None, c1=None, sn=None, g=None,
**options['temporal_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 455.14s
# 2016-05-24: 86.10s
t0 = time.time()
sys.stdout.write("Merging ROIs... ")
sys.stdout.flush()
A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = \
cse.merge_components(
Yr, A, b, C, f, S, sn, options['temporal_params'],
options['spatial_params'], bl=bl, c1=c1, sn=neurons_sn, g=g,
thr=0.7, mx=100, fast_merge=True)
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 702.55s
# 2016-05-24: 11.75s
t0 = time.time()
sys.stdout.write("Updating spatial components... ")
sys.stdout.flush()
A2, b2, C2 = cse.update_spatial_components(
Yr, C_m, f, A_m, sn=sn, **options['spatial_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 77.16s
# 2016-05-24: 99.22s
t0 = time.time()
sys.stdout.write("Updating temporal components... ")
sys.stdout.flush()
C2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = \
cse.update_temporal_components(
Yr, A2, b2, C2, f, bl=None, c1=None, sn=None, g=None,
**options['temporal_params'])
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
# 483.41s
# 2016-05-24: 74.81s
# A: spatial components (ROIs)
# C: denoised [Ca2+]
# YrA: residuals ("noise")
# S: Spikes
savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2})
else:
resdict = loadmat(fn_cnmf)
A2 = resdict["A"]
C2 = resdict["C"]
YrA = resdict["YrA"]
S2 = resdict["S"]
bl2 = resdict["bl"]
proj_fn = haussio_data.dirname_comp + "_proj.npy"
if not os.path.exists(proj_fn):
zproj = utils.zproject(np.transpose(Y, (2, 0, 1)))
np.save(proj_fn, zproj)
else:
zproj = np.load(proj_fn)
# DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2)
t0 = time.time()
sys.stdout.write("Ordering components... ")
sys.stdout.flush()
A_or, C_or, srt = cse.order_components(A2, C2)
sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0))
cse.utilities.stop_server()
polygons = contour(A2, d1, d2, thr=0.9)
rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons])
return rois, C2, zproj, S2, Y, YrA
