'ITER': 2, 'method': 'spgl1', 'p': p, 'n_processes': n_processes, 'backend': 'ipyparallel', 'memory_efficient': False, 'bas_nonneg': True, 'noise_range': [.25, .5], 'noise_method': 'logmexp', 'lags': 5, 'fudge_factor': 1., 'verbosity': False } #%% PREPROCESS DATA t1 = time() Yr, sn, g = cse.preprocess_data(Yr, **preprocess_params) Ain, Cin, b_in, f_in, center = cse.initialize_components(Y, **init_params) print time() - t1 #%% t1 = time() A, b, Cin = cse.update_spatial_components_parallel(Yr, Cin, f_in, Ain, sn=sn, **spatial_params) t_elSPATIAL = time() - t1 print t_elSPATIAL #%% update_temporal_components t1 = time()
def process_data(haussio_data, mask=None, p=2, nrois_init=200): fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' tiffs_to_cnmf(haussio_data, mask) sys.stdout.write('Loading from {0}... '.format( haussio_data.dirname_comp + '_Y*.npy')) Y = np.load(haussio_data.dirname_comp + '_Y.npy', mmap_mode='r') d1, d2, T = Y.shape if not os.path.exists(fn_cnmf): cse.utilities.stop_server() sys.stdout.flush() t0 = time.time() Yr = np.load(haussio_data.dirname_comp + '_Yr.npy', mmap_mode='r') sys.stdout.write('took {0:.2f} s\n'.format(time.time()-t0)) # how to subdivide the work among processes n_pixels_per_process = d1*d2/NCPUS options = cse.utilities.CNMFSetParms(Y, K=nrois_init, p=p, gSig=[9, 9]) options['preprocess_params']['n_processes'] = NCPUS options['preprocess_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['init_params']['nIter'] = 10 options['init_params']['maxIter'] = 10 options['init_params']['use_hals'] = False options['spatial_params']['n_processes'] = NCPUS options['spatial_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['temporal_params']['n_processes'] = NCPUS options['temporal_params'][ 'n_pixels_per_process'] = n_pixels_per_process cse.utilities.start_server(NCPUS) t0 = time.time() sys.stdout.write("Preprocessing... ") sys.stdout.flush() Yr, sn, g = cse.preprocess_data(Yr, **options['preprocess_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 224.94s t0 = time.time() sys.stdout.write("Initializing components... ") sys.stdout.flush() Ain, Cin, b_in, f_in, center = cse.initialize_components( Y, **options['init_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 2281.37s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A, b, Cin = cse.update_spatial_components( Yr, Cin, f_in, Ain, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 252.57s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C, f, S, bl, c1, neurons_sn, g, YrA = \ cse.update_temporal_components( Yr, A, b, Cin, f_in, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 455.14s t0 = time.time() sys.stdout.write("Merging ROIs... ") sys.stdout.flush() A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = \ cse.merge_components( Yr, A, b, C, f, S, sn, options['temporal_params'], options['spatial_params'], bl=bl, c1=c1, sn=neurons_sn, g=g, thr=0.7, mx=100, fast_merge=True) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 702.55s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A2, b2, C2 = cse.update_spatial_components( Yr, C_m, f, A_m, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 77.16s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = \ cse.update_temporal_components( Yr, A2, b2, C2, f, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 483.41s # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise") # S: Spikes savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2}) else: resdict = loadmat(fn_cnmf) A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] bl2 = resdict["bl"] proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(np.transpose(Y, (2, 0, 1))) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) # DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2) t0 = time.time() sys.stdout.write("Ordering components... ") sys.stdout.flush() A_or, C_or, srt = cse.order_components(A2, C2) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) cse.utilities.stop_server() polygons = contour(A2, d1, d2, thr=0.9) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, haussio_data, zproj, S2, Y, YrA
'd1':d1, 'd2':d2, 'dist':3, 'method' : 'ellipse', 'n_processes':n_processes,'n_pixels_per_process':n_pixels_per_process,'backend':'ipyparallel', 'memory_efficient':False } temporal_params = { 'ITER':2, 'method':'spgl1', 'p':p, 'n_processes':n_processes,'backend':'ipyparallel', 'memory_efficient':False, 'bas_nonneg':True, 'noise_range':[.25,.5], 'noise_method':'logmexp', 'lags':5, 'fudge_factor':1., 'verbosity':False } #%% PREPROCESS DATA t1 = time() Yr,sn,g=cse.preprocess_data(Yr,**preprocess_params) Ain, Cin, b_in, f_in, center=cse.initialize_components(Y, **init_params) print time() - t1 plt2 = plt.imshow(Cn,interpolation='None') plt.colorbar() plt.scatter(x=center[:,1], y=center[:,0], c='m', s=40) crd = cse.plot_contours(coo_matrix(Ain[:,::-1]),Cn,thr=0.9) plt.axis((-0.5,d2-0.5,-0.5,d1-0.5)) plt.gca().invert_yaxis() pl.show() #%% t1 = time() A,b,Cin = cse.update_spatial_components_parallel(Yr, Cin, f_in, Ain, sn=sn, **spatial_params) t_elSPATIAL = time() - t1 print t_elSPATIAL
def process_data(haussio_data, mask=None, p=2, nrois_init=400): fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' tiffs_to_cnmf(haussio_data, mask) sys.stdout.write('Loading from {0}... '.format( haussio_data.dirname_comp + '_Y*.npy')) Y = np.load(haussio_data.dirname_comp + '_Y.npy', mmap_mode='r') d1, d2, T = Y.shape if not os.path.exists(fn_cnmf): cse.utilities.stop_server() sys.stdout.flush() t0 = time.time() Yr = np.load(haussio_data.dirname_comp + '_Yr.npy', mmap_mode='r') sys.stdout.write('took {0:.2f} s\n'.format(time.time()-t0)) # how to subdivide the work among processes n_pixels_per_process = d1*d2/NCPUS options = cse.utilities.CNMFSetParms( Y, NCPUS, K=nrois_init, p=p, gSig=[9, 9], ssub=2, tsub=2) options['preprocess_params']['n_processes'] = NCPUS options['preprocess_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['init_params']['nIter'] = 10 options['init_params']['maxIter'] = 10 options['init_params']['use_hals'] = True options['spatial_params']['n_processes'] = NCPUS options['spatial_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['temporal_params']['n_processes'] = NCPUS options['temporal_params'][ 'n_pixels_per_process'] = n_pixels_per_process cse.utilities.start_server() t0 = time.time() sys.stdout.write("Preprocessing... ") sys.stdout.flush() Yr, sn, g, psx = cse.preprocess_data(Yr, **options['preprocess_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 224.94s # 2016-05-24: 146.30s t0 = time.time() sys.stdout.write("Initializing components... ") sys.stdout.flush() Ain, Cin, b_in, f_in, center = cse.initialize_components( Y, **options['init_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 2281.37s # 2016-05-24: 1054.72s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A, b, Cin = cse.update_spatial_components( Yr, Cin, f_in, Ain, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 252.57s # 2016-05-24: 445.95s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C, f, S, bl, c1, neurons_sn, g, YrA = \ cse.update_temporal_components( Yr, A, b, Cin, f_in, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 455.14s # 2016-05-24: 86.10s t0 = time.time() sys.stdout.write("Merging ROIs... ") sys.stdout.flush() A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = \ cse.merge_components( Yr, A, b, C, f, S, sn, options['temporal_params'], options['spatial_params'], bl=bl, c1=c1, sn=neurons_sn, g=g, thr=0.7, mx=100, fast_merge=True) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 702.55s # 2016-05-24: 11.75s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A2, b2, C2 = cse.update_spatial_components( Yr, C_m, f, A_m, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 77.16s # 2016-05-24: 99.22s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = \ cse.update_temporal_components( Yr, A2, b2, C2, f, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 483.41s # 2016-05-24: 74.81s # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise") # S: Spikes savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2}) else: resdict = loadmat(fn_cnmf) A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] bl2 = resdict["bl"] proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(np.transpose(Y, (2, 0, 1))) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) # DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2) t0 = time.time() sys.stdout.write("Ordering components... ") sys.stdout.flush() A_or, C_or, srt = cse.order_components(A2, C2) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) cse.utilities.stop_server() polygons = contour(A2, d1, d2, thr=0.9) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, zproj, S2, Y, YrA