def process_data(haussio_data, mask=None, p=2, nrois_init=200): fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' tiffs_to_cnmf(haussio_data, mask) sys.stdout.write('Loading from {0}... '.format( haussio_data.dirname_comp + '_Y*.npy')) Y = np.load(haussio_data.dirname_comp + '_Y.npy', mmap_mode='r') d1, d2, T = Y.shape if not os.path.exists(fn_cnmf): cse.utilities.stop_server() sys.stdout.flush() t0 = time.time() Yr = np.load(haussio_data.dirname_comp + '_Yr.npy', mmap_mode='r') sys.stdout.write('took {0:.2f} s\n'.format(time.time()-t0)) # how to subdivide the work among processes n_pixels_per_process = d1*d2/NCPUS options = cse.utilities.CNMFSetParms(Y, K=nrois_init, p=p, gSig=[9, 9]) options['preprocess_params']['n_processes'] = NCPUS options['preprocess_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['init_params']['nIter'] = 10 options['init_params']['maxIter'] = 10 options['init_params']['use_hals'] = False options['spatial_params']['n_processes'] = NCPUS options['spatial_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['temporal_params']['n_processes'] = NCPUS options['temporal_params'][ 'n_pixels_per_process'] = n_pixels_per_process cse.utilities.start_server(NCPUS) t0 = time.time() sys.stdout.write("Preprocessing... ") sys.stdout.flush() Yr, sn, g = cse.preprocess_data(Yr, **options['preprocess_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 224.94s t0 = time.time() sys.stdout.write("Initializing components... ") sys.stdout.flush() Ain, Cin, b_in, f_in, center = cse.initialize_components( Y, **options['init_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 2281.37s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A, b, Cin = cse.update_spatial_components( Yr, Cin, f_in, Ain, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 252.57s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C, f, S, bl, c1, neurons_sn, g, YrA = \ cse.update_temporal_components( Yr, A, b, Cin, f_in, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 455.14s t0 = time.time() sys.stdout.write("Merging ROIs... ") sys.stdout.flush() A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = \ cse.merge_components( Yr, A, b, C, f, S, sn, options['temporal_params'], options['spatial_params'], bl=bl, c1=c1, sn=neurons_sn, g=g, thr=0.7, mx=100, fast_merge=True) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 702.55s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A2, b2, C2 = cse.update_spatial_components( Yr, C_m, f, A_m, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 77.16s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = \ cse.update_temporal_components( Yr, A2, b2, C2, f, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 483.41s # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise") # S: Spikes savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2}) else: resdict = loadmat(fn_cnmf) A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] bl2 = resdict["bl"] proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(np.transpose(Y, (2, 0, 1))) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) # DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2) t0 = time.time() sys.stdout.write("Ordering components... ") sys.stdout.flush() A_or, C_or, srt = cse.order_components(A2, C2) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) cse.utilities.stop_server() polygons = contour(A2, d1, d2, thr=0.9) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, haussio_data, zproj, S2, Y, YrA
A,b,Cin = cse.update_spatial_components_parallel(Yr, Cin, f_in, Ain, sn=sn, **spatial_params) t_elSPATIAL = time() - t1 print t_elSPATIAL crd = cse.plot_contours(A,Cn,thr=0.9) #%% update_temporal_components t1 = time() C,f,Y_res,S,bl,c1,neurons_sn,g = cse.update_temporal_components_parallel(Yr,A,b,Cin,f_in,bl=None,c1=None,sn=None,g=None,**temporal_params) t_elTEMPORAL2 = time() - t1 print t_elTEMPORAL2 # took 98 sec #%% merge components corresponding to the same neuron t1 = time() A_m,C_m,nr_m,merged_ROIs,S_m,bl_m,c1_m,sn_m,g_m=cse.mergeROIS_parallel(Y_res,A,b,C,f,S,sn,temporal_params, spatial_params, bl=bl, c1=c1, sn=neurons_sn, g=g, thr=0.8, mx=50) t_elMERGE = time() - t1 print t_elMERGE #%% view results. Notice that in order to scroll components you need to click on the plot A_or, C_or, srt = cse.order_components(A_m,C_m) cse.view_patches(Yr,coo_matrix(A_or),C_or,b,f, d1,d2,secs=0) #%% crd = cse.plot_contours(A_m,Cn,thr=0.9) #%% refine spatial and temporal t1 = time() A2,b2,C2 = cse.update_spatial_components_parallel(Yr, C_m, f, A_m, sn=sn, **spatial_params) #C2,f2,Y_res2,S2,bl2,c12,neurons_sn2,g21 = cse.update_temporal_components_parallel(Yr,A2,b2,C2,f,bl=bl_m,c1=c1_m,sn=sn_m,g=g_m,**temporal_params) C2,f2,Y_res2,S2,bl2,c12,neurons_sn2,g21 = cse.update_temporal_components_parallel(Yr,A2,b2,C2,f,bl=None,c1=None,sn=None,g=None,**temporal_params) print time() - t1 # 100 seconds #%% A_or, C_or, srt = cse.order_components(A2,C2) cse.view_patches(Yr,coo_matrix(A_or),C_or,b2,f2, d1,d2,secs=0) #%% STOP CLUSTER print "Stopping Cluster...." sys.stdout.flush()
def process_data(haussio_data, mask=None, p=2, nrois_init=400): fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat' tiffs_to_cnmf(haussio_data, mask) sys.stdout.write('Loading from {0}... '.format( haussio_data.dirname_comp + '_Y*.npy')) Y = np.load(haussio_data.dirname_comp + '_Y.npy', mmap_mode='r') d1, d2, T = Y.shape if not os.path.exists(fn_cnmf): cse.utilities.stop_server() sys.stdout.flush() t0 = time.time() Yr = np.load(haussio_data.dirname_comp + '_Yr.npy', mmap_mode='r') sys.stdout.write('took {0:.2f} s\n'.format(time.time()-t0)) # how to subdivide the work among processes n_pixels_per_process = d1*d2/NCPUS options = cse.utilities.CNMFSetParms( Y, NCPUS, K=nrois_init, p=p, gSig=[9, 9], ssub=2, tsub=2) options['preprocess_params']['n_processes'] = NCPUS options['preprocess_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['init_params']['nIter'] = 10 options['init_params']['maxIter'] = 10 options['init_params']['use_hals'] = True options['spatial_params']['n_processes'] = NCPUS options['spatial_params'][ 'n_pixels_per_process'] = n_pixels_per_process options['temporal_params']['n_processes'] = NCPUS options['temporal_params'][ 'n_pixels_per_process'] = n_pixels_per_process cse.utilities.start_server() t0 = time.time() sys.stdout.write("Preprocessing... ") sys.stdout.flush() Yr, sn, g, psx = cse.preprocess_data(Yr, **options['preprocess_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 224.94s # 2016-05-24: 146.30s t0 = time.time() sys.stdout.write("Initializing components... ") sys.stdout.flush() Ain, Cin, b_in, f_in, center = cse.initialize_components( Y, **options['init_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 2281.37s # 2016-05-24: 1054.72s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A, b, Cin = cse.update_spatial_components( Yr, Cin, f_in, Ain, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 252.57s # 2016-05-24: 445.95s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C, f, S, bl, c1, neurons_sn, g, YrA = \ cse.update_temporal_components( Yr, A, b, Cin, f_in, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 455.14s # 2016-05-24: 86.10s t0 = time.time() sys.stdout.write("Merging ROIs... ") sys.stdout.flush() A_m, C_m, nr_m, merged_ROIs, S_m, bl_m, c1_m, sn_m, g_m = \ cse.merge_components( Yr, A, b, C, f, S, sn, options['temporal_params'], options['spatial_params'], bl=bl, c1=c1, sn=neurons_sn, g=g, thr=0.7, mx=100, fast_merge=True) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 702.55s # 2016-05-24: 11.75s t0 = time.time() sys.stdout.write("Updating spatial components... ") sys.stdout.flush() A2, b2, C2 = cse.update_spatial_components( Yr, C_m, f, A_m, sn=sn, **options['spatial_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 77.16s # 2016-05-24: 99.22s t0 = time.time() sys.stdout.write("Updating temporal components... ") sys.stdout.flush() C2, f2, S2, bl2, c12, neurons_sn2, g21, YrA = \ cse.update_temporal_components( Yr, A2, b2, C2, f, bl=None, c1=None, sn=None, g=None, **options['temporal_params']) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) # 483.41s # 2016-05-24: 74.81s # A: spatial components (ROIs) # C: denoised [Ca2+] # YrA: residuals ("noise") # S: Spikes savemat(fn_cnmf, {"A": A2, "C": C2, "YrA": YrA, "S": S2, "bl": bl2}) else: resdict = loadmat(fn_cnmf) A2 = resdict["A"] C2 = resdict["C"] YrA = resdict["YrA"] S2 = resdict["S"] bl2 = resdict["bl"] proj_fn = haussio_data.dirname_comp + "_proj.npy" if not os.path.exists(proj_fn): zproj = utils.zproject(np.transpose(Y, (2, 0, 1))) np.save(proj_fn, zproj) else: zproj = np.load(proj_fn) # DF_F, DF = cse.extract_DF_F(Y.reshape(d1*d2, T), A2, C2) t0 = time.time() sys.stdout.write("Ordering components... ") sys.stdout.flush() A_or, C_or, srt = cse.order_components(A2, C2) sys.stdout.write(' took {0:.2f} s\n'.format(time.time()-t0)) cse.utilities.stop_server() polygons = contour(A2, d1, d2, thr=0.9) rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons]) return rois, C2, zproj, S2, Y, YrA