def motion_correct(input_file, gsig, max_shifts, rigid_splits, save_movie,
nprocs):
import matplotlib.pyplot as plt
if input_file.endswith('mat'):
tif = mc.handle_mat_file(input_file)
elif input_file.endswith('tif') or input_file.endswith('tiff'):
tif = input_file
mc_params = dict(gSig_filt=[gsig] * 2,
max_shifts=[max_shifts] * 2,
splits_rig=rigid_splits,
save_movie=save_movie)
dview = util.create_dview(n_procs=nprocs)
corrected = mc.motion_correct(tif, mc_params, dview)
mmapped = util.memmap_file(corrected.fname_tot_rig,
dview=dview,
basename=re.sub(r'\.tif$', '', tif))
# remove unnecessary intermediate
if isinstance(corrected.fname_tot_rig, (list, tuple)):
os.remove(corrected.fname_tot_rig[0])
else:
os.remove(corrected.fname_tot_rig)
Yr, dims, T = cm.load_memmap(mmapped)
Y = Yr.T.reshape((T, ) + dims, order='F')
cn_filt, pnr = cm.summary_images.correlation_pnr(Y[:3000,
max_shifts:-max_shifts,
max_shifts:-max_shifts],
gSig=gsig,
swap_dim=False)
cm.utils.visualization.inspect_correlation_pnr(cn_filt, pnr)
plt.savefig(re.sub(r'\.tif$', '', tif) + '-caiman-corr.png')
cm.stop_server(dview=dview)
def _start_cluster(self):
if 'self.dview' in locals():
cm.stop_server(dview=self.dview)
print('Starting local cluster...', end=' ')
self.c, self.dview, self.n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
print('done.')
def extract_pipeline(input_file, params, out_file, n_procs):
assert input_file.endswith('mmap'), 'Input file needs to be a memmap file!'
dir = os.path.dirname(input_file)
if out_file is None:
search = re.search('_d1_[0-9]{3}_d2_[0-9]{3}_d3', input_file)
if search is not None:
out_file = input_file[:search.start()] + '-extract.h5'
else:
out_file = os.path.splitext(input_file)[0] + '-extract.h5'
if params is None:
params = glob(join(os.path.dirname(input_file), '*caiman*yaml'))[0]
# TODO: if no params are found use default params
with open(params, 'r') as f:
cnmf_options = yaml.load(f, yaml.Loader)
dview = util.create_dview(n_procs=n_procs)
ca_traces, masks, cnmf, dims = extract.extract(input_file,
cnmf_options,
dview=dview)
with h5py.File(out_file, 'w') as f:
f.create_dataset('ca', data=ca_traces, compression='lzf')
f.create_dataset('masks', data=masks, compression='lzf')
util.plot_neurons(ca_traces, masks, join(dir, 'caiman-neurons'), dims)
del cnmf.dview
with open(os.path.splitext(out_file)[0] + '-estimates.dill', 'wb') as f:
dill.dump(cnmf.estimates, f)
with open(os.path.splitext(out_file)[0] + '-cnmf.dill', 'wb') as f:
dill.dump(cnmf, f)
# cnmf.save(os.path.splitext(out_file)[0] + '-cnmf.h5')
print('There are {} neurons, baby!'.format(ca_traces.shape[0]))
cm.stop_server(dview=dview)
def seeded_Caiman_wAgonia(data_path, opts, agonia_th):
'''Run Caiman using as seeds the boxes detected with Agonia and save results
Parameters
----------
data_path : string
path to folder containing the data
opts : caiman object with the options for motion correction
agonia_th : float
threshold for detection confidence for each box'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
#load frames in python format (T x X x Y)
#%% restart cluster to clean up memory
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=12,
single_thread=False)
# Load the boxes in the pickle file into the ROIs arrays.
with open(boxes_path, 'rb') as f:
cajas = pickle.load(f)
f.close()
cajas = cajas[cajas[:, 4] > agonia_th]
#use the average size of the Agonia boxes as the cell size parameter for Caiman
half_width = np.mean(cajas[:, 2] - cajas[:, 0]) / 2
half_height = np.mean(cajas[:, 3] - cajas[:, 1]) / 2
gSig = (half_width.astype(int), half_height.astype(int))
opts_dict = {'gSig': gSig}
opts.change_params(opts_dict)
#Build masks with the Agonia boxes
Ain = np.zeros((np.prod(images.shape[1:]), cajas.shape[0]), dtype=bool)
for i, box in enumerate(cajas):
frame = np.zeros(images.shape[1:])
#note that the coordinates of the boxes are transpose with respect to Caiman objects
frame[box[1].astype('int'):box[3].astype('int'),
box[0].astype('int'):box[2].astype('int')] = 1
Ain[:, i] = frame.flatten('F')
#Run analysis
cnm_seeded = cnmf.CNMF(n_processes, params=opts, dview=dview, Ain=Ain)
try:
print('initiating fit')
cnm_seeded.fit(images)
print('saving results')
#cnm_seeded.estimates.detrend_df_f(quantileMin=50,frames_window=750,detrend_only=False)
cnm_seeded.save(
os.path.join(
data_path,
os.path.splitext(data_name[0])[0] + '_analysis_results.hdf5'))
except:
print('Tenemos un problema...')
cm.stop_server(dview=dview)
def caiman_motion_correct(fnames, opts):
'''do the motion correction of the Caiman pipeline and save results.
Parameters
----------
fnames : string
name of tiff movie
opts : caiman object with the options for motion correction'''
#%% start a cluster for parallel processing (if a cluster already exists it will be closed and a new session will be opened)
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
# first we create a motion correction object with the parameters specified
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
mc.motion_correct(save_movie=True)
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file,
base_name='memmap_',
order='C',
border_to_0=border_to_0,
dview=dview) # exclude borders
cm.stop_server(dview=dview)
def run_fake_expt(self):
"""
Runs the loop over tiffs, chunked by tiff size.
"""
tiff_list = self.make_tiff_groups()
for tiff_group in tiff_list:
self.do_next_group(tiff_group)
#self.do_final_fit()
cm.stop_server(dview=self.dview)
def motion_correct_red(self):
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
self.motion_corrected_images = []
for plane in list(range(self.planes)):
print(f'Starting motion correction plane {plane}')
self.mc = MotionCorrect(self.file_list[plane], dview=dview, **self.opts.get_group('motion'))
self.mc.motion_correct()
self.motion_corrected_images.append(self.mc.total_template_els)
cm.stop_server(dview=dview)
def setup_cluster(self):
"""
Set up the cluster for doing stuff fast.
"""
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
return c, dview, n_processes
def memmap_movie(fnames,load=True):
'''memmap already motion corrected movie using caiman functions.
if load return the memmap shaped ready to fit caiman (using cnm.fit(images))'''
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=12, single_thread=False)
fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C',
border_to_0=0, dview=dview) # exclude borders
cm.stop_server(dview=dview)
if load:
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
return images
# pl.figure()
cnm, Cn2, fname_new = initialize_movie(mc_init, K, gSig, rf, stride, base_name, merge_thresh=1.1,
rval_thr_online=0.9, thresh_fitness_delta_online=-30, thresh_fitness_raw_online=-50,
rval_thr_init=.5, thresh_fitness_delta_init=-20, thresh_fitness_raw_init=-20,
rval_thr_refine=0.995, thresh_fitness_delta_refine=-200, thresh_fitness_raw_refine=-200,
final_frate=2, Npeaks=10, single_thread=False, dview=dview)
cnms.append(cnm)
Cns.append(Cn2)
base_names.append(fname_new)
A, C, b, f, YrA, sn = cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, cnm.sn
crd = plot_contours(scipy.sparse.coo_matrix(A), Cns[-1], thr=0.9)
pl.pause(1)
cm.stop_server()
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
#%% vompute all the Cns
Cns_all = []
templates_all = []
shifts_all = []
corrs_all = []
for (x0, x1, y0, y1, _) in sliding_window_new(m[0], overlaps, strides):
print([x0, y0])
mc, shifts, template, corrs = m[:, x0:x1, y0:y1].copy(
).motion_correct(max_shifts[0], max_shifts[1])
Cns_all.append(mc.local_correlations(
eight_neighbours=True, swap_dim=False))
templates_all.append(shifts)
def main():
pass # For compatibility between running under Spyder and the CLI
c, dview, n_processes =\
cm.cluster.setup_cluster(backend='local', n_processes=None,
single_thread=False)
# %% set up some parameters
fnames = [os.path.join(caiman_datadir(), 'split', 'first3000-ch1.tif'),
os.path.join(caiman_datadir(), 'split', 'second3000-ch1.tif')]
is_patches = True # flag for processing in patches or not
fr = 1.5 # approximate frame rate of data
decay_time = 5.0 # length of transient
if is_patches: # PROCESS IN PATCHES AND THEN COMBINE
rf = 20 # half size of each patch
stride = 4 # overlap between patches
K = 2 # number of components in each patch
else: # PROCESS THE WHOLE FOV AT ONCE
rf = None # setting these parameters to None
stride = None # will run CNMF on the whole FOV
K = 10 # number of neurons expected (in the whole FOV)
gSig = [6, 6] # expected half size of neurons
merge_thresh = 0.80 # merging threshold, max correlation allowed
p = 2 # order of the autoregressive system
gnb = 2 # global background order
params_dict = {'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'rf': rf,
'stride': stride,
'K': K,
'gSig': gSig,
'merge_thr': merge_thresh,
'p': p,
'nb': gnb}
opts = params.CNMFParams(params_dict=params_dict)
# %% Now RUN CaImAn Batch (CNMF)
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
#cnm.estimates.normalize_components()
cnm = cnm.fit_file()
# %% plot contour plots of components
Cn = cm.load(fnames[0], subindices=slice(1000)).local_correlations(swap_dim=False)
cnm.estimates.plot_contours(img=Cn)
# %% load memory mapped file
Yr, dims, T = cm.load_memmap(cnm.mmap_file)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# %% refit
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier (this will pick up only neurons
# and filter out active processes)
min_SNR = 2 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.85 # space correlation threshold (if above this, accept)
use_cnn = False # use the CNN classifier
min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set('quality', {'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': use_cnn,
'min_cnn_thr': min_cnn_thr,
'cnn_lowest': cnn_lowest})
cnm2.estimates.detrend_df_f()
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% visualize selected and rejected components
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
# %% visualize selected components
cnm2.estimates.nb_view_components(images, idx=cnm2.estimates.idx_components, img=Cn)
cnm2.estimates.view_components(images, idx=cnm2.estimates.idx_components_bad, img=Cn)
#%% only select high quality components
cnm2.estimates.select_components(use_object=True)
#%%
cnm2.estimates.plot_contours(img=Cn)
cnm2.estimates.detrend_df_f()
import pickle
f = open("/home/david/zebraHorse/df_f_day55.pkl", "wb")
pickle.dump(cnm2.estimates.F_dff, f)
f.close()
# %% play movie with results (original, reconstructed, amplified residual)
for j in range(10):
cnm2.estimates.play_movie(images, magnification=4.0, frame_range = range(100 * j, 100 * (j + 1)))
#import time
#time.sleep(1000)
# %% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def main():
pass # For compatibility between running under Spyder and the CLI
#%% Select file(s) to be processed (download if not present)
fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fnames = [download_demo(fnames[0])]
#%% First setup some parameters for data and motion correction
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = True
if display_images:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# %% compare with original movie
if display_images:
m_orig = cm.load_movie_chain(fnames)
m_els = cm.load(mc.mmap_file)
ds_ratio = 0.2
moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)], axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
# %% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C',
border_to_0=border_to_0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
# %% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons in pixels
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
ssub = 2 # spatial subsampling during initialization
tsub = 2 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {'fnames': fnames,
'p': p,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub}
opts.change_params(params_dict=opts_dict);
# %% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0). If you want to have
# deconvolution within each patch change params.patch['p_patch'] to a
# nonzero value
#opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% plot contours of found components
Cns = local_correlations_movie_offline(mc.mmap_file[0],
remove_baseline=True, window=1000, stride=1000,
winSize_baseline=100, quantil_min_baseline=10,
dview=dview)
Cn = Cns.max(axis=0)
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
#%% save results
cnm.estimates.Cn = Cn
cnm.save(fname_new[:-5]+'_init.hdf5')
# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # signal to noise ratio for accepting a component
rval_thr = 0.85 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set('quality', {'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% PLOT COMPONENTS
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
# %% VIEW TRACES (accepted and rejected)
if display_images:
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components)
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components_bad)
#%% update object with selected components
cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(img=Cn)
#%%
cnm2.estimates.Cn = Cn
cnm2.save(cnm2.mmap_file[:-4] + 'hdf5')
#%% reconstruct denoised movie (press q to exit)
if display_images:
cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2,
magnification=2,
bpx=border_to_0,
include_bck=False) # background not shown
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def main():
pass # For compatibility between running under Spyder and the CLI
#%%
"""
General parameters
"""
play_movie = 1
plot_extras = 1
plot_extras_cell = 1
compute_mc_metrics = 1
#%% Select file(s) to be processed (download if not present)
"""
Load file
"""
#fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
#if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
# fnames = [download_demo(fnames[0])]
#fnames = ['/home/yuriy/Desktop/Data/rest1_5_9_19_cut.tif']
#f_dir = 'C:\\Users\\rylab_dataPC\\Desktop\\Yuriy\\caiman_data\\short\\'
f_dir = 'G:\\analysis\\190828-calcium_voltage\\soma_dendrites\\pCAG_jREGECO1a_ASAP3_anesth_001\\'
f_name = 'Ch1'
f_ext = 'tif'
fnames = [f_dir + f_name + '.' + f_ext]
#fnames = ['C:/Users/rylab_dataPC/Desktop/Yuriy/caiman_data/rest1_5_9_19_2_cut_ca.hdf5']
#%% First setup some parameters for data and motion correction
"""
Parameters
"""
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 1
#0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
#max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
max_shifts = [6, 6]
# start a new patch for pw-rigid motion correction every x pixels
#strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
strides = [48, 48]
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
if play_movie:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# type "mc."and press TAB to see all interesting associated variables and self. outputs
# interesting outputs
# saved file is mc.fname_tot_els / mc.fname_tot_rig
# mc.x_shifts_els / mc.y_shifts_els: shifts in x/y per frame per patch
# mc.coord_shifts_els: coordinates associated to patches with shifts
# mc.total_template_els: updated template for pw
# mc.total_template_rig: updated template for rigid
# mc.templates_rig: templates for each iteration in rig
#%% # compute metrics for the results (TAKES TIME!!)
if compute_mc_metrics:
# not finished
bord_px_els = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
final_size = np.subtract(
mc.total_template_els.shape,
2 * bord_px_els) # remove pixels in the boundaries
winsize = 100
swap_dim = False
resize_fact_flow = .2 # downsample for computing ROF
tmpl_rig, correlations_orig, flows_orig, norms_orig, crispness_orig = cm.motion_correction.compute_metrics_motion_correction(
fnames[0],
final_size[0],
final_size[1],
swap_dim,
winsize=winsize,
play_flow=False,
resize_fact_flow=resize_fact_flow)
plt.figure()
plt.plot(correlations_orig)
# %% compare with original movie
if play_movie:
m_orig = cm.load_movie_chain(fnames)
m_els = cm.load(mc.mmap_file)
ds_ratio = 0.2
moviehandle = cm.concatenate([
m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)
],
axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
del m_orig
del m_els
if plot_extras:
# plot total template
plt.figure()
plt.imshow(mc.total_template_els)
plt.title('Template after iteration')
# plot x and y corrections
plt.figure()
plt.plot(mc.shifts_rig)
plt.title('Rigid motion correction xy movement')
plt.legend(['x shift', 'y shift'])
plt.xlabel('frames')
# %% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file,
base_name='memmap_',
order='C',
border_to_0=border_to_0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
# %% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
# %% parameters for source extraction and deconvolution
p = 2 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 2 # number of components per patch
gSig = [15, 15] # expected half size of neurons in pixels
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
ssub = 1 # spatial subsampling during initialization
tsub = 1 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {
'fnames': fnames,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub
}
opts.change_params(params_dict=opts_dict)
# %% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
if plot_extras_cell:
num_cell_plot = 51
plt.figure()
plt.plot(cnm.estimates.C[num_cell_plot, :])
plt.title('Temporal component')
plt.legend(['Cell ' + str(num_cell_plot)])
# plot component sptial profile A
# first convert back to dense components
plot_spat_A = cnm.estimates.A[:, num_cell_plot].toarray().reshape(
list(dims))
plt.figure()
plt.imshow(plot_spat_A)
plt.title('Spatial component cell ' + str(num_cell_plot))
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% plot contours of found components
Cn = cm.local_correlations(images, swap_dim=False)
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
if plot_extras:
plt.figure()
plt.imshow(Cn)
plt.title('Local correlations')
# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm.params.change_params({'p': p})
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # signal to noise ratio for accepting a component
rval_thr = 0.90 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set(
'quality', {
'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest
})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% PLOT COMPONENTS
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
plt.suptitle('Component selection: min_SNR=' + str(min_SNR) +
'; rval_thr=' + str(rval_thr) + '; cnn prob range=[' +
str(cnn_lowest) + ' ' + str(cnn_thr) + ']')
# %% VIEW TRACES (accepted and rejected)
if plot_extras:
cnm2.estimates.view_components(images,
img=Cn,
idx=cnm2.estimates.idx_components)
plt.suptitle('Accepted')
cnm2.estimates.view_components(images,
img=Cn,
idx=cnm2.estimates.idx_components_bad)
plt.suptitle('Rejected')
#plt.figure();
#plt.plot(cnm2.estimates.YrA[0,:]+cnm2.estimates.C[0,:])
#
#
#
#
#plt.figure();
#plt.plot(cnm2.estimates.R[0,:]-cnm2.estimates.YrA[0,:]);
#plt.plot();
#plt.show();
#
#
#plt.figure();
#plt.plot(cnm2.estimates.detrend_df_f[1,:])
# these store the good and bad components, and next step sorts them
# cnm2.estimates.idx_components
# cnm2.estimates.idx_components_bad
#%% update object with selected components
#cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(img=Cn)
plt.suptitle("Final results")
#%% Save the mc data as in cmn struct as well
##
#mc_out = dict(
# pw_rigid = mc.pw_rigid,
# fname = mc.fname,
# mmap_file = mc.mmap_file,
# total_template_els = mc.total_template_els,
# total_template_rig = mc.total_template_rig,
# border_nan = mc.border_nan,
# border_to_0 = mc.border_to_0,
# x_shifts_els = mc.x_shifts_els,
# y_shifts_els = mc.y_shifts_els,
# Cn = Cn
# )
#
#
#deepdish.io.save(fnames[0] + '_mc_data.hdf5', mc_out)
#%% reconstruct denoised movie (press q to exit)
if play_movie:
cnm2.estimates.play_movie(images,
q_max=99.9,
gain_res=2,
magnification=2,
bpx=border_to_0,
include_bck=False) # background not shown
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
save_results = True
if save_results:
cnm2.save(fnames[0][:-4] + '_results.hdf5')
print(('using ' + str(n_processes) + ' processes'))
#%% start cluster for efficient computation
single_thread = False
if single_thread:
dview = None
else:
try:
c.close()
except:
print('C was not existing, creating one')
print("Stopping cluster to avoid unnencessary use of memory....")
sys.stdout.flush()
if backend == 'SLURM':
try:
cm.stop_server(is_slurm=True)
except:
print('Nothing to stop')
slurm_script = '/mnt/xfs1/home/agiovann/SOFTWARE/Constrained_NMF/SLURM/slurmStart.sh'
cm.start_server(slurm_script=slurm_script)
pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE']
c = Client(ipython_dir=pdir, profile=profile)
else:
cm.stop_server()
cm.start_server()
c = Client()
print(('Using ' + str(len(c)) + ' processes'))
dview = c[:len(c)]
#%% set parameters and create template by rigid motion correction
t1 = time.time()
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters
# dataset dependent parameters
display_images = False # Set to true to show movies and images
fnames = ['data_endoscope.tif'] # filename to be processed
frate = 10 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
do_motion_correction_nonrigid = True
do_motion_correction_rigid = False # in this case it will also save a rigid motion corrected movie
gSig_filt = (3, 3) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (5, 5) # maximum allowed rigid shift
splits_rig = 10 # for parallelization split the movies in num_splits chuncks across time
strides = (
48, 48
) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (24, 24
) # overlap between pathes (size of patch strides+overlaps)
# for parallelization split the movies in num_splits chuncks across time
# (remember that it should hold that length_movie/num_splits_to_process_rig>100)
splits_els = 10
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None
gSig = 3 # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = 13 # average diameter of a neuron, in general 4*gSig+1
merge_thresh = .7 # merging threshold, max correlation allowed
rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80
stride_cnmf = 20 # amount of overlap between the patches in pixels
# (keep it at least large as gSiz, i.e 4 times the neuron size gSig)
tsub = 2 # downsampling factor in time for initialization,
# increase if you have memory problems
ssub = 1 # downsampling factor in space for initialization,
# increase if you have memory problems
Ain = None # if you want to initialize with some preselected components
# you can pass them here as boolean vectors
low_rank_background = None # None leaves background of each patch intact,
# True performs global low-rank approximation
gnb = -1 # number of background components (rank) if positive,
# else exact ring model with following settings
# gnb=-2: Return background as b and W
# gnb=-1: Return full rank background B
# gnb= 0: Don't return background
nb_patch = -1 # number of background components (rank) per patch,
# use 0 or -1 for exact background of ring model (cf. gnb)
min_corr = .8 # min peak value from correlation image
min_pnr = 10 # min peak to noise ration from PNR image
ssub_B = 2 # additional downsampling factor in space for background
ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor
# parameters for component evaluation
min_SNR = 3 # adaptive way to set threshold on the transient size
r_values_min = 0.85 # threshold on space consistency (if you lower more components
# will be accepted, potentially with worst quality)
#%% start the cluster
try:
cm.stop_server(dview=dview) # stop it if it was running
except:
pass
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', # use this one
n_processes=
24, # number of process to use, if you go out of memory try to reduce this one
single_thread=False)
#%% download demo file
fnames = [download_demo(fnames[0])]
filename_reorder = fnames
#%% MOTION CORRECTION
if do_motion_correction_nonrigid or do_motion_correction_rigid:
# do motion correction rigid
mc = motion_correct_oneP_rigid(
fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
dview=dview,
splits_rig=splits_rig,
save_movie=not (do_motion_correction_nonrigid),
border_nan='copy')
new_templ = mc.total_template_rig
plt.subplot(1, 2, 1)
plt.imshow(new_templ) # % plot template
plt.subplot(1, 2, 2)
plt.plot(mc.shifts_rig) # % plot rigid shifts
plt.legend(['x shifts', 'y shifts'])
plt.xlabel('frames')
plt.ylabel('pixels')
# borders to eliminate from movie because of motion correction
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
filename_reorder = mc.fname_tot_rig
# do motion correction nonrigid
if do_motion_correction_nonrigid:
mc = motion_correct_oneP_nonrigid(
fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
strides=strides,
overlaps=overlaps,
splits_els=splits_els,
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
dview=dview,
splits_rig=None,
save_movie=True, # whether to save movie in memory mapped format
new_templ=new_templ, # template to initialize motion correction
border_nan='copy')
filename_reorder = mc.fname_tot_els
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
# create memory mappable file in the right order on the hard drive (C order)
fname_new = cm.save_memmap(filename_reorder,
base_name='memmap_',
order='C',
border_to_0=bord_px,
dview=dview)
# load memory mappable file
Yr, dims, T = cm.load_memmap(fname_new)
Y = Yr.T.reshape((T, ) + dims, order='F')
#%% compute some summary images (correlation and peak to noise)
# change swap dim if output looks weird, it is a problem with tiffile
cn_filter, pnr = cm.summary_images.correlation_pnr(Y,
gSig=gSig,
swap_dim=False)
# inspect the summary images and set the parameters
inspect_correlation_pnr(cn_filter, pnr)
# print parameters set above, modify them if necessary based on summary images
print(min_corr) # min correlation of peak (from correlation image)
print(min_pnr) # min peak to noise ratio
#%% RUN CNMF ON PATCHES
cnm = cnmf.CNMF(
n_processes=n_processes,
method_init='corr_pnr', # use this for 1 photon
k=K,
gSig=(gSig, gSig),
gSiz=(gSiz, gSiz),
merge_thresh=merge_thresh,
p=p,
dview=dview,
tsub=tsub,
ssub=ssub,
Ain=Ain,
rf=rf,
stride=stride_cnmf,
only_init_patch=True, # just leave it as is
gnb=gnb,
nb_patch=nb_patch,
method_deconvolution='oasis', # could use 'cvxpy' alternatively
low_rank_background=low_rank_background,
update_background_components=
True, # sometimes setting to False improve the results
min_corr=min_corr,
min_pnr=min_pnr,
normalize_init=False, # just leave as is
center_psf=True, # leave as is for 1 photon
ssub_B=ssub_B,
ring_size_factor=ring_size_factor,
del_duplicates=True, # whether to remove duplicates from initialization
border_pix=bord_px) # number of pixels to not consider in the borders
cnm.fit(Y)
#%% DISCARD LOW QUALITY COMPONENTS
cnm.evaluate_components(Y,
min_SNR=min_SNR,
rval_thr=r_values_min,
use_cnn=False,
decay_time=decay_time,
fr=frate)
print(' ***** ')
print('Number of total components: ', len(cnm.C))
print('Number of accepted components: ', len(cnm.idx_components))
#%% PLOT COMPONENTS
cnm.dims = dims
if display_images:
cnm.plot_contours(img=cn_filter, idx=cnm.idx_components)
cnm.view_components(Y, dims, idx=cnm.idx_components)
#%% MOVIES
if display_images:
# fully reconstructed movie
cnm.play_movie(Y, magnification=3, include_bck=True, gain_res=10)
# movie without background
cnm.play_movie(Y, magnification=3, include_bck=False, gain_res=4)
#%% STOP SERVER
cm.stop_server(dview=dview)
def perform_cnmf(video, params, roi_spatial_footprints, roi_temporal_footprints, roi_temporal_residuals, bg_spatial_footprints, bg_temporal_footprints, use_multiprocessing=True):
if use_multiprocessing:
backend = 'multiprocessing'
cm.stop_server()
c, dview, n_processes = cm.cluster.setup_cluster(backend=backend, n_processes=None, single_thread=False)
else:
dview = None
# print("~")
# print(roi_spatial_footprints.shape)
video_path = "video_temp.tif"
tifffile.imsave(video_path, video)
# dataset dependent parameters
fnames = [video_path] # filename to be processed
fr = params['imaging_fps'] # imaging rate in frames per second
decay_time = params['decay_time'] # length of a typical transient in seconds
# parameters for source extraction and deconvolution
p = params['autoregressive_order'] # order of the autoregressive system
gnb = params['num_bg_components'] # number of global background components
merge_thresh = params['merge_threshold'] # merging threshold, max correlation allowed
rf = None # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = roi_spatial_footprints.shape[1] # number of components per patch
gSig = [params['half_size'], params['half_size']] # expected half size of neurons
init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')
rolling_sum = True
rolling_length = 50
is_dendrites = False # flag for analyzing dendritic data
alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF
# parameters for component evaluation
min_SNR = params['min_snr'] # signal to noise ratio for accepting a component
rval_thr = params['min_spatial_corr'] # space correlation threshold for accepting a component
cnn_thr = params['cnn_threshold'] # threshold for CNN based classifier
border_pix = 0
fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C') # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
cnm = cnmf.CNMF(n_processes=8, k=K, gSig=gSig, merge_thresh= merge_thresh,
p = p, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1,
method_init=init_method, alpha_snmf=alpha_snmf, rolling_sum=rolling_sum,
only_init_patch = True, skip_refinement=True, gnb = gnb, border_pix = border_pix, ssub=1, ssub_B=1, tsub=1, Ain=roi_spatial_footprints, Cin=roi_temporal_footprints, b_in=bg_spatial_footprints, f_in=bg_temporal_footprints, do_merge=False)
cnm = cnm.fit(images)
roi_spatial_footprints = cnm.A
roi_temporal_footprints = cnm.C
roi_temporal_residuals = cnm.YrA
bg_spatial_footprints = cnm.b
bg_temporal_footprints = cnm.f
if use_multiprocessing:
if backend == 'multiprocessing':
dview.close()
else:
try:
dview.terminate()
except:
dview.shutdown()
cm.stop_server()
return roi_spatial_footprints, roi_temporal_footprints, roi_temporal_residuals, bg_spatial_footprints, bg_temporal_footprints
print(('using ' + str(n_processes) + ' processes'))
#%% start cluster for efficient computation
single_thread = False
if single_thread:
dview = None
else:
try:
c.close()
except:
print('C was not existing, creating one')
print("Stopping cluster to avoid unnencessary use of memory....")
sys.stdout.flush()
if backend == 'SLURM':
try:
cm.stop_server(is_slurm=True)
except:
print('Nothing to stop')
slurm_script = '/mnt/xfs1/home/agiovann/SOFTWARE/Constrained_NMF/SLURM/slurmStart.sh'
cm.start_server(slurm_script=slurm_script)
pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE']
c = Client(ipython_dir=pdir, profile=profile)
else:
cm.stop_server()
cm.start_server()
c = Client()
print(('Using ' + str(len(c)) + ' processes'))
dview = c[:len(c)]
#%% FOR LOADING ALL TIFF FILES IN A FILE AND SAVING THEM ON A SINGLE MEMORY MAPPABLE FILE
def find_rois_cnmf(video_path, params, mc_borders=None, use_multiprocessing=True):
full_video_path = video_path
directory = os.path.dirname(full_video_path)
filename = os.path.basename(full_video_path)
memmap_video = tifffile.memmap(video_path)
print(memmap_video.shape)
if len(memmap_video.shape) == 5:
num_z = memmap_video.shape[2]
else:
num_z = memmap_video.shape[1]
roi_spatial_footprints = [ None for i in range(num_z) ]
roi_temporal_footprints = [ None for i in range(num_z) ]
roi_temporal_residuals = [ None for i in range(num_z) ]
bg_spatial_footprints = [ None for i in range(num_z) ]
bg_temporal_footprints = [ None for i in range(num_z) ]
# Create the cluster
if use_multiprocessing:
if os.name == 'nt':
backend = 'multiprocessing'
else:
backend = 'ipyparallel'
cm.stop_server()
c, dview, n_processes = cm.cluster.setup_cluster(backend=backend, n_processes=None, single_thread=False)
else:
dview = None
for z in range(num_z):
fname = os.path.splitext(filename)[0] + "_masked_z_{}.tif".format(z)
video_path = os.path.join(directory, fname)
if len(memmap_video.shape) == 5:
tifffile.imsave(video_path, memmap_video[:, :, z, :, :].reshape((-1, memmap_video.shape[3], memmap_video.shape[4])))
else:
tifffile.imsave(video_path, memmap_video[:, z, :, :])
# dataset dependent parameters
fnames = [video_path] # filename to be processed
fr = params['imaging_fps'] # imaging rate in frames per second
decay_time = params['decay_time'] # length of a typical transient in seconds
# parameters for source extraction and deconvolution
p = params['autoregressive_order'] # order of the autoregressive system
gnb = params['num_bg_components'] # number of global background components
merge_thresh = params['merge_threshold'] # merging threshold, max correlation allowed
rf = None # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = params['num_components'] # number of components per patch
gSig = [params['half_size'], params['half_size']] # expected half size of neurons
init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')
rolling_sum = True
rolling_length = 50
is_dendrites = False # flag for analyzing dendritic data
alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF
# parameters for component evaluation
min_SNR = params['min_snr'] # signal to noise ratio for accepting a component
rval_thr = params['min_spatial_corr'] # space correlation threshold for accepting a component
cnn_thr = params['cnn_threshold'] # threshold for CNN based classifier
if mc_borders is not None:
border_pix = mc_borders[z]
else:
border_pix = 0
fname_new = cm.save_memmap(fnames, base_name='memmap_z_{}'.format(z), order='C') # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
cnm = cnmf.CNMF(n_processes=1, k=K, gSig=gSig, merge_thresh= merge_thresh,
p = p, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1,
method_init=init_method, alpha_snmf=alpha_snmf, rolling_sum=rolling_sum,
only_init_patch = False, gnb = gnb, border_pix = border_pix, ssub=1, ssub_B=1, tsub=1)
cnm = cnm.fit(images)
try:
A_in, C_in, b_in, f_in = cnm.A, cnm.C, cnm.b, cnm.f
except:
A_in, C_in, b_in, f_in = cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f
cnm2 = cnmf.CNMF(n_processes=1, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview,
merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in = b_in,
f_in=f_in, rf = None, stride = None, gnb = gnb,
method_deconvolution='oasis', check_nan = True)
cnm2 = cnm2.fit(images)
try:
roi_spatial_footprints[z] = cnm2.A
roi_temporal_footprints[z] = cnm2.C
roi_temporal_residuals[z] = cnm2.YrA
bg_spatial_footprints[z] = cnm2.b
bg_temporal_footprints[z] = cnm2.f
except:
roi_spatial_footprints[z] = cnm2.estimates.A
roi_temporal_footprints[z] = cnm2.estimates.C
roi_temporal_residuals[z] = cnm2.estimates.YrA
bg_spatial_footprints[z] = cnm2.estimates.b
bg_temporal_footprints[z] = cnm2.estimates.f
if os.path.exists(video_path):
os.remove(video_path)
del memmap_video
if use_multiprocessing:
if backend == 'multiprocessing':
dview.close()
else:
try:
dview.terminate()
except:
dview.shutdown()
cm.stop_server()
mmap_files = glob.glob(os.path.join(directory, "*.mmap"))
for mmap_file in mmap_files:
try:
os.remove(mmap_file)
except:
pass
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
return roi_spatial_footprints, roi_temporal_footprints, roi_temporal_residuals, bg_spatial_footprints, bg_temporal_footprints
def demix_and_deconvolve_with_cnmf(scan, num_components=200, AR_order=2,
merge_threshold=0.8, num_processes=20,
num_pixels_per_process=5000, block_size=10000,
num_background_components=4, init_method='greedy_roi',
soma_radius=(5, 5), snmf_alpha=None,
init_on_patches=False, patch_downsampling_factor=None,
percentage_of_patch_overlap=None):
""" Extract spike train activity from multi-photon scans using CNMF.
Uses constrained non-negative matrix factorization to find neurons/components
(locations) and their fluorescence traces (activity) in a timeseries of images, and
deconvolves them using an autoregressive model of the calcium impulse response
function. See Pnevmatikakis et al., 2016 for details.
Default values work alright for somatic images.
:param np.array scan: 3-dimensional scan (image_height, image_width, num_frames).
:param int num_components: An estimate of neurons/spatial components in the scan.
:param int AR_order: Order of the autoregressive process used to model the impulse
response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay.
:param int merge_threshold: Maximal temporal correlation allowed between activity of
overlapping components before merging them.
:param int num_processes: Number of processes to run in parallel. None for as many
processes as available cores.
:param int num_pixels_per_process: Number of pixels that a process handles each
iteration.
:param int block_size: 'number of pixels to process at the same time for dot product'
:param int num_background_components: Number of background components to use.
:param string init_method: Initialization method for the components.
'greedy_roi':Look for a gaussian-shaped patch, apply rank-1 NMF, store components,
calculate residual scan and repeat for num_components.
'sparse_nmf': Regularized non-negative matrix factorization (as impl. in sklearn)
'local_nmf': ...
:param (float, float) soma_radius: Estimated neuron radius (in pixels) in y and x.
Used in'greedy_roi' initialization to define the size of the gaussian window.
:param int snmf_alpha: Regularization parameter (alpha) for the sparse NMF (if used).
:param bool init_on_patches: If True, run the initialization methods on small patches
of the scan rather than on the whole image.
:param int patch_downsampling_factor: Division to the image dimensions to obtain patch
dimensions, e.g., if original size is 256 and factor is 10, patches will be 26x26
:param int percentage_of_patch_overlap: Patches are sampled in a sliding window. This
controls how much overlap is between adjacent patches (0 for none, 0.9 for 90%)
:returns Location matrix (image_height x image_width x num_components). Inferred
location of each component.
:returns Activity matrix (num_components x num_frames). Inferred fluorescence traces
(spike train convolved with the fitted impulse response function).
:returns: Inferred location matrix for background components (image_height x
image_width x num_background_components).
:returns: Inferred activity matrix for background components (image_height x
image_width x num_background_components).
:returns: Raw fluorescence traces (num_components x num_frames) obtained from the
scan minus activity from background and other components.
:returns: Spike matrix (num_components x num_frames). Deconvolved spike activity.
:returns: Autoregressive process coefficients (num_components x AR_order) used to
model the calcium impulse response of each component:
c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ...
..note:: Based on code provided by Andrea Giovanucci.
..note:: The produced number of components is not exactly what you ask for because
some components will be merged or deleted.
..warning:: Computation- and memory-intensive for big scans.
"""
import caiman
from caiman.source_extraction.cnmf import cnmf
# Save as memory mapped file in F order (that's how caiman wants it)
mmap_filename = _save_as_memmap(scan, base_name='/tmp/caiman', order='F').filename
# 'Load' scan
mmap_scan, (image_height, image_width), num_frames = caiman.load_memmap(mmap_filename)
images = np.reshape(mmap_scan.T, (num_frames, image_height, image_width), order='F')
# Start the ipyparallel cluster
client, direct_view, num_processes = caiman.cluster.setup_cluster(
n_processes=num_processes)
# Optionally, run the initialization method in small patches to initialize components
initial_A = None
initial_C = None
initial_f = None
if init_on_patches:
# Calculate patch size (only square patches allowed)
bigger_dimension = max(image_height, image_width)
smaller_dimension = min(image_height, image_width)
patch_size = bigger_dimension / patch_downsampling_factor
patch_size = min(patch_size, smaller_dimension) # if bigger than small dimension
# Calculate num_components_per_patch
num_nonoverlapping_patches = (image_height/patch_size) * (image_width/patch_size)
num_components_per_patch = num_components / num_nonoverlapping_patches
num_components_per_patch = max(num_components_per_patch, 1) # at least 1
# Calculate patch overlap in pixels
overlap_in_pixels = patch_size * percentage_of_patch_overlap
# Make sure they are integers
patch_size = int(round(patch_size))
num_components_per_patch = int(round(num_components_per_patch))
overlap_in_pixels = int(round(overlap_in_pixels))
# Run CNMF on patches (only for initialization, no impulse response modelling p=0)
model = cnmf.CNMF(num_processes, only_init_patch=True, p=0,
rf=int(round(patch_size / 2)), stride=overlap_in_pixels,
k=num_components_per_patch, merge_thresh=merge_threshold,
method_init=init_method, gSig=soma_radius,
alpha_snmf=snmf_alpha, gnb=num_background_components,
n_pixels_per_process=num_pixels_per_process,
block_size=block_size, check_nan=False, dview=direct_view,
method_deconvolution='cvxpy')
model = model.fit(images)
# Delete log files (one per patch)
log_files = glob.glob('caiman*_LOG_*')
for log_file in log_files:
os.remove(log_file)
# Get results
initial_A = model.A
initial_C = model.C
initial_f = model.f
# Run CNMF
model = cnmf.CNMF(num_processes, k=num_components, p=AR_order,
merge_thresh=merge_threshold, gnb=num_background_components,
method_init=init_method, gSig=soma_radius, alpha_snmf=snmf_alpha,
n_pixels_per_process=num_pixels_per_process, block_size=block_size,
check_nan=False, dview=direct_view, Ain=initial_A, Cin=initial_C,
f_in=initial_f, method_deconvolution='cvxpy')
model = model.fit(images)
# Get final results
location_matrix = model.A # pixels x num_components
activity_matrix = model.C # num_components x num_frames
background_location_matrix = model.b # pixels x num_background_components
background_activity_matrix = model.f # num_background_components x num_frames
spikes = model.S # num_components x num_frames, spike_ traces
raw_traces = model.C + model.YrA # num_components x num_frames
AR_coefficients = model.g # AR_order x num_components
# Reshape spatial matrices to be image_height x image_width x num_frames
new_shape = (image_height, image_width, -1)
location_matrix = location_matrix.toarray().reshape(new_shape, order='F')
background_location_matrix = background_location_matrix.reshape(new_shape, order='F')
AR_coefficients = np.array(list(AR_coefficients)) # unwrapping it (num_components x 2)
# Stop ipyparallel cluster
client.close()
caiman.stop_server()
# Delete memory mapped scan
os.remove(mmap_filename)
return (location_matrix, activity_matrix, background_location_matrix,
background_activity_matrix, raw_traces, spikes, AR_coefficients)
def main():
pass # For compatibility between running under Spyder and the CLI
# %% start the cluster
try:
cm.stop_server() # stop it if it was running
except ():
pass
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local',
n_processes=
24, # number of process to use, if you go out of memory try to reduce this one
single_thread=False)
# %% First setup some parameters for motion correction
# dataset dependent parameters
fnames = ['data_endoscope.tif'] # filename to be processed
fnames = [download_demo(fnames[0])] # download file if not already present
filename_reorder = fnames
fr = 10 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
motion_correct = True # flag for motion correction
pw_rigid = False # flag for pw-rigid motion correction
gSig_filt = (3, 3) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (5, 5) # maximum allowed rigid shift
strides = (
48, 48
) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (24, 24
) # overlap between pathes (size of patch strides+overlaps)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
border_nan = 'copy'
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'gSig_filt': gSig_filt,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': border_nan
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% MOTION CORRECTION
# The pw_rigid flag set above, determines where to use rigid or pw-rigid
# motion correction
if motion_correct:
# do motion correction rigid
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
mc.motion_correct(save_movie=True)
fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig
if pw_rigid:
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
else:
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
plt.subplot(1, 2, 1)
plt.imshow(mc.total_template_rig) # % plot template
plt.subplot(1, 2, 2)
plt.plot(mc.shifts_rig) # % plot rigid shifts
plt.legend(['x shifts', 'y shifts'])
plt.xlabel('frames')
plt.ylabel('pixels')
bord_px = 0 if border_nan == 'copy' else bord_px
fname_new = cm.save_memmap(fname_mc,
base_name='memmap_',
order='C',
border_to_0=bord_px)
else: # if no motion correction just memory map the file
fname_new = cm.save_memmap(filename_reorder,
base_name='memmap_',
order='C',
border_to_0=0,
dview=dview)
# load memory mappable file
Yr, dims, T = cm.load_memmap(fname_new)
images = Yr.T.reshape((T, ) + dims, order='F')
# %% Parameters for source extraction and deconvolution (CNMF-E algorithm)
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None for 1p data
gSig = (
3, 3
) # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1
Ain = None # possibility to seed with predetermined binary masks
merge_thr = .7 # merging threshold, max correlation allowed
rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80
stride_cnmf = 20 # amount of overlap between the patches in pixels
# (keep it at least large as gSiz, i.e 4 times the neuron size gSig)
tsub = 2 # downsampling factor in time for initialization,
# increase if you have memory problems
ssub = 1 # downsampling factor in space for initialization,
# increase if you have memory problems
# you can pass them here as boolean vectors
low_rank_background = None # None leaves background of each patch intact,
# True performs global low-rank approximation if gnb>0
gnb = 0 # number of background components (rank) if positive,
# else exact ring model with following settings
# gnb= 0: Return background as b and W
# gnb=-1: Return full rank background B
# gnb<-1: Don't return background
nb_patch = 0 # number of background components (rank) per patch if gnb>0,
# else it is set automatically
min_corr = .8 # min peak value from correlation image
min_pnr = 10 # min peak to noise ration from PNR image
ssub_B = 2 # additional downsampling factor in space for background
ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor
opts.change_params(
params_dict={
'dims': dims,
'method_init': 'corr_pnr', # use this for 1 photon
'K': K,
'gSig': gSig,
'gSiz': gSiz,
'merge_thr': merge_thr,
'p': p,
'tsub': tsub,
'ssub': ssub,
'rf': rf,
'stride': stride_cnmf,
'only_init': True, # set it to True to run CNMF-E
'nb': gnb,
'nb_patch': nb_patch,
'method_deconvolution': 'oasis', # could use 'cvxpy' alternatively
'low_rank_background': low_rank_background,
'update_background_components':
True, # sometimes setting to False improve the results
'min_corr': min_corr,
'min_pnr': min_pnr,
'normalize_init': False, # just leave as is
'center_psf': True, # leave as is for 1 photon
'ssub_B': ssub_B,
'ring_size_factor': ring_size_factor,
'del_duplicates':
True, # whether to remove duplicates from initialization
'border_pix': bord_px
}) # number of pixels to not consider in the borders)
# %% compute some summary images (correlation and peak to noise)
# change swap dim if output looks weird, it is a problem with tiffile
cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1],
gSig=gSig[0],
swap_dim=False)
# if your images file is too long this computation will take unnecessarily
# long time and consume a lot of memory. Consider changing images[::1] to
# images[::5] or something similar to compute on a subset of the data
# inspect the summary images and set the parameters
inspect_correlation_pnr(cn_filter, pnr)
# print parameters set above, modify them if necessary based on summary images
print(min_corr) # min correlation of peak (from correlation image)
print(min_pnr) # min peak to noise ratio
# %% RUN CNMF ON PATCHES
cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=opts)
cnm.fit(images)
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% DISCARD LOW QUALITY COMPONENTS
min_SNR = 2.5 # adaptive way to set threshold on the transient size
r_values_min = 0.85 # threshold on space consistency (if you lower more components
# will be accepted, potentially with worst quality)
cnm.params.set('quality', {
'min_SNR': min_SNR,
'rval_thr': r_values_min,
'use_cnn': False
})
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
print(' ***** ')
print('Number of total components: ', len(cnm.estimates.C))
print('Number of accepted components: ', len(cnm.estimates.idx_components))
# %% PLOT COMPONENTS
cnm.dims = dims
display_images = True # Set to true to show movies and images
if display_images:
cnm.estimates.plot_contours(img=cn_filter,
idx=cnm.estimates.idx_components)
cnm.estimates.view_components(images, idx=cnm.estimates.idx_components)
# %% MOVIES
display_images = False # Set to true to show movies and images
if display_images:
# fully reconstructed movie
cnm.estimates.play_movie(images,
q_max=99.5,
magnification=2,
include_bck=True,
gain_res=10,
bpx=bord_px)
# movie without background
cnm.estimates.play_movie(images,
q_max=99.9,
magnification=2,
include_bck=False,
gain_res=4,
bpx=bord_px)
# %% STOP SERVER
cm.stop_server(dview=dview)
def main():
pass # For compatibility between running under Spyder and the CLI
# %% start a cluster
c, dview, n_processes =\
cm.cluster.setup_cluster(backend='local', n_processes=None,
single_thread=False)
# %% set up some parameters
fnames = [
os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif')
]
# file(s) to be analyzed
is_patches = True # flag for processing in patches or not
fr = 10 # approximate frame rate of data
decay_time = 5.0 # length of transient
if is_patches: # PROCESS IN PATCHES AND THEN COMBINE
rf = 10 # half size of each patch
stride = 4 # overlap between patches
K = 4 # number of components in each patch
else: # PROCESS THE WHOLE FOV AT ONCE
rf = None # setting these parameters to None
stride = None # will run CNMF on the whole FOV
K = 30 # number of neurons expected (in the whole FOV)
gSig = [6, 6] # expected half size of neurons
merge_thresh = 0.80 # merging threshold, max correlation allowed
p = 2 # order of the autoregressive system
gnb = 2 # global background order
params_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'rf': rf,
'stride': stride,
'K': K,
'gSig': gSig,
'merge_thr': merge_thresh,
'p': p,
'nb': gnb
}
opts = params.CNMFParams(params_dict=params_dict)
# %% Now RUN CaImAn Batch (CNMF)
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit_file()
# %% plot contour plots of components
Cns = local_correlations_movie_offline(fnames[0],
remove_baseline=True,
swap_dim=False,
window=1000,
stride=1000,
winSize_baseline=100,
quantil_min_baseline=10,
dview=dview)
Cn = Cns.max(axis=0)
cnm.estimates.plot_contours(img=Cn)
# %% load memory mapped file
Yr, dims, T = cm.load_memmap(cnm.mmap_file)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# %% refit
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier (this will pick up only neurons
# and filter out active processes)
min_SNR = 2 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.85 # space correlation threshold (if above this, accept)
use_cnn = True # use the CNN classifier
min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set(
'quality', {
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': use_cnn,
'min_cnn_thr': min_cnn_thr,
'cnn_lowest': cnn_lowest
})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% visualize selected and rejected components
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
# %% visualize selected components
cnm2.estimates.view_components(images,
idx=cnm2.estimates.idx_components,
img=Cn)
#%% only select high quality components (destructive)
# cnm2.estimates.select_components(use_object=True)
# cnm2.estimates.plot_contours(img=Cn)
#%% save results
cnm2.estimates.Cn = Cn
cnm2.save(cnm2.mmap_file[:-4] + 'hdf5')
# %% play movie with results (original, reconstructed, amplified residual)
cnm2.estimates.play_movie(images, magnification=4)
# %% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters
# dataset dependent parameters
display_images = False # Set this to true to show movies and plots
fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
niter_rig = 1 # number of iterations for rigid motion correction
max_shifts = (6, 6) # maximum allow rigid shift
# for parallelization split the movies in num_splits chuncks across time
splits_rig = 56
# start a new patch for pw-rigid motion correction every x pixels
strides = (48, 48)
# overlap between pathes (size of patch strides+overlaps)
overlaps = (24, 24)
# for parallelization split the movies in num_splits chuncks across time
splits_els = 56
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thresh = 0.8 # merging threshold, max correlation allowed
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
rf = 15
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons
# initialization method (if analyzing dendritic data using 'sparse_nmf')
init_method = 'greedy_roi'
is_dendrites = False # flag for analyzing dendritic data
# sparsity penalty for dendritic data analysis through sparse NMF
alpha_snmf = None
# parameters for component evaluation
min_SNR = 2.5 # signal to noise ratio for accepting a component
rval_thr = 0.8 # space correlation threshold for accepting a component
cnn_thr = 0.8 # threshold for CNN based classifier
#%% download the dataset if it's not present in your folder
if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fname = [download_demo(fname[0])]
#%% play the movie
# playing the movie using opencv. It requires loading the movie in memory. To
# close the video press q
m_orig = cm.load_movie_chain(fname[:1])
downsample_ratio = 0.2
offset_mov = -np.min(m_orig[:100])
moviehandle = m_orig.resize(1, 1, downsample_ratio)
if display_images:
moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2)
#%% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
#%%% MOTION CORRECTION
# first we create a motion correction object with the parameters specified
min_mov = cm.load(fname[0], subindices=range(200)).min()
# this will be subtracted from the movie to make it non-negative
mc = MotionCorrect(fname[0],
min_mov,
dview=dview,
max_shifts=max_shifts,
niter_rig=niter_rig,
splits_rig=splits_rig,
strides=strides,
overlaps=overlaps,
splits_els=splits_els,
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
shifts_opencv=True,
nonneg_movie=True)
# note that the file is not loaded in memory
#%% Run piecewise-rigid motion correction using NoRMCorre
mc.motion_correct_pwrigid(save_movie=True)
m_els = cm.load(mc.fname_tot_els)
bord_px_els = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
# maximum shift to be used for trimming against NaNs
#%% compare with original movie
moviehandle = cm.concatenate([
m_orig.resize(1, 1, downsample_ratio) + offset_mov,
m_els.resize(1, 1, downsample_ratio)
],
axis=2)
display_images = False
if display_images:
moviehandle.play(fr=60, q_max=99.5, magnification=2,
offset=0) # press q to exit
#%% MEMORY MAPPING
# memory map the file in order 'C'
fnames = mc.fname_tot_els # name of the pw-rigidly corrected file.
border_to_0 = bord_px_els # number of pixels to exclude
fname_new = cm.save_memmap(fnames,
base_name='memmap_',
order='C',
border_to_0=bord_px_els) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
#%% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
#%% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
t1 = time.time()
cnm = cnmf.CNMF(n_processes=1,
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=0,
dview=dview,
rf=rf,
stride=stride_cnmf,
memory_fact=1,
method_init=init_method,
alpha_snmf=alpha_snmf,
only_init_patch=False,
gnb=gnb,
border_pix=bord_px_els)
cnm = cnm.fit(images)
#%% plot contours of found components
Cn = cm.local_correlations(images.transpose(1, 2, 0))
Cn[np.isnan(Cn)] = 0
plt.figure()
crd = plot_contours(cnm.A, Cn, thr=0.9)
plt.title('Contour plots of found components')
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=False,
thresh_cnn_min=cnn_thr)
#%% PLOT COMPONENTS
if display_images:
plt.figure()
plt.subplot(121)
crd_good = cm.utils.visualization.plot_contours(cnm.A[:,
idx_components],
Cn,
thr=.8,
vmax=0.75)
plt.title('Contour plots of accepted components')
plt.subplot(122)
crd_bad = cm.utils.visualization.plot_contours(
cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75)
plt.title('Contour plots of rejected components')
#%% VIEW TRACES (accepted and rejected)
if display_images:
view_patches_bar(Yr,
cnm.A.tocsc()[:, idx_components],
cnm.C[idx_components],
cnm.b,
cnm.f,
dims[0],
dims[1],
YrA=cnm.YrA[idx_components],
img=Cn)
view_patches_bar(Yr,
cnm.A.tocsc()[:, idx_components_bad],
cnm.C[idx_components_bad],
cnm.b,
cnm.f,
dims[0],
dims[1],
YrA=cnm.YrA[idx_components_bad],
img=Cn)
#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[
idx_components], cnm.b, cnm.f
cnm2 = cnmf.CNMF(n_processes=1,
k=A_in.shape[-1],
gSig=gSig,
p=p,
dview=dview,
merge_thresh=merge_thresh,
Ain=A_in,
Cin=C_in,
b_in=b_in,
f_in=f_in,
rf=None,
stride=None,
gnb=gnb,
method_deconvolution='oasis',
check_nan=True)
cnm2 = cnm2.fit(images)
#%% Extract DF/F values
F_dff = detrend_df_f(cnm2.A,
cnm2.b,
cnm2.C,
cnm2.f,
YrA=cnm2.YrA,
quantileMin=8,
frames_window=250)
#%% Show final traces
cnm2.view_patches(Yr, dims=dims, img=Cn)
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
#%% reconstruct denoised movie
denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape(
dims + (-1, ), order='F').transpose([2, 0, 1])
#%% play along side original data
moviehandle = cm.concatenate([
m_els.resize(1, 1, downsample_ratio),
denoised.resize(1, 1, downsample_ratio)
],
axis=2)
if display_images:
moviehandle.play(fr=60, gain=15, magnification=2,
offset=0) # press q to exit
print(('using ' + str(n_processes) + ' processes'))
#%% start cluster for efficient computation
single_thread = False
if single_thread:
dview = None
else:
try:
c.close()
except:
print('C was not existing, creating one')
print("Stopping cluster to avoid unnencessary use of memory....")
sys.stdout.flush()
if backend == 'SLURM':
try:
cm.stop_server(is_slurm=True)
except:
print('Nothing to stop')
slurm_script = '/mnt/xfs1/home/agiovann/SOFTWARE/Constrained_NMF/SLURM/slurmStart.sh'
cm.start_server(slurm_script=slurm_script)
pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE']
c = Client(ipython_dir=pdir, profile=profile)
else:
cm.stop_server()
cm.start_server()
c = Client()
print(('Using ' + str(len(c)) + ' processes'))
dview = c[:len(c)]
#%% FOR LOADING ALL TIFF FILES IN A FILE AND SAVING THEM ON A SINGLE MEMORY MAPPABLE FILE
def main():
pass # For compatibility between running under Spyder and the CLI
#%% start a cluster
c, dview, n_processes =\
cm.cluster.setup_cluster(backend='local', n_processes=None,
single_thread=False)
#%% save files to be processed
# This datafile is distributed with Caiman
fnames = [
os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif')
]
# location of dataset (can actually be a list of filed to be concatenated)
add_to_movie = -np.min(cm.load(fnames[0],
subindices=range(200))).astype(float)
# determine minimum value on a small chunk of data
add_to_movie = np.maximum(add_to_movie, 0)
# if minimum is negative subtract to make the data non-negative
base_name = 'Yr'
fname_new = cm.save_memmap(fnames,
dview=dview,
base_name=base_name,
order='C',
add_to_movie=add_to_movie)
#%% LOAD MEMORY MAPPABLE FILE
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
#%% play movie, press q to quit
play_movie = False
if play_movie:
cm.movie(images[1400:]).play(fr=50, magnification=4, gain=3.)
#%% correlation image. From here infer neuron size and density
Cn = cm.movie(images).local_correlations(swap_dim=False)
plt.imshow(Cn, cmap='gray')
plt.title('Correlation Image')
#%% set up some parameters
is_patches = True # flag for processing in patches or not
if is_patches: # PROCESS IN PATCHES AND THEN COMBINE
rf = 10 # half size of each patch
stride = 4 # overlap between patches
K = 4 # number of components in each patch
else: # PROCESS THE WHOLE FOV AT ONCE
rf = None # setting these parameters to None
stride = None # will run CNMF on the whole FOV
K = 30 # number of neurons expected (in the whole FOV)
gSig = [6, 6] # expected half size of neurons
merge_thresh = 0.80 # merging threshold, max correlation allowed
p = 2 # order of the autoregressive system
gnb = 2 # global background order
#%% Now RUN CNMF
cnm = cnmf.CNMF(n_processes,
method_init='greedy_roi',
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=p,
dview=dview,
gnb=gnb,
rf=rf,
stride=stride,
rolling_sum=False)
cnm = cnm.fit(images)
#%% plot contour plots of components
plt.figure()
crd = cm.utils.visualization.plot_contours(cnm.A, Cn, thr=0.9)
plt.title('Contour plots of components')
#%%
A_in, C_in, b_in, f_in = cnm.A[:, :], cnm.C[:], cnm.b, cnm.f
cnm2 = cnmf.CNMF(n_processes=1,
k=A_in.shape[-1],
gSig=gSig,
p=p,
dview=dview,
merge_thresh=merge_thresh,
Ain=A_in,
Cin=C_in,
b_in=b_in,
f_in=f_in,
rf=None,
stride=None,
gnb=gnb,
method_deconvolution='oasis',
check_nan=True)
cnm2 = cnm2.fit(images)
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier (this will pick up only neurons
# and filter out active processes)
fr = 10 # approximate frame rate of data
decay_time = 5.0 # length of transient
min_SNR = 2.5 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.90 # space correlation threshold (if above this, accept)
use_cnn = True # use the CNN classifier
min_cnn_thr = 0.95 # if cnn classifier predicts below this value, reject
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=use_cnn,
thresh_cnn_min=min_cnn_thr)
#%% visualize selected and rejected components
plt.figure()
plt.subplot(1, 2, 1)
cm.utils.visualization.plot_contours(cnm2.A[:, idx_components],
Cn,
thr=0.9)
plt.title('Selected components')
plt.subplot(1, 2, 2)
plt.title('Discaded components')
cm.utils.visualization.plot_contours(cnm2.A[:, idx_components_bad],
Cn,
thr=0.9)
#%%
plt.figure()
crd = cm.utils.visualization.plot_contours(cnm2.A.tocsc()[:,
idx_components],
Cn,
thr=0.9)
plt.title('Contour plots of components')
#%% visualize selected components
cm.utils.visualization.view_patches_bar(Yr,
cnm2.A.tocsc()[:, idx_components],
cnm2.C[idx_components, :],
cnm2.b,
cnm2.f,
dims[0],
dims[1],
YrA=cnm2.YrA[idx_components, :],
img=Cn)
#%% STOP CLUSTER and clean up log files
cm.stop_server()
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def run(
file_path,
n_cpus,
motion_correct: bool = True,
mc_settings: dict = {},
job_name: str = "job",
output_directory: str = "",
):
mkl = os.environ.get("MKL_NUM_THREADS")
blas = os.environ.get("OPENBLAS_NUM_THREADS")
vec = os.environ.get("VECLIB_MAXIMUM_THREADS")
print(f"MKL: {mkl}")
print(f"blas: {blas}")
print(f"vec: {vec}")
# we import the pipeline upon running so they aren't required for all installs
import caiman as cm
from caiman.motion_correction import MotionCorrect
from caiman.source_extraction.cnmf import params as params
# print the directory caiman is imported from
caiman_path = os.path.abspath(cm.__file__)
print(f"caiman path: {caiman_path}")
sys.stdout.flush()
# setup the logger
logger_file = os.path.join(output_directory, "caiman.log")
logging.basicConfig(
format=LOGGER_FORMAT,
filename=logger_file,
filemode="w",
level=logging.DEBUG,
)
# if indices to perform mcorr are set, format them
if "indices" in mc_settings:
indices = mc_settings["indices"]
indices_formatted = ()
for axis_slice in indices:
start = axis_slice[0]
stop = axis_slice[1]
if len(axis_slice) == 3:
step = axis_slice[2]
else:
step = 1
indices_formatted += (slice(start, stop, step), )
mc_settings["indices"] = indices_formatted
# load and update the pipeline settings
mc_parameters = DEFAULT_MCORR_SETTINGS
for k, v in mc_settings.items():
mc_parameters[k] = v
opts = params.CNMFParams(params_dict=mc_parameters)
# get the filenames
if os.path.isfile(file_path):
print(file_path)
fnames = [file_path]
else:
file_pattern = os.path.join(file_path, "*.tif*")
fnames = sorted(glob.glob(file_pattern))
print(fnames)
opts.set("data", {"fnames": fnames})
if n_cpus > 1:
print("starting server")
# start the server
n_proc = np.max([(n_cpus - 1), 1])
c, dview, n_processes = cm.cluster.setup_cluster(backend="local",
n_processes=n_proc,
single_thread=False)
sleep(30)
else:
print("multiprocessing disabled")
dview = None
n_processes = 1
print(n_processes)
print("starting motion correction")
sys.stdout.flush()
mc = MotionCorrect(fnames, dview=dview, **opts.get_group("motion"))
mc.motion_correct(save_movie=True)
pw_rigid = mc_parameters["pw_rigid"]
fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig
if pw_rigid:
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
else:
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
border_nan = mc_parameters["border_nan"]
bord_px = 0 if border_nan == "copy" else bord_px
_ = cm.save_memmap(fname_mc,
base_name="memmap_",
order="C",
border_to_0=bord_px)
# if motion correction was performed, save the file
# we save as hdf5 for better reading performance
# downstream
if motion_correct:
print("saving motion corrected file")
results_filebase = os.path.join(output_directory, job_name)
mcorr_fname = results_filebase + "_mcorr.hdf5"
dataset_name = opts.data["var_name_hdf5"]
# fnames = opts.data["fnames"]
# memmap_files = []
# for f in fnames:
# if isinstance(f, bytes):
# f = f.decode()
# base_file = os.path.splitext(f)[0]
# if pw_rigid:
# memmap_pattern = base_file + "*_els_*"
# else:
# memmap_pattern = base_file + "*_rig_*"
# memmap_files += sorted(glob.glob(memmap_pattern))
if pw_rigid:
memmap_files = mc.fname_tot_els
else:
memmap_files = mc.fname_tot_rig
# get the frame shape
mov = cm.load(memmap_files[0])
frame_shape = mov.shape[-2::]
write_hdf5_movie(
movie_name=mcorr_fname,
memmap_files=memmap_files,
frame_shape=frame_shape,
dataset_name=dataset_name,
compression="gzip",
)
# save the parameters in the same dir as the results
final_params = opts.to_dict()
params_file = os.path.join(output_directory, "all_mcorr_parameters.pkl")
with open(params_file, "wb") as fp:
pickle.dump(final_params, fp)
# deleting mcorr unused memmap files
if pw_rigid:
memmap_files = mc.fname_tot_els
else:
memmap_files = mc.fname_tot_rig
print(f"deleting {memmap_files}")
for f in memmap_files:
os.remove(f)
print("stopping server")
cm.stop_server(dview=dview)
def main():
pass # For compatibility between running under Spyder and the CLI
# %% download and list all files to be processed
#
## folder inside ./example_movies where files will be saved
#fld_name = 'Mesoscope'
#download_demo('Tolias_mesoscope_1.hdf5', fld_name)
#download_demo('Tolias_mesoscope_2.hdf5', fld_name)
#download_demo('Tolias_mesoscope_3.hdf5', fld_name)
#
## folder where files are located
#folder_name = os.path.join(caiman_datadir(), 'example_movies', fld_name)
#extension = 'hdf5' # extension of files
## read all files to be processed
#fnames = glob.glob(folder_name + '/*' + extension)
#
## your list of files should look something like this
#logging.info(fnames)
#
#
start_t = time.time()
#f_dir = 'E:\data\V1\proc_data\\'
#f_dir = 'E:\\data\\Auditory\\caiman_out_multiplane\\'
#f_dir = 'F:\\data\\Auditory\\caiman_out\\movies\\'
f_dir = 'C:\\Users\\ys2605\\Desktop\\stuff\\AC_data\\caiman_data\\movies\\'
#f_name = 'A2_freq_grating1_10_2_18_OA_cut'
#f_dir = 'C:\\Users\\rylab_dataPC\\Desktop\\Yuriy\\DD_data\\proc_data\\'
#f_name = 'vmmn2_9_16_19a_OA_cut'
#f_name = 'ammn_2_dplanes2_10_14_19_OA_mpl1_cut';
f_name = 'A1_ammn_3plt_2plm2_12_27_20_mpl3_cut'
f_ext = 'hdf5'
fnames = [f_dir + f_name + '.' + f_ext]
# %% Set up some parameters
fr = 15.5455 # frame rate (Hz) 3pl + 4ms = 15.5455
decay_time = 0.5 # 2 for s 0.5 for f # approximate length of transient event in seconds
gSig = (5, 5) # expected half size of neurons
p = 2 # order of AR indicator dynamics
min_SNR = 1 # minimum SNR for accepting new components
ds_factor = 1 # spatial downsampling factor (increases speed but may lose some fine structure)
gnb = 2 # number of background components
gSig = tuple(np.ceil(
np.array(gSig) /
ds_factor).astype('int')) # recompute gSig if downsampling is involved
mot_corr = True # flag for online motion correction
pw_rigid = False # flag for pw-rigid motion correction (slower but potentially more accurate)
max_shifts_online = 6 # maximum allowed shift during motion correction
sniper_mode = True # use a CNN to detect new neurons (o/w space correlation)
rval_thr = 0.9 # soace correlation threshold for candidate components
# set up some additional supporting parameters needed for the algorithm
# (these are default values but can change depending on dataset properties)
init_batch = 1000 # number of frames for initialization (presumably from the first file)
K = 2 # initial number of components
epochs = 2 # number of passes over the data
show_movie = True # show the movie as the data gets processed
merge_thr = 0.8
params_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'gSig': gSig,
'p': p,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'merge_thr': merge_thr,
'ds_factor': ds_factor,
'nb': gnb,
'motion_correct': mot_corr,
'init_batch': init_batch,
'init_method': 'bare',
'normalize': True,
'sniper_mode': sniper_mode,
'K': K,
'epochs': epochs,
'max_shifts_online': max_shifts_online,
'pw_rigid': pw_rigid,
'dist_shape_update': True,
'min_num_trial': 10,
'show_movie': show_movie
}
# 'path_to_model': 'C:\Users\ys2605\Anaconda3\envs\caiman2\share\caiman\model',
opts = cnmf.params.CNMFParams(params_dict=params_dict)
# %% fit online
cnm = cnmf.online_cnmf.OnACID(params=opts)
cnm.fit_online()
# %% plot contours (this may take time)
logging.info('Number of components: ' + str(cnm.estimates.A.shape[-1]))
images = cm.load(fnames)
Cn = images.local_correlations(swap_dim=False, frames_per_chunk=500)
cnm.estimates.plot_contours(img=Cn, display_numbers=False)
# %% view components
cnm.estimates.view_components(img=Cn)
# %% plot timing performance (if a movie is generated during processing, timing
# will be severely over-estimated)
T_motion = 1e3 * np.array(cnm.t_motion)
T_detect = 1e3 * np.array(cnm.t_detect)
T_shapes = 1e3 * np.array(cnm.t_shapes)
T_track = 1e3 * np.array(cnm.t_online) - T_motion - T_detect - T_shapes
plt.figure()
plt.stackplot(np.arange(len(T_motion)), T_motion, T_track, T_detect,
T_shapes)
plt.legend(labels=['motion', 'tracking', 'detect', 'shapes'], loc=2)
plt.title('Processing time allocation')
plt.xlabel('Frame #')
plt.ylabel('Processing time [ms]')
#%% RUN IF YOU WANT TO VISUALIZE THE RESULTS (might take time)
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
if opts.online['motion_correct']:
shifts = cnm.estimates.shifts[-cnm.estimates.C.shape[-1]:]
if not opts.motion['pw_rigid']:
memmap_file = cm.motion_correction.apply_shift_online(
images, shifts, save_base_name=(f_dir + f_name + 'MC'))
else:
mc = cm.motion_correction.MotionCorrect(fnames,
dview=dview,
**opts.get_group('motion'))
mc.x_shifts_els = [[sx[0] for sx in sh] for sh in shifts]
mc.y_shifts_els = [[sx[1] for sx in sh] for sh in shifts]
memmap_file = mc.apply_shifts_movie(fnames,
rigid_shifts=False,
save_memmap=True,
save_base_name=(f_dir +
f_name + 'MC'))
else: # To do: apply non-rigid shifts on the fly
memmap_file = images.save(fnames[0][:-4] + 'mmap')
cnm.mmap_file = memmap_file
Yr, dims, T = cm.load_memmap(memmap_file)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
min_SNR = 2 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.85 # space correlation threshold (if above this, accept)
use_cnn = True # use the CNN classifier
min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm.params.set(
'quality', {
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': use_cnn,
'min_cnn_thr': min_cnn_thr,
'cnn_lowest': cnn_lowest
})
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
cnm.estimates.Cn = Cn
cnm.save(
os.path.splitext(fnames[0])[0] + '_' + str(gSig[0]) +
'gsig_results.hdf5')
dview.terminate()
duration = time.time() - start_t
print(duration / 60)
def run(work_dir: str, UUID: str, save_temp_files: str):
logging.basicConfig(
stream=sys.stdout,
level=logging.DEBUG,
format=
"%(relativeCreated)12d [%(filename)s:%(funcName)20s():%(lineno)s] [%(process)d] %(message)s"
)
start_time = time()
batch_dir = os.environ['CURR_BATCH_DIR']
save_temp_files = bool(int(save_temp_files))
output = {'status': 0, 'output_info': ''}
n_processes = int(os.environ['_MESMERIZE_N_THREADS'])
filepath = os.path.join(work_dir, UUID)
imgpath = f'{filepath}_input.tiff'
input_params = pickle.load(open(f'{filepath}.params', 'rb'))
print('******** Creating process pool *********')
c, dview, n_processes = setup_cluster(backend='local',
n_processes=n_processes,
single_thread=False,
ignore_preexisting=True)
try:
if input_params['use_memmap']:
memmap_uuid = input_params['memmap_uuid']
memmap_batchdir = glob(
os.path.join(batch_dir, f'memmap-{memmap_uuid}*.mmap'))
# Check batch dir
if len(memmap_batchdir) > 0:
memmap_path = memmap_batchdir[0]
print(
f'********** Found existing memmap in batch dir: {memmap_path} ********** '
)
# copy to work dir
if not os.path.samefile(batch_dir, work_dir):
print('**** Copying memmap to work dir ****')
shutil.copy(memmap_path, work_dir)
memmap_path = glob(
os.path.join(work_dir,
f'memmap-{memmap_uuid}*.mmap'))[0]
else:
# remake the memmap with the same UUID so that future batch items can rely on it
print(
'********** Memmap not found, re-making memmap with the same UUID **********'
)
memmap_path = cm.save_memmap([imgpath],
base_name=f'memmap-{memmap_uuid}',
is_3D=True,
order='C',
dview=dview)
else:
print('********** Making memmap **********')
memmap_path = cm.save_memmap([imgpath],
base_name=f'memmap-{UUID}',
is_3D=True,
order='C',
dview=dview)
print(f'Using memmap:\n{memmap_path}')
print('********** Loading memmap **********')
Yr, dims, T = cm.load_memmap(memmap_path)
Y = np.reshape(Yr, dims + (T, ), order='F')
images = np.reshape(Yr.T, [T] + list(dims), order='F')
if input_params['use_patches']:
cnm = cnmf.CNMF(n_processes=n_processes,
dview=dview,
only_init_patch=True,
**input_params['cnmf_kwargs'])
else:
cnm = cnmf.CNMF(n_processes,
dview=dview,
**input_params['cnmf_kwargs'])
cnm.fit(images)
print('Number of components:' + str(cnm.estimates.A.shape[-1]))
cnm.params.change_params(
params_dict={
**input_params['eval_kwargs'], 'use_cnn': False
})
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
if input_params['refit']:
cnm.params.set('temporal', {'p': input_params['cnmf_kwargs']['p']})
cnm_ = cnm.refit(images)
else:
cnm_ = cnm
out_filename = f'{UUID}_results.hdf5'
cnm_.save(out_filename)
output_files = [out_filename]
# Save the memmap
if save_temp_files:
print("***** Keeping memmap file *****")
# copy to batch dir if batch_dir != work_dir
if not os.path.samefile(batch_dir, work_dir):
print("***** Copying memmap file to batch dir *****")
shutil.copy(memmap_path, batch_dir)
# Delete the memmap from the work dir
if not os.path.samefile(batch_dir, work_dir):
print("***** Deleting memmap files from work dir *****")
try:
os.remove(memmap_path)
except:
pass
output.update({
'output': UUID,
'status': 1,
'output_files': output_files,
'saved_memmap': save_temp_files
})
except Exception as e:
output.update({'status': 0, 'output_info': traceback.format_exc()})
cm.stop_server(dview=dview)
end_time = time()
processing_time = (end_time - start_time) / 60
output.update({'processing_time': processing_time})
json.dump(output, open(filepath + '.out', 'w'))
print(('using ' + str(n_processes) + ' processes'))
#%% start cluster for efficient computation
single_thread = False
if single_thread:
dview = None
else:
try:
c.close()
except:
print('C was not existing, creating one')
print("Stopping cluster to avoid unnencessary use of memory....")
sys.stdout.flush()
if backend == 'SLURM':
try:
cm.stop_server(is_slurm=True)
except:
print('Nothing to stop')
slurm_script = '/mnt/xfs1/home/agiovann/SOFTWARE/Constrained_NMF/SLURM/slurmStart.sh'
cm.start_server(slurm_script=slurm_script)
pdir, profile = os.environ['IPPPDIR'], os.environ['IPPPROFILE']
c = Client(ipython_dir=pdir, profile=profile)
else:
cm.stop_server()
cm.start_server()
c = Client()
print(('Using ' + str(len(c)) + ' processes'))
dview = c[:len(c)]
#%% set parameters and create template by rigid motion correction
t1 = time.time()
def main():
pass # For compatibility between running under Spyder and the CLI
# %% Load demo movie and ROIs
fnames = download_demo('demo_voltage_imaging.hdf5', 'volpy') # file path to movie file (will download if not present)
path_ROIs = download_demo('demo_voltage_imaging_ROIs.hdf5', 'volpy') # file path to ROIs file (will download if not present)
# %% Setup some parameters for data and motion correction
# dataset parameters
fr = 400 # sample rate of the movie
ROIs = None # Region of interests
index = None # index of neurons
weights = None # reuse spatial weights by
# opts.change_params(params_dict={'weights':vpy.estimates['weights']})
# motion correction parameters
pw_rigid = False # flag for pw-rigid motion correction
gSig_filt = (3, 3) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (5, 5) # maximum allowed rigid shift
strides = (48, 48) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (24, 24) # overlap between pathes (size of patch strides+overlaps)
max_deviation_rigid = 3 # maximum deviation allowed for patch with respect to rigid shifts
border_nan = 'copy'
opts_dict = {
'fnames': fnames,
'fr': fr,
'index': index,
'ROIs': ROIs,
'weights': weights,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'gSig_filt': gSig_filt,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': border_nan
}
opts = volparams(params_dict=opts_dict)
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = False
if display_images:
m_orig = cm.load(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# Create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# Run piecewise rigid motion correction
mc.motion_correct(save_movie=True)
dview.terminate()
# %% motion correction compared with original movie
display_images = False
if display_images:
m_orig = cm.load(fnames)
m_rig = cm.load(mc.mmap_file)
ds_ratio = 0.2
moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
m_rig.resize(1, 1, ds_ratio)], axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
# % movie subtracted from the mean
m_orig2 = (m_orig - np.mean(m_orig, axis=0))
m_rig2 = (m_rig - np.mean(m_rig, axis=0))
moviehandle1 = cm.concatenate([m_orig2.resize(1, 1, ds_ratio),
m_rig2.resize(1, 1, ds_ratio)], axis=2)
moviehandle1.play(fr=60, q_max=99.5, magnification=2)
# %% Memory Mapping
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
border_to_0 = 0 if mc.border_nan == 'copy' else mc.border_to_0
fname_new = cm.save_memmap_join(mc.mmap_file, base_name='memmap_',
add_to_mov=border_to_0, dview=dview, n_chunks=10)
dview.terminate()
# %% change fnames to the new motion corrected one
opts.change_params(params_dict={'fnames': fname_new})
# %% SEGMENTATION
# Create mean and correlation image
use_maskrcnn = True # set to True to predict the ROIs using the mask R-CNN
if not use_maskrcnn: # use manual annotations
with h5py.File(path_ROIs, 'r') as fl:
ROIs = fl['mov'][()] # load ROIs
opts.change_params(params_dict={'ROIs': ROIs,
'index': list(range(ROIs.shape[0])),
'method': 'SpikePursuit'})
else:
m = cm.load(mc.mmap_file[0], subindices=slice(0, 20000))
m.fr = fr
img = m.mean(axis=0)
img = (img-np.mean(img))/np.std(img)
m1 = m.computeDFF(secsWindow=1, in_place=True)[0]
m = m - m1
Cn = m.local_correlations(swap_dim=False, eight_neighbours=True)
img_corr = (Cn-np.mean(Cn))/np.std(Cn)
summary_image = np.stack([img, img, img_corr], axis=2).astype(np.float32)
del m
del m1
# %%
# Mask R-CNN
config = neurons.NeuronsConfig()
class InferenceConfig(config.__class__):
# Run detection on one image at a time
GPU_COUNT = 1
IMAGES_PER_GPU = 1
DETECTION_MIN_CONFIDENCE = 0.7
IMAGE_RESIZE_MODE = "pad64"
IMAGE_MAX_DIM = 512
RPN_NMS_THRESHOLD = 0.7
POST_NMS_ROIS_INFERENCE = 1000
config = InferenceConfig()
config.display()
model_dir = os.path.join(caiman_datadir(), 'model')
DEVICE = "/cpu:0" # /cpu:0 or /gpu:0
with tf.device(DEVICE):
model = modellib.MaskRCNN(mode="inference", model_dir=model_dir,
config=config)
weights_path = download_model('mask_rcnn')
model.load_weights(weights_path, by_name=True)
results = model.detect([summary_image], verbose=1)
r = results[0]
ROIs_mrcnn = r['masks'].transpose([2, 0, 1])
# %% visualize the result
display_result = False
if display_result:
_, ax = plt.subplots(1,1, figsize=(16,16))
visualize.display_instances(summary_image, r['rois'], r['masks'], r['class_ids'],
['BG', 'neurons'], r['scores'], ax=ax,
title="Predictions")
# %% set rois
opts.change_params(params_dict={'ROIs':ROIs_mrcnn,
'index':list(range(ROIs_mrcnn.shape[0])),
'method':'SpikePursuit'})
# %% Trace Denoising and Spike Extraction
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False, maxtasksperchild=1)
vpy = VOLPY(n_processes=n_processes, dview=dview, params=opts)
vpy.fit(n_processes=n_processes, dview=dview)
# %% some visualization
print(np.where(vpy.estimates['passedLocalityTest'])[0]) # neurons that pass locality test
n = 0
# Processed signal and spikes of neurons
plt.figure()
plt.plot(vpy.estimates['trace'][n])
plt.plot(vpy.estimates['spikeTimes'][n],
np.max(vpy.estimates['trace'][n]) * np.ones(vpy.estimates['spikeTimes'][n].shape),
color='g', marker='o', fillstyle='none', linestyle='none')
plt.title('signal and spike times')
plt.show()
# Location of neurons by Mask R-CNN or manual annotation
plt.figure()
if use_maskrcnn:
plt.imshow(ROIs_mrcnn[n])
else:
plt.imshow(ROIs[n])
mv = cm.load(fname_new)
plt.imshow(mv.mean(axis=0),alpha=0.5)
# Spatial filter created by algorithm
plt.figure()
plt.imshow(vpy.estimates['spatialFilter'][n])
plt.colorbar()
plt.title('spatial filter')
plt.show()
# %% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def process_data(haussio_data,
mask=None,
p=2,
nrois_init=400,
roi_iceberg=0.9,
merge_unconnected=None):
if mask is not None:
raise RuntimeError("mask not supported in cnmf.process_data")
fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat'
shapefn = os.path.join(haussio_data.dirname_comp,
haussio.THOR_RAW_FN[:-3] + "shape.npy")
shape = np.load(shapefn)
if len(shape) == 5:
d1, d2 = shape[2], shape[3]
else:
d1, d2 = shape[1], shape[2]
fn_mmap = get_mmap_name(haussio_data.dirname_comp + os.path.sep + 'Yr', d1,
d2, shape[0])
tiffs_to_cnmf(haussio_data)
if os.path.exists(fn_cnmf):
resdict = loadmat(fn_cnmf)
if "dFoF" in resdict.keys():
A2 = resdict["A"]
C2 = resdict["C"]
YrA = resdict["YrA"]
S2 = resdict["S"]
dFoF = resdict["dFoF"]
bl2 = resdict["bl"]
f = resdict["f"]
images = haussio_data.read_raw().squeeze()
else:
dFoF = None
if not os.path.exists(fn_cnmf) or dFoF is None:
c, dview, n_processes = cm.cluster.setup_cluster(
backend='multiprocessing', n_processes=NCPUS, single_thread=False)
Yr, dims, T = cm.load_memmap(fn_mmap, 'r+')
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
fr = 1.0 / haussio_data.dt # imaging rate in frames per second\n",
decay_time = 0.4 # length of a typical transient in seconds\n",
# parameters for source extraction and deconvolution\n",
bord_px_els = 32 # maximum shift to be used for trimming against NaNs
p = 1 # order of the autoregressive system\n",
gnb = 2 # number of global background components\n",
merge_thresh = 0.8 # merging threshold, max correlation allowed\n",
rf = int(
np.round(np.sqrt(d1 * d2) / nrois_init)
) # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50\n",
if rf < 16:
rf = 16
stride_cnmf = 6 # amount of overlap between the patches in pixels\n",
npatches = np.round(d1 / (rf * 2) * d2 / (rf * 2))
K = nrois_init / npatches # number of components per patch\n",
if K < 2:
K = 2
print(rf, npatches, K)
gSig = [8, 8] # expected half size of neurons\n",
init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')\n",
is_dendrites = False # flag for analyzing dendritic data\n",
alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF\n",
# parameters for component evaluation\n",
min_SNR = 2.5 # signal to noise ratio for accepting a component\n",
rval_thr = 0.8 # space correlation threshold for accepting a component\n",
cnn_thr = 0.8 # threshold for CNN based classifier"
cnm = caiman_cnmf.CNMF(n_processes=1,
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=0,
dview=dview,
rf=rf,
stride=stride_cnmf,
memory_fact=1,
method_init=init_method,
alpha_snmf=alpha_snmf,
only_init_patch=False,
gnb=gnb,
border_pix=bord_px_els)
cnm = cnm.fit(images)
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview = dview, min_SNR=min_SNR,
r_values_min = rval_thr, use_cnn = False,
thresh_cnn_lowest = cnn_thr)
A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[
idx_components], cnm.b, cnm.f
cnm2 = caiman_cnmf.CNMF(n_processes=1,
k=A_in.shape[-1],
gSig=gSig,
p=p,
dview=dview,
merge_thresh=merge_thresh,
Ain=A_in,
Cin=C_in,
b_in=b_in,
f_in=f_in,
rf=None,
stride=None,
gnb=gnb,
method_deconvolution='oasis',
check_nan=True)
cnm2 = cnm2.fit(images)
if merge_unconnected is not None:
idx_merge = []
for nroi, ca_roi in enumerate(cnm2.C):
for nroi_compare_counter, ca_roi_compare in enumerate(
cnm2.C[nroi + 1:]):
nroi_compare = nroi_compare_counter + nroi + 1
if nroi_compare not in idx_merge:
correls = np.correlate(ca_roi,
ca_roi_compare,
mode='same')
correls /= np.sqrt(
np.dot(ca_roi, ca_roi) *
np.dot(ca_roi_compare, ca_roi_compare))
if correls.max() > merge_unconnected:
print("Merging ", nroi_compare)
idx_merge.append(nroi_compare)
idx_no_merge = [
idx for idx in range(cnm2.C.shape[0]) if idx not in idx_merge
]
else:
idx_no_merge = range(cnm2.C.shape[0])
A2 = cnm2.A[:, idx_no_merge].tocsc()
C2 = cnm2.C[idx_no_merge]
YrA = cnm2.YrA[idx_no_merge]
S2 = cnm2.S[idx_no_merge]
dFoF = cnm2.detrend_df_f(frames_window=300)[idx_no_merge]
# A: spatial components (ROIs)
# C: denoised [Ca2+]
# YrA: residuals ("noise", i.e. traces = C+YrA)
# S: Spikes
# f: temporal background
savemat(
fn_cnmf, {
"A": A2,
"C": C2,
"YrA": YrA,
"S": S2,
"dFoF": dFoF,
"bl": cnm2.b,
"f": cnm2.f
})
dview.terminate()
cm.stop_server()
proj_fn = haussio_data.dirname_comp + "_proj.npy"
if not os.path.exists(proj_fn):
zproj = utils.zproject(images)
np.save(proj_fn, zproj)
else:
zproj = np.load(proj_fn)
logfiles = glob.glob("*LOG*")
for logfile in logfiles:
try:
os.unlink(logfile)
except OSError:
pass
polygons = contour(A2, images.shape[1], images.shape[2], thr=roi_iceberg)
rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons])
return rois, C2, zproj, S2, images, YrA
def demix_and_deconvolve_with_cnmf(scan, num_components=200, AR_order=2,
merge_threshold=0.8, num_processes=20,
num_pixels_per_process=5000, block_size=10000,
num_background_components=4, init_method='greedy_roi',
soma_radius=(5, 5), snmf_alpha=None,
init_on_patches=False, patch_downsampling_factor=None,
percentage_of_patch_overlap=None):
""" Extract spike train activity from multi-photon scans using CNMF.
Uses constrained non-negative matrix factorization to find neurons/components
(locations) and their fluorescence traces (activity) in a timeseries of images, and
deconvolves them using an autoregressive model of the calcium impulse response
function. See Pnevmatikakis et al., 2016 for details.
Default values work alright for somatic images.
:param np.array scan: 3-dimensional scan (image_height, image_width, num_frames).
:param int num_components: An estimate of neurons/spatial components in the scan.
:param int AR_order: Order of the autoregressive process used to model the impulse
response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay.
:param int merge_threshold: Maximal temporal correlation allowed between activity of
overlapping components before merging them.
:param int num_processes: Number of processes to run in parallel. None for as many
processes as available cores.
:param int num_pixels_per_process: Number of pixels that a process handles each
iteration.
:param int block_size: 'number of pixels to process at the same time for dot product'
:param int num_background_components: Number of background components to use.
:param string init_method: Initialization method for the components.
'greedy_roi':Look for a gaussian-shaped patch, apply rank-1 NMF, store components,
calculate residual scan and repeat for num_components.
'sparse_nmf': Regularized non-negative matrix factorization (as impl. in sklearn)
'local_nmf': ...
:param (float, float) soma_radius: Estimated neuron radius (in pixels) in y and x.
Used in'greedy_roi' initialization to define the size of the gaussian window.
:param int snmf_alpha: Regularization parameter (alpha) for the sparse NMF (if used).
:param bool init_on_patches: If True, run the initialization methods on small patches
of the scan rather than on the whole image.
:param int patch_downsampling_factor: Division to the image dimensions to obtain patch
dimensions, e.g., if original size is 256 and factor is 10, patches will be 26x26
:param int percentage_of_patch_overlap: Patches are sampled in a sliding window. This
controls how much overlap is between adjacent patches (0 for none, 0.9 for 90%)
:returns Location matrix (image_height x image_width x num_components). Inferred
location of each component.
:returns Activity matrix (num_components x num_frames). Inferred fluorescence traces
(spike train convolved with the fitted impulse response function).
:returns: Inferred location matrix for background components (image_height x
image_width x num_background_components).
:returns: Inferred activity matrix for background components (image_height x
image_width x num_background_components).
:returns: Raw fluorescence traces (num_components x num_frames) obtained from the
scan minus activity from background and other components.
:returns: Spike matrix (num_components x num_frames). Deconvolved spike activity.
:returns: Autoregressive process coefficients (num_components x AR_order) used to
model the calcium impulse response of each component:
c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ...
..note:: Based on code provided by Andrea Giovanucci.
..note:: The produced number of components is not exactly what you ask for because
some components will be merged or deleted.
..warning:: Computation- and memory-intensive for big scans.
"""
import caiman
from caiman.source_extraction.cnmf import cnmf
# Save as memory mapped file in F order (that's how caiman wants it)
mmap_filename = _save_as_memmap(scan, base_name='/tmp/caiman', order='F').filename
# 'Load' scan
mmap_scan, (image_height, image_width), num_frames = caiman.load_memmap(mmap_filename)
images = np.reshape(mmap_scan.T, (num_frames, image_height, image_width), order='F')
# Start the ipyparallel cluster
client, direct_view, num_processes = caiman.cluster.setup_cluster(
n_processes=num_processes)
# Optionally, run the initialization method in small patches to initialize components
initial_A = None
initial_C = None
initial_f = None
if init_on_patches:
# Calculate patch size (only square patches allowed)
bigger_dimension = max(image_height, image_width)
smaller_dimension = min(image_height, image_width)
patch_size = bigger_dimension / patch_downsampling_factor
patch_size = min(patch_size, smaller_dimension) # if bigger than small dimension
# Calculate num_components_per_patch
num_nonoverlapping_patches = (image_height/patch_size) * (image_width/patch_size)
num_components_per_patch = num_components / num_nonoverlapping_patches
num_components_per_patch = max(num_components_per_patch, 1) # at least 1
# Calculate patch overlap in pixels
overlap_in_pixels = patch_size * percentage_of_patch_overlap
# Make sure they are integers
patch_size = int(round(patch_size))
num_components_per_patch = int(round(num_components_per_patch))
overlap_in_pixels = int(round(overlap_in_pixels))
# Run CNMF on patches (only for initialization, no impulse response modelling p=0)
model = cnmf.CNMF(num_processes, only_init_patch=True, p=0,
rf=int(round(patch_size / 2)), stride=overlap_in_pixels,
k=num_components_per_patch, merge_thresh=merge_threshold,
method_init=init_method, gSig=soma_radius,
alpha_snmf=snmf_alpha, gnb=num_background_components,
n_pixels_per_process=num_pixels_per_process,
block_size=block_size, check_nan=False, dview=direct_view,
method_deconvolution='cvxpy')
model = model.fit(images)
# Delete log files (one per patch)
log_files = glob.glob('caiman*_LOG_*')
for log_file in log_files:
os.remove(log_file)
# Get results
initial_A = model.A
initial_C = model.C
initial_f = model.f
# Run CNMF
model = cnmf.CNMF(num_processes, k=num_components, p=AR_order,
merge_thresh=merge_threshold, gnb=num_background_components,
method_init=init_method, gSig=soma_radius, alpha_snmf=snmf_alpha,
n_pixels_per_process=num_pixels_per_process, block_size=block_size,
check_nan=False, dview=direct_view, Ain=initial_A, Cin=initial_C,
f_in=initial_f, method_deconvolution='cvxpy')
model = model.fit(images)
# Get final results
location_matrix = model.A # pixels x num_components
activity_matrix = model.C # num_components x num_frames
background_location_matrix = model.b # pixels x num_background_components
background_activity_matrix = model.f # num_background_components x num_frames
spikes = model.S # num_components x num_frames, spike_ traces
raw_traces = model.C + model.YrA # num_components x num_frames
AR_coefficients = model.g # AR_order x num_components
# Reshape spatial matrices to be image_height x image_width x num_frames
new_shape = (image_height, image_width, -1)
location_matrix = location_matrix.toarray().reshape(new_shape, order='F')
background_location_matrix = background_location_matrix.reshape(new_shape, order='F')
AR_coefficients = np.array(list(AR_coefficients)) # unwrapping it (num_components x 2)
# Stop ipyparallel cluster
client.close()
caiman.stop_server()
# Delete memory mapped scan
os.remove(mmap_filename)
return (location_matrix, activity_matrix, background_location_matrix,
background_activity_matrix, raw_traces, spikes, AR_coefficients)
if plot_on:
if m_images.shape[0] < 5000:
Cn = m_images.local_correlations(
swap_dim=params_movie['swap_dim'], frames_per_chunk=1500)
Cn[np.isnan(Cn)] = 0
else:
Cn = np.array(
cm.load(('/'.join(params_movie['gtname'].split('/')[:-2] +
['projections', 'correlation_image.tif'])
))).squeeze()
check_nan = False
# %% start cluster
# TODO: show screenshot 10
try:
cm.stop_server()
dview.terminate()
except:
print('No clusters to stop')
c, dview, n_processes = setup_cluster(backend=backend_patch,
n_processes=n_processes,
single_thread=False)
# %%
params_dict = {
'fnames': [fname_new],
'fr': params_movie['fr'],
'decay_time': params_movie['decay_time'],
'rf': params_movie['rf'],
'stride': params_movie['stride_cnmf'],
'K': params_movie['K'],
corA_cnmfe = np.array(
[np.corrcoef(A_cnmfe.toarray()[:, n], A[:, n])[0, 1] for n in range(N)])
corC_cnmfe_patch = np.array(
[np.corrcoef(C_cnmfe_patch[n], C[n])[0, 1] for n in range(N)])
corA_cnmfe_patch = np.array(
[np.corrcoef(A_cnmfe_patch.toarray()[:, n], A[:, n])[0, 1] for n in range(N)])
else:
corC = np.array([np.corrcoef(C_[mapIdx[n]] + YrA_[mapIdx[n]],
C[n] + YrA_GT[n])[0, 1] for n in range(N)])
corA = np.array([np.corrcoef(A_[:, mapIdx[n]].toarray().squeeze(), A[:, n])[0, 1]
for n in range(N)])
corC_cnmfe = np.array(
[np.corrcoef(Craw_cnmfe[n], C[n] + YrA_GT[n])[0, 1] for n in range(N)])
corA_cnmfe = np.array(
[np.corrcoef(A_cnmfe.toarray()[:, n], A[:, n])[0, 1] for n in range(N)])
corC_cnmfe_patch = np.array(
[np.corrcoef(Craw_cnmfe_patch[n], C[n] + YrA_GT[n])[0, 1] for n in range(N)])
corA_cnmfe_patch = np.array(
[np.corrcoef(A_cnmfe_patch.toarray()[:, n], A[:, n])[0, 1] for n in range(N)])
print(np.median(corC), np.median(corA))
print(np.median(corC_cnmfe), np.median(corA_cnmfe))
print(np.median(corC_cnmfe_patch), np.median(corA_cnmfe_patch))
#%%
crd = cm.utils.visualization.plot_contours(A_, cn_filter, thr=.95, vmax=0.95)
plot_centers(A_, A)
#%%
cm.stop_server(dview=dview)
#%% motion correction with concatenated videos
videoconcat = sys.argv[1]
newpath = os.path.dirname(videoconcat)
ms_ts_name = os.path.join(newpath,"ms_ts.pkl")
if os.path.exists(ms_ts_name):
with open(ms_ts_name, "rb") as f:
ms_ts= pickle.load(f)
else:
print("there is no mt_ts.pkl existed")
fnames=[videoconcat]
m_orig = cm.load_movie_chain(fnames)
# start a cluster for parallel processing (if a cluster already exists it will be closed and a new session will be opened)
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# dataset dependent parameters
fr = 30 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
motion_correct = True # flag for motion correction
# motion correction parameters
pw_rigid = False # flag for pw-rigid motion correction
gSig_filt = (8,8) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (15,15) # maximum allowed rigid shift
strides = (96,96) # start a new patch for pw-rigid motion correction every x pixels
def motion_correct(video_path, max_shift, patch_stride, patch_overlap, use_multiprocessing=True):
full_video_path = video_path
directory = os.path.dirname(full_video_path)
filename = os.path.basename(full_video_path)
memmap_video = tifffile.memmap(video_path)
if use_multiprocessing:
if os.name == 'nt':
backend = 'multiprocessing'
else:
backend = 'ipyparallel'
# Create the cluster
cm.stop_server()
c, dview, n_processes = cm.cluster.setup_cluster(backend=backend, n_processes=None, single_thread=False)
else:
dview = None
z_range = list(range(memmap_video.shape[1]))
new_video_path = os.path.join(directory, "mc_video_temp.tif")
shutil.copyfile(video_path, new_video_path)
mc_video = tifffile.memmap(new_video_path).astype(np.uint16)
mc_borders = [ None for z in z_range ]
counter = 0
for z in z_range:
print("Motion correcting plane z={}...".format(z))
video_path = os.path.join(directory, os.path.splitext(filename)[0] + "_z_{}_temp.tif".format(z))
tifffile.imsave(video_path, memmap_video[:, z, :, :])
mc_video[:, z, :, :] *= 0
# --- PARAMETERS --- #
params_movie = {'fname': video_path,
'max_shifts': (max_shift, max_shift), # maximum allow rigid shift (2,2)
'niter_rig': 3,
'splits_rig': 1, # for parallelization split the movies in num_splits chuncks across time
'num_splits_to_process_rig': None, # if none all the splits are processed and the movie is saved
'strides': (patch_stride, patch_stride), # intervals at which patches are laid out for motion correction
'overlaps': (patch_overlap, patch_overlap), # overlap between pathes (size of patch strides+overlaps)
'splits_els': 1, # for parallelization split the movies in num_splits chuncks across time
'num_splits_to_process_els': [None], # if none all the splits are processed and the movie is saved
'upsample_factor_grid': 4, # upsample factor to avoid smearing when merging patches
'max_deviation_rigid': 3, # maximum deviation allowed for patch with respect to rigid shift
}
# load movie (in memory!)
fname = params_movie['fname']
niter_rig = params_movie['niter_rig']
# maximum allow rigid shift
max_shifts = params_movie['max_shifts']
# for parallelization split the movies in num_splits chuncks across time
splits_rig = params_movie['splits_rig']
# if none all the splits are processed and the movie is saved
num_splits_to_process_rig = params_movie['num_splits_to_process_rig']
# intervals at which patches are laid out for motion correction
strides = params_movie['strides']
# overlap between pathes (size of patch strides+overlaps)
overlaps = params_movie['overlaps']
# for parallelization split the movies in num_splits chuncks across time
splits_els = params_movie['splits_els']
# if none all the splits are processed and the movie is saved
num_splits_to_process_els = params_movie['num_splits_to_process_els']
# upsample factor to avoid smearing when merging patches
upsample_factor_grid = params_movie['upsample_factor_grid']
# maximum deviation allowed for patch with respect to rigid
# shift
max_deviation_rigid = params_movie['max_deviation_rigid']
# --- RIGID MOTION CORRECTION --- #
# Load the original movie
m_orig = tifffile.memmap(fname)
# m_orig = cm.load(fname)
min_mov = np.min(m_orig) # movie must be mostly positive for this to work
offset_mov = -min_mov
# Create motion correction object
mc = MotionCorrect(fname, min_mov,
dview=dview, max_shifts=max_shifts, niter_rig=niter_rig, splits_rig=splits_rig,
num_splits_to_process_rig=num_splits_to_process_rig,
strides= strides, overlaps= overlaps, splits_els=splits_els,
num_splits_to_process_els=num_splits_to_process_els,
upsample_factor_grid=upsample_factor_grid, max_deviation_rigid=max_deviation_rigid,
shifts_opencv = True, nonneg_movie = True, border_nan='min')
# Do rigid motion correction
mc.motion_correct_rigid(save_movie=False)
# --- ELASTIC MOTION CORRECTION --- #
# Do elastic motion correction
mc.motion_correct_pwrigid(save_movie=True, template=mc.total_template_rig, show_template=False)
# # Save elastic shift border
bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
# np.savez(mc.fname_tot_els + "_bord_px_els.npz", bord_px_els)
fnames = mc.fname_tot_els # name of the pw-rigidly corrected file.
border_to_0 = bord_px_els # number of pixels to exclude
fname_new = cm.save_memmap(fnames, base_name='memmap_z_{}'.format(z), order = 'C',
border_to_0 = bord_px_els) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
mc_borders[z] = bord_px_els
# images += np.amin(images)
# print(np.amax(images))
# print(np.amin(images))
# print(type(images))
mc_video[:, z, :, :] = (images - np.amin(images)).astype(np.uint16)
del m_orig
os.remove(video_path)
try:
os.remove(mc.fname_tot_rig)
os.remove(mc.fname_tot_els)
except:
pass
counter += 1
if use_multiprocessing:
if backend == 'multiprocessing':
dview.close()
else:
try:
dview.terminate()
except:
dview.shutdown()
cm.stop_server()
mmap_files = glob.glob(os.path.join(directory, '*.mmap'))
for mmap_file in mmap_files:
try:
os.remove(mmap_file)
except:
pass
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
return mc_video, new_video_path, mc_borders
