def demo(parallel=False):
p = 2 # order of the AR model (in general 1 or 2)
if parallel:
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
else:
n_processes, dview = 2, None
# LOAD MOVIE AND MEMORYMAP
fname_new = cm.save_memmap([os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif')],
base_name='Yr',
order = 'C')
Yr, dims, T = cm.load_memmap(fname_new)
# INIT
cnm = cnmf.CNMF(n_processes, method_init='greedy_roi', k=30, gSig=[4, 4], merge_thresh=.8,
p=p, dview=dview, Ain=None, method_deconvolution='oasis')
# FIT
images = np.reshape(Yr.T, [T] + list(dims), order='F')
cnm = cnm.fit(images)
if parallel:
cm.cluster.stop_server(dview=dview)
# verifying the spatial components
npt.assert_allclose(cnm.A.sum(), 281.1, 1e-2)
# verifying the temporal components
npt.assert_allclose(cnm.C.sum(), 66271668, 1e-2)
try:
dview.terminate()
except:
pass
def demix_and_deconvolve_with_cnmf(scan, num_components=200, AR_order=2,
merge_threshold=0.8, num_processes=20,
num_pixels_per_process=5000, block_size=10000,
num_background_components=4, init_method='greedy_roi',
soma_radius=(5, 5), snmf_alpha=None,
init_on_patches=False, patch_downsampling_factor=None,
percentage_of_patch_overlap=None):
""" Extract spike train activity from multi-photon scans using CNMF.
Uses constrained non-negative matrix factorization to find neurons/components
(locations) and their fluorescence traces (activity) in a timeseries of images, and
deconvolves them using an autoregressive model of the calcium impulse response
function. See Pnevmatikakis et al., 2016 for details.
Default values work alright for somatic images.
:param np.array scan: 3-dimensional scan (image_height, image_width, num_frames).
:param int num_components: An estimate of neurons/spatial components in the scan.
:param int AR_order: Order of the autoregressive process used to model the impulse
response function, e.g., 0 = no modelling; 2 = model rise plus exponential decay.
:param int merge_threshold: Maximal temporal correlation allowed between activity of
overlapping components before merging them.
:param int num_processes: Number of processes to run in parallel. None for as many
processes as available cores.
:param int num_pixels_per_process: Number of pixels that a process handles each
iteration.
:param int block_size: 'number of pixels to process at the same time for dot product'
:param int num_background_components: Number of background components to use.
:param string init_method: Initialization method for the components.
'greedy_roi':Look for a gaussian-shaped patch, apply rank-1 NMF, store components,
calculate residual scan and repeat for num_components.
'sparse_nmf': Regularized non-negative matrix factorization (as impl. in sklearn)
'local_nmf': ...
:param (float, float) soma_radius: Estimated neuron radius (in pixels) in y and x.
Used in'greedy_roi' initialization to define the size of the gaussian window.
:param int snmf_alpha: Regularization parameter (alpha) for the sparse NMF (if used).
:param bool init_on_patches: If True, run the initialization methods on small patches
of the scan rather than on the whole image.
:param int patch_downsampling_factor: Division to the image dimensions to obtain patch
dimensions, e.g., if original size is 256 and factor is 10, patches will be 26x26
:param int percentage_of_patch_overlap: Patches are sampled in a sliding window. This
controls how much overlap is between adjacent patches (0 for none, 0.9 for 90%)
:returns Location matrix (image_height x image_width x num_components). Inferred
location of each component.
:returns Activity matrix (num_components x num_frames). Inferred fluorescence traces
(spike train convolved with the fitted impulse response function).
:returns: Inferred location matrix for background components (image_height x
image_width x num_background_components).
:returns: Inferred activity matrix for background components (image_height x
image_width x num_background_components).
:returns: Raw fluorescence traces (num_components x num_frames) obtained from the
scan minus activity from background and other components.
:returns: Spike matrix (num_components x num_frames). Deconvolved spike activity.
:returns: Autoregressive process coefficients (num_components x AR_order) used to
model the calcium impulse response of each component:
c(t) = c(t-1) * AR_coeffs[0] + c(t-2) * AR_coeffs[1] + ...
..note:: Based on code provided by Andrea Giovanucci.
..note:: The produced number of components is not exactly what you ask for because
some components will be merged or deleted.
..warning:: Computation- and memory-intensive for big scans.
"""
import caiman
from caiman.source_extraction.cnmf import cnmf
# Save as memory mapped file in F order (that's how caiman wants it)
mmap_filename = _save_as_memmap(scan, base_name='/tmp/caiman', order='F').filename
# 'Load' scan
mmap_scan, (image_height, image_width), num_frames = caiman.load_memmap(mmap_filename)
images = np.reshape(mmap_scan.T, (num_frames, image_height, image_width), order='F')
# Start the ipyparallel cluster
client, direct_view, num_processes = caiman.cluster.setup_cluster(
n_processes=num_processes)
# Optionally, run the initialization method in small patches to initialize components
initial_A = None
initial_C = None
initial_f = None
if init_on_patches:
# Calculate patch size (only square patches allowed)
bigger_dimension = max(image_height, image_width)
smaller_dimension = min(image_height, image_width)
patch_size = bigger_dimension / patch_downsampling_factor
patch_size = min(patch_size, smaller_dimension) # if bigger than small dimension
# Calculate num_components_per_patch
num_nonoverlapping_patches = (image_height/patch_size) * (image_width/patch_size)
num_components_per_patch = num_components / num_nonoverlapping_patches
num_components_per_patch = max(num_components_per_patch, 1) # at least 1
# Calculate patch overlap in pixels
overlap_in_pixels = patch_size * percentage_of_patch_overlap
# Make sure they are integers
patch_size = int(round(patch_size))
num_components_per_patch = int(round(num_components_per_patch))
overlap_in_pixels = int(round(overlap_in_pixels))
# Run CNMF on patches (only for initialization, no impulse response modelling p=0)
model = cnmf.CNMF(num_processes, only_init_patch=True, p=0,
rf=int(round(patch_size / 2)), stride=overlap_in_pixels,
k=num_components_per_patch, merge_thresh=merge_threshold,
method_init=init_method, gSig=soma_radius,
alpha_snmf=snmf_alpha, gnb=num_background_components,
n_pixels_per_process=num_pixels_per_process,
block_size=block_size, check_nan=False, dview=direct_view,
method_deconvolution='cvxpy')
model = model.fit(images)
# Delete log files (one per patch)
log_files = glob.glob('caiman*_LOG_*')
for log_file in log_files:
os.remove(log_file)
# Get results
initial_A = model.A
initial_C = model.C
initial_f = model.f
# Run CNMF
model = cnmf.CNMF(num_processes, k=num_components, p=AR_order,
merge_thresh=merge_threshold, gnb=num_background_components,
method_init=init_method, gSig=soma_radius, alpha_snmf=snmf_alpha,
n_pixels_per_process=num_pixels_per_process, block_size=block_size,
check_nan=False, dview=direct_view, Ain=initial_A, Cin=initial_C,
f_in=initial_f, method_deconvolution='cvxpy')
model = model.fit(images)
# Get final results
location_matrix = model.A # pixels x num_components
activity_matrix = model.C # num_components x num_frames
background_location_matrix = model.b # pixels x num_background_components
background_activity_matrix = model.f # num_background_components x num_frames
spikes = model.S # num_components x num_frames, spike_ traces
raw_traces = model.C + model.YrA # num_components x num_frames
AR_coefficients = model.g # AR_order x num_components
# Reshape spatial matrices to be image_height x image_width x num_frames
new_shape = (image_height, image_width, -1)
location_matrix = location_matrix.toarray().reshape(new_shape, order='F')
background_location_matrix = background_location_matrix.reshape(new_shape, order='F')
AR_coefficients = np.array(list(AR_coefficients)) # unwrapping it (num_components x 2)
# Stop ipyparallel cluster
client.close()
caiman.stop_server()
# Delete memory mapped scan
os.remove(mmap_filename)
return (location_matrix, activity_matrix, background_location_matrix,
background_activity_matrix, raw_traces, spikes, AR_coefficients)
from caiman.source_extraction.cnmf.estimates import Estimates
import scipy
estimates = vpy.estimates.copy()
A = np.array(estimates['spatial_filter']).transpose([1,2,0]).reshape((-1, len(estimates['spatial_filter'])),order='F')
A = A / A.max(axis=0)
b = np.zeros((A.shape[0],2))
A = scipy.sparse.csc_matrix(A)
b = scipy.sparse.csc_matrix(b)
C = np.array(estimates['t_rec'])
f = np.zeros((2, C.shape[1]))
R = np.array(estimates['t']) - C
est = Estimates(A=A, C=C, b=b, f=f, R=R, dims=(100,100))
est.YrA = R
#est.plot_contours(img=summary_image[:,:,2])
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
est.dview = dview
#est.view_components(img=summary_image[:,:,2])
est.play_movie(imgs=images, magnification=4)
est = np.load('/home/nel/data/voltage_data/volpy_paper/reconstructed/estimates.npz',allow_pickle=True)['arr_0'].item()
fnames = ['/home/nel/data/voltage_data/volpy_paper/memory/403106_3min_10000._rig__d1_512_d2_128_d3_1_order_F_frames_10000_.mmap']
def main():
pass # For compatibility between running under Spyder and the CLI
#%% Select file(s) to be processed (download if not present)
fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fnames = [download_demo(fnames[0])]
#%% First setup some parameters for data and motion correction
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
# overlap between patches (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = False
if display_images:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# %% compare with original movie
if display_images:
m_orig = cm.load_movie_chain(fnames)
m_els = cm.load(mc.mmap_file)
ds_ratio = 0.2
moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)], axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
# %% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C',
border_to_0=border_to_0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F') # load frames in python format (T x X x Y)
# %% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons in pixels
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
ssub = 2 # spatial subsampling during initialization
tsub = 2 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {'fnames': fnames,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub}
opts.change_params(params_dict=opts_dict)
# %% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% plot contours of found components
Cn = cm.local_correlations(images, swap_dim=False)
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm.params.change_params({'p': p})
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # signal to noise ratio for accepting a component
rval_thr = 0.85 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set('quality', {'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% PLOT COMPONENTS
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
# %% VIEW TRACES (accepted and rejected)
if display_images:
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components)
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components_bad)
#%% update object with selected components
cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(img=Cn)
#%% reconstruct denoised movie (press q to exit)
if display_images:
cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2,
magnification=2,
bpx=border_to_0,
include_bck=False) # background not shown
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def test_general():
""" General Test of pipeline with comparison against ground truth
A shorter version than the demo pipeline that calls comparison for the real test work
Raises:
---------
params_movie
params_cnmf
rig correction
cnmf on patch
cnmf full frame
not able to read the file
no groundtruth
"""
#\bug
#\warning
global params_movie
global params_diplay
fname = params_movie['fname']
niter_rig = params_movie['niter_rig']
max_shifts = params_movie['max_shifts']
splits_rig = params_movie['splits_rig']
num_splits_to_process_rig = params_movie['num_splits_to_process_rig']
cwd = os.getcwd()
fname = download_demo(fname[0])
m_orig = cm.load(fname)
min_mov = m_orig[:400].min()
comp = comparison.Comparison()
comp.dims = np.shape(m_orig)[1:]
################ RIG CORRECTION #################
t1 = time.time()
mc = MotionCorrect(fname,
min_mov,
max_shifts=max_shifts,
niter_rig=niter_rig,
splits_rig=splits_rig,
num_splits_to_process_rig=num_splits_to_process_rig,
shifts_opencv=True,
nonneg_movie=True)
mc.motion_correct_rigid(save_movie=True)
m_rig = cm.load(mc.fname_tot_rig)
bord_px_rig = np.ceil(np.max(mc.shifts_rig)).astype(np.int)
comp.comparison['rig_shifts']['timer'] = time.time() - t1
comp.comparison['rig_shifts']['ourdata'] = mc.shifts_rig
###########################################
if 'max_shifts' not in params_movie:
fnames = params_movie['fname']
border_to_0 = 0
else: # elif not params_movie.has_key('overlaps'):
fnames = mc.fname_tot_rig
border_to_0 = bord_px_rig
m_els = m_rig
idx_xy = None
add_to_movie = -np.nanmin(m_els) + 1 # movie must be positive
remove_init = 0
downsample_factor = 1
base_name = fname[0].split('/')[-1][:-4]
name_new = cm.save_memmap_each(fnames,
base_name=base_name,
resize_fact=(1, 1, downsample_factor),
remove_init=remove_init,
idx_xy=idx_xy,
add_to_movie=add_to_movie,
border_to_0=border_to_0)
name_new.sort()
if len(name_new) > 1:
fname_new = cm.save_memmap_join(name_new,
base_name='Yr',
n_chunks=params_movie['n_chunks'],
dview=None)
else:
logging.warning('One file only, not saving!')
fname_new = name_new[0]
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T, ), order='F')
if np.min(images) < 0:
# TODO: should do this in an automatic fashion with a while loop at the 367 line
raise Exception('Movie too negative, add_to_movie should be larger')
if np.sum(np.isnan(images)) > 0:
# TODO: same here
raise Exception(
'Movie contains nan! You did not remove enough borders')
Cn = cm.local_correlations(Y)
Cn[np.isnan(Cn)] = 0
p = params_movie['p']
merge_thresh = params_movie['merge_thresh']
rf = params_movie['rf']
stride_cnmf = params_movie['stride_cnmf']
K = params_movie['K']
init_method = params_movie['init_method']
gSig = params_movie['gSig']
if params_movie['is_dendrites'] == True:
if params_movie['init_method'] is not 'sparse_nmf':
raise Exception('dendritic requires sparse_nmf')
if params_movie['alpha_snmf'] is None:
raise Exception('need to set a value for alpha_snmf')
################ CNMF PART PATCH #################
t1 = time.time()
cnm = cnmf.CNMF(n_processes=1,
k=K,
gSig=gSig,
merge_thresh=params_movie['merge_thresh'],
p=params_movie['p'],
dview=None,
rf=rf,
stride=stride_cnmf,
memory_fact=params_movie['memory_fact'],
method_init=init_method,
alpha_snmf=100,
only_init_patch=params_movie['only_init_patch'],
gnb=params_movie['gnb'],
method_deconvolution='oasis')
comp.cnmpatch = copy.copy(cnm)
comp.cnmpatch.estimates = None
cnm = cnm.fit(images)
A_tot = cnm.estimates.A
C_tot = cnm.estimates.C
YrA_tot = cnm.estimates.YrA
b_tot = cnm.estimates.b
f_tot = cnm.estimates.f
# DISCARDING
logging.info(('Number of components:' + str(A_tot.shape[-1])))
final_frate = params_movie['final_frate']
# threshold on space consistency
r_values_min = params_movie['r_values_min_patch']
# threshold on time variability
fitness_min = params_movie['fitness_delta_min_patch']
fitness_delta_min = params_movie['fitness_delta_min_patch']
Npeaks = params_movie['Npeaks']
traces = C_tot + YrA_tot
idx_components, idx_components_bad = estimate_components_quality(
traces,
Y,
A_tot,
C_tot,
b_tot,
f_tot,
final_frate=final_frate,
Npeaks=Npeaks,
r_values_min=r_values_min,
fitness_min=fitness_min,
fitness_delta_min=fitness_delta_min)
#######
A_tot = A_tot.tocsc()[:, idx_components]
C_tot = C_tot[idx_components]
comp.comparison['cnmf_on_patch']['timer'] = time.time() - t1
comp.comparison['cnmf_on_patch']['ourdata'] = [A_tot.copy(), C_tot.copy()]
#################### ########################
################ CNMF PART FULL #################
t1 = time.time()
cnm = cnmf.CNMF(n_processes=1,
k=A_tot.shape,
gSig=gSig,
merge_thresh=merge_thresh,
p=p,
Ain=A_tot,
Cin=C_tot,
f_in=f_tot,
rf=None,
stride=None,
method_deconvolution='oasis')
cnm = cnm.fit(images)
# DISCARDING
A, C, b, f, YrA, sn = cnm.estimates.A, cnm.estimates.C, cnm.estimates.b, cnm.estimates.f, cnm.estimates.YrA, cnm.estimates.sn
final_frate = params_movie['final_frate']
# threshold on space consistency
r_values_min = params_movie['r_values_min_full']
# threshold on time variability
fitness_min = params_movie['fitness_delta_min_full']
fitness_delta_min = params_movie['fitness_delta_min_full']
Npeaks = params_movie['Npeaks']
traces = C + YrA
idx_components, idx_components_bad, fitness_raw, fitness_delta, r_values = estimate_components_quality(
traces,
Y,
A,
C,
b,
f,
final_frate=final_frate,
Npeaks=Npeaks,
r_values_min=r_values_min,
fitness_min=fitness_min,
fitness_delta_min=fitness_delta_min,
return_all=True)
##########
A_tot_full = A_tot.tocsc()[:, idx_components]
C_tot_full = C_tot[idx_components]
comp.comparison['cnmf_full_frame']['timer'] = time.time() - t1
comp.comparison['cnmf_full_frame']['ourdata'] = [
A_tot_full.copy(), C_tot_full.copy()
]
#################### ########################
comp.save_with_compare(istruth=False, params=params_movie, Cn=Cn)
log_files = glob.glob('*_LOG_*')
try:
for log_file in log_files:
os.remove(log_file)
except:
logging.warning('Cannot remove log files')
############ assertions ##################
pb = False
if (comp.information['differences']['params_movie']):
logging.error(
"you need to set the same movie parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)"
)
pb = True
if (comp.information['differences']['params_cnm']):
logging.warning(
"you need to set the same cnmf parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)"
)
# pb = True
if (comp.information['diff']['rig']['isdifferent']):
logging.error("the rigid shifts are different from the groundtruth ")
pb = True
if (comp.information['diff']['cnmpatch']['isdifferent']):
logging.error(
"the cnmf on patch produces different results than the groundtruth "
)
pb = True
if (comp.information['diff']['cnmfull']['isdifferent']):
logging.error(
"the cnmf full frame produces different results than the groundtruth "
)
pb = True
assert (not pb)
def substract_neuropil(data_path, agonia_th, neuropil_pctl, signal_pctl):
'''Calculate the neuropil trace as the average fluorescence of every pixel that is not in a box.
subtract it to every box trace, calculated as the activity of the pixels above the signal_pctl.
Parameters
----------
data_path : string
path to folder containing the data
agonia_th : float
threshold for detection confidence for each box
neuropil_pctl : int
percentile of pixels intensity bellow which pixel is considered to calculate neuropil
signal_pctl : int
percentile of pixels intensity above which pixel is considered to calculate trace
Returns
-------
denoised_traces : NxT ndarray
temporal traces after neuropil subtraction for the N boxes
neuropil_trace : array
normalized temporal trace of background noise, length T
neuropil_power : float
mean of neuropil trace before normalization
'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load Caiman results
cnm = cnmf.load_CNMF(results_caiman_path)
# calculate the centers of the CaImAn factors
centers = np.empty((cnm.estimates.A.shape[1], 2))
for i, factor in enumerate(cnm.estimates.A.T):
centers[i] = center_of_mass(factor.toarray().reshape(
cnm.estimates.dims, order='F'))
with open(boxes_path, 'rb') as f:
boxes = pickle.load(f)
f.close()
# keep only cells above confidence threshold
boxes = boxes[boxes[:, 4] > agonia_th].astype('int')
#delete boxes that do not have a caiman cell inside
k = 0
for cell, box in enumerate(boxes):
idx_factor = [
i for i, center in enumerate(centers)
if center[0] > box[1] and center[0] < box[3] and center[1] > box[0]
and center[1] < box[2]
]
if not idx_factor:
boxes = np.delete(boxes, cell - k, axis=0)
k += 1
boxes_traces = np.empty((boxes.shape[0], images.shape[0]))
mask = np.zeros(images.shape[1:], dtype=bool)
for cell, box in enumerate(boxes):
#boxes_traces[cell] = images[:,box[1]:box[3],box[0]:box[2]].mean(axis=(1,2))
med = np.median(images[:, box[1]:box[3], box[0]:box[2]], axis=0)
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
boxes_traces[cell] = box_trace[:,
med > np.percentile(med, signal_pctl
)].mean(axis=1)
mask[box[1].astype('int'):box[3].astype('int'),
box[0].astype('int'):box[2].astype('int')] = 1
mask = (1 - mask).astype('bool')
not_cell = images[:, mask]
#not_cell_med = np.median(not_cell,axis=0)
#neuropil_trace = not_cell[:,not_cell_med<np.percentile(not_cell_med,neuropil_pctl)].mean(axis=1)
#denoised_traces = boxes_traces-neuropil_trace
neuropil_trace = not_cell.mean(axis=1)
neuropil_power = neuropil_trace.mean()
neuropil_trace = neuropil_trace / neuropil_power
denoised_traces = np.array([
boxes_traces[i] - neuropil_trace * boxes_traces[i].mean()
for i in range(boxes_traces.shape[0])
])
return denoised_traces, neuropil_trace, neuropil_power
def trace_correlation(data_path,
agonia_th,
select_cells=False,
plot_results=True,
denoise=False):
'''Calculate the correlation between the mean of the Agonia Box and the CaImAn
factor.
Parameters
----------
data_path : string
path to folder containing the data
agonia_th : float
threshold for detection confidence for each box
select_cells: bool, optional
if True get index of active cells
plot_results: bool, optional
if True do boxplot of correlation values for all cells, if selected_cells,
use only active cells
denoise : bool, optional
if True subtract neuropil
Returns
-------
cell_corr : numpy array
corrcoef value for all cells
idx_active : list
if select_cells=True returns index of active cells, otherwise returns
index of all cells
boxes_traces : NxT ndarray
temporal trace for the N box that has an asociated caiman factor'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
# load Caiman results
cnm = cnmf.load_CNMF(results_caiman_path)
# calculate the centers of the CaImAn factors
centers = np.empty((cnm.estimates.A.shape[1], 2))
for i, factor in enumerate(cnm.estimates.A.T):
centers[i] = center_of_mass(factor.toarray().reshape(
cnm.estimates.dims, order='F'))
# load boxes
with open(boxes_path, 'rb') as f:
boxes = pickle.load(f)
f.close()
# keep only cells above confidence threshold
boxes = boxes[boxes[:, 4] > agonia_th].astype('int')
#delete boxes that do not have a caiman cell inside
k = 0
for cell, box in enumerate(boxes):
idx_factor = [
i for i, center in enumerate(centers)
if center[0] > box[1] and center[0] < box[3] and center[1] > box[0]
and center[1] < box[2]
]
if not idx_factor:
boxes = np.delete(boxes, cell - k, axis=0)
k += 1
# Load video as 3D tensor (each plane is a frame)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
boxes_traces = np.empty((boxes.shape[0], images.shape[0]))
#calculate correlations between boxes and CaImAn factors
cell_corr = np.empty(len(boxes_traces))
neuropil_trace = np.zeros(T)
if denoise:
_, neuropil_trace, neuropil_power = substract_neuropil(
data_path, agonia_th, 100, 80)
for cell, box in enumerate(boxes):
# calculate boxes traces as means over images
#boxes_traces[cell] = images[:,box[1]:box[3],box[0]:box[2]].mean(axis=(1,2))-neuropil_trace
boxes_traces[cell] = images[:, box[1]:box[3],
box[0]:box[2]].mean(axis=(1, 2))
#for using the percentile criteria
med = np.median(images[:, box[1]:box[3], box[0]:box[2]], axis=0)
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
boxes_traces[cell] = box_trace[:,
np.logical_and(
med > np.percentile(med, 80),
med < np.percentile(med, 95))].mean(
axis=1)
boxes_traces[
cell] = boxes_traces[cell] - neuropil_trace * neuropil_power * .7
#boxes_traces[cell] = boxes_traces[cell]-neuropil_trace*boxes_traces[cell].mean()
# get the asociated CaImAn factor by checking if its center of mass is inside the box
idx_factor = [
i for i, center in enumerate(centers)
if center[0] > box[1] and center[0] < box[3] and center[1] > box[0]
and center[1] < box[2]
]
# in case there is more than one center inside the box choose the one closer to the center of the box
if len(idx_factor) > 1:
idx_factor = [
idx_factor[np.argmin([
np.linalg.norm([(box[3] - box[1]) / 2, (box[2] - box[0]) /
2] - c) for c in centers[idx_factor]
])]
]
cell_corr[cell] = np.corrcoef(
[cnm.estimates.C[idx_factor[0]], boxes_traces[cell]])[1, 0]
if select_cells:
#select only active cells using CaImAn criteria
fitness, _, _, _ = compute_event_exceptionality(boxes_traces)
idx_active = [cell for cell, fit in enumerate(fitness) if fit < -20]
else:
idx_active = [cell for cell, _ in enumerate(boxes_traces)]
if plot_results:
corr_toplot = [
corr for id, corr in enumerate(cell_corr)
if id in idx_active and ~np.isnan(corr)
]
fig, ax = plt.subplots()
p = ax.boxplot(corr_toplot)
ax.set_ylim([-1, 1])
plt.text(1.2, 0.8, 'n_cells = {}'.format(len(corr_toplot)))
return cell_corr, idx_active, boxes_traces
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters
# dataset dependent parameters
display_images = False # Set to true to show movies and images
fnames = ['data_endoscope.tif'] # filename to be processed
frate = 10 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
do_motion_correction_nonrigid = True
do_motion_correction_rigid = False # in this case it will also save a rigid motion corrected movie
gSig_filt = (3, 3) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (5, 5) # maximum allowed rigid shift
splits_rig = 10 # for parallelization split the movies in num_splits chuncks across time
strides = (48, 48) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (24, 24) # overlap between pathes (size of patch strides+overlaps)
# for parallelization split the movies in num_splits chuncks across time
# (remember that it should hold that length_movie/num_splits_to_process_rig>100)
splits_els = 10
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None
gSig = 3 # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = 13 # average diameter of a neuron, in general 4*gSig+1
merge_thresh = .7 # merging threshold, max correlation allowed
rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80
stride_cnmf = 20 # amount of overlap between the patches in pixels
# (keep it at least large as gSiz, i.e 4 times the neuron size gSig)
tsub = 2 # downsampling factor in time for initialization,
# increase if you have memory problems
ssub = 1 # downsampling factor in space for initialization,
# increase if you have memory problems
Ain = None # if you want to initialize with some preselected components
# you can pass them here as boolean vectors
low_rank_background = None # None leaves background of each patch intact,
# True performs global low-rank approximation
gnb = -1 # number of background components (rank) if positive,
# else exact ring model with following settings
# gnb=-2: Return background as b and W
# gnb=-1: Return full rank background B
# gnb= 0: Don't return background
nb_patch = -1 # number of background components (rank) per patch,
# use 0 or -1 for exact background of ring model (cf. gnb)
min_corr = .8 # min peak value from correlation image
min_pnr = 10 # min peak to noise ration from PNR image
ssub_B = 2 # additional downsampling factor in space for background
ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor
# parameters for component evaluation
min_SNR = 3 # adaptive way to set threshold on the transient size
r_values_min = 0.85 # threshold on space consistency (if you lower more components
# will be accepted, potentially with worst quality)
#%% start the cluster
try:
cm.stop_server(dview=dview) # stop it if it was running
except:
pass
c, dview, n_processes = cm.cluster.setup_cluster(backend='local', # use this one
n_processes=24, # number of process to use, if you go out of memory try to reduce this one
single_thread=False)
#%% download demo file
fnames = [download_demo(fnames[0])]
filename_reorder = fnames
#%% MOTION CORRECTION
if do_motion_correction_nonrigid or do_motion_correction_rigid:
# do motion correction rigid
mc = motion_correct_oneP_rigid(fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
dview=dview,
splits_rig=splits_rig,
save_movie=not(do_motion_correction_nonrigid)
)
new_templ = mc.total_template_rig
plt.subplot(1, 2, 1)
plt.imshow(new_templ) # % plot template
plt.subplot(1, 2, 2)
plt.plot(mc.shifts_rig) # % plot rigid shifts
plt.legend(['x shifts', 'y shifts'])
plt.xlabel('frames')
plt.ylabel('pixels')
# borders to eliminate from movie because of motion correction
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
filename_reorder = mc.fname_tot_rig
# do motion correction nonrigid
if do_motion_correction_nonrigid:
mc = motion_correct_oneP_nonrigid(
fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
strides=strides,
overlaps=overlaps,
splits_els=splits_els,
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
dview=dview,
splits_rig=None,
save_movie=True, # whether to save movie in memory mapped format
new_templ=new_templ # template to initialize motion correction
)
filename_reorder = mc.fname_tot_els
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
# create memory mappable file in the right order on the hard drive (C order)
fname_new = cm.save_memmap_each(
filename_reorder,
base_name='memmap_',
order='C',
border_to_0=bord_px,
dview=dview)
fname_new = cm.save_memmap_join(fname_new, base_name='memmap_', dview=dview)
# load memory mappable file
Yr, dims, T = cm.load_memmap(fname_new)
Y = Yr.T.reshape((T,) + dims, order='F')
#%% compute some summary images (correlation and peak to noise)
# change swap dim if output looks weird, it is a problem with tiffile
cn_filter, pnr = cm.summary_images.correlation_pnr(Y, gSig=gSig, swap_dim=False)
# inspect the summary images and set the parameters
inspect_correlation_pnr(cn_filter, pnr)
# print parameters set above, modify them if necessary based on summary images
print(min_corr) # min correlation of peak (from correlation image)
print(min_pnr) # min peak to noise ratio
#%% RUN CNMF ON PATCHES
cnm = cnmf.CNMF(
n_processes=n_processes,
method_init='corr_pnr', # use this for 1 photon
k=K,
gSig=(gSig, gSig),
gSiz=(gSiz, gSiz),
merge_thresh=merge_thresh,
p=p,
dview=dview,
tsub=tsub,
ssub=ssub,
Ain=Ain,
rf=rf,
stride=stride_cnmf,
only_init_patch=True, # just leave it as is
gnb=gnb,
nb_patch=nb_patch,
method_deconvolution='oasis', # could use 'cvxpy' alternatively
low_rank_background=low_rank_background,
update_background_components=True, # sometimes setting to False improve the results
min_corr=min_corr,
min_pnr=min_pnr,
normalize_init=False, # just leave as is
center_psf=True, # leave as is for 1 photon
ssub_B=ssub_B,
ring_size_factor=ring_size_factor,
del_duplicates=True, # whether to remove duplicates from initialization
border_pix=bord_px) # number of pixels to not consider in the borders
cnm.fit(Y)
#%% DISCARD LOW QUALITY COMPONENTS
idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = \
estimate_components_quality_auto(
Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate,
decay_time, gSig, dims, dview=dview,
min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False)
print(' ***** ')
print((len(cnm.C)))
print((len(idx_components)))
cm.stop_server(dview=dview)
#%% PLOT COMPONENTS
if display_images:
plt.figure(figsize=(12, 6))
plt.subplot(121)
crd_good = cm.utils.visualization.plot_contours(
cnm.A[:, idx_components], cn_filter, thr=.8, vmax=0.95)
plt.title('Contour plots of accepted components')
plt.subplot(122)
crd_bad = cm.utils.visualization.plot_contours(
cnm.A[:, idx_components_bad], cn_filter, thr=.8, vmax=0.95)
plt.title('Contour plots of rejected components')
#%% VISUALIZE IN DETAILS COMPONENTS
cm.utils.visualization.view_patches_bar(
Yr, cnm.A[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f,
dims[0], dims[1], YrA=cnm.YrA[idx_components], img=cn_filter)
#%% MOVIES
if display_images:
B = cnm.b.dot(cnm.f)
if 'sparse' in str(type(B)):
B = B.toarray()
# denoised movie
cm.movie(np.reshape(cnm.A.tocsc()[:, idx_components].dot(cnm.C[idx_components]) + B,
dims + (-1,), order='F').transpose(2, 0, 1)).play(magnification=3, gain=1.)
# only neurons
cm.movie(np.reshape(cnm.A.tocsc()[:, idx_components].dot(
cnm.C[idx_components]), dims + (-1,), order='F').transpose(2, 0, 1)
).play(magnification=3, gain=10.)
# only the background
cm.movie(np.reshape(B, dims + (-1,), order='F').transpose(2, 0, 1)
).play(magnification=3, gain=1.)
# residuals
cm.movie(np.array(Y) - np.reshape(cnm.A.tocsc()[:, :].dot(cnm.C[:]) + B,
dims + (-1,), order='F').transpose(2, 0, 1)
).play(magnification=3, gain=10., fr=10)
# eventually, you can rerun the algorithm on the residuals
plt.imshow(cm.movie(np.array(Y) - np.reshape(cnm.A.tocsc()[:, :].dot(cnm.C[:]) + B,
dims + (-1,), order='F').transpose(2, 0, 1)
).local_correlations(swap_dim=False))
def main():
pass # For compatibility between running under Spyder and the CLI
# %% start the cluster
try:
cm.stop_server() # stop it if it was running
except ():
pass
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local',
n_processes=
24, # number of process to use, if you go out of memory try to reduce this one
single_thread=False)
# %% First setup some parameters for motion correction
# dataset dependent parameters
fnames = ['data_endoscope.tif'] # filename to be processed
fnames = [download_demo(fnames[0])] # download file if not already present
filename_reorder = fnames
fr = 10 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
motion_correct = True # flag for motion correction
pw_rigid = False # flag for pw-rigid motion correction
gSig_filt = (3, 3) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (5, 5) # maximum allowed rigid shift
strides = (
48, 48
) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (24, 24
) # overlap between pathes (size of patch strides+overlaps)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
border_nan = 'copy'
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'gSig_filt': gSig_filt,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': border_nan
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% MOTION CORRECTION
# The pw_rigid flag set above, determines where to use rigid or pw-rigid
# motion correction
if motion_correct:
# do motion correction rigid
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
mc.motion_correct(save_movie=True)
fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig
if pw_rigid:
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
else:
# marker
if show_chart:
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
plt.subplot(1, 2, 1)
plt.imshow(mc.total_template_rig) # % plot template
plt.subplot(1, 2, 2)
plt.plot(mc.shifts_rig) # % plot rigid shifts
plt.legend(['x shifts', 'y shifts'])
plt.xlabel('frames')
plt.ylabel('pixels')
bord_px = 0 if border_nan is 'copy' else bord_px
fname_new = cm.save_memmap(fname_mc,
base_name='memmap_',
order='C',
border_to_0=bord_px)
else: # if no motion correction just memory map the file
fname_new = cm.save_memmap(filename_reorder,
base_name='memmap_',
order='C',
border_to_0=0,
dview=dview)
# load memory mappable file
Yr, dims, T = cm.load_memmap(fname_new)
images = Yr.T.reshape((T, ) + dims, order='F')
# %% Parameters for source extraction and deconvolution (CNMF-E algorithm)
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None for 1p data
gSig = (
3, 3
) # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1
Ain = None # possibility to seed with predetermined binary masks
merge_thr = .7 # merging threshold, max correlation allowed
rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80
stride_cnmf = 20 # amount of overlap between the patches in pixels
# (keep it at least large as gSiz, i.e 4 times the neuron size gSig)
tsub = 2 # downsampling factor in time for initialization,
# increase if you have memory problems
ssub = 1 # downsampling factor in space for initialization,
# increase if you have memory problems
# you can pass them here as boolean vectors
low_rank_background = None # None leaves background of each patch intact,
# True performs global low-rank approximation if gnb>0
gnb = 0 # number of background components (rank) if positive,
# else exact ring model with following settings
# gnb= 0: Return background as b and W
# gnb=-1: Return full rank background B
# gnb<-1: Don't return background
nb_patch = 0 # number of background components (rank) per patch if gnb>0,
# else it is set automatically
min_corr = .8 # min peak value from correlation image
min_pnr = 10 # min peak to noise ration from PNR image
ssub_B = 2 # additional downsampling factor in space for background
ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor
opts.change_params(
params_dict={
'dims': dims,
'method_init': 'corr_pnr', # use this for 1 photon
'K': K,
'gSig': gSig,
'gSiz': gSiz,
'merge_thr': merge_thr,
'p': p,
'tsub': tsub,
'ssub': ssub,
'rf': rf,
'stride': stride_cnmf,
'only_init': True, # set it to True to run CNMF-E
'nb': gnb,
'nb_patch': nb_patch,
'method_deconvolution': 'oasis', # could use 'cvxpy' alternatively
'low_rank_background': low_rank_background,
'update_background_components':
True, # sometimes setting to False improve the results
'min_corr': min_corr,
'min_pnr': min_pnr,
'normalize_init': False, # just leave as is
'center_psf': True, # leave as is for 1 photon
'ssub_B': ssub_B,
'ring_size_factor': ring_size_factor,
'del_duplicates':
True, # whether to remove duplicates from initialization
'border_pix': bord_px
}) # number of pixels to not consider in the borders)
# %% compute some summary images (correlation and peak to noise)
# change swap dim if output looks weird, it is a problem with tiffile
cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1],
gSig=gSig[0],
swap_dim=False)
# if your images file is too long this computation will take unnecessarily
# long time and consume a lot of memory. Consider changing images[::1] to
# images[::5] or something similar to compute on a subset of the data
# inspect the summary images and set the parameters
inspect_correlation_pnr(cn_filter, pnr)
# print parameters set above, modify them if necessary based on summary images
print(min_corr) # min correlation of peak (from correlation image)
print(min_pnr) # min peak to noise ratio
# %% RUN CNMF ON PATCHES
cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=opts)
cnm.fit(images)
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% DISCARD LOW QUALITY COMPONENTS
min_SNR = 2.5 # adaptive way to set threshold on the transient size
r_values_min = 0.85 # threshold on space consistency (if you lower more components
# will be accepted, potentially with worst quality)
cnm.params.set('quality', {
'min_SNR': min_SNR,
'rval_thr': r_values_min,
'use_cnn': False
})
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
print(' ***** ')
print('Number of total components: ', len(cnm.estimates.C))
print('Number of accepted components: ', len(cnm.estimates.idx_components))
# %% PLOT COMPONENTS
cnm.dims = dims
display_images = False # Set to true to show movies and images
if display_images:
cnm.estimates.plot_contours(img=cn_filter,
idx=cnm.estimates.idx_components)
cnm.estimates.view_components(images, idx=cnm.estimates.idx_components)
# %% MOVIES
display_images = False # Set to true to show movies and images
if display_images:
# fully reconstructed movie
cnm.estimates.play_movie(images,
q_max=99.5,
magnification=2,
include_bck=True,
gain_res=10,
bpx=bord_px)
# movie without background
cnm.estimates.play_movie(images,
q_max=99.9,
magnification=2,
include_bck=False,
gain_res=4,
bpx=bord_px)
# %% STOP SERVER
cm.stop_server(dview=dview)
def run_CaImAn_session(pathSession, suffix="", use_parallel=True):
pass # For compatibility between running under Spyder and the CLI
### %% set paths
#pathSession = "/media/wollex/Analyze_AS3/Data/879/Session01/"
#pathSession = "/home/wollex/Data/Documents/Uni/2016-XXXX_PhD/Japan/Work/Data/M879/Session01"
#sv_dir = "/home/wollex/Data/Documents/Uni/2016-XXXX_PhD/Japan/Work/Data/tmp/"
sv_dir = "/home/aschmidt/Documents/Data/tmp/"
fname = None
for f in os.listdir(pathSession):
if f.startswith("thy") or f.startswith("shank"):
fname = pathSession + f
if f.endswith('.h5'):
break
if not fname or not os.path.exists(fname):
print("No file here to process :(")
return
svname = pathSession + "results_OnACID.mat"
#if os.path.exists(svname):
#print("Processed file already present - skipping")
#return
svname_h5 = pathSession + "results_OnACID.hdf5"
t_start = time.time()
border_thr = 5 # minimal distance of centroid to border
# set up CNMF parameters
params_dict = {
#general data
'fnames': [fname],
'fr': 15,
'decay_time': 0.47,
'gSig': [6, 6], # expected half size of neurons
#model/analysis
'rf': 64 // 2, # size of patch
'K': 200, # max number of components
'nb': 2, # number of background components per patch
'p': 0, # order of AR indicator dynamics
'stride': 8,
#'simultaneously': True,
# init
'ssub': 2, # spatial subsampling during initialization
'tsub': 5, # temporal subsampling during initialization
#motion
'motion_correct': False,
'pw_rigid': False,
'strides': 96,
'max_shifts': 12, # maximum allowed rigid shift in pixels
'overlaps':
24, # overlap between patches (size of patch in pixels: strides+overlaps)
'max_deviation_rigid':
12, # maximum deviation allowed for patch with respect to rigid shifts
#'only_init': False, # whether to run only the initialization
#online
'init_batch': 300, # number of frames for initialization
'init_method': 'bare', # initialization method
'update_freq':
2000, # update every shape at least once every update_freq steps
'use_dense': False,
#'dist_shape_update': True,
#make things more memory efficient
'memory_efficient': False,
'block_size_temp': 5000,
'num_blocks_per_run_temp': 20,
'block_size_spat': 5000,
'num_blocks_per_run_spat': 20,
#quality
'min_SNR': 2.5, # minimum SNR for accepting candidate components
'rval_thr':
0.85, # space correlation threshold for accepting a component
'rval_lowest': 0,
'sniper_mode': True, # flag for using CNN for detecting new components
#'test_both': True, # use CNN and correlation to test for new components
'use_cnn': True,
'thresh_CNN_noisy': 0.6, # CNN threshold for candidate components
'min_cnn_thr': 0.8, # threshold for CNN based classifier
'cnn_lowest':
0.3, # neurons with cnn probability lower than this value are rejected
#display
'show_movie': False,
'save_online_movie': False,
'movie_name_online': "test_mp4v.avi"
}
opts = cnmf.params.CNMFParams(params_dict=params_dict)
print("Start writing memmapped file @t = " + time.ctime())
if use_parallel:
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
else:
dview = None
n_processes = 1
fname_memmap = cm.save_memmap([fname],
base_name='memmap%s_' % suffix,
save_dir=sv_dir,
n_chunks=20,
order='C',
dview=dview) # exclude borders
if use_parallel:
cm.stop_server(dview=dview) ## restart server to clean up memory
print(fname_memmap)
if not fname.endswith('.h5'):
print('perform motion correction')
# first we create a motion correction object with the specified parameters
#mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion'))
#%% Run (piecewise-rigid motion) correction using NoRMCorre
#mc.motion_correct(save_movie=True)
#fname_memmap = cm.save_memmap(mc.mmap_file, base_name='memmap_', save_dir=sv_dir, n_chunks=20, order='C', dview=dview) # exclude borders
#os.remove(mc.mmap_file[0])
opts.change_params({'motion_correct': True, 'pw_rigid': True})
print("Done @t = %s, (time passed: %s)" %
(time.ctime(), str(time.time() - t_start)))
#fname_memmap = sv_dir + "memmap__d1_512_d2_512_d3_1_order_C_frames_8989_.mmap"
opts.change_params({'fnames': [fname_memmap]})
Cn = cm.load(fname_memmap,
subindices=slice(0, None,
5)).local_correlations(swap_dim=False)
Yr, dims, T = cm.load_memmap(fname_memmap)
Y = np.reshape(Yr.T, [T] + list(dims), order='F')
### ------------------ 1st run ------------------ ###
### %% fit with online object on memmapped data
cnm = cnmf.online_cnmf.OnACID(params=opts)
cnm.fit_online()
print('Number of components found:' + str(cnm.estimates.A.shape[-1]))
plt.close('all')
### %% evaluate components (CNN, SNR, correlation, border-proximity)
if use_parallel:
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
else:
dview = None
n_processes = 1
cnm.estimates.evaluate_components(Y, opts, dview)
#cnm.estimates.view_components(img=Cn)
#plt.close('all')
cnm.estimates.plot_contours(
img=Cn, idx=cnm.estimates.idx_components,
crd=None) ## plot contours, need that one to get the coordinates
plt.draw()
plt.pause(1)
idx_border = []
for n in cnm.estimates.idx_components:
if (cnm.estimates.coordinates[n]['CoM'] < border_thr).any() or (
cnm.estimates.coordinates[n]['CoM'] >
(cnm.estimates.dims[0] - border_thr)).any():
idx_border.append(n)
cnm.estimates.idx_components = np.setdiff1d(cnm.estimates.idx_components,
idx_border)
cnm.estimates.idx_components_bad = np.union1d(
cnm.estimates.idx_components_bad, idx_border)
cnm.estimates.select_components(
use_object=True, save_discarded_components=False
) #%% update object with selected components
print('Number of components left after evaluation:' +
str(cnm.estimates.A.shape[-1]))
### %% save file to allow loading as CNMF- (instead of OnACID-) file
#cnm.estimates = clear_cnm(cnm.estimates,remove=['shifts','discarded_components'])
cnm.estimates = clear_cnm(
cnm.estimates,
retain=['A', 'C', 'S', 'b', 'f', 'YrA', 'dims', 'coordinates', 'sn'])
cnm.save(svname_h5)
### -------------------2nd run --------------------- ###
### %% run a refit on the whole data
if use_parallel:
cm.stop_server(dview=dview) ## restart server to clean up memory
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
else:
dview = None
n_processes = 1
cnm = cnmf.cnmf.load_CNMF(svname_h5, n_processes, dview)
cnm.params.change_params({'p': 1})
cnm.estimates.dims = Cn.shape # gets lost for some reason
print(
"merge & update spatial + temporal & deconvolve @t = %s, (time passed: %s)"
% (time.ctime(), str(time.time() - t_start)))
cnm.update_temporal(
Yr) # need this to calculate noise per component for merging purposes
cnm.merge_comps(Yr, mx=1000, fast_merge=False)
cnm.estimates.C[np.where(
np.isnan(cnm.estimates.C)
)] = 0 ## for some reason, there are NaNs in it -> cant process this
cnm.update_temporal(Yr) # update temporal trace after merging
cnm.params.change_params({'n_pixels_per_process':
1000}) ## for some reason this one gets lost
cnm.estimates.sn, _ = cnmf.pre_processing.get_noise_fft(
Yr[:, :2000].astype(np.float32))
cnm.update_spatial(Yr) # update shapes a last time
cnm.update_temporal(Yr) # update temporal trace a last time
cnm.deconvolve()
if use_parallel:
cm.stop_server(dview=dview)
print("Done @t = %s, (time passed: %s)" %
(time.ctime(), str(time.time() - t_start)))
print('Number of components left after merging:' +
str(cnm.estimates.A.shape[-1]))
### ------------------- store results ------------------- ###
###%% store results in matlab array for further processing
results = dict(A=cnm.estimates.A,
C=cnm.estimates.C,
S=cnm.estimates.S,
Cn=Cn,
b=cnm.estimates.b,
f=cnm.estimates.f)
savemat(svname, results)
#hdf5storage.write(results, '.', svname, matlab_compatible=True)
#cnm.estimates.coordinates = None
#cnm.estimates.plot_contours(img=Cn, crd=None)
#cnm.estimates.view_components(img=Cn)
#plt.draw()
#plt.pause(1)
### %% save only items that are needed to save disk-space
#cnm.estimates = clear_cnm(cnm.estimates,retain=['A','C','S','b','f','YrA'])
#cnm.save(svname_h5)
print("Total time taken: " + str(time.time() - t_start))
os.remove(fname_memmap)
def run(batch_dir: str, UUID: str):
logging.basicConfig(
stream=sys.stdout,
level=logging.DEBUG,
format=
"%(relativeCreated)12d [%(filename)s:%(funcName)20s():%(lineno)s] [%(process)d] %(message)s"
)
start_time = time()
output = {'status': 0, 'output_info': ''}
n_processes = os.environ['_MESMERIZE_N_THREADS']
n_processes = int(n_processes)
file_path = os.path.join(batch_dir, UUID)
filename = [file_path + '_input.tiff']
input_params = pickle.load(open(file_path + '.params', 'rb'))
# If Ain is specified
if input_params['do_cnmfe']:
Ain = None
item_uuid = input_params['cnmfe_kwargs'].pop('Ain')
if item_uuid:
print('>> Ain specified, looking for cnm-A file <<')
parent_batch_dir = os.environ['CURR_BATCH_DIR']
item_out_file = os.path.join(parent_batch_dir, f'{item_uuid}.out')
t0 = time()
timeout = 60
while not os.path.isfile(item_out_file):
print('>>> cnm-A not found, waiting for 15 seconds <<<')
sleep(15)
if time() - t0 > timeout:
output.update({
'status':
0,
'output_info':
'Timeout exceeding in waiting for Ain input file'
})
raise TimeoutError(
'Timeout exceeding in waiting for Ain input file')
if os.path.isfile(item_out_file):
if json.load(open(item_out_file, 'r'))['status']:
Ain_path = os.path.join(parent_batch_dir,
item_uuid + '_results.hdf5')
Ain = load_dict_from_hdf5(Ain_path)['estimates']['A']
print('>>> Found Ain file <<<')
else:
raise FileNotFoundError(
'>>> Could not find specified Ain file <<<')
print('*********** Creating Process Pool ***********')
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=n_processes,
single_thread=False,
ignore_preexisting=True)
try:
print('Creating memmap')
memmap_path = cm.save_memmap(
filename,
base_name=f'memmap-{UUID}',
order='C',
dview=dview,
border_to_0=input_params['border_pix'],
)
Yr, dims, T = cm.load_memmap(memmap_path)
Y = Yr.T.reshape((T, ) + dims, order='F')
if input_params['do_cnmfe']:
gSig = input_params['cnmfe_kwargs']['gSig'][0]
else:
gSig = input_params['corr_pnr_kwargs']['gSig']
cn_filter, pnr = cm.summary_images.correlation_pnr(Y,
swap_dim=False,
gSig=gSig)
if not input_params['do_cnmfe'] and input_params['do_corr_pnr']:
pickle.dump(cn_filter,
open(UUID + '_cn_filter.pikl', 'wb'),
protocol=4)
pickle.dump(pnr, open(UUID + '_pnr.pikl', 'wb'), protocol=4)
output_file_list = \
[
UUID + '_pnr.pikl',
UUID + '_cn_filter.pikl',
]
output.update({
'output': UUID,
'status': 1,
'output_info': 'inspect correlation & pnr',
'output_files': output_file_list
})
dview.terminate()
for mf in glob(batch_dir + '/memmap-*'):
try:
os.remove(mf)
except: # Windows doesn't like removing the memmaps
pass
end_time = time()
processing_time = (end_time - start_time) / 60
output.update({'processing_time': processing_time})
json.dump(output, open(file_path + '.out', 'w'))
return
cnmf_params_dict = \
{
"method_init": 'corr_pnr',
"n_processes": n_processes,
"only_init_patch": True, # for 1p
"center_psf": True, # for 1p
"normalize_init": False, # for 1p
}
cnmf_params_dict.update(**input_params['cnmfe_kwargs'])
cnm = cnmf.CNMF(
n_processes=n_processes,
dview=dview,
Ain=Ain,
params=CNMFParams(params_dict=cnmf_params_dict),
)
cnm.fit(Y)
# DISCARD LOW QUALITY COMPONENTS
cnm.params.set('quality', {
'use_cnn': False,
**input_params['eval_kwargs']
})
cnm.estimates.evaluate_components(Y, cnm.params, dview=dview)
out_filename = f'{UUID}_results.hdf5'
cnm.save(out_filename)
pickle.dump(pnr, open(UUID + '_pnr.pikl', 'wb'), protocol=4)
pickle.dump(cn_filter,
open(UUID + '_cn_filter.pikl', 'wb'),
protocol=4)
output.update({
'output':
filename[:-5],
'status':
1,
'output_files': [
out_filename,
UUID + '_pnr.pikl',
UUID + '_cn_filter.pikl',
]
})
except Exception as e:
output.update({
'status': 0,
'Y.shape': Y.shape,
'output_info': traceback.format_exc()
})
dview.terminate()
for mf in glob(batch_dir + '/memmap-*'):
try:
os.remove(mf)
except: # Windows doesn't like removing the memmaps
pass
end_time = time()
processing_time = (end_time - start_time) / 60
output.update({'processing_time': processing_time})
json.dump(output, open(file_path + '.out', 'w'))
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters
# dataset dependent parameters
display_images = False # Set to true to show movies and images
fnames = ['data_endoscope.tif'] # filename to be processed
fr = 10 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
do_motion_correction_nonrigid = False
do_motion_correction_rigid = True # choose motion correction type
gSig_filt = (3, 3) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (5, 5) # maximum allowed rigid shift
strides = (
48, 48
) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (24, 24
) # overlap between pathes (size of patch strides+overlaps)
# for parallelization split the movies in num_splits chuncks across time
# (make sure that length_movie/num_splits_to_process_rig>100)
splits_rig = 10
splits_els = 10
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
#%% start the cluster
try:
cm.stop_server() # stop it if it was running
except ():
pass
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local',
n_processes=
24, # number of process to use, if you go out of memory try to reduce this one
single_thread=False)
#%% download demo file
fnames = [download_demo(fnames[0])]
filename_reorder = fnames
#%% MOTION CORRECTION
if do_motion_correction_nonrigid or do_motion_correction_rigid:
# do motion correction rigid
mc = motion_correct_oneP_rigid(
fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
dview=dview,
splits_rig=splits_rig,
save_movie=not (do_motion_correction_nonrigid),
border_nan='copy')
new_templ = mc.total_template_rig
plt.subplot(1, 2, 1)
plt.imshow(new_templ) # % plot template
plt.subplot(1, 2, 2)
plt.plot(mc.shifts_rig) # % plot rigid shifts
plt.legend(['x shifts', 'y shifts'])
plt.xlabel('frames')
plt.ylabel('pixels')
# borders to eliminate from movie because of motion correction
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
filename_reorder = mc.fname_tot_rig
# do motion correction nonrigid
if do_motion_correction_nonrigid:
mc = motion_correct_oneP_nonrigid(
fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
strides=strides,
overlaps=overlaps,
splits_els=splits_els,
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
dview=dview,
splits_rig=None,
save_movie=True, # whether to save movie in memory mapped format
new_templ=new_templ, # template to initialize motion correction
border_nan='copy')
filename_reorder = mc.fname_tot_els
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
# create memory mappable file in the right order on the hard drive (C order)
fname_new = cm.save_memmap(filename_reorder,
base_name='memmap_',
order='C',
border_to_0=bord_px,
dview=dview)
# load memory mappable file
Yr, dims, T = cm.load_memmap(fname_new)
Y = Yr.T.reshape((T, ) + dims, order='F')
#%% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None
gSig = (
3, 3
) # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1
Ain = None # possibility to seed with predetermined binary masks
merge_thresh = .7 # merging threshold, max correlation allowed
rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80
stride_cnmf = 20 # amount of overlap between the patches in pixels
# (keep it at least large as gSiz, i.e 4 times the neuron size gSig)
tsub = 2 # downsampling factor in time for initialization,
# increase if you have memory problems
ssub = 1 # downsampling factor in space for initialization,
# increase if you have memory problems
# you can pass them here as boolean vectors
low_rank_background = None # None leaves background of each patch intact,
# True performs global low-rank approximation if gnb>0
gnb = 0 # number of background components (rank) if positive,
# else exact ring model with following settings
# gnb= 0: Return background as b and W
# gnb=-1: Return full rank background B
# gnb<-1: Don't return background
nb_patch = 0 # number of background components (rank) per patch if gnb>0,
# else it is set automatically
min_corr = .8 # min peak value from correlation image
min_pnr = 10 # min peak to noise ration from PNR image
ssub_B = 2 # additional downsampling factor in space for background
ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor
# parameters for component evaluation
min_SNR = 3 # adaptive way to set threshold on the transient size
r_values_min = 0.85 # threshold on space consistency (if you lower more components
# will be accepted, potentially with worst quality)
opts = params.CNMFParams(
dims=dims,
fr=fr,
decay_time=decay_time,
method_init='corr_pnr', # use this for 1 photon
k=K,
gSig=gSig,
gSiz=gSiz,
merge_thresh=merge_thresh,
p=p,
tsub=tsub,
ssub=ssub,
rf=rf,
stride=stride_cnmf,
only_init_patch=True, # set it to True to run CNMF-E
gnb=gnb,
nb_patch=nb_patch,
method_deconvolution='oasis', # could use 'cvxpy' alternatively
low_rank_background=low_rank_background,
update_background_components=
True, # sometimes setting to False improve the results
min_corr=min_corr,
min_pnr=min_pnr,
normalize_init=False, # just leave as is
center_psf=True, # leave as is for 1 photon
ssub_B=ssub_B,
ring_size_factor=ring_size_factor,
del_duplicates=True, # whether to remove duplicates from initialization
border_pix=bord_px) # number of pixels to not consider in the borders)
#%% compute some summary images (correlation and peak to noise)
# change swap dim if output looks weird, it is a problem with tiffile
cn_filter, pnr = cm.summary_images.correlation_pnr(Y,
gSig=gSig[0],
swap_dim=False)
# inspect the summary images and set the parameters
inspect_correlation_pnr(cn_filter, pnr)
# print parameters set above, modify them if necessary based on summary images
print(min_corr) # min correlation of peak (from correlation image)
print(min_pnr) # min peak to noise ratio
#%% RUN CNMF ON PATCHES
cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=opts)
cnm.fit(Y)
#%% DISCARD LOW QUALITY COMPONENTS
cnm.params.set('quality', {
'min_SNR': min_SNR,
'rval_thr': r_values_min,
'use_cnn': False
})
cnm.estimates.evaluate_components(Y, cnm.params, dview=dview)
print(' ***** ')
print('Number of total components: ', len(cnm.estimates.C))
print('Number of accepted components: ', len(cnm.estimates.idx_components))
#%% PLOT COMPONENTS
cnm.dims = dims
if display_images:
cnm.estimates.plot_contours(img=cn_filter,
idx=cnm.estimates.idx_components)
cnm.estimates.view_components(Y, idx=cnm.estimates.idx_components)
#%% MOVIES
if display_images:
# fully reconstructed movie
cnm.estimates.play_movie(Y,
q_max=99.9,
magnification=2,
include_bck=True,
gain_res=10,
bpx=bord_px)
# movie without background
cnm.estimates.play_movie(Y,
q_max=99.9,
magnification=2,
include_bck=False,
gain_res=4,
bpx=bord_px)
#%% STOP SERVER
cm.stop_server(dview=dview)
def caimanAlign(fnames, frameRate=20):
"""
fnames: list of file path
"""
startSeconds = time.time()
opts = caimanOptions.createParams(fnames, frameRate=frameRate)
#start a cluster for parallel processing (if a cluster already exists it will be closed and a new session will be opened)
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
#
# create a motion correction object with the parameters specified.
#Note that the file is not loaded in memory
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# set mmap_file
'''
tmpPath, tmpExt = os.path.splitext(fnames[0])
tmpPath += '.mmap'
print('setting mc.mmap_file to tmpPath:', tmpPath)
mc.mmap_file = tmpPath
'''
#print('mc.mmap_file:', mc.mmap_file)
#
# perform motion correction
# We will perform piecewise rigid motion correction using the NoRMCorre algorithm.
# This has already been selected by setting pw_rigid=True when defining the parameters object.
print(' performing motion correction ... please wait ...', fnames[0])
mc.motion_correct(save_movie=True)
m_els = cm.load(mc.fname_tot_els)
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
print(' border_to_0:', border_to_0)
# maximum shift to be used for trimming against NaNs
#
# memory mapping
print(' memory mapping')
print(' save_memmap mc.mmap_file:', mc.mmap_file)
fname_new = cm.save_memmap(mc.mmap_file,
base_name='memmap_',
order='C',
border_to_0=border_to_0,
dview=dview) # exclude borders
print(' mc.mmap_file:', mc.mmap_file)
print(' fname_new:', fname_new)
# make a copy of mmap file with a sane name
# does not work, seems you can not 'copy' a mmap file like a normal file
'''
tmpSavePath, tmpExt = os.path.splitext(fnames[0])
tmpSavePath += '.mmap' # need to be _aligned_ch1.tif
shutil.copyfile(fname_new, tmpSavePath)
'''
# save a _mmap.txt file with the name of the memory map file
'''
tmpFile, tmpFilename = os.path.splitext(fnames[0])
tmpFile += '_mmap.txt'
with open(tmpFile, 'w') as myTmpFile:
myTmpFile.write(fname_new)
'''
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
#load frames in python format (T x X x Y)
# save as .tif
images_8bit = ((images - images.min()) / (images.ptp() / 255.0)).astype(
np.uint8) # map the data range to 0 - 255
tmpSavePath, tmpExt = os.path.splitext(fnames[0])
tmpSavePath += '_aligned.tif' # need to be _aligned_ch1.tif
print(' saving aligned .tif', images_8bit.shape, images_8bit.dtype,
'tmpSavePath:', tmpSavePath)
tifffile.imsave(tmpSavePath, images_8bit, bigtiff=True)
# shut down the cluster
if 'dview' in locals():
cm.stop_server(dview=dview)
# remove mmap file fname_new
if os.path.isfile(fname_new):
print(' removing mmap file:', fname_new)
os.remove(fname_new)
oneFile = mc.mmap_file[0]
if os.path.isfile(oneFile):
print(' removing mmap file mc.mmap_file[0]:', oneFile)
os.remove(oneFile)
# done
stopSeconds = time.time()
elapsedSeconds = round(stopSeconds - startSeconds, 3)
elapsedMinutes = round(elapsedSeconds / 60, 3)
print(f' caimanAlign() done in {elapsedMinutes} minutes')
def volspike(pars):
""" Function for finding spikes of single neuron with given ROI in
voltage imaging. Use function denoise_spikes to find spikes
from one dimensional signal, and use ridge regression to find the
best weight. Do these two steps iteratively to find
best spike times.
Args:
pars: list
fnames: str
name of the memory mapping file in C order
fr: int
frame rate of the movie
cell_n: int
index of the cell processing
ROIs: 3-d array
all regions of interests
weights: 3-d array
spatial weights of different cells generated by previous data blocks as initialization
args: dictionary
context_size: int
number of pixels surrounding the ROI to use as context
censor_size: int
number of pixels surrounding the ROI to censor from the background PCA; roughly
the spatial scale of scattered/dendritic neural signals, in pixels
visualize_ROI: boolean
whether to visualize the region of interest inside the context region
flip_signal: boolean
whether to flip signal upside down for spike detection
True for voltron, False for others
hp_freq_pb: float
high-pass frequency for removing photobleaching
nPC_bg: int
number of principle components used for background subtraction
ridge_bg: float
regularization strength for ridge regression in background removal
hp_freq: float
high-pass cutoff frequency to filter the signal after computing the trace
clip: int
maximum number of spikes for producing templates
threshold_method: str
adaptive_threshold or simple method for thresholding signals
adaptive_threshold method threshold based on estimated peak distribution
simple method threshold based on estimated noise level
min_spikes: int
minimal number of spikes to be detected
pnorm: float
a variable decides spike count chosen for adaptive threshold method
threshold: float
threshold for spike detection in simple threshold method
The real threshold is the value multiplied by the estimated noise level
sigmas: 1-d array
spatial smoothing radius imposed on high-pass filtered
movie only for finding weights
n_iter: int
number of iterations alternating between estimating spike times
and spatial filters
weight_update: str
ridge or NMF for weight update
do_plot: boolean
if Ture, plot trace of signals and spiketimes,
peak triggered average, histogram of heights in the last iteration
do_cross_val: boolean
whether to use cross validation to optimize regression regularization parameters
sub_freq: float
frequency for subthreshold extraction
Returns:
output: dictionary
cell_n: int
index of cell
t: 1-d array
trace without applying whitened matched filter
ts: 1-d array
trace after applying whitened matched filter
t_rec: 1-d array
reconstructed signal of the neuron
t_sub: 1-d array
subthreshold signal of the neuron
spikes: 1-d array
spike time of the neuron
num_spikes: list
number of spikes detected in each iteration
low_spikes: boolean
True if detected number spikes is less than min_spikes
template: 1-d array
temporal template of the neuron
snr: float
signal to noise ratio of the processed signal
thresh: float
threshold of the signal
weights: 2-d array
ridge regression coefficients for fitting reconstructed signal
locality: boolean
False if the maximum of spatial filter is not in the initial ROI
context_coord: 2-d array
boundary of context region in x,y coordinates
mean_im: 1-d array
mean of the signal in ROI after removing photobleaching, used for
producing F0
F0: 1-d array
baseline signal
dFF: 1-d array
scaled signal
rawROI: dictionary
including the result after the first spike extraction
"""
# load parameters
fnames = pars[0]
fr = pars[1]
cell_n = pars[2]
bw = pars[3]
weights_init = pars[4]
args = pars[5]
window_length = fr * 0.02 # window length for temporal templates
output = {}
output['rawROI'] = {}
print(f'Now processing cell number {cell_n}')
# load the movie in C-order mermory mapping file
Yr, dims, T = cm.load_memmap(fnames)
if bw.shape == dims:
images = np.reshape(Yr.T, [T] + list(dims), order='F')
else:
raise Exception('Dimensions of movie and ROIs do not accord')
# extract the context region from the entire movie
bwexp = dilation(bw,
np.ones([args['context_size'], args['context_size']]),
shift_x=True,
shift_y=True)
Xinds = np.where(np.any(bwexp > 0, axis=1) > 0)[0]
Yinds = np.where(np.any(bwexp > 0, axis=0) > 0)[0]
bw = bw[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1]
notbw = 1 - dilation(bw, disk(args['censor_size']))
data = np.array(images[:, Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1])
bw = (bw > 0)
notbw = (notbw > 0)
ref = np.median(data[:500, :, :], axis=0)
bwexp[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1] = True
# visualize the ROI
if args['visualize_ROI']:
fig = plt.figure()
plt.subplot(131)
plt.imshow(ref)
plt.axis('image')
plt.xlabel('mean Intensity')
plt.subplot(132)
plt.imshow(bw)
plt.axis('image')
plt.xlabel('initial ROI')
plt.subplot(133)
plt.imshow(notbw)
plt.axis('image')
plt.xlabel('background')
fig.suptitle('ROI selection')
plt.show()
# flip the signal if necessary
if args['flip_signal'] == True:
data = -data
# remove the photobleaching effect by high-pass filtering the signal
output['mean_im'] = np.mean(data, axis=0)
data = np.reshape(data, (data.shape[0], -1))
data = data - np.mean(data, 0)
data = data - np.mean(data, 0) #do again because of numeric issues
data_hp = signal_filter(data.T, args['hp_freq_pb'], fr).T
data_lp = data - data_hp
# compute the initial trace
if weights_init is None:
t0 = np.nanmean(data_hp[:, bw.ravel()], 1)
else:
print('Reuse weights')
t0 = np.matmul(data_hp, weights_init.ravel()) # reuse weights
t0 = t0 - np.mean(t0)
# remove any variance in trace that can be predicted from the background principal components
data_svd = data_hp[:, notbw.ravel()]
if data_svd.shape[1] < args['nPC_bg'] + 1:
raise Exception(
f'Too few pixels ({data_svd.shape[1]}) for background extraction (at least {args["nPC_bg"]} needed);'
f'please decrease context_size and censor_size')
Ub, Sb, Vb = svds(data_svd, args['nPC_bg'])
alpha = args['nPC_bg'] * args[
'ridge_bg'] # square of F-norm of Ub is equal to number of principal components
reg = Ridge(alpha=alpha, fit_intercept=False, solver='lsqr').fit(Ub, t0)
t0 = np.double(t0 - np.matmul(Ub, reg.coef_))
# spike detection for the initial trace
ts, spikes, t_rec, templates, low_spikes, thresh = denoise_spikes(
t0,
window_length,
fr,
hp_freq=args['hp_freq'],
clip=args['clip'],
threshold_method=args['threshold_method'],
pnorm=args['pnorm'],
threshold=args['threshold'],
min_spikes=args['min_spikes'],
do_plot=False)
output['rawROI']['t'] = t0.copy()
output['rawROI']['ts'] = ts.copy()
output['rawROI']['spikes'] = spikes.copy()
if weights_init is None:
output['rawROI']['weights'] = bw.copy()
else:
output['rawROI']['weights'] = weights_init.copy()
output['rawROI']['t'] = output['rawROI']['t'] * np.mean(
t0[output['rawROI']['spikes']]) / np.mean(output['rawROI']['t'][
output['rawROI']['spikes']]) # correct shrinkage
output['rawROI']['templates'] = templates
num_spikes = [spikes.shape[0]]
# prebuild the regression matrix generate a predictor for ridge regression
pred = np.empty_like(data_hp)
pred[:] = data_hp
pred = np.hstack(
(np.ones((data_hp.shape[0], 1), dtype=np.single),
np.reshape(
movie.gaussian_blur_2D(np.reshape(
pred, (data_hp.shape[0], ref.shape[0], ref.shape[1])),
kernel_size_x=7,
kernel_size_y=7,
kernel_std_x=1.5,
kernel_std_y=1.5,
borderType=cv2.BORDER_REPLICATE),
data_hp.shape)))
# cross-validation of regularized regression parameters
lambdamax = np.single(np.linalg.norm(pred[:, 1:], ord='fro')**2)
lambdas = lambdamax * np.logspace(-4, -2, 3)
if args['do_cross_val']:
# need to add
logging.warning('doing cross validation')
raise Exception('cross validation option is not availble yet')
else:
s_max = 1
l_max = 2
sigma = args['sigmas'][s_max]
recon = np.empty_like(data_hp)
recon[:] = data_hp
recon = np.hstack(
(np.ones((data_hp.shape[0], 1), dtype=np.single),
np.reshape(
movie.gaussian_blur_2D(
np.reshape(recon,
(data_hp.shape[0], ref.shape[0], ref.shape[1])),
kernel_size_x=np.int(2 * np.ceil(2 * sigma) + 1),
kernel_size_y=np.int(2 * np.ceil(2 * sigma) + 1),
kernel_std_x=sigma,
kernel_std_y=sigma,
borderType=cv2.BORDER_REPLICATE), data_hp.shape)))
# refine weights and estimate spike times for several iterations
for iteration in range(args['n_iter']):
if iteration == args['n_iter'] - 1:
do_plot = args['do_plot']
else:
do_plot = False
# update weights
tr = np.single(t_rec.copy())
if args['weight_update'] == 'NMF':
C = np.array([tr, np.ones_like(tr)
]) # constant baselines as 2nd component
CCt = C.dot(C.T)
CY = C.dot(recon[:, 1:])
A = np.maximum(np.linalg.inv(CCt).dot(CY), 0)
for _ in range(5):
for m in range(2):
A[m] += (CY[m] - CCt[m].dot(A)) / CCt[m, m]
if m == 0:
A[m] = np.maximum(A[m], 0)
weights = np.concatenate([[0], A[0]])
elif args['weight_update'] == 'ridge':
Ri = Ridge(alpha=lambdas[l_max], fit_intercept=True, solver='lsqr')
Ri.fit(recon, tr)
weights = Ri.coef_
weights[0] = Ri.intercept_
# update the signal
t = np.matmul(recon, weights)
t = t - np.mean(t)
# ridge regression to remove background components
b = Ridge(alpha=alpha, fit_intercept=False, solver='lsqr').fit(Ub,
t).coef_
t = t - np.matmul(Ub, b)
# correct shrinkage
weights = weights * np.mean(t0[spikes]) / np.mean(t[spikes])
t = np.double(t * np.mean(t0[spikes]) / np.mean(t[spikes]))
# estimate spike times
ts, spikes, t_rec, templates, low_spikes, thresh = denoise_spikes(
t,
window_length,
fr,
hp_freq=args['hp_freq'],
clip=args['clip'],
threshold_method=args['threshold_method'],
pnorm=args['pnorm'],
threshold=args['threshold'],
min_spikes=args['min_spikes'],
do_plot=do_plot)
num_spikes.append(spikes.shape[0])
# compute SNR
if len(spikes) > 0:
t = t - np.median(t)
selectSpikes = np.zeros(t.shape)
selectSpikes[spikes] = 1
sgn = np.mean(t[selectSpikes > 0])
ff1 = -t * (t < 0)
Ns = np.sum(ff1 > 0)
noise = np.sqrt(np.divide(np.sum(ff1**2), Ns))
snr = sgn / noise
else:
snr = 0
# locality test
matrix = np.matmul(np.transpose(pred[:, 1:]), t_rec)
sigmax = np.sqrt(np.sum(np.multiply(pred[:, 1:], pred[:, 1:]), axis=0))
sigmay = np.sqrt(np.dot(t_rec, t_rec))
IMcorr = matrix / sigmax / sigmay
maxCorrInROI = np.max(IMcorr[bw.ravel()])
if np.any(IMcorr[notbw.ravel()] > maxCorrInROI):
locality = False
else:
locality = True
# weights in the FOV
weights = np.reshape(weights[1:], ref.shape, order='C')
weights_FOV = np.zeros(images.shape[1:])
weights_FOV[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1] = weights
# subthreshold activity extraction
t_sub = t.copy() - t_rec
t_sub = signal_filter(t_sub, args['sub_freq'], fr, order=5, mode='low')
# output
output['cell_n'] = cell_n
output['t'] = t
output['ts'] = ts
output['t_rec'] = t_rec
output['t_sub'] = t_sub
output['spikes'] = spikes
output['low_spikes'] = low_spikes
output['num_spikes'] = num_spikes
output['templates'] = templates
output['snr'] = snr
output['thresh'] = thresh
output['weights'] = weights_FOV
output['locality'] = locality
output['context_coord'] = np.transpose(
np.vstack((Xinds[[0, -1]], Yinds[[0, -1]])))
output['F0'] = np.abs(
np.nanmean(
data_lp[:, bw.flatten()] + output['mean_im'][bw][np.newaxis, :],
1))
output['dFF'] = t / output['F0']
output['rawROI']['dFF'] = output['rawROI']['t'] / output['F0']
return output
def make_movie(self):
"""
Load memmaps and make the movie.
"""
Yr, dims, T = cm.load_memmap(self.memmap)
self.movie = np.reshape(Yr.T, [T] + list(dims), order='F')
def volspike(pars):
""" Main function for finding spikes of one single neuron with given ROI in
voltage imaging. Using function denoiseSpikes to find spikes
of one dimensional signal, using ridge regression to find the
best spatial filters. Do these two steps iteratively to find
best spike time.
Args:
pars: list
fnames: str
name of the memory map file
fr: int
cellN: int
number of cell processing
ROIs: 3-d array
all region of interests
weights: 3-d array
spatial weights of different cells generated by previous data blocks as initialization
args: dictionary
doCrossVal: boolean
whether to use cross validation to optimize regression regularization parameters
doGlobalSubtract: boolean
whether to subtract the signal which can be predicted by the entire video
contextSize: int
number of pixels surrounding the ROI to use as context
censorSize: int
number of pixels surrounding the ROI to censor from the background PCA; roughly
the spatial scale of scattered/dendritic neural signals, in pixels
nPC_bg: int
number of principle components used for background subtraction
tau_lp: int
time window for lowpass filter (seconds); signals slower than this will be ignored
tau_pred: int
time window in seconds for high pass filtering to make predictor for regression
sigmas: 1-d array
spatial smoothing radius imposed on spatial filter
nIter: int
number of iterations alternating between estimating temporal and spatial filters
localAlign: boolean
globalAlign: boolean
highPassRegression: boolean
whether to regress on a high-passed version of the data. Slightly improves detection of spikes,
but makes subthreshold unreliable
use_ridge: boolean
whether use ridge regression for removing background signal. If not, the algorithm will use
linear regression
Ridge_bg_coef: float
regularization strength for ridge regression in background removal.
Returns:
output: dictionary
rawROI: dictionary
including the result after the first spike extraction
bwexp: 2-d array
expansion area of ROI, used for reconstructing signal
meanIM: 1-d array
trace after high-pass filter
num_spikes: list
number of spikes in each iteration
passedLocalityTest: boolean
False if the maximum of spatial filter is not in the initial ROI
snr: float
signal to noise ratio of the processed signal
y: 1-d array
processed signal of the video
yFilt: 1-d array
spike train of the signal
ROIbw: 2-d array
region of interest
recons_signal: 1-d array
reconstructed signal of the neuron
spatialFilter: 2-d array
spatial filter of the neuron
falsePosRate: float
possibility of misclassify noise as real spikes
detectionRate: float
possibility of real spikes being detected
template: 1-d array
spike template of the neuron
spikeTimes: 1-d array
spike time of the neuron
thresh: float
threshold of the signal
F0: 1-d array
initial signal
dFF: 1-d array
scaled signal
bg_pc: 2-d array
background principal components extracted from svd
low_spk: boolean
whether there are few spikes(less than 30) in the signal
weights: 2-d array
ridge regression coefficient for updating spatial filter
cellN: int
index of cell
"""
fnames = pars[0]
sampleRate = pars[1]
cellN = pars[2]
bw = pars[3]
weights_init = pars[4]
args = pars[5]
print('Now processing cell number {0}'.format(cellN))
doCrossVal = args['doCrossVal']
doGlobalSubtract = args['doGlobalSubtract']
contextSize = args['contextSize']
censorSize = args['censorSize']
nPC_bg = args['nPC_bg']
tau_lp = args['tau_lp']
tau_pred = args['tau_pred']
sigmas = args['sigmas']
nIter = args['nIter']
localAlign = args['localAlign']
globalAlign = args['globalAlign']
highPassRegression = args['highPassRegression']
use_Ridge = args['use_Ridge']
Ridge_bg_coef = args['Ridge_bg_coef']
windowLength = sampleRate * 0.02 # window length for spike templates
output = {}
output['rawROI'] = {}
Yr, dims, T = cm.load_memmap(fnames)
if bw.shape == dims:
images = np.reshape(Yr.T, [T] + list(dims), order='F')
elif bw.shape == dims[::-1]:
images = np.reshape(Yr.T, [T] + list(dims),
order='F').transpose([0, 2, 1])
else:
print('size of ROI and video does not accrod')
# extract relevant region and align
bwexp = dilation(bw,
np.ones([contextSize, contextSize]),
shift_x=True,
shift_y=True)
Xinds = np.where(np.any(bwexp > 0, axis=1) > 0)[0]
Yinds = np.where(np.any(bwexp > 0, axis=0) > 0)[0]
bw = bw[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1]
notbw = 1 - dilation(bw, disk(censorSize))
data = np.array(images[:, Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1])
bw = (bw > 0)
notbw = (notbw > 0)
ref = np.median(data[:500, :, :], axis=0)
bwexp[Xinds[0]:Xinds[-1] + 1, Yinds[0]:Yinds[-1] + 1] = True
output['bwexp'] = bwexp
# visualize ROI
# fig = plt.figure()
# plt.subplot(131);plt.imshow(ref);plt.axis('image');plt.xlabel('mean Intensity')
# plt.subplot(132);plt.imshow(bw);plt.axis('image');plt.xlabel('initial ROI')
# plt.subplot(133);plt.imshow(notbw);plt.axis('image');plt.xlabel('background')
# fig.suptitle('ROI selection')
# plt.show()
output['meanIM'] = np.mean(data, axis=0)
data = np.reshape(data, (data.shape[0], -1))
data = data - np.mean(data, 0)
data = data - np.mean(data, 0)
# remove low frequency components
data_hp = highpassVideo(data.T, 1 / tau_lp, sampleRate).T
"""
data_pred = np.empty_like(data_hp)
if highPassRegression:
data_pred[:] = highpassVideo(data, 1 / tau_pred, sampleRate)
else:
data_pred[:] = data_hp
"""
# initial trace
if weights_init is None:
t = np.nanmean(data_hp[:, bw.ravel()], 1)
else:
t = -np.matmul(data_hp, weights_init[1:]) # weights are negative
t = t - np.mean(t)
# remove any variance in trace that can be predicted from the background principal components
Ub, Sb, Vb = svds(data_hp[:, notbw.ravel()], nPC_bg)
if use_Ridge:
alpha = nPC_bg * Ridge_bg_coef # = np.single(np.linalg.norm(Ub, ord='fro') ** 2) * Ridge_bg_coef
reg = Ridge(alpha=alpha, fit_intercept=False, solver='lsqr').fit(Ub, t)
else:
reg = LinearRegression(fit_intercept=False).fit(Ub, t)
t = np.double(t - np.matmul(Ub, reg.coef_))
# find out spikes of initial trace
Xspikes, spikeTimes, guessData, output['rawROI']['falsePosRate'], output['rawROI']['detectionRate'], \
output['rawROI']['templates'], low_spk, _ = denoiseSpikes(-t, windowLength, sampleRate, False, 100)
Xspikes = -Xspikes
output['rawROI']['X'] = t.copy()
output['rawROI']['Xspikes'] = Xspikes.copy()
output['rawROI']['spikeTimes'] = spikeTimes.copy()
output['rawROI']['spatialFilter'] = bw.copy()
output['rawROI']['X'] = output['rawROI']['X'] * np.mean(
t[output['rawROI']['spikeTimes']]) / np.mean(output['rawROI']['X'][
output['rawROI']['spikeTimes']]) # correct shrinkage
output['num_spikes'] = [spikeTimes.shape[0]]
templates = output['rawROI']['templates']
selectSpikes = np.zeros(Xspikes.shape)
selectSpikes[spikeTimes] = 1
sgn = np.mean(Xspikes[selectSpikes > 0])
noise = np.std(Xspikes[selectSpikes == 0])
snr = sgn / noise
# prebuild the regression matrix
# generate a predictor for ridge regression
pred = np.empty_like(data_hp)
pred[:] = data_hp
pred = np.hstack(
(np.ones((data_hp.shape[0], 1), dtype=np.single),
np.reshape(
movie.gaussian_blur_2D(np.reshape(
pred, (data_hp.shape[0], ref.shape[0], ref.shape[1])),
kernel_size_x=7,
kernel_size_y=7,
kernel_std_x=1.5,
kernel_std_y=1.5,
borderType=cv2.BORDER_REPLICATE),
data_hp.shape)))
# Cross-validation of regularized regression parameters
lambdamax = np.single(np.linalg.norm(pred[:, 1:], ord='fro')**2)
lambdas = lambdamax * np.logspace(-4, -2, 3)
I0 = np.eye(pred.shape[1], dtype=np.single)
I0[0, 0] = 0
if doCrossVal:
# need to add
print('doing cross validation')
else:
s_max = 1
l_max = 2
lambd = lambdas[l_max]
sigma = sigmas[s_max]
lambda_ix = l_max
selectPred = np.ones(data_hp.shape[0])
if highPassRegression:
selectPred[:np.int16(sampleRate / 2 + 1)] = 0
selectPred[-1 - np.int16(sampleRate / 2):] = 0
sigma = sigmas[s_max]
"""
pred = np.empty_like(data_pred)
pred[:] = data_pred
pred = np.hstack((np.ones((data_pred.shape[0], 1), dtype=np.single), np.reshape
(movie.gaussian_blur_2D(np.reshape(pred,
(data_pred.shape[0], ref.shape[0], ref.shape[1])),
kernel_size_x=np.int(2 * np.ceil(2 * sigma) + 1),
kernel_size_y=np.int(2 * np.ceil(2 * sigma) + 1),
kernel_std_x=sigma, kernel_std_y=sigma,
borderType=cv2.BORDER_REPLICATE), data_pred.shape)))
"""
recon = np.empty_like(data_hp)
recon[:] = data_hp
recon = np.hstack(
(np.ones((data_hp.shape[0], 1), dtype=np.single),
np.reshape(
movie.gaussian_blur_2D(
np.reshape(recon,
(data_hp.shape[0], ref.shape[0], ref.shape[1])),
kernel_size_x=np.int(2 * np.ceil(2 * sigma) + 1),
kernel_size_y=np.int(2 * np.ceil(2 * sigma) + 1),
kernel_std_x=sigma,
kernel_std_y=sigma,
borderType=cv2.BORDER_REPLICATE), data_hp.shape)))
# Identify spatial filters with regularized regression
for j in range(3 if args['weight_update'] == 'NMF' else 1):
if j == 0:
initialGuess = guessData[:]
else:
guessData = initialGuess[:]
for iteration in range(nIter):
doPlot = False
if iteration == nIter - 1:
doPlot = False
# print('Identifying spatial filters')
# print(iteration)
gD = np.single(guessData[selectPred > 0])
if args['weight_update'] == 'NMF':
C = np.array([gD, np.ones_like(gD)
]) # constant baselines as 2nd component
CCt = C.dot(C.T)
CY = C.dot(recon[:, 1:])
A = np.minimum(np.linalg.inv(CCt).dot(CY), 0)
for _ in range(5):
for m in range(2):
A[m] += (CY[m] - CCt[m].dot(A)) / CCt[m, m]
if m == 0:
A[m] = np.minimum(A[m], 0)
weights = np.concatenate([[0], A[0]])
else:
Ri = Ridge(alpha=lambdas[l_max],
fit_intercept=True,
solver='lsqr')
Ri.fit(recon, gD)
weights = Ri.coef_
weights[0] = Ri.intercept_
X = np.matmul(recon, weights)
X = X - np.mean(X)
# for i in np.where(np.array([np.corrcoef(X, Ub[:,i])[0,1] for i in range(nPC_bg)]) > .4)[0]:
# Ub[:, i] -= Ub[:, i].dot(X) / X.dot(X) * X
spatialFilter = np.empty_like(weights)
spatialFilter[:] = weights
spatialFilter = movie.gaussian_blur_2D(
np.reshape(spatialFilter[1:], ref.shape,
order='C')[np.newaxis, :, :],
kernel_size_x=np.int(2 * np.ceil(2 * sigma) + 1),
kernel_size_y=np.int(2 * np.ceil(2 * sigma) + 1),
kernel_std_x=sigma,
kernel_std_y=sigma,
borderType=cv2.BORDER_REPLICATE)[0]
if use_Ridge:
#alpha = np.single(np.linalg.norm(Ub, ord='fro') ** 2) * 0.01
b = Ridge(alpha=alpha, fit_intercept=False,
solver='lsqr').fit(Ub, X).coef_
else:
b = LinearRegression(fit_intercept=False).fit(Ub, X).coef_
"""
if doPlot:
plt.figure()
plt.plot(X)
plt.plot(np.matmul(Ub, b))
plt.title('Denoised trace vs background')
plt.show()
"""
X = X - np.matmul(Ub, b)
# correct shrinkage
X = np.double(X * np.mean(t[spikeTimes]) / np.mean(X[spikeTimes]))
# generate the new trace and the new denoised trace
Xspikes, spikeTimes, guessData, falsePosRate, detectionRate, templates, _, thresh = denoiseSpikes(
-X, windowLength, sampleRate, doPlot=False)
selectSpikes = np.zeros(Xspikes.shape)
selectSpikes[spikeTimes] = 1
sgn = np.mean(Xspikes[selectSpikes > 0])
noise = np.std(Xspikes[selectSpikes == 0])
snr = sgn / noise
output['num_spikes'].append(spikeTimes.shape[0])
# ensure that the maximum of the spatial filter is within the ROI
matrix = np.matmul(np.transpose(pred[:, 1:]), -guessData)
sigmax = np.sqrt(np.sum(np.multiply(pred[:, 1:], pred[:, 1:]), axis=0))
sigmay = np.sqrt(np.dot(guessData, guessData))
IMcorr = matrix / sigmax / sigmay
maxCorrInROI = np.max(IMcorr[bw.ravel()])
if np.any(IMcorr[notbw.ravel()] > maxCorrInROI):
output['passedLocalityTest'] = False
recon -= np.outer(recon.dot(weights),
weights) / weights.dot(weights)
else:
output['passedLocalityTest'] = True
break
# compute SNR
selectSpikes = np.zeros(Xspikes.shape)
selectSpikes[spikeTimes] = 1
sgn = np.mean(Xspikes[selectSpikes > 0])
noise = np.std(Xspikes[selectSpikes == 0])
snr = sgn / noise
output['snr'] = snr
del pred
del recon
data_lp = data - data_hp
# output
output['y'] = X
output['yFilt'] = -Xspikes
output['ROI'] = np.transpose(np.vstack((Xinds[[0, -1]], Yinds[[0, -1]])))
output['ROIbw'] = bw
output['recons_signal'] = guessData
output['spatialFilter'] = -spatialFilter
output['falsePosRate'] = falsePosRate
output['detectionRate'] = detectionRate
output['templates'] = templates
output['spikeTimes'] = spikeTimes
output['thresh'] = thresh
output['F0'] = np.nanmean(
data_lp[:, bw.flatten()] + output['meanIM'][bw][np.newaxis, :], 1)
output['dFF'] = X / output['F0']
output['rawROI']['dFF'] = output['rawROI']['X'] / output['F0']
output['bg_pc'] = Ub # background components
output['low_spk'] = low_spk
output['weights'] = weights
output['cellN'] = cellN
return output
single_thread=False)
# In[ ]:
## path to motion corrected tif file
folder = '/projects/p30771/miniscope/data/GRIN011/1_24_2019/H10_M19_S59/TIFs/full_movie_caiman_analysis/'
#mc_file = 'memmap__d1_480_d2_752_d3_1_order_C_frames_202_.mmap'
memory_map_file = sys.argv[1]
# In[4]:
# load memory mappable file
print('loading file:')
print(memory_map_file)
Yr, dims, T = cm.load_memmap(memory_map_file)
images = Yr.T.reshape((T, ) + dims, order='F')
#
opts = params.CNMFParams(params_dict={})
#
bord_px = 0
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None
gSig = (
3, 3
) # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1
Ain = None # possibility to seed with predetermined binary masks
merge_thresh = .7 # merging threshold, max correlation allowed
# Jan 42
folder = '/mnt/ceph/users/agiovann/ImagingData/LABELLING/Ben_labelled/Jan_Convnet/Jan42_2016_04_06_Green'
file_mmap = 'Yr_d1_512_d2_512_d3_1_order_C_frames_3699_.mmap'
ben_zip = '/mnt/ceph/users/agiovann/ImagingData/LABELLING/Ben_labelled/Jan_Convnet/Jan42_2016_04_06_Green/All_9.zip'
princeton_zip = '/mnt/xfs1/home/agiovann/labeling/Jan42_exp4_001/regions/princeton_regions.mat'
thresh_noise = -20
#%%
import pylab as pl
pl.ion()
with np.load(os.path.join(folder,'results_analysis.npz')) as ld:
locals().update(ld)
A=scipy.sparse.coo_matrix(A)
Yr, dims, T = cm.load_memmap(os.path.join(folder,file_mmap))
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T,), order='F')
gSig = [8, 8] # expected half size of neurons
final_frate=1
with np.load(os.path.join(folder,'results_blobs.npz')) as ld:
print((list(ld.keys())))
locals().update(ld)
masks_cnmf=masks[idx_blob]
#%%
crd = plot_contours(A.tocsc()[:, idx_components], Cn, thr=0.9)
#%%
masks_ben=cm.base.rois.nf_read_roi_zip(ben_zip,dims)
selected_rows = db.select(states_df,
'source_extraction',
mouse=mouse_number,
session=session,
trial=6,
is_rest=is_rest,
cropping_v=cropping_version,
motion_correction_v=motion_correction_version,
alignment_v=0,
source_extraction_v=1)
row = selected_rows.iloc[1]
#input_mmap_file_path = eval(row.loc['alignment_output'])['main']
input_mmap_file_path = eval(row.loc['motion_correction_output'])['main']
Yr, dims, T = cm.load_memmap(input_mmap_file_path)
# logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.')
images = Yr.T.reshape((T, ) + dims, order='F')
images = cm.movie(images)
output = eval(row.loc['source_extraction_output'])
cnm_file_path = output['main']
cnm = load_CNMF(db.get_file(cnm_file_path))
W, b0 = cm.source_extraction.cnmf.initialization.compute_W(
Yr,
cnm.estimates.A.toarray(),
cnm.estimates.C,
cnm.estimates.dims,
1.4 * 5,
ssub=2)
cnm.estimates.W = W
def run_caiman_pipeline(data_path,
opts,
refit=False,
component_evaluation=False,
fr=14.913,
rf=15,
decay_time=0.6):
'''Run the caiman pipeline out of the box, without using Agonia seeds.
Parameters
----------
data_path : string
path to folder containing the data
opts : caiman object with option parameters
refit : bool, optional
if True re-run CNMF seeded on the selected components from the fitting
component_evaluation : bool, optional
if True filter components using shape and signal criteria
fr : float
frame rate of video
rf : int
half-size of the patches in pixels (in seeded is None)
decay_time = float
decay time of calcium indicator
Returns
-------
cnm : out of the box results of caiman detection
cnm2 : refited-filtered results if refit, component_evaluations flags are True'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
opts_dict = {
'only_init': True,
'rf': rf, #15
'fr': fr, #14.913,
'decay_time': decay_time
} #0.6
opts.change_params(opts_dict)
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
if refit:
cnm2 = cnm.refit(images, dview=dview)
if component_evaluation:
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
#cnm2.estimates.select_components(use_object=True)
if 'dview' in locals():
cm.stop_server(dview=dview)
return cnm, cnm2
info = dict()
info['experiment_title'] = experiment_title
info['experiment_date'] = experiment_date
info['animal_name'] = animal_name
info['spatial_downsampling'] = spatial_downsampling
info['downsample_subpath'] = downsample_subpath
info['local_rootdir'] = local_rootdir
session_fpaths = miniscope_file.list_session_dirs(local_miniscope_path,
experiment_date, animal_name)
info['session_fpaths'] = session_fpaths
timestamp_files = []
session_lengths = []
for s_fpath in session_fpaths:
vids_fpath = miniscope_file.list_vidfiles(s_fpath, vid_prefix)
total_frames = 0
memmap_files = miniscope_file.get_memmap_files(s_fpath, doPwRigid,
vid_prefix)
for memmap_file in memmap_files:
Yr, dim, T = cm.load_memmap(memmap_file, 'r')
total_frames += T
session_lengths.append(total_frames)
timestamp_files.append(miniscope_file.get_timestamps_fpath(s_fpath))
info['session_lengths'] = session_lengths
info['timestamp_files'] = timestamp_files
with open(caiman_result_dir + '/session_info.yaml', 'w') as f:
yaml.dump(info, f)
def signal_to_noise(data_path,
agonia_th,
ground_truth=None,
neurofinder=False):
'''Calculate signal to noise ratio of each box
Parameters
----------
data_path : string
path to folder containing the data
agonia_th : float
threshold for detection confidence for each box'''
data_name, median_projection, fnames, fname_new, results_caiman_path, boxes_path = get_files_names(
data_path)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load Caiman results
cnm = cnmf.load_CNMF(results_caiman_path)
# calculate the centers of the CaImAn factors
centers = np.empty((cnm.estimates.A.shape[1], 2))
for i, factor in enumerate(cnm.estimates.A.T):
centers[i] = center_of_mass(factor.toarray().reshape(
cnm.estimates.dims, order='F'))
with open(boxes_path, 'rb') as f:
boxes = pickle.load(f)
f.close()
# keep only cells above confidence threshold
boxes = boxes[boxes[:, 4] > agonia_th].astype('int')
k = 0
for cell, box in enumerate(boxes):
idx_factor = [
i for i, center in enumerate(centers)
if center[0] > box[1] and center[0] < box[3] and center[1] > box[0]
and center[1] < box[2]
]
if not idx_factor:
boxes = np.delete(boxes, cell - k, axis=0)
k += 1
if ground_truth is not None:
stats = ag.compute_stats(np.array(ground_truth),
boxes,
iou_threshold=0.5)
stnr = [] #np.zeros(boxes.shape[0])
if neurofinder:
for cell, box in enumerate(boxes):
if ground_truth is not None:
if cell in stats['true_positives'][:, 1]:
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
box_trace_arr = np.array(images[:, box[1]:box[3],
box[0]:box[2]])
box_trace_arr_cut = box_trace_arr.copy()
box_trace_arr_cut[box_trace < 30] = np.nan
mean_box = np.nanmean(box_trace_arr_cut, axis=(1, 2))
trace_flat = box_trace[box_trace > 30]
noise_box = np.std(
trace_flat[trace_flat < np.percentile(trace_flat, 25)])
amp_trace = max(mean_box) - min(mean_box)
stnr.append(amp_trace / noise_box)
else:
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
box_trace_arr = np.array(images[:, box[1]:box[3],
box[0]:box[2]])
box_trace_arr_cut = box_trace_arr.copy()
box_trace_arr_cut[box_trace < 30] = np.nan
mean_box = np.nanmean(box_trace_arr_cut, axis=(1, 2))
trace_flat = box_trace[box_trace > 30]
noise_box = np.std(
trace_flat[trace_flat < np.percentile(trace_flat, 25)])
amp_trace = max(mean_box) - min(mean_box)
stnr.append(amp_trace / noise_box)
else:
for cell, box in enumerate(boxes):
if ground_truth is not None:
if cell in stats['true_positives'][:, 1]:
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
mean_box = box_trace.mean(axis=(1, 2))
amp_trace = max(mean_box) - min(mean_box)
noise_box = np.std(box_trace[box_trace < np.percentile(
box_trace[box_trace != 0], 25)])
stnr.append(amp_trace / noise_box)
else:
box_trace = images[:, box[1]:box[3], box[0]:box[2]]
mean_box = box_trace.mean(axis=(1, 2))
amp_trace = max(mean_box) - min(mean_box)
noise_box = np.std(box_trace[
box_trace < np.percentile(box_trace[box_trace != 0], 25)])
stnr.append(amp_trace / noise_box)
return np.array(stnr)
def run_component_evaluation(input_file,
session_wise=False,
equalization=False):
sql = "SELECT source_extraction_session_wise,min_SNR,alignment_main,equalization_main,motion_correction_main,rval_thr,use_cnn FROM Analysis WHERE source_extraction_main=?"
val = [
input_file,
]
mycursor.execute(sql, val)
myresult = mycursor.fetchall()
data = []
aux = []
for x in myresult:
aux = x
for y in aux:
data.append(y)
if session_wise:
input_mmap_file_path = data[2]
elif equalization:
input_mmap_file_path = data[3]
else:
input_mmap_file_path = data[4]
parameters = {'min_SNR': data[1], 'rval_thr': data[5], 'use_cnn': data[6]}
data_dir = os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/session_wise/' if \
data[0] else os.environ['DATA_DIR_LOCAL'] + 'data/interim/component_evaluation/trial_wise/'
sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,alignment_v,equalization_v,source_extraction_v,component_evaluation_v,input,home_path,decoding_main FROM Analysis WHERE source_extraction_main=?"
val = [
input_file,
]
mycursor.execute(sql, val)
result = mycursor.fetchall()
data = []
inter = []
for x in result:
inter = x
for y in inter:
data.append(y)
# Update the database
if data[10] == 0:
data[10] = 1
file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}"
output_file_path = f'main/{file_name}.hdf5'
sql1 = "UPDATE Analysis SET component_evaluation_main=?,component_evaluation_v=? WHERE source_extraction_main=? "
val1 = [output_file_path, data[10], input_file]
mycursor.execute(sql1, val1)
else:
data[10] += 1
file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}.{data[7]}.{data[8]}.{data[9]}.{data[10]}"
output_file_path = f'main/{file_name}.hdf5'
sql2 = "INSERT INTO Analysis (component_evaluation_main,component_evaluation_v) VALUES (?,?)"
val2 = [output_file_path, data[10]]
mycursor.execute(sql2, val2)
database.commit()
output_file_path_full = data_dir + output_file_path
# Load CNMF object
cnm = load_CNMF(input_mmap_file_path)
# Load the original movie
Yr, dims, T = cm.load_memmap(input_mmap_file_path)
images = Yr.T.reshape((T, ) + dims, order='F')
# Set the parmeters
cnm.params.set('quality', parameters)
# Stop the cluster if one exists
n_processes = psutil.cpu_count()
try:
cm.cluster.stop_server()
except:
pass
# Start a new cluster
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local',
n_processes=n_processes,
# number of process to use, if you go out of memory try to reduce this one
single_thread=False)
# Evaluate components
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
logging.debug('Number of total components: ', len(cnm.estimates.C))
logging.debug('Number of accepted components: ',
len(cnm.estimates.idx_components))
# Stop the cluster
dview.terminate()
# Save CNMF object
cnm.save(output_file_path_full)
return output_file_path
def main():
pass # For compatibility between running under Spyder and the CLI
# %% start a cluster
c, dview, n_processes =\
cm.cluster.setup_cluster(backend='local', n_processes=None,
single_thread=False)
# %% set up some parameters
fnames = [
os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif')
]
# file(s) to be analyzed
is_patches = True # flag for processing in patches or not
fr = 10 # approximate frame rate of data
decay_time = 5.0 # length of transient
if is_patches: # PROCESS IN PATCHES AND THEN COMBINE
rf = 10 # half size of each patch
stride = 4 # overlap between patches
K = 4 # number of components in each patch
else: # PROCESS THE WHOLE FOV AT ONCE
rf = None # setting these parameters to None
stride = None # will run CNMF on the whole FOV
K = 30 # number of neurons expected (in the whole FOV)
gSig = [6, 6] # expected half size of neurons
merge_thresh = 0.80 # merging threshold, max correlation allowed
p = 2 # order of the autoregressive system
gnb = 2 # global background order
params_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'rf': rf,
'stride': stride,
'K': K,
'gSig': gSig,
'merge_thr': merge_thresh,
'p': p,
'nb': gnb
}
opts = params.CNMFParams(params_dict=params_dict)
# %% Now RUN CaImAn Batch (CNMF)
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit_file()
# %% plot contour plots of components
Cns = local_correlations_movie_offline(fnames[0],
remove_baseline=True,
swap_dim=False,
window=1000,
stride=1000,
winSize_baseline=100,
quantil_min_baseline=10,
dview=dview)
Cn = Cns.max(axis=0)
cnm.estimates.plot_contours(img=Cn)
# %% load memory mapped file
Yr, dims, T = cm.load_memmap(cnm.mmap_file)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# %% refit
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier (this will pick up only neurons
# and filter out active processes)
min_SNR = 2 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.85 # space correlation threshold (if above this, accept)
use_cnn = True # use the CNN classifier
min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set(
'quality', {
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': use_cnn,
'min_cnn_thr': min_cnn_thr,
'cnn_lowest': cnn_lowest
})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% visualize selected and rejected components
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
# %% visualize selected components
cnm2.estimates.view_components(images,
idx=cnm2.estimates.idx_components,
img=Cn)
#%% only select high quality components (destructive)
# cnm2.estimates.select_components(use_object=True)
# cnm2.estimates.plot_contours(img=Cn)
#%% save results
cnm2.estimates.Cn = Cn
cnm2.save(cnm2.mmap_file[:-4] + 'hdf5')
# %% play movie with results (original, reconstructed, amplified residual)
cnm2.estimates.play_movie(images, magnification=4)
# %% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def load_images(memmap_fpath):
# load memory mappable file
Yr, dims, T = cm.load_memmap(memmap_fpath)
images = Yr.T.reshape((T,) + dims, order='F')
return images
def run(work_dir: str, UUID: str, save_temp_files: str):
logging.basicConfig(
stream=sys.stdout,
level=logging.DEBUG,
format=
"%(relativeCreated)12d [%(filename)s:%(funcName)20s():%(lineno)s] [%(process)d] %(message)s"
)
start_time = time()
batch_dir = os.environ['CURR_BATCH_DIR']
save_temp_files = bool(int(save_temp_files))
output = {'status': 0, 'output_info': ''}
n_processes = int(os.environ['_MESMERIZE_N_THREADS'])
filepath = os.path.join(work_dir, UUID)
imgpath = f'{filepath}_input.tiff'
input_params = pickle.load(open(f'{filepath}.params', 'rb'))
print('******** Creating process pool *********')
c, dview, n_processes = setup_cluster(backend='local',
n_processes=n_processes,
single_thread=False,
ignore_preexisting=True)
try:
if input_params['use_memmap']:
memmap_uuid = input_params['memmap_uuid']
memmap_batchdir = glob(
os.path.join(batch_dir, f'memmap-{memmap_uuid}*.mmap'))
# Check batch dir
if len(memmap_batchdir) > 0:
memmap_path = memmap_batchdir[0]
print(
f'********** Found existing memmap in batch dir: {memmap_path} ********** '
)
# copy to work dir
if not os.path.samefile(batch_dir, work_dir):
print('**** Copying memmap to work dir ****')
shutil.copy(memmap_path, work_dir)
memmap_path = glob(
os.path.join(work_dir,
f'memmap-{memmap_uuid}*.mmap'))[0]
else:
# remake the memmap with the same UUID so that future batch items can rely on it
print(
'********** Memmap not found, re-making memmap with the same UUID **********'
)
memmap_path = cm.save_memmap([imgpath],
base_name=f'memmap-{memmap_uuid}',
is_3D=True,
order='C',
dview=dview)
else:
print('********** Making memmap **********')
memmap_path = cm.save_memmap([imgpath],
base_name=f'memmap-{UUID}',
is_3D=True,
order='C',
dview=dview)
print(f'Using memmap:\n{memmap_path}')
print('********** Loading memmap **********')
Yr, dims, T = cm.load_memmap(memmap_path)
Y = np.reshape(Yr, dims + (T, ), order='F')
images = np.reshape(Yr.T, [T] + list(dims), order='F')
if input_params['use_patches']:
cnm = cnmf.CNMF(n_processes=n_processes,
dview=dview,
only_init_patch=True,
**input_params['cnmf_kwargs'])
else:
cnm = cnmf.CNMF(n_processes,
dview=dview,
**input_params['cnmf_kwargs'])
cnm.fit(images)
print('Number of components:' + str(cnm.estimates.A.shape[-1]))
cnm.params.change_params(
params_dict={
**input_params['eval_kwargs'], 'use_cnn': False
})
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
if input_params['refit']:
cnm.params.set('temporal', {'p': input_params['cnmf_kwargs']['p']})
cnm_ = cnm.refit(images)
else:
cnm_ = cnm
out_filename = f'{UUID}_results.hdf5'
cnm_.save(out_filename)
output_files = [out_filename]
# Save the memmap
if save_temp_files:
print("***** Keeping memmap file *****")
# copy to batch dir if batch_dir != work_dir
if not os.path.samefile(batch_dir, work_dir):
print("***** Copying memmap file to batch dir *****")
shutil.copy(memmap_path, batch_dir)
# Delete the memmap from the work dir
if not os.path.samefile(batch_dir, work_dir):
print("***** Deleting memmap files from work dir *****")
try:
os.remove(memmap_path)
except:
pass
output.update({
'output': UUID,
'status': 1,
'output_files': output_files,
'saved_memmap': save_temp_files
})
except Exception as e:
output.update({'status': 0, 'output_info': traceback.format_exc()})
cm.stop_server(dview=dview)
end_time = time()
processing_time = (end_time - start_time) / 60
output.update({'processing_time': processing_time})
json.dump(output, open(filepath + '.out', 'w'))
def select_corr_pnr_threshold(mouse_row, parameters_source_extraction):
'''
Plots the summary images correlation and pnr. Also the pointwise product between them (used in Caiman paper Zhou
et al 2018)
:param mouse_row:
:param parameters_source_extraction: parameters that will be used for source
extraction. the relevant parameter here are min_corr and min_pnr because the source extraction algorithm is
initialized (initial cell templates) in all values that surpasses that threshold
:return: max_combined, max_pnr, max_corr: threshold for corr*pnr, and corresponding values of corr and pnr
'''
input_mmap_file_path = eval(
mouse_row.loc['motion_correction_output'])['main']
# Load memory mappable input file
if os.path.isfile(input_mmap_file_path):
Yr, dims, T = cm.load_memmap(input_mmap_file_path)
# logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.')
images = Yr.T.reshape((T, ) + dims, order='F')
else:
logging.warning(
f'{mouse_row.name} .mmap file does not exist. Cancelling')
# Determine output paths
step_index = db.get_step_index('motion_correction')
data_dir = 'data/interim/source_extraction/trial_wise/'
# Check if the summary images are already there
gSig = parameters_source_extraction['gSig'][0]
corr_npy_file_path, pnr_npy_file_path = fm.get_corr_pnr_path(
mouse_row.name, gSig_abs=(gSig, gSig))
if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path):
# Already computed summary images
logging.info(f'{mouse_row.name} Already computed summary images')
cn_filter = np.load(corr_npy_file_path)
pnr = np.load(pnr_npy_file_path)
else:
# Compute summary images
t0 = datetime.datetime.today()
logging.info(f'{mouse_row.name} Computing summary images')
cn_filter, pnr = cm.summary_images.correlation_pnr(
images[::1],
gSig=parameters_source_extraction['gSig'][0],
swap_dim=False)
# Saving summary images as npy files
corr_npy_file_path = data_dir + f'meta/corr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.npy'
pnr_npy_file_path = data_dir + f'meta/pnr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.npy'
with open(corr_npy_file_path, 'wb') as f:
np.save(f, cn_filter)
with open(pnr_npy_file_path, 'wb') as f:
np.save(f, pnr)
combination = cn_filter * pnr # this is as defined in Zhou et al 2018 (definition of R, P and L, eq 14)
max_combined = np.argmax(combination)
row = int(np.floor(max_combined / cn_filter.shape[1]))
column = int(max_combined - row * cn_filter.shape[1])
max_corr = cn_filter[row, column]
max_pnr = pnr[row, column]
return max_combined, max_corr, max_pnr
def initialize_movie_online(Y,
K,
gSig,
rf,
stride,
base_name,
p=1,
merge_thresh=0.95,
rval_thr_online=0.9,
thresh_fitness_delta_online=-30,
thresh_fitness_raw_online=-50,
rval_thr_init=.5,
thresh_fitness_delta_init=-20,
thresh_fitness_raw_init=-20,
rval_thr_refine=0.95,
thresh_fitness_delta_refine=-100,
thresh_fitness_raw_refine=-100,
final_frate=10,
Npeaks=10,
single_thread=True,
dview=None,
n_processes=None):
"""
Initialize movie using CNMF on minibatch. See CNMF parameters
"""
_, d1, d2 = Y.shape
dims = (d1, d2)
Yr = Y.to_2D().T
# merging threshold, max correlation allowed
# order of the autoregressive system
#T = Y.shape[0]
base_name = base_name + '.mmap'
fname_new = Y.save(base_name, order='C')
#%
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
Y = np.reshape(Yr, dims + (T, ), order='F')
Cn2 = cm.local_correlations(Y)
# pl.imshow(Cn2)
#%
#% RUN ALGORITHM ON PATCHES
# pl.close('all')
cnm_init = cm.source_extraction.cnmf.CNMF(
n_processes,
method_init='greedy_roi',
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=0,
dview=dview,
Ain=None,
rf=rf,
stride=stride,
method_deconvolution='oasis',
skip_refinement=False,
normalize_init=False,
options_local_NMF=None,
minibatch_shape=100,
minibatch_suff_stat=5,
update_num_comps=True,
rval_thr=rval_thr_online,
thresh_fitness_delta=thresh_fitness_delta_online,
thresh_fitness_raw=thresh_fitness_raw_online,
batch_update_suff_stat=True,
max_comp_update_shape=5)
cnm_init = cnm_init.fit(images)
A_tot = cnm_init.A
C_tot = cnm_init.C
YrA_tot = cnm_init.YrA
b_tot = cnm_init.b
f_tot = cnm_init.f
print(('Number of components:' + str(A_tot.shape[-1])))
#%
traces = C_tot + YrA_tot
# traces_a=traces-scipy.ndimage.percentile_filter(traces,8,size=[1,np.shape(traces)[-1]/5])
# traces_b=np.diff(traces,axis=1)
fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = cm.components_evaluation.evaluate_components(
Y,
traces,
A_tot,
C_tot,
b_tot,
f_tot,
final_frate,
remove_baseline=True,
N=5,
robust_std=False,
Athresh=0.1,
Npeaks=Npeaks,
thresh_C=0.3)
idx_components_r = np.where(r_values >= rval_thr_init)[0]
idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_init)[0]
idx_components_delta = np.where(
fitness_delta < thresh_fitness_delta_init)[0]
idx_components = np.union1d(idx_components_r, idx_components_raw)
idx_components = np.union1d(idx_components, idx_components_delta)
idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components)
print(('Keeping ' + str(len(idx_components)) + ' and discarding ' +
str(len(idx_components_bad))))
A_tot = A_tot.tocsc()[:, idx_components]
C_tot = C_tot[idx_components]
#%
cnm_refine = cm.source_extraction.cnmf.CNMF(
n_processes,
method_init='greedy_roi',
k=A_tot.shape,
gSig=gSig,
merge_thresh=merge_thresh,
rf=None,
stride=None,
p=p,
dview=dview,
Ain=A_tot,
Cin=C_tot,
f_in=f_tot,
method_deconvolution='oasis',
skip_refinement=True,
normalize_init=False,
options_local_NMF=None,
minibatch_shape=100,
minibatch_suff_stat=5,
update_num_comps=True,
rval_thr=rval_thr_refine,
thresh_fitness_delta=thresh_fitness_delta_refine,
thresh_fitness_raw=thresh_fitness_raw_refine,
batch_update_suff_stat=True,
max_comp_update_shape=5)
cnm_refine = cnm_refine.fit(images)
#%
A, C, b, f, YrA, sn = cnm_refine.A, cnm_refine.C, cnm_refine.b, cnm_refine.f, cnm_refine.YrA, cnm_refine.sn
#%
final_frate = 10
Npeaks = 10
traces = C + YrA
fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = \
cm.components_evaluation.evaluate_components(Y, traces, A, C, b, f, final_frate, remove_baseline=True,
N=5, robust_std=False, Athresh=0.1, Npeaks=Npeaks, thresh_C=0.3)
idx_components_r = np.where(r_values >= rval_thr_refine)[0]
idx_components_raw = np.where(fitness_raw < thresh_fitness_raw_refine)[0]
idx_components_delta = np.where(
fitness_delta < thresh_fitness_delta_refine)[0]
idx_components = np.union1d(idx_components_r, idx_components_raw)
idx_components = np.union1d(idx_components, idx_components_delta)
idx_components_bad = np.setdiff1d(list(range(len(traces))), idx_components)
print(' ***** ')
print((len(traces)))
print((len(idx_components)))
#%
cnm_refine.idx_components = idx_components
cnm_refine.idx_components_bad = idx_components_bad
cnm_refine.r_values = r_values
cnm_refine.fitness_raw = fitness_raw
cnm_refine.fitness_delta = fitness_delta
cnm_refine.Cn2 = Cn2
#%
# cnm_init.dview = None
# save_object(cnm_init,fls[0][:-4]+ '_DS_' + str(ds)+ '_init.pkl')
return cnm_refine, Cn2, fname_new
def run_source_extraction(row, parameters, dview, session_wise = False):
'''
This is the function for source extraction.
Its goal is to take in a .mmap file,
perform source extraction on it using cnmf-e and save the cnmf object as a .pkl file.
Args:
row: pd.DataFrame object
The row corresponding to the analysis state to be source extracted.
Returns:
row: pd.DataFrame object
The row corresponding to the source extracted analysis state.
'''
step_index = 5
row_local = row.copy()
row_local.loc['source_extraction_parameters'] = str(parameters)
row_local = db.set_version_analysis('source_extraction',row_local,session_wise)
index = row_local.name
# Determine input path
if parameters['session_wise']:
input_mmap_file_path = eval(row_local.loc['alignment_output'])['main']
if parameters['equalization']:
input_mmap_file_path =eval(row_local['equalization_output'])['main']
else:
input_mmap_file_path = eval(row_local.loc['motion_correction_output'])['main']
if not os.path.isfile(input_mmap_file_path):
logging.error('Input file does not exist. Cancelling.')
return row_local
# Determine output paths
file_name = db.create_file_name(step_index, index)
if parameters['session_wise']:
data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/session_wise/'
else:
data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/trial_wise/'
output_file_path = data_dir + f'main/{file_name}.hdf5'
# Create a dictionary with parameters
output = {
'main': output_file_path,
'meta':{
'analysis' : {
'analyst' : os.environ['ANALYST'],
'date' : datetime.datetime.today().strftime("%m-%d-%Y"),
'time' : datetime.datetime.today().strftime("%H:%M:%S"),
},
'duration': {}
}
}
# Load memmory mappable input file
if os.path.isfile(input_mmap_file_path):
Yr, dims, T = cm.load_memmap(input_mmap_file_path)
# logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.')
images = Yr.T.reshape((T,) + dims, order='F')
else:
logging.warning(f'{index} .mmap file does not exist. Cancelling')
return row_local
# SOURCE EXTRACTION
# Check if the summary images are already there
corr_npy_file_path, pnr_npy_file_path = fm.get_corr_pnr_path(index, gSig_abs = parameters['gSig'][0])
if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path):
# Already computed summary images
logging.info(f'{index} Already computed summary images')
cn_filter = np.load(corr_npy_file_path)
pnr = np.load(pnr_npy_file_path)
else:
# Compute summary images
t0 = datetime.datetime.today()
logging.info(f'{index} Computing summary images')
cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1], gSig = parameters['gSig'][0], swap_dim=False)
dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes
output['meta']['duration']['summary_images'] = dt
logging.info(f'{index} Computed summary images. dt = {dt} min')
# Saving summary images as npy files
gSig = parameters['gSig'][0]
corr_npy_file_path = data_dir + f'/meta/corr/{db.create_file_name(3, index)}_gSig_{gSig}.npy'
pnr_npy_file_path = data_dir + f'/meta/pnr/{db.create_file_name(3, index)}_gSig_{gSig}.npy'
with open(corr_npy_file_path, 'wb') as f:
np.save(f, cn_filter)
with open(pnr_npy_file_path, 'wb') as f:
np.save(f, pnr)
# Store the paths in the meta dictionary
output['meta']['corr'] = {'main': corr_npy_file_path, 'meta': {}}
output['meta']['pnr'] = {'main': pnr_npy_file_path, 'meta': {}}
# Calculate min, mean, max value for cn_filter and pnr
corr_min, corr_mean, corr_max = cn_filter.min(), cn_filter.mean(), cn_filter.max()
output['meta']['corr']['meta'] = {'min': corr_min, 'mean': corr_mean, 'max': corr_max}
pnr_min, pnr_mean, pnr_max = pnr.min(), pnr.mean(), pnr.max()
output['meta']['pnr']['meta'] = {'min': pnr_min, 'mean': pnr_mean, 'max': pnr_max}
# If min_corr and min_pnr are specified via a linear equation, calculate
# this value
if type(parameters['min_corr']) == list:
min_corr = parameters['min_corr'][0]*corr_mean + parameters['min_corr'][1]
parameters['min_corr'] = min_corr
logging.info(f'{index} Automatically setting min_corr = {min_corr}')
if type(parameters['min_pnr']) == list:
min_pnr = parameters['min_pnr'][0]*pnr_mean + parameters['min_pnr'][1]
parameters['min_pnr'] = min_pnr
logging.info(f'{index} Automatically setting min_pnr = {min_pnr}')
# Set the parameters for caiman
opts = params.CNMFParams(params_dict = parameters)
# SOURCE EXTRACTION
logging.info(f'{index} Performing source extraction')
t0 = datetime.datetime.today()
n_processes = psutil.cpu_count()
logging.info(f'{index} n_processes: {n_processes}')
cnm = cnmf.CNMF(n_processes = n_processes, dview = dview, params = opts)
cnm.fit(images)
cnm.estimates.dims = dims
# Store the number of neurons
output['meta']['K'] = len(cnm.estimates.C)
# Calculate the center of masses
cnm.estimates.center = caiman.base.rois.com(cnm.estimates.A, images.shape[1], images.shape[2])
# Save the cnmf object as a hdf5 file
logging.info(f'{index} Saving cnmf object')
cnm.save(output_file_path)
dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes
output['meta']['duration']['source_extraction'] = dt
logging.info(f'{index} Source extraction finished. dt = {dt} min')
# Write necessary variables in row and return
row_local.loc['source_extraction_parameters'] = str(parameters)
row_local.loc['source_extraction_output'] = str(output)
return row_local
def plot_corr_pnr(mouse_row, parameters_source_extraction):
'''
Plots the summary images correlation and pnr. Also the pointwise product between them (used in Caiman paper Zhou
et al 2018)
:param mouse_row:
:param parameters_source_extraction: parameters that will be used for source
extraction. the relevant parameter here are min_corr and min_pnr because the source extraction algorithm is
initialized (initial cell templates) in all values that surpasses that threshold
:return: figure
'''
input_mmap_file_path = eval(mouse_row.loc['motion_correction_output'])['main']
# Load memory mappable input file
if os.path.isfile(input_mmap_file_path):
Yr, dims, T = cm.load_memmap(input_mmap_file_path)
# logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.')
images = Yr.T.reshape((T,) + dims, order='F')
else:
logging.warning(f'{mouse_row.name} .mmap file does not exist. Cancelling')
# Determine output paths
step_index = db.get_step_index('motion_correction')
data_dir = 'data/interim/source_extraction/trial_wise/'
# Check if the summary images are already there
gSig = parameters_source_extraction['gSig'][0]
corr_npy_file_path, pnr_npy_file_path = fm.get_corr_pnr_path(mouse_row.name, gSig_abs=(gSig, gSig))
if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path):
# Already computed summary images
logging.info(f'{mouse_row.name} Already computed summary images')
cn_filter = np.load(corr_npy_file_path)
pnr = np.load(pnr_npy_file_path)
else:
# Compute summary images
t0 = datetime.datetime.today()
logging.info(f'{mouse_row.name} Computing summary images')
cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1], gSig=parameters_source_extraction['gSig'][0],
swap_dim=False)
# Saving summary images as npy files
corr_npy_file_path = data_dir + f'meta/corr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.npy'
pnr_npy_file_path = data_dir + f'meta/pnr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.npy'
with open(corr_npy_file_path, 'wb') as f:
np.save(f, cn_filter)
with open(pnr_npy_file_path, 'wb') as f:
np.save(f, pnr)
fig = plt.figure(figsize=(15, 15))
min_corr = round(parameters_source_extraction['min_corr'], 2)
min_pnr = round(parameters_source_extraction['min_pnr'], 1)
max_corr = round(cn_filter.max(), 2)
max_pnr= 20
# continuous
cmap = 'viridis'
fig, axes = plt.subplots(1, 3, sharex=True)
corr_fig = axes[0].imshow(np.clip(cn_filter,min_corr,max_corr), cmap=cmap)
axes[0].set_title('Correlation')
fig.colorbar(corr_fig, ax=axes[0])
pnr_fig = axes[1].imshow(np.clip(pnr,min_pnr,max_pnr), cmap=cmap)
axes[1].set_title('PNR')
fig.colorbar(pnr_fig, ax=axes[1])
combined = cn_filter*pnr
max_combined = 10
min_combined = np.min(combined)
corr_pnr_fig = axes[2].imshow(np.clip(cn_filter*pnr,min_combined,max_combined), cmap=cmap)
axes[2].set_title('Corr * PNR')
fig.colorbar(corr_pnr_fig, ax=axes[2])
fig_dir = 'data/interim/source_extraction/trial_wise/meta/'
fig_name= fig_dir + f'figures/corr_pnr/{db.create_file_name(3, mouse_row.name)}_gSig_{gSig}.png'
fig.savefig(fig_name)
return fig
def run(batch_dir: str, UUID: str):
output = {'status': 0, 'output_info': ''}
n_processes = os.environ['_MESMERIZE_N_THREADS']
n_processes = int(n_processes)
file_path = batch_dir + '/' + UUID
filename = [file_path + '.tiff']
input_params = pickle.load(open(file_path + '.params', 'rb'))
fr = input_params['fr']
p = input_params['p']
gnb = input_params['gnb']
merge_thresh = input_params['merge_thresh']
rf = input_params['rf']
stride_cnmf = input_params['stride_cnmf']
K = input_params['k']
gSig = input_params['gSig']
gSig = [gSig, gSig]
min_SNR = input_params['min_SNR']
rval_thr = input_params['rval_thr']
cnn_thr = input_params['cnn_thr']
decay_time = input_params['decay_time']
bord_px = input_params['bord_px']
refit = input_params['refit']
print('*********** Creating Process Pool ***********')
c, dview, np = cm.cluster.setup_cluster(backend='local', n_processes=n_processes, single_thread=False)
try:
print('Creating memmap')
fname_new = cm.save_memmap_each(
filename,
base_name='memmap_' + UUID,
order='C',
border_to_0=bord_px,
dview=dview)
fname_new = cm.save_memmap_join(fname_new, base_name='memmap_' + UUID, dview=dview)
Yr, dims, T = cm.load_memmap(fname_new)
Y = Yr.T.reshape((T,) + dims, order='F')
cnm = cnmf.CNMF(n_processes=n_processes, k=K, gSig=gSig, merge_thresh=merge_thresh,
p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1,
method_init='greedy_roi', alpha_snmf=None,
only_init_patch=False, gnb=gnb, border_pix=bord_px)
cnm.fit(Y)
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(Y, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=False,
thresh_cnn_lowest=cnn_thr)
if refit:
A_in, C_in, b_in, f_in = cnm.A[:,
idx_components], cnm.C[idx_components], cnm.b, cnm.f
cnm2 = cnmf.CNMF(n_processes=n_processes, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview,
merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in,
f_in=f_in, rf=None, stride=None, gnb=gnb,
method_deconvolution='oasis', check_nan=True)
cnm2 = cnm2.fit(Y)
cnmA = cnm2.A
cnmb = cnm2.b
cnmC = cnm2.C
cnm_f = cnm2.f
cnmYrA = cnm2.YrA
else:
cnmA = cnm.A
cnmb = cnm.b
cnmC = cnm.C
cnm_f = cnm.f
cnmYrA = cnm.YrA
pickle.dump(Yr, open(UUID + '_Yr.pikl', 'wb'), protocol=4)
pickle.dump(cnmA, open(UUID + '_cnm-A.pikl', 'wb'), protocol=4)
pickle.dump(cnmb, open(UUID + '_cnm-b.pikl', 'wb'), protocol=4)
pickle.dump(cnmC, open(UUID + '_cnm-C.pikl', 'wb'), protocol=4)
pickle.dump(cnm_f, open(UUID + '_cnm-f.pikl', 'wb'), protocol=4)
pickle.dump(idx_components, open(UUID + '_idx_components.pikl', 'wb'), protocol=4)
pickle.dump(cnmYrA, open(UUID + '_cnm-YrA.pikl', 'wb'), protocol=4)
pickle.dump(dims, open(UUID + '_dims.pikl', 'wb'), protocol=4)
output_file_list = [UUID + '_cnm-A.pikl',
UUID + '_Yr.pikl',
UUID + '_cnm-b.pikl',
UUID + '_cnm-C.pikl',
UUID + '_cnm-f.pikl',
UUID + '_idx_components.pikl',
UUID + '_cnm-YrA.pikl',
UUID + '_dims.pikl',
UUID + '.out'
]
output.update({'output': UUID,
'status': 1,
'output_files': output_file_list
})
except Exception:
output.update({'status': 0, 'output_info': traceback.format_exc()})
dview.terminate()
for mf in glob(batch_dir + '/memmap_*'):
os.remove(mf)
json.dump(output, open(file_path + '.out', 'w'))
