def extract(mmap_file, cnmf_opts, nprocs=8, dview=None):
yr, dims, t = cm.load_memmap(mmap_file)
y = yr.T.reshape((t, *dims), order='F')
cnmf = source.cnmf.CNMF(n_processes=nprocs,
method_init=cnmf_opts['init-method'],
gSiz=[cnmf_opts['gsiz']] * 2,
gSig=[cnmf_opts['gsig']] * 2,
merge_thresh=cnmf_opts['merge-thresh'],
Ain=None,
k=None,
tsub=cnmf_opts['t-sub'],
ssub=cnmf_opts['s-sub'],
rf=cnmf_opts['half-patch-size'],
p=1,
dview=dview,
stride=cnmf_opts['patch-overlap'],
only_init_patch=True,
method_deconvolution='oasis',
nb_patch=-1,
gnb=cnmf_opts['background-components'],
low_rank_background=None,
update_background_components=True,
min_corr=cnmf_opts['min-corr'],
min_pnr=cnmf_opts['min-pnr'],
normalize_init=False,
center_psf=True,
ring_size_factor=cnmf_opts['ring-size-factor'],
del_duplicates=True,
ssub_B=2,
border_pix=cnmf_opts['border-px'])
cnmf.fit(y)
estimates = cnmf.estimates
good_idx, bad_idx, _, _, _ = evaluate.estimate_components_quality_auto(
Y=y,
A=estimates.A,
C=estimates.C,
b=estimates.b,
f=estimates.f,
YrA=estimates.YrA,
frate=cnmf_opts['fps'],
decay_time=cnmf_opts['decay-time'],
gSig=cnmf_opts['gsig'],
dims=dims,
dview=None,
min_SNR=cnmf_opts['min-snr'],
use_cnn=False)
return estimates.C[good_idx], estimates.A.toarray()[:,
good_idx], cnmf, dims
def components_eval(images, cnm, dview, dims):
'''Evaluating the components being found
Args:
images:
The input video
shape: (frames, dims, dims)
cnm:
Object obtained from CNMF algorithm with C,A,S,b,f
dview:
Direct View object for parallelization pruposes when using ipyparallel
dims:
Dimension of each frame
shape: (521, 521)
Returns:
idx_components:
list of ids of components that are considered to be a neuron
'''
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = estimate_components_quality_auto(
images,
cnm.A,
cnm.C,
cnm.b,
cnm.f,
cnm.YrA,
fr,
decay_time,
gSig,
dims,
dview=dview,
min_SNR=min_SNR,
r_values_min=rval_thr,
use_cnn=use_cnn,
thresh_cnn_lowest=min_cnn_thr)
return idx_components
def main():
pass # For compatibility between running under Spyder and the CLI
#%% start a cluster
c, dview, n_processes =\
cm.cluster.setup_cluster(backend='local', n_processes=None,
single_thread=False)
#%% save files to be processed
# This datafile is distributed with Caiman
fnames = [
os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif')
]
# location of dataset (can actually be a list of filed to be concatenated)
add_to_movie = -np.min(cm.load(fnames[0],
subindices=range(200))).astype(float)
# determine minimum value on a small chunk of data
add_to_movie = np.maximum(add_to_movie, 0)
# if minimum is negative subtract to make the data non-negative
base_name = 'Yr'
fname_new = cm.save_memmap(fnames,
dview=dview,
base_name=base_name,
order='C',
add_to_movie=add_to_movie)
#%% LOAD MEMORY MAPPABLE FILE
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
#%% play movie, press q to quit
play_movie = False
if play_movie:
cm.movie(images[1400:]).play(fr=50, magnification=4, gain=3.)
#%% correlation image. From here infer neuron size and density
Cn = cm.movie(images).local_correlations(swap_dim=False)
plt.imshow(Cn, cmap='gray')
plt.title('Correlation Image')
#%% set up some parameters
is_patches = True # flag for processing in patches or not
if is_patches: # PROCESS IN PATCHES AND THEN COMBINE
rf = 10 # half size of each patch
stride = 4 # overlap between patches
K = 4 # number of components in each patch
else: # PROCESS THE WHOLE FOV AT ONCE
rf = None # setting these parameters to None
stride = None # will run CNMF on the whole FOV
K = 30 # number of neurons expected (in the whole FOV)
gSig = [6, 6] # expected half size of neurons
merge_thresh = 0.80 # merging threshold, max correlation allowed
p = 2 # order of the autoregressive system
gnb = 2 # global background order
#%% Now RUN CNMF
cnm = cnmf.CNMF(n_processes,
method_init='greedy_roi',
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=p,
dview=dview,
gnb=gnb,
rf=rf,
stride=stride,
rolling_sum=False)
cnm = cnm.fit(images)
#%% plot contour plots of components
plt.figure()
crd = cm.utils.visualization.plot_contours(cnm.A, Cn, thr=0.9)
plt.title('Contour plots of components')
#%%
A_in, C_in, b_in, f_in = cnm.A[:, :], cnm.C[:], cnm.b, cnm.f
cnm2 = cnmf.CNMF(n_processes=1,
k=A_in.shape[-1],
gSig=gSig,
p=p,
dview=dview,
merge_thresh=merge_thresh,
Ain=A_in,
Cin=C_in,
b_in=b_in,
f_in=f_in,
rf=None,
stride=None,
gnb=gnb,
method_deconvolution='oasis',
check_nan=True)
cnm2 = cnm2.fit(images)
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier (this will pick up only neurons
# and filter out active processes)
fr = 10 # approximate frame rate of data
decay_time = 5.0 # length of transient
min_SNR = 2.5 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.90 # space correlation threshold (if above this, accept)
use_cnn = True # use the CNN classifier
min_cnn_thr = 0.95 # if cnn classifier predicts below this value, reject
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=use_cnn,
thresh_cnn_min=min_cnn_thr)
#%% visualize selected and rejected components
plt.figure()
plt.subplot(1, 2, 1)
cm.utils.visualization.plot_contours(cnm2.A[:, idx_components],
Cn,
thr=0.9)
plt.title('Selected components')
plt.subplot(1, 2, 2)
plt.title('Discaded components')
cm.utils.visualization.plot_contours(cnm2.A[:, idx_components_bad],
Cn,
thr=0.9)
#%%
plt.figure()
crd = cm.utils.visualization.plot_contours(cnm2.A.tocsc()[:,
idx_components],
Cn,
thr=0.9)
plt.title('Contour plots of components')
#%% visualize selected components
cm.utils.visualization.view_patches_bar(Yr,
cnm2.A.tocsc()[:, idx_components],
cnm2.C[idx_components, :],
cnm2.b,
cnm2.f,
dims[0],
dims[1],
YrA=cnm2.YrA[idx_components, :],
img=Cn)
#%% STOP CLUSTER and clean up log files
cm.stop_server()
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def filter_rois(video_paths, roi_spatial_footprints, roi_temporal_footprints, roi_temporal_residuals, bg_spatial_footprints, bg_temporal_footprints, params):
directory = os.path.dirname(video_paths[0])
final_video_path = os.path.join(directory, "final_video_temp.tif")
with tifffile.TiffWriter(final_video_path, bigtiff=True) as tif:
for i in range(len(video_paths)):
video_path = video_paths[i]
video = tifffile.memmap(video_path)
if len(video.shape) == 3:
# add a z dimension
video = video[:, np.newaxis, :, :]
tif.save(video)
del video
final_video = tifffile.memmap(final_video_path).astype(np.uint16)
if len(final_video.shape) == 5:
final_video_path_2 = os.path.join(directory, "final_video_temp_2.tif")
tifffile.imsave(final_video_path_2, final_video.reshape((final_video.shape[0]*final_video.shape[1], final_video.shape[2], final_video.shape[3], final_video.shape[4])))
if os.path.exists(final_video_path):
os.remove(final_video_path)
else:
final_video_path_2 = final_video_path
filtered_out_rois = []
memmap_video = tifffile.memmap(final_video_path_2)
if len(memmap_video.shape) == 5:
n_frames = memmap_video.shape[1]
num_z = memmap_video.shape[2]
height = memmap_video.shape[3]
width = memmap_video.shape[4]
else:
n_frames = memmap_video.shape[0]
num_z = memmap_video.shape[1]
height = memmap_video.shape[2]
width = memmap_video.shape[3]
for z in range(num_z):
directory = os.path.dirname(final_video_path_2)
filename = os.path.basename(final_video_path_2)
fname = os.path.splitext(filename)[0] + "_masked_z_{}_d1_{}_d2_{}_d3_1_order_C_frames_{}_.mmap".format(z, height, width, n_frames)
video_path = os.path.join(directory, fname)
if len(memmap_video.shape) == 5:
tifffile.imsave(video_path, memmap_video[:, :, z, :, :].reshape((-1, memmap_video.shape[3], memmap_video.shape[4])).transpose([1, 2, 0]).reshape(-1, memmap_video.shape[0]).astype(np.float32))
else:
tifffile.imsave(video_path, memmap_video[:, z, :, :].transpose([1, 2, 0]).reshape(-1, memmap_video.shape[0]).astype(np.float32))
video = tifffile.memmap(video_path)
print(video.shape)
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(video, roi_spatial_footprints[z], roi_temporal_footprints[z], bg_spatial_footprints[z], bg_temporal_footprints[z],
roi_temporal_residuals[z], params['imaging_fps']/num_z, params['decay_time'], params['half_size'], (video.shape[-2], video.shape[-1]),
dview = None, min_SNR=params['min_snr'],
r_values_min = params['min_spatial_corr'], use_cnn = params['use_cnn'],
thresh_cnn_lowest = params['cnn_threshold'], gSig_range=[ (i, i) for i in range(max(1, params['half_size']-5), params['half_size']+5) ])
filtered_out_rois.append(list(idx_components_bad))
if isinstance(roi_spatial_footprints[z], scipy.sparse.coo_matrix):
f = roi_spatial_footprints[z].toarray()
else:
f = roi_spatial_footprints[z]
for i in range(f.shape[-1]):
area = np.sum(f[:, i] > 0)
if (area < params['min_area'] or area > params['max_area']) and i not in filtered_out_rois[-1]:
filtered_out_rois[-1].append(i)
if os.path.exists(video_path):
try:
os.remove(video_path)
except:
pass
del memmap_video
if os.path.exists(final_video_path_2):
os.remove(final_video_path_2)
mmap_files = glob.glob(os.path.join(directory, "*.mmap"))
for mmap_file in mmap_files:
try:
os.remove(mmap_file)
except:
pass
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
return filtered_out_rois
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters
# dataset dependent parameters
display_images = False # Set this to true to show movies and plots
fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
niter_rig = 1 # number of iterations for rigid motion correction
max_shifts = (6, 6) # maximum allow rigid shift
# for parallelization split the movies in num_splits chuncks across time
splits_rig = 56
# start a new patch for pw-rigid motion correction every x pixels
strides = (48, 48)
# overlap between pathes (size of patch strides+overlaps)
overlaps = (24, 24)
# for parallelization split the movies in num_splits chuncks across time
splits_els = 56
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thresh = 0.8 # merging threshold, max correlation allowed
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
rf = 15
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons
# initialization method (if analyzing dendritic data using 'sparse_nmf')
init_method = 'greedy_roi'
is_dendrites = False # flag for analyzing dendritic data
# sparsity penalty for dendritic data analysis through sparse NMF
alpha_snmf = None
# parameters for component evaluation
min_SNR = 2.5 # signal to noise ratio for accepting a component
rval_thr = 0.8 # space correlation threshold for accepting a component
cnn_thr = 0.8 # threshold for CNN based classifier
#%% download the dataset if it's not present in your folder
if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fname = [download_demo(fname[0])]
#%% play the movie
# playing the movie using opencv. It requires loading the movie in memory. To
# close the video press q
m_orig = cm.load_movie_chain(fname[:1])
downsample_ratio = 0.2
offset_mov = -np.min(m_orig[:100])
moviehandle = m_orig.resize(1, 1, downsample_ratio)
if display_images:
moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2)
#%% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
#%%% MOTION CORRECTION
# first we create a motion correction object with the parameters specified
min_mov = cm.load(fname[0], subindices=range(200)).min()
# this will be subtracted from the movie to make it non-negative
mc = MotionCorrect(fname[0],
min_mov,
dview=dview,
max_shifts=max_shifts,
niter_rig=niter_rig,
splits_rig=splits_rig,
strides=strides,
overlaps=overlaps,
splits_els=splits_els,
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
shifts_opencv=True,
nonneg_movie=True)
# note that the file is not loaded in memory
#%% Run piecewise-rigid motion correction using NoRMCorre
mc.motion_correct_pwrigid(save_movie=True)
m_els = cm.load(mc.fname_tot_els)
bord_px_els = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
# maximum shift to be used for trimming against NaNs
#%% compare with original movie
moviehandle = cm.concatenate([
m_orig.resize(1, 1, downsample_ratio) + offset_mov,
m_els.resize(1, 1, downsample_ratio)
],
axis=2)
display_images = False
if display_images:
moviehandle.play(fr=60, q_max=99.5, magnification=2,
offset=0) # press q to exit
#%% MEMORY MAPPING
# memory map the file in order 'C'
fnames = mc.fname_tot_els # name of the pw-rigidly corrected file.
border_to_0 = bord_px_els # number of pixels to exclude
fname_new = cm.save_memmap(fnames,
base_name='memmap_',
order='C',
border_to_0=bord_px_els) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
#%% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
#%% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
t1 = time.time()
cnm = cnmf.CNMF(n_processes=1,
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=0,
dview=dview,
rf=rf,
stride=stride_cnmf,
memory_fact=1,
method_init=init_method,
alpha_snmf=alpha_snmf,
only_init_patch=False,
gnb=gnb,
border_pix=bord_px_els)
cnm = cnm.fit(images)
#%% plot contours of found components
Cn = cm.local_correlations(images.transpose(1, 2, 0))
Cn[np.isnan(Cn)] = 0
plt.figure()
crd = plot_contours(cnm.A, Cn, thr=0.9)
plt.title('Contour plots of found components')
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=False,
thresh_cnn_min=cnn_thr)
#%% PLOT COMPONENTS
if display_images:
plt.figure()
plt.subplot(121)
crd_good = cm.utils.visualization.plot_contours(cnm.A[:,
idx_components],
Cn,
thr=.8,
vmax=0.75)
plt.title('Contour plots of accepted components')
plt.subplot(122)
crd_bad = cm.utils.visualization.plot_contours(
cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75)
plt.title('Contour plots of rejected components')
#%% VIEW TRACES (accepted and rejected)
if display_images:
view_patches_bar(Yr,
cnm.A.tocsc()[:, idx_components],
cnm.C[idx_components],
cnm.b,
cnm.f,
dims[0],
dims[1],
YrA=cnm.YrA[idx_components],
img=Cn)
view_patches_bar(Yr,
cnm.A.tocsc()[:, idx_components_bad],
cnm.C[idx_components_bad],
cnm.b,
cnm.f,
dims[0],
dims[1],
YrA=cnm.YrA[idx_components_bad],
img=Cn)
#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[
idx_components], cnm.b, cnm.f
cnm2 = cnmf.CNMF(n_processes=1,
k=A_in.shape[-1],
gSig=gSig,
p=p,
dview=dview,
merge_thresh=merge_thresh,
Ain=A_in,
Cin=C_in,
b_in=b_in,
f_in=f_in,
rf=None,
stride=None,
gnb=gnb,
method_deconvolution='oasis',
check_nan=True)
cnm2 = cnm2.fit(images)
#%% Extract DF/F values
F_dff = detrend_df_f(cnm2.A,
cnm2.b,
cnm2.C,
cnm2.f,
YrA=cnm2.YrA,
quantileMin=8,
frames_window=250)
#%% Show final traces
cnm2.view_patches(Yr, dims=dims, img=Cn)
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
#%% reconstruct denoised movie
denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape(
dims + (-1, ), order='F').transpose([2, 0, 1])
#%% play along side original data
moviehandle = cm.concatenate([
m_els.resize(1, 1, downsample_ratio),
denoised.resize(1, 1, downsample_ratio)
],
axis=2)
if display_images:
moviehandle.play(fr=60, gain=15, magnification=2,
offset=0) # press q to exit
cnm = cnm.fit(images)
t_refine = time.time() - t1
A, C, b, f, YrA, sn = cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, cnm.sn
# %% again recheck quality of components, stricter criteria
idx_components, idx_components_bad, comp_SNR, comp_SNR_delta, r_values, predictionsCNN = estimate_components_quality_auto(
Y,
A,
C,
b,
f,
YrA,
params_movie['fr'],
params_movie['decay_time'],
gSig,
dims,
dview=dview,
min_SNR=global_params['min_SNR'],
r_values_min=global_params['rval_thr'],
r_values_lowest=global_params['min_rval_thr_rejected'],
Npeaks=global_params['Npeaks'],
use_cnn=True,
thresh_cnn_min=global_params['min_cnn_thresh'],
thresh_cnn_lowest=global_params[
'max_classifier_probability_rejected'],
thresh_fitness_delta=global_params['max_fitness_delta_accepted'])
# N_samples = np.ceil(params_movie['fr']*params_movie['decay_time']).astype(np.int) # number of timesteps to consider when testing new neuron candidates
# min_SNR = global_params['min_SNR'] # adaptive way to set threshold (will be equal to min_SNR)
# #pr_inc = 1 - scipy.stats.norm.cdf(global_params['min_SNR']) # inclusion probability of noise transient
# #thresh_fitness_raw = np.log(pr_inc)*N_samples # event exceptionality threshold
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
# parameters for component evaluation
min_SNR = 1 # signal to noise ratio for accepting a component
rval_thr = 0.8 # space correlation threshold for accepting a component
cnn_thr = 0.5 # threshold for CNN based classifier
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm2.A, cnm2.C, cnm2.b, cnm2.f,
cnm2.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=False,
thresh_cnn_min=cnn_thr, gSig_range = [list(np.add(i*2,a)) for i,a in enumerate([gSig]*5)])
#%% PLOT COMPONENTS
plt.figure()
plt.subplot(121)
crd_good = cm.utils.visualization.plot_contours(
cnm2.A[:, idx_components], Cn, thr=.95, vmax=0.25)
plt.title('Contour plots of accepted components')
plt.subplot(122)
crd_bad = cm.utils.visualization.plot_contours(
cnm2.A[:, idx_components_bad], Cn, thr=.95, vmax=0.25)
plt.title('Contour plots of rejected components')
#%% VIEW TRACES (accepted and rejected)
# roi_all_match = roi_1_match#np.concatenate([roi_all_match, roi_1_match],axis = 0)
A_gt_thr = roi_all_match.transpose([1, 2, 0]).reshape(
(np.prod(dims), -1), order='F')
A_gt = A_gt_thr
else:
if filter_SNR:
final_frate = params_movie['fr']
Npeaks = global_params['Npeaks']
traces_gt = C_gt + YrA_gt
idx_filter_snr, idx_filter_snr_bad, fitness_raw_gt, fitness_delta_gt, r_values_gt = estimate_components_quality_auto(
traces_gt,
Y,
A_gt,
C_gt,
b_gt,
f_gt,
final_frate=final_frate,
Npeaks=Npeaks,
r_values_min=1,
fitness_min=-10,
fitness_delta_min=-10,
return_all=True)
print(len(idx_filter_snr))
print(len(idx_filter_snr_bad))
A_gt, C_gt, b_gt, f_gt, traces_gt, YrA_gt = A_gt.tocsc(
)[:,
idx_filter_snr], C_gt[idx_filter_snr], b_gt, f_gt, traces_gt[
idx_filter_snr], YrA_gt[idx_filter_snr]
A_gt_thr = cm.source_extraction.cnmf.spatial.threshold_components(
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters
# dataset dependent parameters
display_images = False # Set to true to show movies and images
fnames = ['data_endoscope.tif'] # filename to be processed
frate = 10 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
do_motion_correction_nonrigid = True
do_motion_correction_rigid = False # in this case it will also save a rigid motion corrected movie
gSig_filt = (3, 3) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (5, 5) # maximum allowed rigid shift
splits_rig = 10 # for parallelization split the movies in num_splits chuncks across time
strides = (48, 48) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (24, 24) # overlap between pathes (size of patch strides+overlaps)
# for parallelization split the movies in num_splits chuncks across time
# (remember that it should hold that length_movie/num_splits_to_process_rig>100)
splits_els = 10
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None
gSig = 3 # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = 13 # average diameter of a neuron, in general 4*gSig+1
merge_thresh = .7 # merging threshold, max correlation allowed
rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80
stride_cnmf = 20 # amount of overlap between the patches in pixels
# (keep it at least large as gSiz, i.e 4 times the neuron size gSig)
tsub = 2 # downsampling factor in time for initialization,
# increase if you have memory problems
ssub = 1 # downsampling factor in space for initialization,
# increase if you have memory problems
Ain = None # if you want to initialize with some preselected components
# you can pass them here as boolean vectors
low_rank_background = None # None leaves background of each patch intact,
# True performs global low-rank approximation
gnb = -1 # number of background components (rank) if positive,
# else exact ring model with following settings
# gnb=-2: Return background as b and W
# gnb=-1: Return full rank background B
# gnb= 0: Don't return background
nb_patch = -1 # number of background components (rank) per patch,
# use 0 or -1 for exact background of ring model (cf. gnb)
min_corr = .8 # min peak value from correlation image
min_pnr = 10 # min peak to noise ration from PNR image
ssub_B = 2 # additional downsampling factor in space for background
ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor
# parameters for component evaluation
min_SNR = 3 # adaptive way to set threshold on the transient size
r_values_min = 0.85 # threshold on space consistency (if you lower more components
# will be accepted, potentially with worst quality)
#%% start the cluster
try:
cm.stop_server(dview=dview) # stop it if it was running
except:
pass
c, dview, n_processes = cm.cluster.setup_cluster(backend='local', # use this one
n_processes=24, # number of process to use, if you go out of memory try to reduce this one
single_thread=False)
#%% download demo file
fnames = [download_demo(fnames[0])]
filename_reorder = fnames
#%% MOTION CORRECTION
if do_motion_correction_nonrigid or do_motion_correction_rigid:
# do motion correction rigid
mc = motion_correct_oneP_rigid(fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
dview=dview,
splits_rig=splits_rig,
save_movie=not(do_motion_correction_nonrigid)
)
new_templ = mc.total_template_rig
plt.subplot(1, 2, 1)
plt.imshow(new_templ) # % plot template
plt.subplot(1, 2, 2)
plt.plot(mc.shifts_rig) # % plot rigid shifts
plt.legend(['x shifts', 'y shifts'])
plt.xlabel('frames')
plt.ylabel('pixels')
# borders to eliminate from movie because of motion correction
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
filename_reorder = mc.fname_tot_rig
# do motion correction nonrigid
if do_motion_correction_nonrigid:
mc = motion_correct_oneP_nonrigid(
fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
strides=strides,
overlaps=overlaps,
splits_els=splits_els,
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
dview=dview,
splits_rig=None,
save_movie=True, # whether to save movie in memory mapped format
new_templ=new_templ # template to initialize motion correction
)
filename_reorder = mc.fname_tot_els
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
# create memory mappable file in the right order on the hard drive (C order)
fname_new = cm.save_memmap_each(
filename_reorder,
base_name='memmap_',
order='C',
border_to_0=bord_px,
dview=dview)
fname_new = cm.save_memmap_join(fname_new, base_name='memmap_', dview=dview)
# load memory mappable file
Yr, dims, T = cm.load_memmap(fname_new)
Y = Yr.T.reshape((T,) + dims, order='F')
#%% compute some summary images (correlation and peak to noise)
# change swap dim if output looks weird, it is a problem with tiffile
cn_filter, pnr = cm.summary_images.correlation_pnr(Y, gSig=gSig, swap_dim=False)
# inspect the summary images and set the parameters
inspect_correlation_pnr(cn_filter, pnr)
# print parameters set above, modify them if necessary based on summary images
print(min_corr) # min correlation of peak (from correlation image)
print(min_pnr) # min peak to noise ratio
#%% RUN CNMF ON PATCHES
cnm = cnmf.CNMF(
n_processes=n_processes,
method_init='corr_pnr', # use this for 1 photon
k=K,
gSig=(gSig, gSig),
gSiz=(gSiz, gSiz),
merge_thresh=merge_thresh,
p=p,
dview=dview,
tsub=tsub,
ssub=ssub,
Ain=Ain,
rf=rf,
stride=stride_cnmf,
only_init_patch=True, # just leave it as is
gnb=gnb,
nb_patch=nb_patch,
method_deconvolution='oasis', # could use 'cvxpy' alternatively
low_rank_background=low_rank_background,
update_background_components=True, # sometimes setting to False improve the results
min_corr=min_corr,
min_pnr=min_pnr,
normalize_init=False, # just leave as is
center_psf=True, # leave as is for 1 photon
ssub_B=ssub_B,
ring_size_factor=ring_size_factor,
del_duplicates=True, # whether to remove duplicates from initialization
border_pix=bord_px) # number of pixels to not consider in the borders
cnm.fit(Y)
#%% DISCARD LOW QUALITY COMPONENTS
idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = \
estimate_components_quality_auto(
Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate,
decay_time, gSig, dims, dview=dview,
min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False)
print(' ***** ')
print((len(cnm.C)))
print((len(idx_components)))
cm.stop_server(dview=dview)
#%% PLOT COMPONENTS
if display_images:
plt.figure(figsize=(12, 6))
plt.subplot(121)
crd_good = cm.utils.visualization.plot_contours(
cnm.A[:, idx_components], cn_filter, thr=.8, vmax=0.95)
plt.title('Contour plots of accepted components')
plt.subplot(122)
crd_bad = cm.utils.visualization.plot_contours(
cnm.A[:, idx_components_bad], cn_filter, thr=.8, vmax=0.95)
plt.title('Contour plots of rejected components')
#%% VISUALIZE IN DETAILS COMPONENTS
cm.utils.visualization.view_patches_bar(
Yr, cnm.A[:, idx_components], cnm.C[idx_components], cnm.b, cnm.f,
dims[0], dims[1], YrA=cnm.YrA[idx_components], img=cn_filter)
#%% MOVIES
if display_images:
B = cnm.b.dot(cnm.f)
if 'sparse' in str(type(B)):
B = B.toarray()
# denoised movie
cm.movie(np.reshape(cnm.A.tocsc()[:, idx_components].dot(cnm.C[idx_components]) + B,
dims + (-1,), order='F').transpose(2, 0, 1)).play(magnification=3, gain=1.)
# only neurons
cm.movie(np.reshape(cnm.A.tocsc()[:, idx_components].dot(
cnm.C[idx_components]), dims + (-1,), order='F').transpose(2, 0, 1)
).play(magnification=3, gain=10.)
# only the background
cm.movie(np.reshape(B, dims + (-1,), order='F').transpose(2, 0, 1)
).play(magnification=3, gain=1.)
# residuals
cm.movie(np.array(Y) - np.reshape(cnm.A.tocsc()[:, :].dot(cnm.C[:]) + B,
dims + (-1,), order='F').transpose(2, 0, 1)
).play(magnification=3, gain=10., fr=10)
# eventually, you can rerun the algorithm on the residuals
plt.imshow(cm.movie(np.array(Y) - np.reshape(cnm.A.tocsc()[:, :].dot(cnm.C[:]) + B,
dims + (-1,), order='F').transpose(2, 0, 1)
).local_correlations(swap_dim=False))
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier (this will pick up only neurons
# and filter out active processes)
fr = 3 # approximate frame rate of data
decay_time = 1 # length of transient
min_SNR = 2.5 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.90 # space correlation threshold (if above this, accept)
use_cnn = False # use the CNN classifier
min_cnn_thr = 0.95 # if cnn classifier predicts below this value, reject
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm2.A, cnm2.C, cnm2.b, cnm2.f,
cnm2.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=use_cnn,
thresh_cnn_min=min_cnn_thr, thresh_cnn_lowest=0)
#%%
cm.utils.visualization.plot_contours(cnm2.A, Cn, thr=0.9)
#%% visualize selected and rejected components
plt.figure()
plt.subplot(1, 2, 1)
cm.utils.visualization.plot_contours(cnm2.A[:, idx_components], Cn, thr=0.9)
plt.title('Selected components')
plt.subplot(1, 2, 2)
plt.title('Discaded components')
cm.utils.visualization.plot_contours(cnm2.A[:, idx_components_bad],
Cn,
thr=0.9)
min_pnr=min_pnr, # min peak to noise ration from PNR image
normalize_init=False, # just leave as is
center_psf=True, # leave as is for 1 photon
del_duplicates=True) # whether to remove duplicates from initialization
cnm.fit(Y)
# %% DISCARD LOW QUALITY COMPONENTS
idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = estimate_components_quality_auto(
Y,
cnm.A,
cnm.C,
cnm.b,
cnm.f,
cnm.YrA,
frate,
decay_time,
gSig,
dims,
dview=dview,
min_SNR=min_SNR,
r_values_min=r_values_min,
use_cnn=False)
print(' ***** ')
print((len(cnm.C)))
print((len(idx_components)))
#%% PLOT ALL COMPONENTS
crd = cm.utils.visualization.plot_contours(cnm.A, cn_filter, thr=.8, vmax=0.95)
#%% PLOT ONLY GOOD QUALITY COMPONENTS
crd = cm.utils.visualization.plot_contours(cnm.A.tocsc()[:, idx_components],
def caiman_main_light_weight(fr, fnames, z=0, dend=False):
"""
Main function to compute the caiman algorithm. For more details see github and papers
fpath(str): Folder where to store the plots
fr(int): framerate
fnames(list-str): list with the names of the files to be computed together
z(array): vector with the values of z relative to y
dend(bool): Boleean to change parameters to look for neurons or dendrites
display_images(bool): to display and save different plots
returns
F_dff(array): array with the dff of the components
com(array): matrix with the position values of the components as given by caiman
cnm(struct): struct with different stimates and returns from caiman"""
# parameters
decay_time = 0.4 # length of a typical transient in seconds
# Look for the best parameters for this 2p system and never change them again :)
# motion correction parameters
niter_rig = 1 # number of iterations for rigid motion correction
max_shifts = (3, 3) # maximum allow rigid shift
splits_rig = 10 # for parallelization split the movies in num_splits chuncks across time
strides = (
96, 96
) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (48, 48
) # overlap between pathes (size of patch strides+overlaps)
splits_els = 10 # for parallelization split the movies in num_splits chuncks across time
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
max_deviation_rigid = 3 # maximum deviation allowed for patch with respect to rigid shifts
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thresh = 0.8 # merging threshold, max correlation allowed
rf = 25 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 10 # amount of overlap between the patches in pixels
K = 25 # number of components per patch
if dend:
gSig = [1, 1] # expected half size of neurons
init_method = 'sparse_nmf' # initialization method (if analyzing dendritic data using 'sparse_nmf')
alpha_snmf = 1e-6 # sparsity penalty for dendritic data analysis through sparse NMF
else:
gSig = [3, 3] # expected half size of neurons
init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')
alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF
# parameters for component evaluation
min_SNR = 2.5 # signal to noise ratio for accepting a component
rval_thr = 0.8 # space correlation threshold for accepting a component
cnn_thr = 0.8 # threshold for CNN based classifier
dview = None # parallel processing keeps crashing.
print('***************Starting motion correction*************')
print('files:')
print(fnames)
# %% start a cluster for parallel processing
# c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the parameters specified
min_mov = cm.load(fnames[0]).min()
# this will be subtracted from the movie to make it non-negative
mc = MotionCorrect(fnames,
min_mov,
dview=dview,
max_shifts=max_shifts,
niter_rig=niter_rig,
splits_rig=splits_rig,
strides=strides,
overlaps=overlaps,
splits_els=splits_els,
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
shifts_opencv=True,
nonneg_movie=True)
# note that the file is not loaded in memory
# %% Run piecewise-rigid motion correction using NoRMCorre
mc.motion_correct_pwrigid(save_movie=True)
bord_px_els = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
totdes = [np.nansum(mc.x_shifts_els), np.nansum(mc.y_shifts_els)]
print('***************Motion correction has ended*************')
# maximum shift to be used for trimming against NaNs
# %% MEMORY MAPPING
# memory map the file in order 'C'
fnames = mc.fname_tot_els # name of the pw-rigidly corrected file.
fname_new = cm.save_memmap(fnames,
base_name='memmap_',
order='C',
border_to_0=bord_px_els) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
# %% restart cluster to clean up memory
# cm.stop_server(dview=dview)
# c, dview, n_processes = cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False)
# %% RUN CNMF ON PATCHES
print('***************Running CNMF...*************')
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
cnm = cnmf.CNMF(n_processes=1,
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=0,
dview=dview,
rf=rf,
stride=stride_cnmf,
memory_fact=1,
method_init=init_method,
alpha_snmf=alpha_snmf,
only_init_patch=False,
gnb=gnb,
border_pix=bord_px_els)
cnm = cnm.fit(images)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.estimates.A, cnm.estimates.C, cnm.estimates.b,
cnm.estimates.f,
cnm.estimates.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=False,
thresh_cnn_min=cnn_thr)
# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
A_in, C_in, b_in, f_in = cnm.estimates.A[:, idx_components], cnm.estimates.C[
idx_components], cnm.estimates.b, cnm.estimates.f
cnm2 = cnmf.CNMF(n_processes=1,
k=A_in.shape[-1],
gSig=gSig,
p=p,
dview=dview,
merge_thresh=merge_thresh,
Ain=A_in,
Cin=C_in,
b_in=b_in,
f_in=f_in,
rf=None,
stride=None,
gnb=gnb,
method_deconvolution='oasis',
check_nan=True)
print('***************Fit*************')
cnm2 = cnm2.fit(images)
print('***************Extractind DFFs*************')
# %% Extract DF/F values
# cm.stop_server(dview=dview)
try:
F_dff = detrend_df_f(cnm2.estimates.A,
cnm2.estimates.b,
cnm2.estimates.C,
cnm2.estimates.f,
YrA=cnm2.estimates.YrA,
quantileMin=8,
frames_window=250)
# F_dff = detrend_df_f(cnm.A, cnm.b, cnm.C, cnm.f, YrA=cnm.YrA, quantileMin=8, frames_window=250)
except:
F_dff = cnm2.estimates.C * np.nan
print('WHAAT went wrong again?')
print('***************stopping cluster*************')
# %% STOP CLUSTER and clean up log files
# cm.stop_server(dview=dview)
# ***************************************************************************************
# Preparing output data
# F_dff -> DFF values, is a matrix [number of neurons, length recording]
# com --> center of mass, is a matrix [number of neurons, 2]
print('***************preparing output data*************')
if len(dims) <= 2:
if len(z) == 1:
com = np.concatenate(
(cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]),
np.zeros((cnm2.estimates.A.shape[1], 1)) + z), 1)
elif len(z) == dims[0]:
auxcom = cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1])
zy = np.zeros((auxcom.shape[0], 1))
for y in np.arange(auxcom.shape[0]):
zy[y, 0] = z[int(auxcom[y, 0])]
com = np.concatenate((auxcom, zy), 1)
else:
print(
'WARNING: Z value was not correctly defined, only X and Y values on file, z==zeros'
)
print(['length of z was: ' + str(len(z))])
com = np.concatenate(
(cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1]),
np.zeros((cnm2.estimates.A.shape[1], 1))), 1)
else:
com = cm.base.rois.com(cnm2.estimates.A, dims[0], dims[1], dims[2])
return F_dff, com, cnm2, totdes, SNR_comp[idx_components]
def CNMF_PROCESS(tif_movie, _k, _g, _merge):
"""
Inputs .tif movie (transforming from time series images)
Applys constrained nonnegative matrix factorization
Outputs selected neurons sparse matrix and dimension of movie
"""
dataset_name = tif_movie.replace('.tif', '')
# start a cluster
c, dview, n_processes =\
cm.cluster.setup_cluster(backend='local', n_processes=None, single_thread=False)
# process movie file
fnames = [tif_movie]
# location of dataset (can actually be a list of filed to be concatenated)
add_to_movie = -np.min(cm.load(fnames[0],
subindices=range(200))).astype(float)
# determine minimum value on a small chunk of data
add_to_movie = np.maximum(add_to_movie, 0)
# if minimum is negative subtract to make the data non-negative
base_name = 'Yr'
name_new = cm.save_memmap_each(fnames,
dview=dview,
base_name=base_name,
add_to_movie=add_to_movie)
name_new.sort()
fname_new = cm.save_memmap_join(name_new, base_name='Yr', dview=dview)
### LOAD MEMORY MAPPABLE FILE
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# ### play movie, press q to quit
# play_movie = False
# if play_movie:
# cm.movie(images[1400:]).play(fr=50, magnification=4, gain=3.)
### correlation image. From here infer neuron size and density
Cn = cm.movie(images).local_correlations(swap_dim=False)
plt.imshow(Cn, cmap='gray')
plt.title('Correlation Image')
# plt.show()
plt.savefig('figures/correlation_' + dataset_name + '.png')
### set up some parameters
is_patches = False # flag for processing in patches or not
if is_patches: # PROCESS IN PATCHES AND THEN COMBINE
rf = 10 # half size of each patch
stride = 4 # overlap between patches
K = 3 # number of components in each patch
else: # PROCESS THE WHOLE FOV AT ONCE
rf = None # setting these parameters to None
stride = None # will run CNMF on the whole FOV
K = _k # number of neurons expected (in the whole FOV)
gSig = [_g, _g] # expected half size of neurons
merge_thresh = _merge # merging threshold, max correlation allowed
p = 2 # order of the autoregressive system
gnb = 2 # global background order
### Now RUN CNMF
cnm = cnmf.CNMF(n_processes,
method_init='greedy_roi',
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=p,
dview=dview,
gnb=gnb,
rf=rf,
stride=stride,
rolling_sum=False)
cnm = cnm.fit(images)
### plot contour plots of components
plt.figure()
crd = cm.utils.visualization.plot_contours(cnm.A, Cn, thr=0.9)
plt.title('Contour plots of components')
# plt.show()
plt.savefig('figures/contour_' + dataset_name + '.png')
### COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier (this will pick up only neurons
# and filter out active processes)
fr = 10 # approximate frame rate of data
decay_time = 5.0 # length of transient
min_SNR = 2.5 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.90 # space correlation threshold (if above this, accept)
use_cnn = True # use the CNN classifier
min_cnn_thr = 0.10 # if cnn classifier predicts below this value, reject
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=use_cnn,
thresh_cnn_lowest=min_cnn_thr)
### visualize selected and rejected components
plt.figure()
plt.subplot(1, 2, 1)
cm.utils.visualization.plot_contours(cnm.A[:, idx_components], Cn, thr=0.9)
plt.title('Selected components')
plt.savefig('figures/selected_' + dataset_name + '.png')
plt.subplot(1, 2, 2)
plt.title('Discaded components')
cm.utils.visualization.plot_contours(cnm.A[:, idx_components_bad],
Cn,
thr=0.9)
plt.savefig('figures/discaded_' + dataset_name + '.png')
# plt.show(block=True)
### visualize selected components
cm.utils.visualization.view_patches_bar(Yr,
cnm.A.tocsc()[:, idx_components],
cnm.C[idx_components, :],
cnm.b,
cnm.f,
dims[0],
dims[1],
YrA=cnm.YrA[idx_components, :],
img=Cn)
### STOP CLUSTER and clean up log files
cm.stop_server()
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
dview.terminate()
return cnm.A.tocsc()[:, idx_components]
def run(batch_dir: str, UUID: str):
start_time = time()
output = {'status': 0, 'output_info': ''}
n_processes = os.environ['_MESMERIZE_N_THREADS']
n_processes = int(n_processes)
file_path = batch_dir + '/' + UUID
filename = file_path + '.tiff'
input_params = pickle.load(open(file_path + '.params', 'rb'))
frate = input_params['frate']
gSig = input_params['gSig']
gSiz = 3 * gSig + 1
min_corr = input_params['min_corr']
min_pnr = input_params['min_pnr']
min_SNR = input_params['min_SNR']
r_values_min = input_params['r_values_min']
decay_time = input_params['decay_time']
rf = input_params['rf']
stride = input_params['stride']
gnb = input_params['gnb']
nb_patch = input_params['nb_patch']
k = input_params['k']
if 'Ain' in input_params.keys():
if input_params['Ain']:
print('>> Ain specified, looking for cnm-A file <<')
item_uuid = input_params['Ain']
parent_batch_dir = os.environ['CURR_BATCH_DIR']
item_out_file = os.path.join(parent_batch_dir, f'{item_uuid}.out')
t0 = time()
timeout = 60
while not os.path.isfile(item_out_file):
print('>>> cnm-A not found, waiting for 15 seconds <<<')
sleep(15)
if time() - t0 > timeout:
output.update({'status': 0, 'output_info': 'Timeout exceeding in waiting for Ain input file'})
raise TimeoutError('Timeout exceeding in waiting for Ain input file')
if os.path.isfile(item_out_file):
if json.load(open(item_out_file, 'r'))['status']:
Ain_file = os.path.join(parent_batch_dir, item_uuid + '_cnm-A.pikl')
Ain = pickle.load(open(Ain_file, 'rb'))
print('>>> Found Ain file <<<')
else:
raise FileNotFoundError('>>> Could not find specified Ain file <<<')
else:
Ain = None
else:
Ain = None
if 'method_deconvolution' in input_params.keys():
method_deconvolution = input_params['method_deconvolution']
else:
method_deconvolution = 'oasis'
if 'deconv_flag' in input_params.keys():
deconv_flag = input_params['deconv_flag']
else:
deconv_flag = True
filename = [filename]
print('*********** Creating Process Pool ***********')
c, dview, n_processes = cm.cluster.setup_cluster(backend='local', # use this one
n_processes=n_processes,
single_thread=False)
if 'bord_px' in input_params.keys():
bord_px = input_params['bord_px']
else:
bord_px = 6
try:
print('Creating memmap')
fname_new = cm.save_memmap_each(
filename,
base_name='memmap_' + UUID,
order='C',
border_to_0=bord_px,
dview=dview)
fname_new = cm.save_memmap_join(fname_new, base_name='memmap_' + UUID, dview=dview)
# load memory mappable file
Yr, dims, T = cm.load_memmap(fname_new)
Y = Yr.T.reshape((T,) + dims, order='F')
# compute some summary images (correlation and peak to noise)
# change swap dim if output looks weird, it is a problem with tiffile
cn_filter, pnr = cm.summary_images.correlation_pnr(
Y, gSig=gSig, swap_dim=False)
if not input_params['do_cnmfe'] and input_params['do_corr_pnr']:
pickle.dump(cn_filter, open(UUID + '_cn_filter.pikl', 'wb'), protocol=4)
pickle.dump(pnr, open(UUID + '_pnr.pikl', 'wb'), protocol=4)
output_file_list = [UUID + '_pnr.pikl',
UUID + '_cn_filter.pikl',
UUID + '_dims.pikl',
UUID + '.out'
]
output.update({'output': UUID,
'status': 1,
'output_info': 'inspect correlation & pnr',
'output_files': output_file_list
})
dview.terminate()
for mf in glob(batch_dir + '/memmap_*'):
os.remove(mf)
end_time = time()
processing_time = (end_time - start_time) / 60
output.update({'processing_time': processing_time})
json.dump(output, open(file_path + '.out', 'w'))
return
cnm = cnmf.CNMF(n_processes=n_processes,
method_init='corr_pnr', # use this for 1 photon
k=k, # neurons per patch
gSig=(gSig, gSig), # half size of neuron
gSiz=(gSiz, gSiz), # in general 3*gSig+1
merge_thresh=.3, # threshold for merging
p=1, # order of autoregressive process to fit
dview=dview, # if None it will run on a single thread
# downsampling factor in time for initialization, increase if you have memory problems
tsub=2,
# downsampling factor in space for initialization, increase if you have memory problems
ssub=2,
# if you want to initialize with some preselcted components you can pass them here as boolean vectors
Ain=Ain,
# half size of the patch (final patch will be 100x100)
rf=(rf, rf),
# overlap among patches (keep it at least large as 4 times the neuron size)
stride=(stride, stride),
only_init_patch=True, # just leave it as is
gnb=gnb, # number of background components
nb_patch=nb_patch, # number of background components per patch
method_deconvolution=method_deconvolution, # could use 'cvxpy' alternatively
deconv_flag=deconv_flag,
low_rank_background=True, # leave as is
# sometimes setting to False improve the results
update_background_components=True,
min_corr=min_corr, # min peak value from correlation image
min_pnr=min_pnr, # min peak to noise ration from PNR image
normalize_init=False, # just leave as is
center_psf=True, # leave as is for 1 photon
del_duplicates=True, # whether to remove duplicates from initialization
border_pix=bord_px) # number of pixels to not consider in the borders
cnm.fit(Y)
# DISCARD LOW QUALITY COMPONENTS
idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = estimate_components_quality_auto(
Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate,
decay_time, gSig, dims, dview=dview,
min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False)
# np.save(filename[:-5] + '_curves.npy', cnm.C)
pickle.dump(Yr, open(UUID + '_Yr.pikl', 'wb'), protocol=4)
pickle.dump(cnm.A, open(UUID + '_cnm-A.pikl', 'wb'), protocol=4)
pickle.dump(cnm.b, open(UUID + '_cnm-b.pikl', 'wb'), protocol=4)
pickle.dump(cnm.C, open(UUID + '_cnm-C.pikl', 'wb'), protocol=4)
pickle.dump(cnm.f, open(UUID + '_cnm-f.pikl', 'wb'), protocol=4)
pickle.dump(idx_components, open(UUID + '_idx_components.pikl', 'wb'), protocol=4)
pickle.dump(cnm.YrA, open(UUID + '_cnm-YrA.pikl', 'wb'), protocol=4)
pickle.dump(pnr, open(UUID + '_pnr.pikl', 'wb'), protocol=4)
pickle.dump(cn_filter, open(UUID + '_cn_filter.pikl', 'wb'), protocol=4)
pickle.dump(dims, open(UUID + '_dims.pikl', 'wb'), protocol=4)
output_file_list = [UUID + '_cnm-A.pikl',
UUID + '_Yr.pikl',
UUID + '_cnm-b.pikl',
UUID + '_cnm-C.pikl',
UUID + '_cnm-f.pikl',
UUID + '_idx_components.pikl',
UUID + '_cnm-YrA.pikl',
UUID + '_pnr.pikl',
UUID + '_cn_filter.pikl',
UUID + '_dims.pikl',
UUID + '.out'
]
output.update({'output': filename[:-5],
'status': 1,
'output_files': output_file_list
})
except Exception as e:
output.update({'status': 0, 'output_info': traceback.format_exc()})
dview.terminate()
for mf in glob(batch_dir + '/memmap_*'):
os.remove(mf)
end_time = time()
processing_time = (end_time - start_time) / 60
output.update({'processing_time': processing_time})
json.dump(output, open(file_path + '.out', 'w'))
Cn = cm.local_correlations(images.transpose(1, 2, 0))
Cn[np.isnan(Cn)] = 0
plt.figure()
crd = plot_contours(cnm.A, Cn, thr=0.9)
plt.title('Contour plots of found components')
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=False,
thresh_cnn_min=cnn_thr)
#%% PLOT COMPONENTS
if display_images:
plt.figure()
plt.subplot(121)
crd_good = cm.utils.visualization.plot_contours(cnm.A[:, idx_components],
Cn,
thr=.8,
vmax=0.75)
plt.title('Contour plots of accepted components')
plt.subplot(122)
crd_bad = cm.utils.visualization.plot_contours(cnm.A[:,
update_background_components=True, # sometimes setting to False improve the results
min_corr=min_corr,
min_pnr=min_pnr,
normalize_init=False, # just leave as is
center_psf=True, # leave as is for 1 photon
ssub_B=ssub_B,
ring_size_factor=ring_size_factor,
del_duplicates=True, # whether to remove duplicates from initialization
border_pix=bord_px) # number of pixels to not consider in the borders
cnm.fit(Y)
#%% DISCARD LOW QUALITY COMPONENTS
idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = \
estimate_components_quality_auto(
Y, cnm.A, cnm.C, cnm.b, cnm.f, cnm.YrA, frate,
decay_time, gSig, dims, dview=dview,
min_SNR=min_SNR, r_values_min=r_values_min, use_cnn=False)
print(' ***** ')
print((len(cnm.C)))
print((len(idx_components)))
cm.stop_server(dview=dview)
#%% PLOT COMPONENTS
if display_images:
plt.figure(figsize=(12, 6))
plt.subplot(121)
crd_good = cm.utils.visualization.plot_contours(
cnm.A[:, idx_components], cn_filter, thr=.8, vmax=0.95)
def run_cnmfe(tiff_files, param_file, output_file):
""" Run the CNMFe algorithm through CaImAn.
:param tiff_files: A list of .tiff files corresponding to a calcium imaging movie.
:param param_file: A .yaml parameter file, containing values for the following parameters:
num_processes : int
The number of processes to run in parallel. The more parallel processes, the more memory that is used.
rf : array-like
An array [half-width, half-height] that specifies the size of a patch.
stride : int
The amount of overlap in pixels between patches.
K : int
The maximum number of cells per patch.
gSiz : int
The expected diameter of a neuron in pixels.
gSig : int
The standard deviation a high pass Gaussian filter applied to the movie prior to seed pixel search, roughly
equal to the half-size of the neuron in pixels.
min_pnr : float
The minimum peak-to-noise ratio that is taken into account when searching for seed pixels.
min_corr : float
The minimum pixel correlation that is taken into account when searching for seed pixels.
min_SNR : float
Cells with an signal-to-noise (SNR) less than this are rejected.
rval_thr : float
Cells with a spatial correlation of greater than this are accepted.
decay_time : float
The expected decay time of a calcium event in seconds.
ssub_B : int
The spatial downsampling factor used on the background term.
merge_threshold : float
Cells that are spatially close with a temporal correlation of greater than merge_threshold are automatically merged.
:param output_file: The path to a .hdf5 file that will be written to contain the traces, footprints, and deconvolved
events identified by CNMFe.
"""
for tiff_file in tiff_files:
if not os.path.exists(tiff_file):
raise FileNotFoundError(tiff_file)
if not os.path.exists(param_file):
raise FileNotFoundError(param_file)
with open(param_file, 'r') as f:
params = yaml.load(f)
expected_params = [
'gSiz', 'gSig', 'K', 'min_corr', 'min_pnr', 'rf', 'stride',
'decay_time', 'min_SNR', 'rval_thr', 'merge_threshold', 'ssub_B',
'frame_rate', 'num_rows', 'num_cols', 'num_frames', 'num_processes'
]
for pname in expected_params:
if pname not in params:
raise ValueError('Missing parameter {} in file {}'.format(
pname, param_file))
gSiz = params['gSiz']
gSig = params['gSig']
K = params['K']
min_corr = params['min_corr']
min_pnr = params['min_pnr']
rf = params['rf']
stride = params['stride']
decay_time = params['decay_time']
min_SNR = params['min_SNR']
rval_thr = params['rval_thr']
merge_threshold = params['merge_threshold']
ssub_B = params['ssub_B']
frame_rate = params['frame_rate']
num_rows = params['num_rows']
num_cols = params['num_cols']
num_frames = params['num_frames']
num_processes = params['num_processes']
# write memmapped file
print('Exporting .isxd to memmap file...')
mmap_file = _export_movie_to_memmap(tiff_files,
num_frames,
num_rows,
num_cols,
overwrite=False)
print('Wrote .mmap file to: {}'.format(mmap_file))
# open memmapped file
Yr, dims, T = load_memmap(mmap_file)
Y = Yr.T.reshape((T, ) + dims, order='F')
# grab parallel IPython handle
dview = None
if num_processes > 1:
import ipyparallel as ipp
c = ipp.Client()
dview = c[:]
print('Running using parallel IPython, # clusters = {}'.format(
len(c.ids)))
num_processes = len(c.ids)
# initialize CNMFE parameter object and set user params
cnmfe_params = CNMFParams()
if gSiz is None:
raise ValueError(
'You must set gSiz to an integer, ideally roughly equal to the expected half-cell width.'
)
gSiz = _turn_into_array(gSiz)
if gSig is None:
raise ValueError(
'You must set gSig to a non-zero integer. The default value is 5.')
gSig = _turn_into_array(gSig)
cnmfe_params.set('preprocess', {'p': 1})
cnmfe_params.set(
'init', {
'K': K,
'min_corr': min_corr,
'min_pnr': min_pnr,
'gSiz': gSiz,
'gSig': gSig
})
if rf is None:
cnmfe_params.set('patch', {'rf': None, 'stride': 1})
else:
cnmfe_params.set('patch', {'rf': np.array(rf), 'stride': stride})
cnmfe_params.set('data', {'decay_time': decay_time})
cnmfe_params.set('quality', {'min_SNR': min_SNR, 'rval_thr': rval_thr})
cnmfe_params.set('merging', {'merge_thr': merge_threshold})
# set parameters that force CNMF into one-photon mode with no temporal or spatial downsampling,
# except for the background term
cnmfe_params.set(
'init', {
'center_psf': True,
'method_init': 'corr_pnr',
'normalize_init': False,
'nb': -1,
'ssub_B': ssub_B,
'tsub': 1,
'ssub': 1
})
cnmfe_params.set(
'patch', {
'only_init': True,
'low_rank_background': None,
'nb_patch': -1,
'p_tsub': 1,
'p_ssub': 1
})
cnmfe_params.set('spatial', {
'nb': -1,
'update_background_components': False
})
cnmfe_params.set('temporal', {'nb': -1, 'p': 1})
# construct and run CNMFE
print('Running CNMFe...')
cnmfe = CNMF(num_processes, dview=dview, params=cnmfe_params)
cnmfe.fit(Y)
# run auto accept/reject
print('Estimating component quality...')
idx_components, idx_components_bad, comp_SNR, r_values, pred_CNN = estimate_components_quality_auto(
Y,
cnmfe.estimates.A,
cnmfe.estimates.C,
cnmfe.estimates.b,
cnmfe.estimates.f,
cnmfe.estimates.YrA,
frame_rate,
cnmfe_params.get('data', 'decay_time'),
cnmfe_params.get('init', 'gSiz'),
cnmfe.dims,
dview=None,
min_SNR=cnmfe_params.get('quality', 'min_SNR'),
use_cnn=False)
save_cnmfe(cnmfe, output_file, good_idx=idx_components)
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters
# num processes
n_proc = 12
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
niter_rig = 1 # number of iterations for rigid motion correction
max_shifts = (6, 6) # maximum allow rigid shift
# for parallelization split the movies in num_splits chuncks across time
splits_rig = 56
# start a new patch for pw-rigid motion correction every x pixels
strides = (48, 48)
# overlap between pathes (size of patch strides+overlaps)
overlaps = (24, 24)
# for parallelization split the movies in num_splits chuncks across time
splits_els = 56
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thresh = 0.8 # merging threshold, max correlation allowed
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
rf = 15
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons
# initialization method (if analyzing dendritic data using 'sparse_nmf')
init_method = 'greedy_roi'
is_dendrites = False # flag for analyzing dendritic data
# sparsity penalty for dendritic data analysis through sparse NMF
alpha_snmf = None
# parameters for component evaluation
min_SNR = 2.5 # signal to noise ratio for accepting a component
rval_thr = 0.8 # space correlation threshold for accepting a component
cnn_thr = 0.8 # threshold for CNN based classifier
#%% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=n_proc, single_thread=False)
print('Parallel processing initialized.')
print('Beginning motion correction')
#%%% MOTION CORRECTION
t_ms = time.time()
# first we create a motion correction object with the parameters specified
min_mov = cm.load(fname[0], subindices=range(200)).min()
# this will be subtracted from the movie to make it non-negative
if not nomc:
mc = MotionCorrect(fname[0], min_mov,
dview=dview, max_shifts=max_shifts, niter_rig=niter_rig,splits_rig=splits_rig,strides=strides, overlaps=overlaps, splits_els=splits_els,upsample_factor_grid=upsample_factor_grid,max_deviation_rigid=max_deviation_rigid,shifts_opencv=True, nonneg_movie=True, use_cuda=use_cuda)
# note that the file is not loaded in memory
#%% Run piecewise-rigid motion correction using NoRMCorre
mc.motion_correct_pwrigid(save_movie=True)
m_els = cm.load(mc.fname_tot_els)
bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)),np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
t_mf = time.time()
print('Motion correction complete in ', int(t_mf - t_ms),' seconds')
# maximum shift to be used for trimming against NaNs
#%% compare with original movie
#%% MEMORY MAPPING
# memory map the file in order 'C'
fnames = mc.fname_tot_els # name of the pw-rigidly corrected file.
border_to_0 = bord_px_els # number of pixels to exclude
fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C',
border_to_0=bord_px_els) # exclude borders
bord_px_els = 5
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
#%% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=n_proc, single_thread=False)
#%% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
print('Beginning initial CNMF fit')
t1 = time.time()
cnm = cnmf.CNMF(n_processes=n_proc, k=K, gSig=gSig, merge_thresh=merge_thresh,
p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1,
method_init=init_method, alpha_snmf=alpha_snmf,
only_init_patch=False, gnb=gnb, border_pix=bord_px_els)
cnm = cnm.fit(images)
t2 = time.time()
print('Initial CNMF fit complete in ', int(t2-t1), 'seconds.')
Cn = cm.local_correlations(images.transpose(1, 2, 0))
Cn[np.isnan(Cn)] = 0
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
t3 = time.time()
print('Estimating component quality')
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=False,
thresh_cnn_min=cnn_thr)
t4 = time.time()
print('Component quality estimation complete in ', int(t4-t3),' seconds')
#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
print('Re-running CNMF to refine and deconvolve')
t5 = time.time()
A_in, C_in, b_in, f_in = cnm.A[:,
idx_components], cnm.C[idx_components], cnm.b, cnm.f
cnm2 = cnmf.CNMF(n_processes=n_proc, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview,
merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in,
f_in=f_in, rf=None, stride=None, gnb=gnb,
method_deconvolution='oasis', check_nan=True)
cnm2 = cnm2.fit(images)
t6 = time.time()
print('CNMF re-run complete in ', int(t6-t5), ' seconds')
#%% Extract DF/F values
# F_dff = detrend_df_f(cnm2.A, cnm2.b, cnm2.C, cnm2.f, YrA=cnm2.YrA,
# quantileMin=8, frames_window=250)
save_results = True
if save_results:
if os.path.isdir(infile[0]):
outfile = os.path.join(infile[0],'caiman_output.npz')
else:
outfile = os.path.join(os.path.split(infile[0])[0],'caiman_output.npz')
np.savez_compressed(outfile,Cn=Cn, A=cnm2.A.todense(), C=cnm2.C,b=cnm2.b, f=cnm2.f, YrA=cnm2.YrA, d1=d1, d2=d2,idx_components=idx_components, idx_components_bad=idx_components_bad)
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
def process_data(haussio_data,
mask=None,
p=2,
nrois_init=400,
roi_iceberg=0.9,
merge_unconnected=None):
if mask is not None:
raise RuntimeError("mask not supported in cnmf.process_data")
fn_cnmf = haussio_data.dirname_comp + '_cnmf.mat'
shapefn = os.path.join(haussio_data.dirname_comp,
haussio.THOR_RAW_FN[:-3] + "shape.npy")
shape = np.load(shapefn)
if len(shape) == 5:
d1, d2 = shape[2], shape[3]
else:
d1, d2 = shape[1], shape[2]
fn_mmap = get_mmap_name(haussio_data.dirname_comp + os.path.sep + 'Yr', d1,
d2, shape[0])
tiffs_to_cnmf(haussio_data)
if os.path.exists(fn_cnmf):
resdict = loadmat(fn_cnmf)
if "dFoF" in resdict.keys():
A2 = resdict["A"]
C2 = resdict["C"]
YrA = resdict["YrA"]
S2 = resdict["S"]
dFoF = resdict["dFoF"]
bl2 = resdict["bl"]
f = resdict["f"]
images = haussio_data.read_raw().squeeze()
else:
dFoF = None
if not os.path.exists(fn_cnmf) or dFoF is None:
c, dview, n_processes = cm.cluster.setup_cluster(
backend='multiprocessing', n_processes=NCPUS, single_thread=False)
Yr, dims, T = cm.load_memmap(fn_mmap, 'r+')
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
fr = 1.0 / haussio_data.dt # imaging rate in frames per second\n",
decay_time = 0.4 # length of a typical transient in seconds\n",
# parameters for source extraction and deconvolution\n",
bord_px_els = 32 # maximum shift to be used for trimming against NaNs
p = 1 # order of the autoregressive system\n",
gnb = 2 # number of global background components\n",
merge_thresh = 0.8 # merging threshold, max correlation allowed\n",
rf = int(
np.round(np.sqrt(d1 * d2) / nrois_init)
) # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50\n",
if rf < 16:
rf = 16
stride_cnmf = 6 # amount of overlap between the patches in pixels\n",
npatches = np.round(d1 / (rf * 2) * d2 / (rf * 2))
K = nrois_init / npatches # number of components per patch\n",
if K < 2:
K = 2
print(rf, npatches, K)
gSig = [8, 8] # expected half size of neurons\n",
init_method = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')\n",
is_dendrites = False # flag for analyzing dendritic data\n",
alpha_snmf = None # sparsity penalty for dendritic data analysis through sparse NMF\n",
# parameters for component evaluation\n",
min_SNR = 2.5 # signal to noise ratio for accepting a component\n",
rval_thr = 0.8 # space correlation threshold for accepting a component\n",
cnn_thr = 0.8 # threshold for CNN based classifier"
cnm = caiman_cnmf.CNMF(n_processes=1,
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=0,
dview=dview,
rf=rf,
stride=stride_cnmf,
memory_fact=1,
method_init=init_method,
alpha_snmf=alpha_snmf,
only_init_patch=False,
gnb=gnb,
border_pix=bord_px_els)
cnm = cnm.fit(images)
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview = dview, min_SNR=min_SNR,
r_values_min = rval_thr, use_cnn = False,
thresh_cnn_lowest = cnn_thr)
A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[
idx_components], cnm.b, cnm.f
cnm2 = caiman_cnmf.CNMF(n_processes=1,
k=A_in.shape[-1],
gSig=gSig,
p=p,
dview=dview,
merge_thresh=merge_thresh,
Ain=A_in,
Cin=C_in,
b_in=b_in,
f_in=f_in,
rf=None,
stride=None,
gnb=gnb,
method_deconvolution='oasis',
check_nan=True)
cnm2 = cnm2.fit(images)
if merge_unconnected is not None:
idx_merge = []
for nroi, ca_roi in enumerate(cnm2.C):
for nroi_compare_counter, ca_roi_compare in enumerate(
cnm2.C[nroi + 1:]):
nroi_compare = nroi_compare_counter + nroi + 1
if nroi_compare not in idx_merge:
correls = np.correlate(ca_roi,
ca_roi_compare,
mode='same')
correls /= np.sqrt(
np.dot(ca_roi, ca_roi) *
np.dot(ca_roi_compare, ca_roi_compare))
if correls.max() > merge_unconnected:
print("Merging ", nroi_compare)
idx_merge.append(nroi_compare)
idx_no_merge = [
idx for idx in range(cnm2.C.shape[0]) if idx not in idx_merge
]
else:
idx_no_merge = range(cnm2.C.shape[0])
A2 = cnm2.A[:, idx_no_merge].tocsc()
C2 = cnm2.C[idx_no_merge]
YrA = cnm2.YrA[idx_no_merge]
S2 = cnm2.S[idx_no_merge]
dFoF = cnm2.detrend_df_f(frames_window=300)[idx_no_merge]
# A: spatial components (ROIs)
# C: denoised [Ca2+]
# YrA: residuals ("noise", i.e. traces = C+YrA)
# S: Spikes
# f: temporal background
savemat(
fn_cnmf, {
"A": A2,
"C": C2,
"YrA": YrA,
"S": S2,
"dFoF": dFoF,
"bl": cnm2.b,
"f": cnm2.f
})
dview.terminate()
cm.stop_server()
proj_fn = haussio_data.dirname_comp + "_proj.npy"
if not os.path.exists(proj_fn):
zproj = utils.zproject(images)
np.save(proj_fn, zproj)
else:
zproj = np.load(proj_fn)
logfiles = glob.glob("*LOG*")
for logfile in logfiles:
try:
os.unlink(logfile)
except OSError:
pass
polygons = contour(A2, images.shape[1], images.shape[2], thr=roi_iceberg)
rois = ROIList([sima.ROI.ROI(polygons=poly) for poly in polygons])
return rois, C2, zproj, S2, images, YrA
Cn = cm.local_correlations(images.transpose(1,2,0))
Cn[np.isnan(Cn)] = 0
plt.figure(); crd = plot_contours(cnm.A, Cn, thr=0.9)
plt.title('Contour plots of found components')
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
idx_components, idx_components_bad, SNR_comp, SNR_comp_delta, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview = dview, min_SNR=min_SNR,
r_values_min = rval_thr, use_cnn = False,
thresh_cnn_lowest = cnn_thr)
#%% PLOT COMPONENTS
plt.figure();
plt.subplot(121); crd_good = cm.utils.visualization.plot_contours(cnm.A[:,idx_components], Cn, thr=.8, vmax=0.75)
plt.title('Contour plots of accepted components')
plt.subplot(122); crd_bad = cm.utils.visualization.plot_contours(cnm.A[:,idx_components_bad], Cn, thr=.8, vmax=0.75)
plt.title('Contour plots of rejected components')
#%% VIEW TRACES (accepted and rejected)
view_patches_bar(Yr, cnm.A.tocsc()[:, idx_components], cnm.C[idx_components],
cnm.b, cnm.f, dims[0], dims[1],YrA=cnm.YrA[idx_components],
img=Cn)
def run(batch_dir: str, UUID: str):
output = {'status': 0, 'output_info': ''}
n_processes = os.environ['_MESMERIZE_N_THREADS']
n_processes = int(n_processes)
file_path = batch_dir + '/' + UUID
filename = [file_path + '.tiff']
input_params = pickle.load(open(file_path + '.params', 'rb'))
fr = input_params['fr']
p = input_params['p']
gnb = input_params['gnb']
merge_thresh = input_params['merge_thresh']
rf = input_params['rf']
stride_cnmf = input_params['stride_cnmf']
K = input_params['k']
gSig = input_params['gSig']
gSig = [gSig, gSig]
min_SNR = input_params['min_SNR']
rval_thr = input_params['rval_thr']
cnn_thr = input_params['cnn_thr']
decay_time = input_params['decay_time']
bord_px = input_params['bord_px']
refit = input_params['refit']
print('*********** Creating Process Pool ***********')
c, dview, np = cm.cluster.setup_cluster(backend='local', n_processes=n_processes, single_thread=False)
try:
print('Creating memmap')
fname_new = cm.save_memmap_each(
filename,
base_name='memmap_' + UUID,
order='C',
border_to_0=bord_px,
dview=dview)
fname_new = cm.save_memmap_join(fname_new, base_name='memmap_' + UUID, dview=dview)
Yr, dims, T = cm.load_memmap(fname_new)
Y = Yr.T.reshape((T,) + dims, order='F')
cnm = cnmf.CNMF(n_processes=n_processes, k=K, gSig=gSig, merge_thresh=merge_thresh,
p=0, dview=dview, rf=rf, stride=stride_cnmf, memory_fact=1,
method_init='greedy_roi', alpha_snmf=None,
only_init_patch=False, gnb=gnb, border_pix=bord_px)
cnm.fit(Y)
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(Y, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=False,
thresh_cnn_lowest=cnn_thr)
if refit:
A_in, C_in, b_in, f_in = cnm.A[:,
idx_components], cnm.C[idx_components], cnm.b, cnm.f
cnm2 = cnmf.CNMF(n_processes=n_processes, k=A_in.shape[-1], gSig=gSig, p=p, dview=dview,
merge_thresh=merge_thresh, Ain=A_in, Cin=C_in, b_in=b_in,
f_in=f_in, rf=None, stride=None, gnb=gnb,
method_deconvolution='oasis', check_nan=True)
cnm2 = cnm2.fit(Y)
cnmA = cnm2.A
cnmb = cnm2.b
cnmC = cnm2.C
cnm_f = cnm2.f
cnmYrA = cnm2.YrA
else:
cnmA = cnm.A
cnmb = cnm.b
cnmC = cnm.C
cnm_f = cnm.f
cnmYrA = cnm.YrA
pickle.dump(Yr, open(UUID + '_Yr.pikl', 'wb'), protocol=4)
pickle.dump(cnmA, open(UUID + '_cnm-A.pikl', 'wb'), protocol=4)
pickle.dump(cnmb, open(UUID + '_cnm-b.pikl', 'wb'), protocol=4)
pickle.dump(cnmC, open(UUID + '_cnm-C.pikl', 'wb'), protocol=4)
pickle.dump(cnm_f, open(UUID + '_cnm-f.pikl', 'wb'), protocol=4)
pickle.dump(idx_components, open(UUID + '_idx_components.pikl', 'wb'), protocol=4)
pickle.dump(cnmYrA, open(UUID + '_cnm-YrA.pikl', 'wb'), protocol=4)
pickle.dump(dims, open(UUID + '_dims.pikl', 'wb'), protocol=4)
output_file_list = [UUID + '_cnm-A.pikl',
UUID + '_Yr.pikl',
UUID + '_cnm-b.pikl',
UUID + '_cnm-C.pikl',
UUID + '_cnm-f.pikl',
UUID + '_idx_components.pikl',
UUID + '_cnm-YrA.pikl',
UUID + '_dims.pikl',
UUID + '.out'
]
output.update({'output': UUID,
'status': 1,
'output_files': output_file_list
})
except Exception:
output.update({'status': 0, 'output_info': traceback.format_exc()})
dview.terminate()
for mf in glob(batch_dir + '/memmap_*'):
os.remove(mf)
json.dump(output, open(file_path + '.out', 'w'))
min_pnr=min_pnr, # min peak to noise ration from PNR image
normalize_init=False, # just leave as is
center_psf=True, # leave as is for 1 photon
del_duplicates=True) # whether to remove duplicates from initialization
cnm.fit(Y)
# %% DISCARD LOW QUALITY COMPONENTS
idx_components, idx_components_bad, fitness_raw, fitness_delta, r_values = estimate_components_quality_auto(
Y,
cnm.A,
cnm.C,
cnm.b,
cnm.f,
cnm.YrA,
frate,
decay_time,
gSig,
dims,
dview=dview,
min_SNR=min_SNR,
r_values_min=r_values_min,
use_cnn=False)
print(' ***** ')
print((len(cnm.C)))
print((len(idx_components)))
#%% PLOT ALL COMPONENTS
crd = cm.utils.visualization.plot_contours(cnm.A, cn_filter, thr=.8, vmax=0.95)
#%% PLOT ONLY GOOD QUALITY COMPONENTS
crd = cm.utils.visualization.plot_contours(cnm.A.tocsc()[:, idx_components],
