def init_mc_params(self, **kwargs):
"""
Define parameters.
"""
#Build the dict. These are defaults.
params_dict = {
'fnames': self.fnames, # file names
'fr': 20, # frame rate
'decay_time': .4, # transient decay time
'pw_rigid':
False, # flag for performing piecewise-rigid motion correction (otherwise just rigid)
'max_shifts': (5, 5), # maximum allowed rigid shift
'gSig_filt':
(3, 3), # size of high pass spatial filtering, used in 1p data
'strides': (
48, 48
), # start a new patch for pw-rigid motion correction every x pixels
'overlaps':
(24,
24), # overlap between pathes (size of patch strides+overlaps)
'max_deviation_rigid':
3, # maximum deviation allowed for patch with respect to rigid shifts
'border_nan': 'copy', # replicate values along the boundaries
}
# If any parameters were modified from default, this line reflects those changes.
for key, value in kwargs.items():
params_dict[key] = value
opts = params.CNMFParams(params_dict=params_dict)
return opts
def set_opts(fnames, fr, decay_time, opts_dict):
"""Parameters not defined in the dictionary will assume their default
values. The resulting params object is a collection of subdictionaries
pertaining to the dataset to be analyzed (params.data), motion correction
(params.motion), data pre-processing (params.preprocess), initialization
(params.init), patch processing (params.patch), spatial and temporal
component (params.spatial), (params.temporal), quality evaluation
(params.quality) and online processing (params.online)"""
opts_dict = {
**opts_dict, 'fnames': fnames,
'fr': fr,
'decay_time': decay_time
}
opts = params.CNMFParams(params_dict=opts_dict)
return opts
def __init__(self, mc_opts, tiff, channels, planes, x_start, x_end):
self.tiff = tiff
self.channels = channels
self.planes = planes
self.x_start = x_start
self.x_end = x_end
self.xslice = slice(x_start, x_end)
self.mc_opts = mc_opts
self.opts = params.CNMFParams(params_dict=mc_opts)
self.c, self.dview, self.n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
self.file_list = []
self.images = []
self.motion_corrected_images = []
self.masks = []
def motion_correct_file(folder, dview):
import caiman as cm
from caiman.motion_correction import MotionCorrect
from caiman.source_extraction.cnmf import params as params
print(folder)
mc_dict = {
'fnames': [os.path.join(folder, 'movie_total.hdf5')],
'fr': 7,
'decay_time': 1.5,
'dxy': [1.15, 1.15],
'pw_rigid': False,
'max_shifts': [5,5],
'strides': None,
'overlaps': None,
'max_deviation_rigid': None,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
mc = MotionCorrect(mc_dict['fnames'], dview=dview, **opts.get_group('motion'))
mc.motion_correct(save_movie=True)
def __init__(self,
caiman_params,
channels,
planes,
x_start,
x_end,
folder,
batch_size=15):
self.channels = channels
self.planes = planes
self.x_start = x_start
self.x_end = x_end
self.folder = folder
self.caiman_params = caiman_params
# derived params
self.folder_tiffs = folder + '*.tif*'
self.save_folder = folder + 'out/'
print('Setting up caiman...')
self.opts = params.CNMFParams(params_dict=self.caiman_params)
self.batch_size = batch_size # can be overridden by expt runner
self.fnumber = 0
self._splits = None
self._json = None
self.times = None
self.cond = None
self.vis_cond = None
# other init things to do
# start server
self._start_cluster()
# cleanup
cleanup(self.folder, 'mmap')
cleanup(self.save_folder, 'hdf5')
cleanup(self.save_folder, 'json')
cleanup(os.getcwd(), 'npz')
def __init__(self, tiff, channels, planes, x_start, x_end, mc_opts, use_green_ch=False,):
self.tiff = tiff
self.channels = channels
self.planes = planes
self.x_start = x_start
self.x_end = x_end
self.xslice = slice(x_start, x_end)
self.mc_opts = mc_opts
self.use_green_ch = use_green_ch
if self.use_green_ch == True:
self.channel2use = 0
else:
self.channel2use = 1
if self.mc_opts:
self.opts = params.CNMFParams(params_dict=mc_opts)
self.file_list = []
self.images = []
self.motion_corrected_images = []
self.masks = []
self.coords = None
self.corr_images = []
def cnmf_neurofinder(params_dict):
import numpy as np
from caiman.source_extraction.cnmf import cnmf
from caiman.source_extraction.cnmf import params
fname_new = params_dict['fnames'][0]
print(fname_new)
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
opts = params.CNMFParams(params_dict=params_dict)
dview = params_dict['dview']
print('Starting CNMF')
opts.set('temporal', {'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
cnm.params.change_params({'update_background_components': True,
'skip_refinement': False,
'n_pixels_per_process': 4000, 'dview': dview})
opts.set('temporal', {'p': params_dict['p']})
cnm2 = cnm.refit(images, dview=dview)
cnm2.save(fname_new[:-5] + '_cnmf.hdf5')
return cnm2
def __init__(self, caiman_params, channels, planes, x_start, x_end,
folder):
self.channels = channels
self.planes = planes
self.x_start = x_start
self.x_end = x_end
self.folder = folder
self.caiman_params = caiman_params
self.trial_lengths = []
# derived params
self.folder_tiffs = folder + '*.tif*'
try:
if not os.path.exists(folder + 'out/'):
os.mkdir(folder + 'out/')
except OSError:
print("can't make the save path for some reason :( ")
self.save_folder = folder + 'out/'
print('Setting up caiman...')
self.opts = params.CNMFParams(params_dict=self.caiman_params)
self.batch_size = 15 # can be overridden by expt runner
self.fnumber = 0
self.bad_tiff_size = 10
self._splits = None
self._json = None
# other init things to do
# start server
self._start_cluster()
# cleanup
cleanup_mmaps(self.folder)
cleanup_hdf5(self.save_folder)
cleanup_json(self.save_folder)
self._everything_is_OK = True
#print("param_dict param does not exist, with error: {}".format(e))
raise OSError("no param dict!")
try:
cnn = configparams["cnn"]
download(bucketname, cnn, locname + "/")
path_cnn = os.path.join(locname, os.path.basename(path_paramdict))
paramdict["online"]["path_to_model"] = path_cnn
print(paramdict)
except Exception as e:
print("cnn param does not exist!, error: {}".format(e))
paramdict["online"][
"path_to_model"] = "/home/ubuntu/caiman_data/model/cnn_model_online.h5"
## If param_mode is simple, write the given parameters into a new caiman parameter dictionary.
elif param_mode == "simple":
paramset = params.CNMFParams()
try:
other_params = configparams["params"]
for key in other_params:
paramset.change_params({key: other_params[key]})
paramdict = paramset.to_dict()
paramdict["online"][
"path_to_model"] = "/home/ubuntu/caiman_data/model/cnn_model_online.h5"
except Exception as e:
print("params not given")
raise OSError("params not given correctly.")
else:
print("param mode {} not recognized. exiting.".format(param_mode))
raise OSError("param dict not given correctly.")
def run(
file_path,
n_cpus,
motion_correct: bool = True,
mc_settings: dict = {},
job_name: str = "job",
output_directory: str = "",
):
mkl = os.environ.get("MKL_NUM_THREADS")
blas = os.environ.get("OPENBLAS_NUM_THREADS")
vec = os.environ.get("VECLIB_MAXIMUM_THREADS")
print(f"MKL: {mkl}")
print(f"blas: {blas}")
print(f"vec: {vec}")
# we import the pipeline upon running so they aren't required for all installs
import caiman as cm
from caiman.motion_correction import MotionCorrect
from caiman.source_extraction.cnmf import params as params
# print the directory caiman is imported from
caiman_path = os.path.abspath(cm.__file__)
print(f"caiman path: {caiman_path}")
sys.stdout.flush()
# setup the logger
logger_file = os.path.join(output_directory, "caiman.log")
logging.basicConfig(
format=LOGGER_FORMAT,
filename=logger_file,
filemode="w",
level=logging.DEBUG,
)
# if indices to perform mcorr are set, format them
if "indices" in mc_settings:
indices = mc_settings["indices"]
indices_formatted = ()
for axis_slice in indices:
start = axis_slice[0]
stop = axis_slice[1]
if len(axis_slice) == 3:
step = axis_slice[2]
else:
step = 1
indices_formatted += (slice(start, stop, step), )
mc_settings["indices"] = indices_formatted
# load and update the pipeline settings
mc_parameters = DEFAULT_MCORR_SETTINGS
for k, v in mc_settings.items():
mc_parameters[k] = v
opts = params.CNMFParams(params_dict=mc_parameters)
# get the filenames
if os.path.isfile(file_path):
print(file_path)
fnames = [file_path]
else:
file_pattern = os.path.join(file_path, "*.tif*")
fnames = sorted(glob.glob(file_pattern))
print(fnames)
opts.set("data", {"fnames": fnames})
if n_cpus > 1:
print("starting server")
# start the server
n_proc = np.max([(n_cpus - 1), 1])
c, dview, n_processes = cm.cluster.setup_cluster(backend="local",
n_processes=n_proc,
single_thread=False)
sleep(30)
else:
print("multiprocessing disabled")
dview = None
n_processes = 1
print(n_processes)
print("starting motion correction")
sys.stdout.flush()
mc = MotionCorrect(fnames, dview=dview, **opts.get_group("motion"))
mc.motion_correct(save_movie=True)
pw_rigid = mc_parameters["pw_rigid"]
fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig
if pw_rigid:
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
else:
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
border_nan = mc_parameters["border_nan"]
bord_px = 0 if border_nan == "copy" else bord_px
_ = cm.save_memmap(fname_mc,
base_name="memmap_",
order="C",
border_to_0=bord_px)
# if motion correction was performed, save the file
# we save as hdf5 for better reading performance
# downstream
if motion_correct:
print("saving motion corrected file")
results_filebase = os.path.join(output_directory, job_name)
mcorr_fname = results_filebase + "_mcorr.hdf5"
dataset_name = opts.data["var_name_hdf5"]
# fnames = opts.data["fnames"]
# memmap_files = []
# for f in fnames:
# if isinstance(f, bytes):
# f = f.decode()
# base_file = os.path.splitext(f)[0]
# if pw_rigid:
# memmap_pattern = base_file + "*_els_*"
# else:
# memmap_pattern = base_file + "*_rig_*"
# memmap_files += sorted(glob.glob(memmap_pattern))
if pw_rigid:
memmap_files = mc.fname_tot_els
else:
memmap_files = mc.fname_tot_rig
# get the frame shape
mov = cm.load(memmap_files[0])
frame_shape = mov.shape[-2::]
write_hdf5_movie(
movie_name=mcorr_fname,
memmap_files=memmap_files,
frame_shape=frame_shape,
dataset_name=dataset_name,
compression="gzip",
)
# save the parameters in the same dir as the results
final_params = opts.to_dict()
params_file = os.path.join(output_directory, "all_mcorr_parameters.pkl")
with open(params_file, "wb") as fp:
pickle.dump(final_params, fp)
# deleting mcorr unused memmap files
if pw_rigid:
memmap_files = mc.fname_tot_els
else:
memmap_files = mc.fname_tot_rig
print(f"deleting {memmap_files}")
for f in memmap_files:
os.remove(f)
print("stopping server")
cm.stop_server(dview=dview)
def main():
pass # For compatibility between running under Spyder and the CLI
# %% start a cluster
c, dview, n_processes =\
cm.cluster.setup_cluster(backend='local', n_processes=None,
single_thread=False)
# %% set up some parameters
fnames = [
os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif')
]
# file(s) to be analyzed
is_patches = True # flag for processing in patches or not
fr = 10 # approximate frame rate of data
decay_time = 5.0 # length of transient
if is_patches: # PROCESS IN PATCHES AND THEN COMBINE
rf = 10 # half size of each patch
stride = 4 # overlap between patches
K = 4 # number of components in each patch
else: # PROCESS THE WHOLE FOV AT ONCE
rf = None # setting these parameters to None
stride = None # will run CNMF on the whole FOV
K = 30 # number of neurons expected (in the whole FOV)
gSig = [6, 6] # expected half size of neurons
merge_thresh = 0.80 # merging threshold, max correlation allowed
p = 2 # order of the autoregressive system
gnb = 2 # global background order
params_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'rf': rf,
'stride': stride,
'K': K,
'gSig': gSig,
'merge_thr': merge_thresh,
'p': p,
'nb': gnb
}
opts = params.CNMFParams(params_dict=params_dict)
# %% Now RUN CaImAn Batch (CNMF)
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit_file()
# %% plot contour plots of components
Cns = local_correlations_movie_offline(fnames[0],
remove_baseline=True,
swap_dim=False,
window=1000,
stride=1000,
winSize_baseline=100,
quantil_min_baseline=10,
dview=dview)
Cn = Cns.max(axis=0)
cnm.estimates.plot_contours(img=Cn)
# %% load memory mapped file
Yr, dims, T = cm.load_memmap(cnm.mmap_file)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# %% refit
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier (this will pick up only neurons
# and filter out active processes)
min_SNR = 2 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.85 # space correlation threshold (if above this, accept)
use_cnn = True # use the CNN classifier
min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set(
'quality', {
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': use_cnn,
'min_cnn_thr': min_cnn_thr,
'cnn_lowest': cnn_lowest
})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% visualize selected and rejected components
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
# %% visualize selected components
cnm2.estimates.view_components(images,
idx=cnm2.estimates.idx_components,
img=Cn)
#%% only select high quality components (destructive)
# cnm2.estimates.select_components(use_object=True)
# cnm2.estimates.plot_contours(img=Cn)
#%% save results
cnm2.estimates.Cn = Cn
cnm2.save(cnm2.mmap_file[:-4] + 'hdf5')
# %% play movie with results (original, reconstructed, amplified residual)
cnm2.estimates.play_movie(images, magnification=4)
# %% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def run_source_extraction(row, parameters, dview, session_wise = False):
'''
This is the function for source extraction.
Its goal is to take in a .mmap file,
perform source extraction on it using cnmf-e and save the cnmf object as a .pkl file.
Args:
row: pd.DataFrame object
The row corresponding to the analysis state to be source extracted.
Returns:
row: pd.DataFrame object
The row corresponding to the source extracted analysis state.
'''
step_index = 5
row_local = row.copy()
row_local.loc['source_extraction_parameters'] = str(parameters)
row_local = db.set_version_analysis('source_extraction',row_local,session_wise)
index = row_local.name
# Determine input path
if parameters['session_wise']:
input_mmap_file_path = eval(row_local.loc['alignment_output'])['main']
if parameters['equalization']:
input_mmap_file_path =eval(row_local['equalization_output'])['main']
else:
input_mmap_file_path = eval(row_local.loc['motion_correction_output'])['main']
if not os.path.isfile(input_mmap_file_path):
logging.error('Input file does not exist. Cancelling.')
return row_local
# Determine output paths
file_name = db.create_file_name(step_index, index)
if parameters['session_wise']:
data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/session_wise/'
else:
data_dir = os.environ['DATA_DIR'] + 'data/interim/source_extraction/trial_wise/'
output_file_path = data_dir + f'main/{file_name}.hdf5'
# Create a dictionary with parameters
output = {
'main': output_file_path,
'meta':{
'analysis' : {
'analyst' : os.environ['ANALYST'],
'date' : datetime.datetime.today().strftime("%m-%d-%Y"),
'time' : datetime.datetime.today().strftime("%H:%M:%S"),
},
'duration': {}
}
}
# Load memmory mappable input file
if os.path.isfile(input_mmap_file_path):
Yr, dims, T = cm.load_memmap(input_mmap_file_path)
# logging.debug(f'{index} Loaded movie. dims = {dims}, T = {T}.')
images = Yr.T.reshape((T,) + dims, order='F')
else:
logging.warning(f'{index} .mmap file does not exist. Cancelling')
return row_local
# SOURCE EXTRACTION
# Check if the summary images are already there
corr_npy_file_path, pnr_npy_file_path = fm.get_corr_pnr_path(index, gSig_abs = parameters['gSig'][0])
if corr_npy_file_path != None and os.path.isfile(corr_npy_file_path):
# Already computed summary images
logging.info(f'{index} Already computed summary images')
cn_filter = np.load(corr_npy_file_path)
pnr = np.load(pnr_npy_file_path)
else:
# Compute summary images
t0 = datetime.datetime.today()
logging.info(f'{index} Computing summary images')
cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1], gSig = parameters['gSig'][0], swap_dim=False)
dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes
output['meta']['duration']['summary_images'] = dt
logging.info(f'{index} Computed summary images. dt = {dt} min')
# Saving summary images as npy files
gSig = parameters['gSig'][0]
corr_npy_file_path = data_dir + f'/meta/corr/{db.create_file_name(3, index)}_gSig_{gSig}.npy'
pnr_npy_file_path = data_dir + f'/meta/pnr/{db.create_file_name(3, index)}_gSig_{gSig}.npy'
with open(corr_npy_file_path, 'wb') as f:
np.save(f, cn_filter)
with open(pnr_npy_file_path, 'wb') as f:
np.save(f, pnr)
# Store the paths in the meta dictionary
output['meta']['corr'] = {'main': corr_npy_file_path, 'meta': {}}
output['meta']['pnr'] = {'main': pnr_npy_file_path, 'meta': {}}
# Calculate min, mean, max value for cn_filter and pnr
corr_min, corr_mean, corr_max = cn_filter.min(), cn_filter.mean(), cn_filter.max()
output['meta']['corr']['meta'] = {'min': corr_min, 'mean': corr_mean, 'max': corr_max}
pnr_min, pnr_mean, pnr_max = pnr.min(), pnr.mean(), pnr.max()
output['meta']['pnr']['meta'] = {'min': pnr_min, 'mean': pnr_mean, 'max': pnr_max}
# If min_corr and min_pnr are specified via a linear equation, calculate
# this value
if type(parameters['min_corr']) == list:
min_corr = parameters['min_corr'][0]*corr_mean + parameters['min_corr'][1]
parameters['min_corr'] = min_corr
logging.info(f'{index} Automatically setting min_corr = {min_corr}')
if type(parameters['min_pnr']) == list:
min_pnr = parameters['min_pnr'][0]*pnr_mean + parameters['min_pnr'][1]
parameters['min_pnr'] = min_pnr
logging.info(f'{index} Automatically setting min_pnr = {min_pnr}')
# Set the parameters for caiman
opts = params.CNMFParams(params_dict = parameters)
# SOURCE EXTRACTION
logging.info(f'{index} Performing source extraction')
t0 = datetime.datetime.today()
n_processes = psutil.cpu_count()
logging.info(f'{index} n_processes: {n_processes}')
cnm = cnmf.CNMF(n_processes = n_processes, dview = dview, params = opts)
cnm.fit(images)
cnm.estimates.dims = dims
# Store the number of neurons
output['meta']['K'] = len(cnm.estimates.C)
# Calculate the center of masses
cnm.estimates.center = caiman.base.rois.com(cnm.estimates.A, images.shape[1], images.shape[2])
# Save the cnmf object as a hdf5 file
logging.info(f'{index} Saving cnmf object')
cnm.save(output_file_path)
dt = int((datetime.datetime.today() - t0).seconds/60) # timedelta in minutes
output['meta']['duration']['source_extraction'] = dt
logging.info(f'{index} Source extraction finished. dt = {dt} min')
# Write necessary variables in row and return
row_local.loc['source_extraction_parameters'] = str(parameters)
row_local.loc['source_extraction_output'] = str(output)
return row_local
def main():
pass # For compatibility between running under Spyder and the CLI
#%% Select file(s) to be processed (download if not present)
# fnames = [os.path.join(caiman_datadir(), 'example_movies/sampled3dMovieRigid.nwb')]
fnames = [
os.path.join(caiman_datadir(), 'example_movies/sampled3dMovie2.nwb')
]
# filename to be created or processed
# dataset dependent parameters
fr = 5 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
starting_time = 0.
#%% load the file and save it in the NWB format (if it doesn't exist already)
if not os.path.exists(fnames[0]):
# fnames_orig = [os.path.join(caiman_datadir(), 'example_movies/sampled3dMovieRigid.h5')] # filename to be processed
fnames_orig = [
os.path.join(caiman_datadir(), 'example_movies/sampled3dMovie2.h5')
] # filename to be processed
orig_movie = cm.load(fnames_orig, fr=fr, is3D=True)
# orig_movie = cm.load_movie_chain(fnames_orig,fr=fr,is3D=True)
# save file in NWB format with various additional info
orig_movie.save(fnames[0],
sess_desc='test',
identifier='demo 3d',
exp_desc='demo movie',
imaging_plane_description='multi plane',
emission_lambda=520.0,
indicator='none',
location='visual cortex',
starting_time=starting_time,
experimenter='NAOMi',
lab_name='Tank Lab',
institution='Princeton U',
experiment_description='Experiment Description',
session_id='Session 1',
var_name_hdf5='TwoPhotonSeries')
#%% First setup some parameters for data and motion correction
# motion correction parameters
dxy = (1., 1., 5.) # spatial resolution in x, y, and z in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (10., 10., 10.) # maximum shift in um
patch_motion_um = (50., 50., 30.
) # patch size for non-rigid correction in um
# pw_rigid = False # flag to select rigid vs pw_rigid motion correction
niter_rig = 1
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a / b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a / b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24, 4)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
is3D = True
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'niter_rig': niter_rig,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy',
'var_name_hdf5': 'acquisition/TwoPhotonSeries',
'is3D': is3D,
'splits_els': 12,
'splits_rig': 12
}
opts = params.CNMFParams(
params_dict=mc_dict
) #NOTE: default adjustments of parameters are not set yet, manually setting them now
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = True
if display_images:
m_orig = cm.load_movie_chain(fnames,
var_name_hdf5=opts.data['var_name_hdf5'],
is3D=True)
T, h, w, z = m_orig.shape # Time, plane, height, weight
m_orig = np.reshape(np.transpose(m_orig, (3, 0, 1, 2)), (T * z, h, w))
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
# NOTE: ignore dview right now for debugging purposes
# c, dview, n_processes = cm.cluster.setup_cluster(
# backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames,
dview=None,
var_name_hdf5=opts.data['var_name_hdf5'],
**opts.get_group('motion'))
# mc = MotionCorrect(fnames, dview=dview, var_name_hdf5=opts.data['var_name_hdf5'], **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# %% compare with original movie
if display_images:
m_orig = cm.load_movie_chain(fnames,
var_name_hdf5=opts.data['var_name_hdf5'],
is3D=True)
T, h, w, z = m_orig.shape # Time, plane, height, weight
m_orig = np.reshape(np.transpose(m_orig, (3, 0, 1, 2)), (T * z, h, w))
m_els = cm.load(mc.mmap_file, is3D=True)
m_els = np.reshape(np.transpose(m_els, (3, 0, 1, 2)), (T * z, h, w))
ds_ratio = 0.2
moviehandle = cm.concatenate([
m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)
],
axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
def main():
pass # For compatibility between running under Spyder and the CLI
#%% Select file(s) to be processed (download if not present)
root = '/Users/hheiser/Desktop/testing data/chronic_M2N3/0d_baseline/channel1'
fnames = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\1\file_00003.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\2\file_00004.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\3\file_00005.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\4\file_00006.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\5\file_00007.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\6\file_00008.tif']
fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fnames = [download_demo(fnames[0])]
#%% First setup some parameters for data and motion correction
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
#%% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = True
if display_images:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=30, magnification=1, do_loop=False)
#%% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
#%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
#%% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
#%% compare with original movie
if display_images:
m_orig = cm.load_movie_chain(fnames[:3])
m_els = cm.load(mmap_file[:3])
ds_ratio = 0.2
moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)], axis=2)
moviehandle.play(fr=15, q_max=99.5, magnification=2) # press q to exit
#%% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan == 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
mmap_file = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\1\file_00001_els__d1_512_d2_512_d3_1_order_F_frames_1147_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\2\file_00002_els__d1_512_d2_512_d3_1_order_F_frames_2520_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\3\file_00003_els__d1_512_d2_512_d3_1_order_F_frames_3814_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\4\file_00004_els__d1_512_d2_512_d3_1_order_F_frames_5154_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\5\file_00005_els__d1_512_d2_512_d3_1_order_F_frames_2677_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\6\file_00006_els__d1_512_d2_512_d3_1_order_F_frames_3685_.mmap']
fname_new = r'E:\PhD\Data\DG\M14_20191014\N1\memmap__d1_512_d2_512_d3_1_order_C_frames_34939_.mmap'
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C',
border_to_0=0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
#%% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
#%% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons in pixels
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
ssub = 2 # spatial subsampling during initialization
tsub = 2 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {'fnames': fnames,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub}
opts.change_params(params_dict=opts_dict)
#%% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
#opts.change_params({'p': 1,'rf':None, 'only_init':False})
opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
#%% RUN CNMF SEEDED WITH MANUAL MASK
# load mask
mask = np.asarray(imageio.imread('/Users/hheiser/Desktop/testing data/file_00020_no_motion/avg_mask_fixed.png'), dtype=bool)
# get component ROIs from the mask and plot them
Ain, labels, mR = cm.base.rois.extract_binary_masks(mask)
# plot original mask and extracted labels to check mask
fig, ax = plt.subplots(1,2)
ax[0].imshow(-mR,cmap='binary')
ax[0].set_title('Original mask')
ax[1].imshow(labels)
ax[1].set_title('Extracted labelled ROIs')
""""
plt.figure()
crd = cm.utils.visualization.plot_contours(
Ain.astype('float32'), mR, thr=0.99, display_numbers=True) # todo check if this is important for the pipeline
plt.title('Contour plots of detected ROIs in the structural channel')
"""
opts.change_params({'rf': None, 'only_init': False})
# run CNMF seeded with this mask
cnm_corr = cnmf.CNMF(n_processes, params=opts, dview=dview, Ain=Ain)
cnm_corr_seed = cnm_corr_seed.fit(images)
#cnm_seed = cnm_seed.fit_file(motion_correct=False)
#%% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
#%% plot contours of found components
Cn = cm.local_correlations(images, swap_dim=False)
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm.params.change_params({'p': p})
cnm2 = cnm.refit(images, dview=dview)
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # signal to noise ratio for accepting a component
rval_thr = 0.85 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set('quality', {'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest})
cnm2.estimates.evaluate_components(images, params=cnm2.params, dview=dview)
#%% PLOT COMPONENTS
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
#%% VIEW TRACES (accepted and rejected)
if display_images:
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components)
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components_bad)
#%% update object with selected components
#### -> will delete rejected components!
cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(img=Cn)
#%% reconstruct denoised movie (press q to exit)
if display_images:
cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2,
magnification=1,
bpx=border_to_0,
include_bck=True) # background not shown
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
#%% save results
dirname = fnames[0][:-4] + "_results.hdf5"
cnm2.estimates.Cn = Cn
cnm2.save(dirname)
#load results
cnm2 = cnmf.load_CNMF(dirname)
mov_name = fnames[0][:-4] + "_movie_restored_2_gain.tif"
helper.save_movie(cnm2.estimates,images,mov_name,frame_range=range(200),include_bck=True)
def run_alignmnet(selected_rows, parameters, dview):
'''
This is the main function for the alignment step. It applies methods
from the CaImAn package used originally in motion correction
to do alignment.
Args:
df: pd.DataFrame
A dataframe containing the analysis states you want to have aligned.
parameters: dict
The alignment parameters.
dview: object
The dview object
Returns:
df: pd.DataFrame
A dataframe containing the aligned analysis states.
'''
# Sort the dataframe correctly
df = selected_rows.copy()
df = df.sort_values(by=paths.multi_index_structure)
# Determine the mouse and session of the dataset
index = df.iloc[0].name
mouse, session, *r = index
# alignment_v = index[len(paths.data_structure) + step_index]
alignment_v = len(df)
alignment_index = (mouse, session, alignment_v)
# Determine the output .mmap file name
file_name = f'mouse_{mouse}_session_{session}_v{alignment_v}'
output_mmap_file_path = os.environ['DATA_DIR'] + f'data/interim/alignment/main/{file_name}.mmap'
try:
df.reset_index()[['session','trial', 'is_rest']].set_index(['session','trial', 'is_rest'], verify_integrity=True)
except ValueError:
logging.error('You passed multiple of the same trial in the dataframe df')
return df
output = {
'meta': {
'analysis': {
'analyst': os.environ['ANALYST'],
'date': datetime.datetime.today().strftime("%m-%d-%Y"),
'time': datetime.datetime.today().strftime("%H:%M:%S")
},
'duration': {}
}
}
# Get necessary parameters
motion_correction_parameters_list = []
motion_correction_output_list = []
input_mmap_file_list = []
trial_index_list = []
x_ = []
_x = []
y_ = []
_y = []
for idx, row in df.iterrows():
motion_correction_parameters_list.append(eval(row.loc['motion_correction_parameters']))
motion_correction_output = eval(row.loc['motion_correction_output'])
motion_correction_output_list.append(motion_correction_output)
input_mmap_file_list.append(motion_correction_output['main'])
trial_index_list.append(db.get_trial_name(idx[2], idx[3]))
[x1,x2,y1,y2] = motion_correction_output['meta']['cropping_points']
x_.append(x1)
_x.append(x2)
y_.append(y1)
_y.append(y2)
new_x1 = max(x_)
new_x2 = max(_x)
new_y1 = max(y_)
new_y2 = max(_y)
m_list = []
for i in range(len(input_mmap_file_list)):
m = cm.load(input_mmap_file_list[i])
motion_correction_output = eval(df.iloc[i].loc['motion_correction_output'])
[x1,x2,y1,y2] = motion_correction_output['meta']['cropping_points']
m = m.crop(new_x1 - x1, new_x2 - x2, new_y1 - y1, new_y2 - y2, 0, 0)
m_list.append(m)
# Concatenate them using the concat function
m_concat = cm.concatenate(m_list, axis=0)
data_dir = os.environ['DATA_DIR'] + 'data/interim/alignment/main/'
file_name = db.create_file_name(step_index, index)
fname= m_concat.save(data_dir + file_name + '.mmap', order='C')
#meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y]
#output['meta']['cropping_points'] = [x_, _x, y_, _y]
# Save the movie
#fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' + '.mmap', order='C')
#logging.info(f'{index} Cropped and saved rigid movie as {fname_tot_els}')
# MOTION CORRECTING EACH INDIVIDUAL MOVIE WITH RESPECT TO A TEMPLATE MADE OF THE FIRST MOVIE
logging.info(f'{alignment_index} Performing motion correction on all movies with respect to a template made of \
the first movie.')
t0 = datetime.datetime.today()
# Create a template of the first movie
template_index = trial_index_list.index(parameters['make_template_from_trial'])
m0 = cm.load(input_mmap_file_list[template_index ])
[x1, x2, y1, y2] = motion_correction_output_list[template_index]['meta']['cropping_points']
m0 = m0.crop(new_x1 - x1, new_x2 - x2, new_y1 - y1, new_y2 - y2, 0, 0)
m0_filt = cm.movie(
np.array([high_pass_filter_space(m_, parameters['gSig_filt']) for m_ in m0]))
template0 = cm.motion_correction.bin_median(
m0_filt.motion_correct(5, 5, template=None)[0]) # may be improved in the future
# Setting the parameters
opts = params.CNMFParams(params_dict=parameters)
# Create a motion correction object
mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion'))
# Perform non-rigid motion correction
mc.motion_correct(template=template0, save_movie=True)
# Cropping borders
x_ = math.ceil(abs(np.array(mc.shifts_rig)[:, 1].max()) if np.array(mc.shifts_rig)[:, 1].max() > 0 else 0)
_x = math.ceil(abs(np.array(mc.shifts_rig)[:, 1].min()) if np.array(mc.shifts_rig)[:, 1].min() < 0 else 0)
y_ = math.ceil(abs(np.array(mc.shifts_rig)[:, 0].max()) if np.array(mc.shifts_rig)[:, 0].max() > 0 else 0)
_y = math.ceil(abs(np.array(mc.shifts_rig)[:, 0].min()) if np.array(mc.shifts_rig)[:, 0].min() < 0 else 0)
# Load the motion corrected movie into memory
movie= cm.load(mc.fname_tot_rig[0])
# Crop all movies to those border pixels
movie.crop(x_, _x, y_, _y, 0, 0)
output['meta']['cropping_points'] = [x_, _x, y_, _y]
#save motion corrected and cropped movie
output_mmap_file_path_tot = movie.save(data_dir + file_name + '.mmap', order='C')
logging.info(f'{index} Cropped and saved rigid movie as {output_mmap_file_path_tot}')
# Save the path in teh output dictionary
output['main'] = output_mmap_file_path_tot
# Remove the remaining non-cropped movie
os.remove(mc.fname_tot_rig[0])
# Create a timeline and store it
timeline = [[trial_index_list[0], 0]]
timepoints = [0]
for i in range(1, len(m_list)):
m = m_list[i]
timeline.append([trial_index_list[i], timeline[i - 1][1] + m.shape[0]])
timepoints.append(timepoints[i-1]+ m.shape[0])
timeline_pkl_file_path = os.environ['DATA_DIR'] + f'data/interim/alignment/meta/timeline/{file_name}.pkl'
with open(timeline_pkl_file_path,'wb') as f:
pickle.dump(timeline,f)
output['meta']['timeline'] = timeline_pkl_file_path
timepoints.append(movie.shape[0])
dt = int((datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes
output['meta']['duration']['concatenation'] = dt
logging.info(f'{alignment_index} Performed concatenation. dt = {dt} min.')
for idx, row in df.iterrows():
df.loc[idx, 'alignment_output'] = str(output)
df.loc[idx, 'alignment_parameters'] = str(parameters)
## modify all motion correction file to the aligned version
data_dir = os.environ['DATA_DIR'] + 'data/interim/motion_correction/main/'
for i in range(len(input_mmap_file_list)):
row = df.iloc[i].copy()
motion_correction_output_list.append(motion_correction_output)
aligned_movie = movie[timepoints[i]:timepoints[i+1]]
file_name = db.create_file_name(2, selected_rows.iloc[i].name)
motion_correction_output_aligned = aligned_movie.save(data_dir + file_name + '_els' + '.mmap', order='C')
new_output= {'main' : motion_correction_output_aligned }
new_dict = eval(row['motion_correction_output'])
new_dict.update(new_output)
row['motion_correction_output'] = str(new_dict)
df = db.append_to_or_merge_with_states_df(df, row)
# # Delete the motion corrected movies
# for fname in mc.fname_tot_rig:
# os.remove(fname)
return df
None, # half-size of patch in pixels. If None, no patches are constructed and the whole FOV is processed jointly
#'stride': stride,
'update_num_comps': False,
'motion_correct': False,
'sniper_mode':
True, # whether to use the online CNN classifier for screening candidate components (otherwise space correlation is used)
'thresh_CNN_noisy': thresh_CNN_noisy,
'K': K,
'expected_comps': K,
'update_num_comps': False, # whether to search for new components
'min_num_trial': new_K,
'method_deconvolution': deconv_method,
'show_movie': True
}
allParams = params.CNMFParams(params_dict=initialParamsDict
) # define parameters in the params.CNMFParams
caimanResults = cnmf.online_cnmf.OnACID(
params=allParams) # pass parameters to caiman object
timer.toc()
# %% ********* Wait for initialization trigger message from MicroManager: *********
print("Now waiting for MicroManager to capture " + str(initFrames) +
" initialization frames..")
print("*** Starting Initialization protocol with " + initMethod_online +
" method ***")
caimanResults.initialize_online() # initialize model
timer.toc()
# %% ********* Visualize results of initialization: *********
print(
def main():
pass # For compatibility between running under Spyder and the CLI
#%% Select file(s) to be processed (download if not present)
fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fnames = [download_demo(fnames[0])]
#%% First setup some parameters for data and motion correction
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = True
if display_images:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# %% compare with original movie
if display_images:
m_orig = cm.load_movie_chain(fnames)
m_els = cm.load(mc.mmap_file)
ds_ratio = 0.2
moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)], axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
# %% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C',
border_to_0=border_to_0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
# %% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons in pixels
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
ssub = 2 # spatial subsampling during initialization
tsub = 2 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {'fnames': fnames,
'p': p,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub}
opts.change_params(params_dict=opts_dict);
# %% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0). If you want to have
# deconvolution within each patch change params.patch['p_patch'] to a
# nonzero value
#opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% plot contours of found components
Cns = local_correlations_movie_offline(mc.mmap_file[0],
remove_baseline=True, window=1000, stride=1000,
winSize_baseline=100, quantil_min_baseline=10,
dview=dview)
Cn = Cns.max(axis=0)
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
#%% save results
cnm.estimates.Cn = Cn
cnm.save(fname_new[:-5]+'_init.hdf5')
# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # signal to noise ratio for accepting a component
rval_thr = 0.85 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set('quality', {'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% PLOT COMPONENTS
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
# %% VIEW TRACES (accepted and rejected)
if display_images:
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components)
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components_bad)
#%% update object with selected components
cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(img=Cn)
#%%
cnm2.estimates.Cn = Cn
cnm2.save(cnm2.mmap_file[:-4] + 'hdf5')
#%% reconstruct denoised movie (press q to exit)
if display_images:
cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2,
magnification=2,
bpx=border_to_0,
include_bck=False) # background not shown
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
opts = params.CNMFParams(
params_dict={
'fr': frate,
'dims': dims,
'decay_time': decay_time,
'method_init': 'corr_pnr', # use this for 1 photon
'K': K,
'gSig': gSig,
'gSiz': gSiz,
'merge_thr': merge_thr,
'p': p,
'tsub': tsub,
'ssub': ssub,
'rf': rf,
'stride': stride_cnmf,
'only_init': True, # set it to True to run CNMF-E
'nb': gnb,
'nb_patch': nb_patch,
'method_deconvolution': 'oasis', # could use 'cvxpy' alternatively
'low_rank_background': low_rank_background,
'update_background_components':
True, # sometimes setting to False improve the results
'min_corr': min_corr,
'min_pnr': min_pnr,
'normalize_init': False, # just leave as is
'center_psf': True, # leave as is for 1 photon
'ssub_B': ssub_B,
'ring_size_factor': ring_size_factor,
'del_duplicates':
True, # whether to remove duplicates from initialization
'border_pix': 0 # number of pixels to not consider in the borders)
})
def extract_components(self, images,
fname) -> Tuple[cnmf.CNMF, cnmf.CNMF, dict]:
"""
Uses constrained NNMF to extract spatial and temporal components, performs deconvolution and validates
extracted components
:param images: The tyx stack from which components should be extracted
:param fname: The name of the original file from which <images> originated
:return:
[0]: Cnmf object after initial extraction
[1]: Cnmf object after deconvolution and subsequent component validation
[2]: Wrapped CaImAn parameter dictionary
"""
resolution = self.fov_um / images.shape[1]
fr = 1 / self.time_per_frame # frame-rate
dxy = (resolution, resolution) # spatial resolution in um/pixel
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
try:
# set up parameters for source extraction
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
# TODO: Extraction fails if patch size is too small relative to neuron size (K<1).
# Add intelligent computation of patch size
# size patch to roughly have an edge length of 8 cells
n_rad_pixels = self.neuron_radius / resolution
rf = int(n_rad_pixels * 8) # half-size of the patches in pixels
stride_cnmf = 6 # amount of overlap between the patches in pixels
patch_area_um = (
rf * 2 * dxy[0]
)**2 # we use this to calculate expected number of components per patch
neur_area_um = np.pi * self.neuron_radius**2
K = int(patch_area_um /
neur_area_um) # number of components (~neurons) per patch
# expected half size of neurons in pixels
gSig = [
int(self.neuron_radius / dxy[0]),
int(self.neuron_radius / dxy[1])
]
method_init = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')
ssub = 2 # spatial subsampling during initialization
tsub = 1 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {
'fnames': [
fname
], # NOTE: This parameter seems only necessary to allow contour extraction
'fr': fr,
'decay_time': self.decay_time,
'dxy': dxy,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes':
int(n_processes
), # return type is numpy.int64 which can't json serialize
'only_init': True,
'ssub': ssub,
'tsub': tsub
}
opts = params.CNMFParams(params_dict=opts_dict)
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
# rerun seeded CNMF on accepted patches to refine and perform deconvolution
cnm.params.change_params({'p': p})
cnm2 = cnm.refit(images, dview=dview)
# Validate components
val_dict = {
'decay_time': self.decay_time,
'min_SNR': self.min_snr,
'SNR_lowest': self.snr_lowest,
'rval_thr': self.rval_thr,
'rval_lowest': self.rval_lowest,
'use_cnn': self.use_cnn,
'min_cnn_thr': self.cnn_thr,
'cnn_lowest': self.cnn_lowest
}
cnm2.params.set('quality', val_dict)
cnm2.estimates.evaluate_components(images,
cnm2.params,
dview=dview)
# update object with selected components
cnm2.estimates.select_components(use_object=True)
# extract DF/F values
cnm2.estimates.detrend_df_f(**self.detrend_dff_params)
finally:
cm.stop_server(dview=dview)
return cnm, cnm2, {"CNMF": opts_dict, "Validation": val_dict}
def motion_correct(
self, fname: str,
co_fname: str) -> Tuple[np.ndarray, dict, Optional[np.ndarray]]:
"""
Uses caiman non-rigid motion correction to remove/reduce motion artefacts
Note: ALways saves an intermediate mem-map representation in order C of the corrected 32-bit stack
:param fname: The filename of the source file
:param co_fname: Filename of a stack that should be co-aligned or None
:return:
[0]: Corrected stack as a memmap
[1]: Wrapped CaImAn parameter dictionary
[2]: Corrected co-stack as a memmap or None
"""
cont_folder = path.dirname(fname) # the containing folder
stack_name = path.split(fname)[1]
save_dir = cont_folder + f"/{self.ana_dir}"
if not path.exists(save_dir):
makedirs(save_dir)
print("Created analysis directory", flush=True)
out_name = save_dir + '/' + stack_name
if co_fname is not None:
co_stack_name = path.split(co_fname)[1]
co_out_name = save_dir + '/' + co_stack_name
else:
co_out_name = None
test_image = imread(
fname, key=0) # load first frame of stack to compute resolution
assert test_image.shape[0] == test_image.shape[1]
resolution = self.fov_um / test_image.shape[0]
fr = 1 / self.time_per_frame # frame-rate
dxy = (resolution, resolution) # spatial resolution in um/pixel
max_shift_um = (self.neuron_radius * 4, self.neuron_radius * 4
) # maximally allow shift by ~2 cell diameters
# maximum allowed rigid shift in pixels
max_shifts = [int(a / b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple(
[int(a / b) for a, b in zip(self.patch_motion_um, dxy)])
# overlap between patches (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts (unit unclear - likely pixels as it is int)
max_deviation_rigid = 3
# create parameter dictionary
mc_dict = {
'fnames': [fname],
'fr': fr,
'decay_time': self.decay_time,
'dxy': dxy,
'pw_rigid': self.pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
try:
# motion correction
mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion'))
# Run (piecewise-rigid motion) correction using NoRMCorre
# if we don't save the movie into a memmap, there doesn't seem to be
# any possibility to get at the corrected data later???
mc.motion_correct(save_movie=True)
# memory mapping
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file,
base_name=out_name,
order='C',
border_to_0=border_to_0)
# delete the original mem-map
if mc.fname_tot_els is None:
[remove(fn) for fn in mc.fname_tot_rig]
else:
[remove(fn) for fn in mc.fname_tot_els]
# now load the new file and transform output into [z,y,x] stack
yr, dims, n_t = cm.load_memmap(fname_new)
images = np.reshape(yr.T, [n_t] + list(dims), order='F')
if self.save_projection:
# save anatomical projection as 16bit tif
anat_projection = images.copy()
anat_projection = np.sum(anat_projection, 0)
anat_projection -= np.min(anat_projection)
anat_projection /= np.max(anat_projection)
anat_projection *= (2**16 - 1)
anat_projection[anat_projection < 0] = 0
anat_projection[anat_projection > (2**16 - 1)] = (2**16 - 1)
anat_projection = anat_projection.astype(np.uint16)
imsave(out_name,
anat_projection,
imagej=True,
resolution=(1 / dxy[0], 1 / dxy[1]),
metadata={
'axes': 'YX',
'unit': 'um'
})
# Repeat for the co-stack if present
if co_fname is not None:
co_aligned_images = np.array(mc.apply_shifts_movie(co_fname))
if self.save_projection:
# save anatomical projection as 16bit tif
anat_projection = np.sum(co_aligned_images, 0)
anat_projection -= np.min(anat_projection)
anat_projection /= np.max(anat_projection)
anat_projection *= (2**16 - 1)
anat_projection[anat_projection < 0] = 0
anat_projection[anat_projection > (2**16 - 1)] = (2**16 -
1)
anat_projection = anat_projection.astype(np.uint16)
imsave(co_out_name,
anat_projection,
imagej=True,
resolution=(1 / dxy[0], 1 / dxy[1]),
metadata={
'axes': 'YX',
'unit': 'um'
})
else:
co_aligned_images = None
finally:
cm.stop_server(dview=dview)
return images, {"Motion Correction": mc_dict}, co_aligned_images
def main():
pass # For compatibility between running under Spyder and the CLI
# %% start the cluster
try:
cm.stop_server() # stop it if it was running
except ():
pass
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local',
n_processes=
24, # number of process to use, if you go out of memory try to reduce this one
single_thread=False)
# %% First setup some parameters for motion correction
# dataset dependent parameters
fnames = ['data_endoscope.tif'] # filename to be processed
fnames = [download_demo(fnames[0])] # download file if not already present
filename_reorder = fnames
fr = 10 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
motion_correct = True # flag for motion correction
pw_rigid = False # flag for pw-rigid motion correction
gSig_filt = (3, 3) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (5, 5) # maximum allowed rigid shift
strides = (
48, 48
) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (24, 24
) # overlap between pathes (size of patch strides+overlaps)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
border_nan = 'copy'
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'gSig_filt': gSig_filt,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': border_nan
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% MOTION CORRECTION
# The pw_rigid flag set above, determines where to use rigid or pw-rigid
# motion correction
if motion_correct:
# do motion correction rigid
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
mc.motion_correct(save_movie=True)
fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig
if pw_rigid:
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
else:
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
plt.subplot(1, 2, 1)
plt.imshow(mc.total_template_rig) # % plot template
plt.subplot(1, 2, 2)
plt.plot(mc.shifts_rig) # % plot rigid shifts
plt.legend(['x shifts', 'y shifts'])
plt.xlabel('frames')
plt.ylabel('pixels')
bord_px = 0 if border_nan == 'copy' else bord_px
fname_new = cm.save_memmap(fname_mc,
base_name='memmap_',
order='C',
border_to_0=bord_px)
else: # if no motion correction just memory map the file
fname_new = cm.save_memmap(filename_reorder,
base_name='memmap_',
order='C',
border_to_0=0,
dview=dview)
# load memory mappable file
Yr, dims, T = cm.load_memmap(fname_new)
images = Yr.T.reshape((T, ) + dims, order='F')
# %% Parameters for source extraction and deconvolution (CNMF-E algorithm)
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None for 1p data
gSig = (
3, 3
) # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1
Ain = None # possibility to seed with predetermined binary masks
merge_thr = .7 # merging threshold, max correlation allowed
rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80
stride_cnmf = 20 # amount of overlap between the patches in pixels
# (keep it at least large as gSiz, i.e 4 times the neuron size gSig)
tsub = 2 # downsampling factor in time for initialization,
# increase if you have memory problems
ssub = 1 # downsampling factor in space for initialization,
# increase if you have memory problems
# you can pass them here as boolean vectors
low_rank_background = None # None leaves background of each patch intact,
# True performs global low-rank approximation if gnb>0
gnb = 0 # number of background components (rank) if positive,
# else exact ring model with following settings
# gnb= 0: Return background as b and W
# gnb=-1: Return full rank background B
# gnb<-1: Don't return background
nb_patch = 0 # number of background components (rank) per patch if gnb>0,
# else it is set automatically
min_corr = .8 # min peak value from correlation image
min_pnr = 10 # min peak to noise ration from PNR image
ssub_B = 2 # additional downsampling factor in space for background
ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor
opts.change_params(
params_dict={
'dims': dims,
'method_init': 'corr_pnr', # use this for 1 photon
'K': K,
'gSig': gSig,
'gSiz': gSiz,
'merge_thr': merge_thr,
'p': p,
'tsub': tsub,
'ssub': ssub,
'rf': rf,
'stride': stride_cnmf,
'only_init': True, # set it to True to run CNMF-E
'nb': gnb,
'nb_patch': nb_patch,
'method_deconvolution': 'oasis', # could use 'cvxpy' alternatively
'low_rank_background': low_rank_background,
'update_background_components':
True, # sometimes setting to False improve the results
'min_corr': min_corr,
'min_pnr': min_pnr,
'normalize_init': False, # just leave as is
'center_psf': True, # leave as is for 1 photon
'ssub_B': ssub_B,
'ring_size_factor': ring_size_factor,
'del_duplicates':
True, # whether to remove duplicates from initialization
'border_pix': bord_px
}) # number of pixels to not consider in the borders)
# %% compute some summary images (correlation and peak to noise)
# change swap dim if output looks weird, it is a problem with tiffile
cn_filter, pnr = cm.summary_images.correlation_pnr(images[::1],
gSig=gSig[0],
swap_dim=False)
# if your images file is too long this computation will take unnecessarily
# long time and consume a lot of memory. Consider changing images[::1] to
# images[::5] or something similar to compute on a subset of the data
# inspect the summary images and set the parameters
inspect_correlation_pnr(cn_filter, pnr)
# print parameters set above, modify them if necessary based on summary images
print(min_corr) # min correlation of peak (from correlation image)
print(min_pnr) # min peak to noise ratio
# %% RUN CNMF ON PATCHES
cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=opts)
cnm.fit(images)
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% DISCARD LOW QUALITY COMPONENTS
min_SNR = 2.5 # adaptive way to set threshold on the transient size
r_values_min = 0.85 # threshold on space consistency (if you lower more components
# will be accepted, potentially with worst quality)
cnm.params.set('quality', {
'min_SNR': min_SNR,
'rval_thr': r_values_min,
'use_cnn': False
})
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
print(' ***** ')
print('Number of total components: ', len(cnm.estimates.C))
print('Number of accepted components: ', len(cnm.estimates.idx_components))
# %% PLOT COMPONENTS
cnm.dims = dims
display_images = True # Set to true to show movies and images
if display_images:
cnm.estimates.plot_contours(img=cn_filter,
idx=cnm.estimates.idx_components)
cnm.estimates.view_components(images, idx=cnm.estimates.idx_components)
# %% MOVIES
display_images = False # Set to true to show movies and images
if display_images:
# fully reconstructed movie
cnm.estimates.play_movie(images,
q_max=99.5,
magnification=2,
include_bck=True,
gain_res=10,
bpx=bord_px)
# movie without background
cnm.estimates.play_movie(images,
q_max=99.9,
magnification=2,
include_bck=False,
gain_res=4,
bpx=bord_px)
# %% STOP SERVER
cm.stop_server(dview=dview)
#'min_SNR': 8,
#'SNR_lowest': 2.5,
'min_SNR': 6,
'SNR_lowest': 2.5,
}
max_thr = 0.45
vca1_neuron_sizes = {'max': 200, 'min': 10}
dca1_neuron_sizes = {
'max': 110,
#'min': 20
'min': 5
}
neuron_size_params = vca1_neuron_sizes
opts = params.CNMFParams(params_dict=eval_params)
A = cnm_obj.estimates.A
frames = session_trace_offset + range((vid_index - vid_start_index) * 1000,
(vid_index - vid_start_index + 1) * 1000)
images = video.load_images(local_mmap_fpath)
if not filtered:
cnm_obj.estimates.threshold_spatial_components(maxthr=max_thr)
cnm_obj.estimates.remove_small_large_neurons(
min_size_neuro=neuron_size_params['min'],
max_size_neuro=neuron_size_params['max'])
idx_components_bad = cnm_obj.estimates.idx_components_bad
if idx_components_bad is None:
idx_components_bad = []
if reevaluate:
def createParams(fnames, frameRate=20):
# dataset dependent parameters
#fr = 30 # imaging rate in frames per second
# VIDEO
#fr = 20 # imaging rate in frames per second
# 2p
#fr = 15 # imaging rate in frames per second
fr = frameRate
# first pass analysis was this
#decay_time = 1.2 #0.4 # length of a typical transient in seconds
# this is in folder 'slower'
decay_time = 0.4 #2.0 #0.4 # length of a typical transient in seconds
# VIDEO
#myScaleFactor=5 # first run was 7
#myScaleFactor2 = 2
# 2P
myScaleFactor = 1 # first run was 7
myScaleFactor2 = 1
# motion correction parameters
strides = (
48 * myScaleFactor, 48 * myScaleFactor
) #(48, 48) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (
24 * myScaleFactor2, 24 * myScaleFactor2
) #(24, 24) # overlap between pathes (size of patch strides+overlaps)
max_shifts = (6 * myScaleFactor, 6 * myScaleFactor
) #(6,6) # maximum allowed rigid shifts (in pixels)
max_deviation_rigid = 3 * myScaleFactor #3 # maximum shifts deviation allowed for patch with respect to rigid shifts
pw_rigid = True # flag for performing non-rigid motion correction
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15 * myScaleFactor #15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 * myScaleFactor #6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4 * myScaleFactor, 4 * myScaleFactor
] #[4, 4] # expected half size of neurons in pixels
method_init = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')
ssub = 1 # spatial subsampling during initialization
tsub = 1 # temporal subsampling during intialization
# parameters for component evaluation
# VIDEO
#min_SNR = 1.15 #2.0 # signal to noise ratio for accepting a component
# 2P
min_SNR = 2.0 # signal to noise ratio for accepting a component
rval_thr = 0.85 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
print('============================================================')
print('============================================================')
print(' These are options manually set by cudmore')
print(
' Be sure to double check them and be sure to decide on "video" or "2p"'
)
print(' caimanOptions.createParams()')
print(' fr:', fr)
print(' decay_time:', decay_time)
print(' myScaleFactor:', myScaleFactor)
print(' myScaleFactor2:', myScaleFactor2)
print(' min_SNR (for component evaluation):', min_SNR)
print('============================================================')
print('============================================================')
##
##
opts_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'strides': strides,
'overlaps': overlaps,
'max_shifts': max_shifts,
'max_deviation_rigid': max_deviation_rigid,
'pw_rigid': pw_rigid,
'p': p,
'nb': gnb,
'rf': rf,
'K': K,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'only_init': True,
'ssub': ssub,
'tsub': tsub,
'merge_thr': merge_thr,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest
}
opts = params.CNMFParams(params_dict=opts_dict)
return opts
def run_alignment(mouse, sessions, motion_correction_v, cropping_v, dview):
"""
This is the main function for the alignment step. It applies methods
from the CaImAn package used originally in motion correction
to do alignment.
"""
for session in sessions:
# Update the database
file_name = f"mouse_{mouse}_session_{session}_alignment"
sql1 = "UPDATE Analysis SET alignment_main=? WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=? "
val1 = [file_name, mouse, session, motion_correction_v, cropping_v]
cursor.execute(sql1, val1)
# Determine the output .mmap file name
output_mmap_file_path = os.environ[
'DATA_DIR_LOCAL'] + f'data/interim/alignment/main/{file_name}.mmap'
sql = "SELECT motion_correction_main FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=? "
val = [mouse, session, motion_correction_v, cropping_v]
cursor.execute(sql, val)
result = cursor.fetchall()
input_mmap_file_list = []
inter = []
for x in result:
inter += x
for y in inter:
input_mmap_file_list.append(y)
sql = "SELECT motion_correction_cropping_points_x1 FROM Analysis WHERE mouse = ? AND session=?AND motion_correction_v =? AND cropping_v=? "
val = [mouse, session, motion_correction_v, cropping_v]
cursor.execute(sql, val)
result = cursor.fetchall()
x_ = []
inter = []
for i in result:
inter += i
for j in range(0, len(inter)):
x_.append(inter[j])
sql = "SELECT motion_correction_cropping_points_x2 FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=? "
val = [mouse, session, motion_correction_v, cropping_v]
cursor.execute(sql, val)
result = cursor.fetchall()
_x = []
inter = []
for i in result:
inter += i
for j in range(0, len(inter)):
_x.append(inter[j])
sql = "SELECT motion_correction_cropping_points_y1 FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?"
val = [mouse, session, motion_correction_v, cropping_v]
cursor.execute(sql, val)
result = cursor.fetchall()
_y = []
inter = []
for i in result:
inter += i
for j in range(0, len(inter)):
_y.append(inter[j])
sql = "SELECT motion_correction_cropping_points_y2 FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?"
val = [mouse, session, motion_correction_v, cropping_v]
cursor.execute(sql, val)
result = cursor.fetchall()
y_ = []
inter = []
for i in result:
inter += i
for j in range(0, len(inter)):
y_.append(inter[j])
new_x1 = max(x_)
new_x2 = max(_x)
new_y1 = max(y_)
new_y2 = max(_y)
m_list = []
for i in range(len(input_mmap_file_list)):
m = cm.load(input_mmap_file_list[i])
m = m.crop(new_x1 - x_[i], new_x2 - _x[i], new_y1 - y_[i],
new_y2 - _y[i], 0, 0)
m_list.append(m)
# Concatenate them using the concat function
m_concat = cm.concatenate(m_list, axis=0)
fname = m_concat.save(output_mmap_file_path, order='C')
# MOTION CORRECTING EACH INDIVIDUAL MOVIE WITH RESPECT TO A TEMPLATE MADE OF THE FIRST MOVIE
logging.info(
'Performing motion correction on all movies with respect to a template made of the first movie.'
)
t0 = datetime.datetime.today()
# parameters alignment
sql5 = "SELECT make_template_from_trial,gSig_filt,max_shifts,niter_rig,strides,overlaps,upsample_factor_grid,num_frames_split,max_deviation_rigid,shifts_opencv,use_conda,nonneg_movie, border_nan FROM Analysis WHERE alignment_main=? "
val5 = [
file_name,
]
cursor.execute(sql5, val5)
myresult = cursor.fetchall()
para = []
aux = []
for x in myresult:
aux = x
for y in aux:
para.append(y)
parameters = {
'make_template_from_trial': para[0],
'gSig_filt': (para[1], para[1]),
'max_shifts': (para[2], para[2]),
'niter_rig': para[3],
'strides': (para[4], para[4]),
'overlaps': (para[5], para[5]),
'upsample_factor_grid': para[6],
'num_frames_split': para[7],
'max_deviation_rigid': para[8],
'shifts_opencv': para[9],
'use_cuda': para[10],
'nonneg_movie': para[11],
'border_nan': para[12]
}
# Create a template of the first movie
template_index = parameters['make_template_from_trial']
m0 = cm.load(input_mmap_file_list[1])
[x1, x2, y1, y2] = [x_, _x, y_, _y]
for i in range(len(input_mmap_file_list)):
m0 = m0.crop(new_x1 - x_[i], new_x2 - _x[i], new_y1 - y_[i],
new_y2 - _y[i], 0, 0)
m0_filt = cm.movie(
np.array([
high_pass_filter_space(m_, parameters['gSig_filt'])
for m_ in m0
]))
template0 = cm.motion_correction.bin_median(
m0_filt.motion_correct(
5, 5, template=None)[0]) # may be improved in the future
# Setting the parameters
opts = params.CNMFParams(params_dict=parameters)
# Create a motion correction object
mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion'))
# Perform non-rigid motion correction
mc.motion_correct(template=template0, save_movie=True)
# Cropping borders
x_ = math.ceil(
abs(np.array(mc.shifts_rig)[:, 1].max()
) if np.array(mc.shifts_rig)[:, 1].max() > 0 else 0)
_x = math.ceil(
abs(np.array(mc.shifts_rig)[:, 1].min()
) if np.array(mc.shifts_rig)[:, 1].min() < 0 else 0)
y_ = math.ceil(
abs(np.array(mc.shifts_rig)[:, 0].max()
) if np.array(mc.shifts_rig)[:, 0].max() > 0 else 0)
_y = math.ceil(
abs(np.array(mc.shifts_rig)[:, 0].min()
) if np.array(mc.shifts_rig)[:, 0].min() < 0 else 0)
# Load the motion corrected movie into memory
movie = cm.load(mc.fname_tot_rig[0])
# Crop all movies to those border pixels
movie.crop(x_, _x, y_, _y, 0, 0)
sql1 = "UPDATE Analysis SET alignment_x1=?, alignment_x2 =?, alignment_y1=?, alignment_y2=? WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?"
val1 = [
x_, _x, y_, _y, mouse, session, motion_correction_v, cropping_v
]
cursor.execute(sql1, val1)
# save motion corrected and cropped movie
output_mmap_file_path_tot = movie.save(
os.environ['DATA_DIR_LOCAL'] +
f'data/interim/alignment/main/{file_name}.mmap',
order='C')
logging.info(
f' Cropped and saved rigid movie as {output_mmap_file_path_tot}')
# Remove the remaining non-cropped movie
os.remove(mc.fname_tot_rig[0])
# Create a timeline and store it
sql = "SELECT trial FROM Analysis WHERE mouse = ? AND session=? AND motion_correction_v =? AND cropping_v=?"
val = [mouse, session, motion_correction_v, cropping_v]
cursor.execute(sql, val)
result = cursor.fetchall()
trial_index_list = []
inter = []
for i in result:
inter += i
for j in range(0, len(inter)):
trial_index_list.append(inter[j])
timeline = [[trial_index_list[0], 0]]
timepoints = [0]
for i in range(1, len(m_list)):
m = m_list[i]
timeline.append(
[trial_index_list[i], timeline[i - 1][1] + m.shape[0]])
timepoints.append(timepoints[i - 1] + m.shape[0])
timeline_pkl_file_path = os.environ[
'DATA_DIR'] + f'data/interim/alignment/meta/timeline/{file_name}.pkl'
with open(timeline_pkl_file_path, 'wb') as f:
pickle.dump(timeline, f)
sql1 = "UPDATE Analysis SET alignment_timeline=? WHERE mouse = ? AND session=?AND motion_correction_v =? AND cropping_v=? "
val1 = [
timeline_pkl_file_path, mouse, session, motion_correction_v,
cropping_v
]
cursor.execute(sql1, val1)
timepoints.append(movie.shape[0])
dt = int((datetime.datetime.today() - t0).seconds /
60) # timedelta in minutes
sql1 = "UPDATE Analysis SET alignment_duration_concatenation=? WHERE mouse = ? AND session=?AND motion_correction_v =? AND cropping_v=? "
val1 = [dt, mouse, session, motion_correction_v, cropping_v]
cursor.execute(sql1, val1)
logging.info(f' Performed concatenation. dt = {dt} min.')
## modify all motion correction file to the aligned version
data_dir = os.environ[
'DATA_DIR'] + 'data/interim/motion_correction/main/'
for i in range(len(input_mmap_file_list)):
aligned_movie = movie[timepoints[i]:timepoints[i + 1]]
motion_correction_output_aligned = aligned_movie.save(
data_dir + file_name + '_els' + '.mmap', order='C')
sql1 = "UPDATE Analysis SET motion_correct_align=? WHERE motion_correction_meta=? AND motion_correction_v"
val1 = [
motion_correction_output_aligned, input_mmap_file_list[i],
motion_correction_v
]
cursor.execute(sql1, val1)
database.commit()
return
def main():
pass # For compatibility between running under Spyder and the CLI
#%%
"""
General parameters
"""
play_movie = 1
plot_extras = 1
plot_extras_cell = 1
compute_mc_metrics = 1
#%% Select file(s) to be processed (download if not present)
"""
Load file
"""
#fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
#if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
# fnames = [download_demo(fnames[0])]
#fnames = ['/home/yuriy/Desktop/Data/rest1_5_9_19_cut.tif']
#f_dir = 'C:\\Users\\rylab_dataPC\\Desktop\\Yuriy\\caiman_data\\short\\'
f_dir = 'G:\\analysis\\190828-calcium_voltage\\soma_dendrites\\pCAG_jREGECO1a_ASAP3_anesth_001\\'
f_name = 'Ch1'
f_ext = 'tif'
fnames = [f_dir + f_name + '.' + f_ext]
#fnames = ['C:/Users/rylab_dataPC/Desktop/Yuriy/caiman_data/rest1_5_9_19_2_cut_ca.hdf5']
#%% First setup some parameters for data and motion correction
"""
Parameters
"""
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 1
#0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
#max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
max_shifts = [6, 6]
# start a new patch for pw-rigid motion correction every x pixels
#strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
strides = [48, 48]
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
if play_movie:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# type "mc."and press TAB to see all interesting associated variables and self. outputs
# interesting outputs
# saved file is mc.fname_tot_els / mc.fname_tot_rig
# mc.x_shifts_els / mc.y_shifts_els: shifts in x/y per frame per patch
# mc.coord_shifts_els: coordinates associated to patches with shifts
# mc.total_template_els: updated template for pw
# mc.total_template_rig: updated template for rigid
# mc.templates_rig: templates for each iteration in rig
#%% # compute metrics for the results (TAKES TIME!!)
if compute_mc_metrics:
# not finished
bord_px_els = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
final_size = np.subtract(
mc.total_template_els.shape,
2 * bord_px_els) # remove pixels in the boundaries
winsize = 100
swap_dim = False
resize_fact_flow = .2 # downsample for computing ROF
tmpl_rig, correlations_orig, flows_orig, norms_orig, crispness_orig = cm.motion_correction.compute_metrics_motion_correction(
fnames[0],
final_size[0],
final_size[1],
swap_dim,
winsize=winsize,
play_flow=False,
resize_fact_flow=resize_fact_flow)
plt.figure()
plt.plot(correlations_orig)
# %% compare with original movie
if play_movie:
m_orig = cm.load_movie_chain(fnames)
m_els = cm.load(mc.mmap_file)
ds_ratio = 0.2
moviehandle = cm.concatenate([
m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)
],
axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
del m_orig
del m_els
if plot_extras:
# plot total template
plt.figure()
plt.imshow(mc.total_template_els)
plt.title('Template after iteration')
# plot x and y corrections
plt.figure()
plt.plot(mc.shifts_rig)
plt.title('Rigid motion correction xy movement')
plt.legend(['x shift', 'y shift'])
plt.xlabel('frames')
# %% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file,
base_name='memmap_',
order='C',
border_to_0=border_to_0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
# %% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
# %% parameters for source extraction and deconvolution
p = 2 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 2 # number of components per patch
gSig = [15, 15] # expected half size of neurons in pixels
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
ssub = 1 # spatial subsampling during initialization
tsub = 1 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {
'fnames': fnames,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub
}
opts.change_params(params_dict=opts_dict)
# %% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
if plot_extras_cell:
num_cell_plot = 51
plt.figure()
plt.plot(cnm.estimates.C[num_cell_plot, :])
plt.title('Temporal component')
plt.legend(['Cell ' + str(num_cell_plot)])
# plot component sptial profile A
# first convert back to dense components
plot_spat_A = cnm.estimates.A[:, num_cell_plot].toarray().reshape(
list(dims))
plt.figure()
plt.imshow(plot_spat_A)
plt.title('Spatial component cell ' + str(num_cell_plot))
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% plot contours of found components
Cn = cm.local_correlations(images, swap_dim=False)
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
if plot_extras:
plt.figure()
plt.imshow(Cn)
plt.title('Local correlations')
# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm.params.change_params({'p': p})
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # signal to noise ratio for accepting a component
rval_thr = 0.90 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set(
'quality', {
'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest
})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% PLOT COMPONENTS
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
plt.suptitle('Component selection: min_SNR=' + str(min_SNR) +
'; rval_thr=' + str(rval_thr) + '; cnn prob range=[' +
str(cnn_lowest) + ' ' + str(cnn_thr) + ']')
# %% VIEW TRACES (accepted and rejected)
if plot_extras:
cnm2.estimates.view_components(images,
img=Cn,
idx=cnm2.estimates.idx_components)
plt.suptitle('Accepted')
cnm2.estimates.view_components(images,
img=Cn,
idx=cnm2.estimates.idx_components_bad)
plt.suptitle('Rejected')
#plt.figure();
#plt.plot(cnm2.estimates.YrA[0,:]+cnm2.estimates.C[0,:])
#
#
#
#
#plt.figure();
#plt.plot(cnm2.estimates.R[0,:]-cnm2.estimates.YrA[0,:]);
#plt.plot();
#plt.show();
#
#
#plt.figure();
#plt.plot(cnm2.estimates.detrend_df_f[1,:])
# these store the good and bad components, and next step sorts them
# cnm2.estimates.idx_components
# cnm2.estimates.idx_components_bad
#%% update object with selected components
#cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(img=Cn)
plt.suptitle("Final results")
#%% Save the mc data as in cmn struct as well
##
#mc_out = dict(
# pw_rigid = mc.pw_rigid,
# fname = mc.fname,
# mmap_file = mc.mmap_file,
# total_template_els = mc.total_template_els,
# total_template_rig = mc.total_template_rig,
# border_nan = mc.border_nan,
# border_to_0 = mc.border_to_0,
# x_shifts_els = mc.x_shifts_els,
# y_shifts_els = mc.y_shifts_els,
# Cn = Cn
# )
#
#
#deepdish.io.save(fnames[0] + '_mc_data.hdf5', mc_out)
#%% reconstruct denoised movie (press q to exit)
if play_movie:
cnm2.estimates.play_movie(images,
q_max=99.9,
gain_res=2,
magnification=2,
bpx=border_to_0,
include_bck=False) # background not shown
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
save_results = True
if save_results:
cnm2.save(fnames[0][:-4] + '_results.hdf5')
def run_motion_correction(cropping_file, dview):
"""
This is the function for motion correction. Its goal is to take in a decoded and
cropped .tif file, perform motion correction, and save the result as a .mmap file.
This function is only runnable on the cn76 server because it requires parallel processing.
Args:
cropping_file: tif file after cropping
dview: cluster
Returns:
row: pd.DataFrame object
The row corresponding to the motion corrected analysis state.
"""
# Get output file paths
data_dir = os.environ['DATA_DIR_LOCAL'] + 'data/interim/motion_correction/'
sql = "SELECT mouse,session,trial,is_rest,decoding_v,cropping_v,motion_correction_v,input,home_path,decoding_main FROM Analysis WHERE cropping_main=? ORDER BY motion_correction_v"
val = [
cropping_file,
]
cursor.execute(sql, val)
result = cursor.fetchall()
data = []
inter = []
for x in result:
inter = x
for y in inter:
data.append(y)
# Update the database
if data[6] == 0:
data[6] = 1
file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}"
output_meta_pkl_file_path = f'meta/metrics/{file_name}.pkl'
sql1 = "UPDATE Analysis SET motion_correction_meta=?,motion_correction_v=? WHERE cropping_main=? "
val1 = [output_meta_pkl_file_path, data[6], cropping_file]
cursor.execute(sql1, val1)
else:
data[6] += 1
file_name = f"mouse_{data[0]}_session_{data[1]}_trial_{data[2]}.{data[3]}.v{data[4]}.{data[5]}.{data[6]}"
output_meta_pkl_file_path = f'meta/metrics/{file_name}.pkl'
sql2 = "INSERT INTO Analysis (motion_correction_meta,motion_correction_v) VALUES (?,?)"
val2 = [output_meta_pkl_file_path, data[6]]
cursor.execute(sql2, val2)
database.commit()
sql3 = "UPDATE Analysis SET decoding_main=?,decoding_v=?,mouse=?,session=?,trial=?,is_rest=?,input=?,home_path=?,cropping_v=?,cropping_main=? WHERE motion_correction_meta=? AND motion_correction_v=?"
val3 = [
data[9], data[4], data[0], data[1], data[2], data[3], data[7],
data[8], data[5], cropping_file, output_meta_pkl_file_path, data[6]
]
cursor.execute(sql3, val3)
database.commit()
output_meta_pkl_file_path_full = data_dir + output_meta_pkl_file_path
# Calculate movie minimum to subtract from movie
cropping_file_full = os.environ['DATA_DIR_LOCAL'] + cropping_file
min_mov = np.min(cm.load(cropping_file_full))
# Apply the parameters to the CaImAn algorithm
sql5 = "SELECT motion_correct,pw_rigid,save_movie_rig,gSig_filt,max_shifts,niter_rig,strides,overlaps,upsample_factor_grid,num_frames_split,max_deviation_rigid,shifts_opencv,use_conda,nonneg_movie, border_nan FROM Analysis WHERE cropping_main=? "
val5 = [
cropping_file,
]
cursor.execute(sql5, val5)
myresult = cursor.fetchall()
para = []
aux = []
for x in myresult:
aux = x
for y in aux:
para.append(y)
parameters = {
'motion_correct': para[0],
'pw_rigid': para[1],
'save_movie_rig': para[2],
'gSig_filt': (para[3], para[3]),
'max_shifts': (para[4], para[4]),
'niter_rig': para[5],
'strides': (para[6], para[6]),
'overlaps': (para[7], para[7]),
'upsample_factor_grid': para[8],
'num_frames_split': para[9],
'max_deviation_rigid': para[10],
'shifts_opencv': para[11],
'use_cuda': para[12],
'nonneg_movie': para[13],
'border_nan': para[14]
}
caiman_parameters = parameters.copy()
caiman_parameters['min_mov'] = min_mov
opts = params.CNMFParams(params_dict=caiman_parameters)
# Rigid motion correction (in both cases)
logging.info('Performing rigid motion correction')
t0 = datetime.datetime.today()
# Create a MotionCorrect object
mc = MotionCorrect([cropping_file_full],
dview=dview,
**opts.get_group('motion'))
# Perform rigid motion correction
mc.motion_correct_rigid(save_movie=parameters['save_movie_rig'],
template=None)
dt = int(
(datetime.datetime.today() - t0).seconds / 60) # timedelta in minutes
logging.info(f' Rigid motion correction finished. dt = {dt} min')
# Obtain template, rigid shifts and border pixels
total_template_rig = mc.total_template_rig
shifts_rig = mc.shifts_rig
# Save template, rigid shifts and border pixels in a dictionary
meta_pkl_dict = {
'rigid': {
'template': total_template_rig,
'shifts': shifts_rig,
}
}
sql = "UPDATE Analysis SET duration_rigid=? WHERE motion_correction_meta=? AND motion_correction_v=? "
val = [dt, output_meta_pkl_file_path, data[6]]
cursor.execute(sql, val)
if parameters['save_movie_rig'] == 1:
# Load the movie saved by CaImAn, which is in the wrong
# directory and is not yet cropped
logging.info(f' Loading rigid movie for cropping')
m_rig = cm.load(mc.fname_tot_rig[0])
logging.info(f' Loaded rigid movie for cropping')
# Get the cropping points determined by the maximal rigid shifts
x_, _x, y_, _y = get_crop_from_rigid_shifts(shifts_rig)
# Crop the movie
logging.info(
f' Cropping and saving rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}'
)
m_rig = m_rig.crop(x_, _x, y_, _y, 0, 0)
meta_pkl_dict['rigid']['cropping_points'] = [x_, _x, y_, _y]
sql = "UPDATE Analysis SET motion_correction_cropping_points_x1=?,motion_correction_cropping_points_x2=?,motion_correction_cropping_points_y1=?,motion_correction_cropping_points_y2=? WHERE motion_correction_meta=? AND motion_correction_v=? "
val = [x_, _x, y_, _y, output_meta_pkl_file_path, data[6]]
cursor.execute(sql, val)
# Save the movie
rig_role = 'alternate' if parameters['pw_rigid'] else 'main'
fname_tot_rig = m_rig.save(data_dir + rig_role + '/' + file_name +
'_rig' + '.mmap',
order='C')
logging.info(f' Cropped and saved rigid movie as {fname_tot_rig}')
# Remove the remaining non-cropped movie
os.remove(mc.fname_tot_rig[0])
sql = "UPDATE Analysis SET motion_correction_rig_role=? WHERE motion_correction_meta=? AND motion_correction_v=? "
val = [fname_tot_rig, output_meta_pkl_file_path, data[6]]
cursor.execute(sql, val)
database.commit()
# If specified in the parameters, apply piecewise-rigid motion correction
if parameters['pw_rigid'] == 1:
logging.info(f' Performing piecewise-rigid motion correction')
t0 = datetime.datetime.today()
# Perform non-rigid (piecewise rigid) motion correction. Use the rigid result as a template.
mc.motion_correct_pwrigid(save_movie=True, template=total_template_rig)
# Obtain template and filename
total_template_els = mc.total_template_els
fname_tot_els = mc.fname_tot_els[0]
dt = int((datetime.datetime.today() - t0).seconds /
60) # timedelta in minutes
meta_pkl_dict['pw_rigid'] = {
'template': total_template_els,
'x_shifts': mc.x_shifts_els,
'y_shifts': mc.
y_shifts_els # removed them initially because they take up space probably
}
logging.info(
f' Piecewise-rigid motion correction finished. dt = {dt} min')
# Load the movie saved by CaImAn, which is in the wrong
# directory and is not yet cropped
logging.info(f' Loading pw-rigid movie for cropping')
m_els = cm.load(fname_tot_els)
logging.info(f' Loaded pw-rigid movie for cropping')
# Get the cropping points determined by the maximal rigid shifts
x_, _x, y_, _y = get_crop_from_pw_rigid_shifts(
np.array(mc.x_shifts_els), np.array(mc.y_shifts_els))
# Crop the movie
logging.info(
f' Cropping and saving pw-rigid movie with cropping points: [x_, _x, y_, _y] = {[x_, _x, y_, _y]}'
)
m_els = m_els.crop(x_, _x, y_, _y, 0, 0)
meta_pkl_dict['pw_rigid']['cropping_points'] = [x_, _x, y_, _y]
# Save the movie
fname_tot_els = m_els.save(data_dir + 'main/' + file_name + '_els' +
'.mmap',
order='C')
logging.info(f'Cropped and saved rigid movie as {fname_tot_els}')
# Remove the remaining non-cropped movie
os.remove(mc.fname_tot_els[0])
sql = "UPDATE Analysis SET motion_correction_main=?, motion_correction_cropping_points_x1=?,motion_correction_cropping_points_x2=?,motion_correction_cropping_points_y1=?,motion_correction_cropping_points_y2=?,duration_pw_rigid=? WHERE motion_correction_meta=? AND motion_correction_v=? "
val = [
fname_tot_els, x_, _x, y_, _y, dt, output_meta_pkl_file_path,
data[6]
]
cursor.execute(sql, val)
database.commit()
# Write meta results dictionary to the pkl file
pkl_file = open(output_meta_pkl_file_path_full, 'wb')
pickle.dump(meta_pkl_dict, pkl_file)
pkl_file.close()
return fname_tot_els, data[6]
def main():
pass # For compatibility between running under Spyder and the CLI
c, dview, n_processes =\
cm.cluster.setup_cluster(backend='local', n_processes=None,
single_thread=False)
# %% set up some parameters
fnames = [os.path.join(caiman_datadir(), 'split', 'first3000-ch1.tif'),
os.path.join(caiman_datadir(), 'split', 'second3000-ch1.tif')]
is_patches = True # flag for processing in patches or not
fr = 1.5 # approximate frame rate of data
decay_time = 5.0 # length of transient
if is_patches: # PROCESS IN PATCHES AND THEN COMBINE
rf = 20 # half size of each patch
stride = 4 # overlap between patches
K = 2 # number of components in each patch
else: # PROCESS THE WHOLE FOV AT ONCE
rf = None # setting these parameters to None
stride = None # will run CNMF on the whole FOV
K = 10 # number of neurons expected (in the whole FOV)
gSig = [6, 6] # expected half size of neurons
merge_thresh = 0.80 # merging threshold, max correlation allowed
p = 2 # order of the autoregressive system
gnb = 2 # global background order
params_dict = {'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'rf': rf,
'stride': stride,
'K': K,
'gSig': gSig,
'merge_thr': merge_thresh,
'p': p,
'nb': gnb}
opts = params.CNMFParams(params_dict=params_dict)
# %% Now RUN CaImAn Batch (CNMF)
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
#cnm.estimates.normalize_components()
cnm = cnm.fit_file()
# %% plot contour plots of components
Cn = cm.load(fnames[0], subindices=slice(1000)).local_correlations(swap_dim=False)
cnm.estimates.plot_contours(img=Cn)
# %% load memory mapped file
Yr, dims, T = cm.load_memmap(cnm.mmap_file)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# %% refit
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier (this will pick up only neurons
# and filter out active processes)
min_SNR = 2 # peak SNR for accepted components (if above this, acept)
rval_thr = 0.85 # space correlation threshold (if above this, accept)
use_cnn = False # use the CNN classifier
min_cnn_thr = 0.99 # if cnn classifier predicts below this value, reject
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set('quality', {'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': use_cnn,
'min_cnn_thr': min_cnn_thr,
'cnn_lowest': cnn_lowest})
cnm2.estimates.detrend_df_f()
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% visualize selected and rejected components
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
# %% visualize selected components
cnm2.estimates.nb_view_components(images, idx=cnm2.estimates.idx_components, img=Cn)
cnm2.estimates.view_components(images, idx=cnm2.estimates.idx_components_bad, img=Cn)
#%% only select high quality components
cnm2.estimates.select_components(use_object=True)
#%%
cnm2.estimates.plot_contours(img=Cn)
cnm2.estimates.detrend_df_f()
import pickle
f = open("/home/david/zebraHorse/df_f_day55.pkl", "wb")
pickle.dump(cnm2.estimates.F_dff, f)
f.close()
# %% play movie with results (original, reconstructed, amplified residual)
for j in range(10):
cnm2.estimates.play_movie(images, magnification=4.0, frame_range = range(100 * j, 100 * (j + 1)))
#import time
#time.sleep(1000)
# %% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('Yr*_LOG_*')
for log_file in log_files:
os.remove(log_file)
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters
# dataset dependent parameters
display_images = False # Set to true to show movies and images
fnames = ['data_endoscope.tif'] # filename to be processed
fr = 10 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
do_motion_correction_nonrigid = False
do_motion_correction_rigid = True # choose motion correction type
gSig_filt = (3, 3) # size of filter, in general gSig (see below),
# change this one if algorithm does not work
max_shifts = (5, 5) # maximum allowed rigid shift
strides = (
48, 48
) # start a new patch for pw-rigid motion correction every x pixels
overlaps = (24, 24
) # overlap between pathes (size of patch strides+overlaps)
# for parallelization split the movies in num_splits chuncks across time
# (make sure that length_movie/num_splits_to_process_rig>100)
splits_rig = 10
splits_els = 10
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
#%% start the cluster
try:
cm.stop_server() # stop it if it was running
except ():
pass
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local',
n_processes=
24, # number of process to use, if you go out of memory try to reduce this one
single_thread=False)
#%% download demo file
fnames = [download_demo(fnames[0])]
filename_reorder = fnames
#%% MOTION CORRECTION
if do_motion_correction_nonrigid or do_motion_correction_rigid:
# do motion correction rigid
mc = motion_correct_oneP_rigid(
fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
dview=dview,
splits_rig=splits_rig,
save_movie=not (do_motion_correction_nonrigid),
border_nan='copy')
new_templ = mc.total_template_rig
plt.subplot(1, 2, 1)
plt.imshow(new_templ) # % plot template
plt.subplot(1, 2, 2)
plt.plot(mc.shifts_rig) # % plot rigid shifts
plt.legend(['x shifts', 'y shifts'])
plt.xlabel('frames')
plt.ylabel('pixels')
# borders to eliminate from movie because of motion correction
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
filename_reorder = mc.fname_tot_rig
# do motion correction nonrigid
if do_motion_correction_nonrigid:
mc = motion_correct_oneP_nonrigid(
fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
strides=strides,
overlaps=overlaps,
splits_els=splits_els,
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
dview=dview,
splits_rig=None,
save_movie=True, # whether to save movie in memory mapped format
new_templ=new_templ, # template to initialize motion correction
border_nan='copy')
filename_reorder = mc.fname_tot_els
bord_px = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
# create memory mappable file in the right order on the hard drive (C order)
fname_new = cm.save_memmap(filename_reorder,
base_name='memmap_',
order='C',
border_to_0=bord_px,
dview=dview)
# load memory mappable file
Yr, dims, T = cm.load_memmap(fname_new)
Y = Yr.T.reshape((T, ) + dims, order='F')
#%% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None
gSig = (
3, 3
) # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1
Ain = None # possibility to seed with predetermined binary masks
merge_thresh = .7 # merging threshold, max correlation allowed
rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80
stride_cnmf = 20 # amount of overlap between the patches in pixels
# (keep it at least large as gSiz, i.e 4 times the neuron size gSig)
tsub = 2 # downsampling factor in time for initialization,
# increase if you have memory problems
ssub = 1 # downsampling factor in space for initialization,
# increase if you have memory problems
# you can pass them here as boolean vectors
low_rank_background = None # None leaves background of each patch intact,
# True performs global low-rank approximation if gnb>0
gnb = 0 # number of background components (rank) if positive,
# else exact ring model with following settings
# gnb= 0: Return background as b and W
# gnb=-1: Return full rank background B
# gnb<-1: Don't return background
nb_patch = 0 # number of background components (rank) per patch if gnb>0,
# else it is set automatically
min_corr = .8 # min peak value from correlation image
min_pnr = 10 # min peak to noise ration from PNR image
ssub_B = 2 # additional downsampling factor in space for background
ring_size_factor = 1.4 # radius of ring is gSiz*ring_size_factor
# parameters for component evaluation
min_SNR = 3 # adaptive way to set threshold on the transient size
r_values_min = 0.85 # threshold on space consistency (if you lower more components
# will be accepted, potentially with worst quality)
opts = params.CNMFParams(
dims=dims,
fr=fr,
decay_time=decay_time,
method_init='corr_pnr', # use this for 1 photon
k=K,
gSig=gSig,
gSiz=gSiz,
merge_thresh=merge_thresh,
p=p,
tsub=tsub,
ssub=ssub,
rf=rf,
stride=stride_cnmf,
only_init_patch=True, # set it to True to run CNMF-E
gnb=gnb,
nb_patch=nb_patch,
method_deconvolution='oasis', # could use 'cvxpy' alternatively
low_rank_background=low_rank_background,
update_background_components=
True, # sometimes setting to False improve the results
min_corr=min_corr,
min_pnr=min_pnr,
normalize_init=False, # just leave as is
center_psf=True, # leave as is for 1 photon
ssub_B=ssub_B,
ring_size_factor=ring_size_factor,
del_duplicates=True, # whether to remove duplicates from initialization
border_pix=bord_px) # number of pixels to not consider in the borders)
#%% compute some summary images (correlation and peak to noise)
# change swap dim if output looks weird, it is a problem with tiffile
cn_filter, pnr = cm.summary_images.correlation_pnr(Y,
gSig=gSig[0],
swap_dim=False)
# inspect the summary images and set the parameters
inspect_correlation_pnr(cn_filter, pnr)
# print parameters set above, modify them if necessary based on summary images
print(min_corr) # min correlation of peak (from correlation image)
print(min_pnr) # min peak to noise ratio
#%% RUN CNMF ON PATCHES
cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=opts)
cnm.fit(Y)
#%% DISCARD LOW QUALITY COMPONENTS
cnm.params.set('quality', {
'min_SNR': min_SNR,
'rval_thr': r_values_min,
'use_cnn': False
})
cnm.estimates.evaluate_components(Y, cnm.params, dview=dview)
print(' ***** ')
print('Number of total components: ', len(cnm.estimates.C))
print('Number of accepted components: ', len(cnm.estimates.idx_components))
#%% PLOT COMPONENTS
cnm.dims = dims
if display_images:
cnm.estimates.plot_contours(img=cn_filter,
idx=cnm.estimates.idx_components)
cnm.estimates.view_components(Y, idx=cnm.estimates.idx_components)
#%% MOVIES
if display_images:
# fully reconstructed movie
cnm.estimates.play_movie(Y,
q_max=99.9,
magnification=2,
include_bck=True,
gain_res=10,
bpx=bord_px)
# movie without background
cnm.estimates.play_movie(Y,
q_max=99.9,
magnification=2,
include_bck=False,
gain_res=4,
bpx=bord_px)
#%% STOP SERVER
cm.stop_server(dview=dview)
border_nan = 'copy'
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'gSig_filt': gSig_filt,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': border_nan
}
opts = params.CNMFParams(params_dict=mc_dict)
if motion_correct:
# do motion correction rigid
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
mc.motion_correct(save_movie=True)
fname_mc = mc.fname_tot_els if pw_rigid else mc.fname_tot_rig
if pw_rigid:
bord_px = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
else:
bord_px = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
plt.subplot(1, 2, 1); plt.imshow(mc.total_template_rig) # % plot template
plt.subplot(1, 2, 2); plt.plot(mc.shifts_rig) # % plot rigid shifts
plt.legend(['x shifts', 'y shifts'])
plt.xlabel('frames')
## path to motion corrected tif file
folder = '/projects/p30771/miniscope/data/GRIN011/1_24_2019/H10_M19_S59/TIFs/full_movie_caiman_analysis/'
#mc_file = 'memmap__d1_480_d2_752_d3_1_order_C_frames_202_.mmap'
memory_map_file = sys.argv[1]
# In[4]:
# load memory mappable file
print('loading file:')
print(memory_map_file)
Yr, dims, T = cm.load_memmap(memory_map_file)
images = Yr.T.reshape((T, ) + dims, order='F')
#
opts = params.CNMFParams(params_dict={})
#
bord_px = 0
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
K = None # upper bound on number of components per patch, in general None
gSig = (
3, 3
) # gaussian width of a 2D gaussian kernel, which approximates a neuron
gSiz = (13, 13) # average diameter of a neuron, in general 4*gSig+1
Ain = None # possibility to seed with predetermined binary masks
merge_thresh = .7 # merging threshold, max correlation allowed
rf = 40 # half-size of the patches in pixels. e.g., if rf=40, patches are 80x80
stride_cnmf = 20 # amount of overlap between the patches in pixels
# (keep it at least large as gSiz, i.e 4 times the neuron size gSig)
tsub = 2 # downsampling factor in time for initialization,
