def mov_load_func(x):
with warnings.catch_warnings():
if os.path.isdir(x):
return load_movie_chain(
sorted(glob.glob(os.path.join(x, '*.tif*'))))
else:
return load_movie_chain([x])
def play_movie(fnames, ds_ratio=0.2, q_max=99.5, fr=30, mag=2):
"""Play the movie. This will require loading the movie in memory which
in general is not needed by the pipeline. Displaying the movie uses the
OpenCV library. Press q to close the video panel."""
m_orig = cm.load_movie_chain(fnames)
m_orig.resize(1, 1, ds_ratio).play(q_max=q_max, fr=fr, magnification=mag)
return
def create_correlation_image(fl_group):
import caiman as cm
cimg = cm.load_movie_chain(fl_group).local_correlations(
eight_neighbours=True, swap_dim=False)
# vm = (np.percentile(cimg[~np.isnan(cimg)],90))
# pl.imshow(cimg,vmax = vm)
# pl.pause(1)
return cimg
def preprocess_neurofinder(folder):
import caiman as cm
import os
fls = glob.glob(os.path.join(folder, 'images/*.tiff'))
fls.sort()
print(fls[:5])
m = cm.load_movie_chain(fls)
m.save(os.path.join(folder, 'movie_total.hdf5'))
def create_correlation_image(fl_group):
import caiman as cm
cimg = cm.load_movie_chain(fl_group).local_correlations(
eight_neighbours=True, swap_dim=False)
# vm = (np.percentile(cimg[~np.isnan(cimg)],90))
# pl.imshow(cimg,vmax = vm)
# pl.pause(1)
return cimg
def __init__(self, file_name, max_shifts, strides, overlaps,
upsample_factor_grid, max_deviation_rigid):
self.name_orig = [file_name]
self.data_orig = cm.load_movie_chain(self.name_orig)
self.name_rig, self.name_pwrig, self.shifts_rig, self.x_shifts_pwrig, self.y_shifts_pwrig, self.template_shape = self._run_motion_correction(
file_name, max_shifts, strides, overlaps, upsample_factor_grid,
max_deviation_rigid)
self.data_rig = cm.load(self.name_rig)
self.data_pwrig = cm.load(self.name_pwrig)
def merge_denoised_tiff_files(movie, loaddir, savedir):
#%%
cpu_num = 2
#cpu_num_spikepursuit = 1
filenames = os.listdir(loaddir)
counter = 0
filenames_final = list()
residualnames = list()
while 'denoised_{}.tif'.format(counter) in filenames:
m_new_denoised = cm.load(
os.path.join(loaddir,
'denoised_{}.tif'.format(counter))).transpose(
2, 0, 1)
i_new_sn = imio.imread(
os.path.join(loaddir, 'Sn_image_{}.tif'.format(counter)))[:, :, 0]
m_new_trend = cm.load(
os.path.join(loaddir,
'trend_{}.tif'.format(counter))).transpose(2, 0, 1)
movief = m_new_denoised * i_new_sn + m_new_trend
movief.save(os.path.join(loaddir, 'movie{}.tif'.format(counter)))
filenames_final.append(
os.path.join(loaddir, 'movie{}.tif'.format(counter)))
residualnames.append(
os.path.join(loaddir, 'PMD_residual_{}.tif'.format(counter)))
counter += 1
print(counter)
#%%
residuals_movie = cm.load_movie_chain(residualnames)
residuals_movie.save(os.path.join(savedir, 'PMD_residuals.tif'))
#movie_big = cm.load_movie_chain(filenames_final)
# %% Memory Mapping
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=cpu_num,
single_thread=False)
fname_new = cm.save_memmap(filenames_final,
base_name=movie['movie_name'],
dview=dview,
n_chunks=10,
order='C')
dview.terminate()
fname = pathlib.Path(fname_new).name
shutil.move(fname_new, os.path.join(savedir, fname))
print('done')
def motion_corr(fnames, dview, opts, disp_movie, is_3d=False):
"""Perform motion correction"""
# Create a motion correction object with the parameters specified. Note
# that the file is not loaded in memory
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# Run piecewise-rigid motion correction using NoRMCorre
mc.motion_correct(save_movie=True)
# Determine maximum shift to be used for trimming against NaNs
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# Compare with original movie
if disp_movie and not is_3d:
m_els = cm.load(mc.fname_tot_els)
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
cm.concatenate([
m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)
],
axis=2).play(fr=60, gain=15, magnification=2,
offset=0) # press q to exit
return mc, border_to_0
def main():
pass # For compatibility between running under Spyder and the CLI
#%% Select file(s) to be processed (download if not present)
# fnames = [os.path.join(caiman_datadir(), 'example_movies/sampled3dMovieRigid.nwb')]
fnames = [
os.path.join(caiman_datadir(), 'example_movies/sampled3dMovie2.nwb')
]
# filename to be created or processed
# dataset dependent parameters
fr = 5 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
starting_time = 0.
#%% load the file and save it in the NWB format (if it doesn't exist already)
if not os.path.exists(fnames[0]):
# fnames_orig = [os.path.join(caiman_datadir(), 'example_movies/sampled3dMovieRigid.h5')] # filename to be processed
fnames_orig = [
os.path.join(caiman_datadir(), 'example_movies/sampled3dMovie2.h5')
] # filename to be processed
orig_movie = cm.load(fnames_orig, fr=fr, is3D=True)
# orig_movie = cm.load_movie_chain(fnames_orig,fr=fr,is3D=True)
# save file in NWB format with various additional info
orig_movie.save(fnames[0],
sess_desc='test',
identifier='demo 3d',
exp_desc='demo movie',
imaging_plane_description='multi plane',
emission_lambda=520.0,
indicator='none',
location='visual cortex',
starting_time=starting_time,
experimenter='NAOMi',
lab_name='Tank Lab',
institution='Princeton U',
experiment_description='Experiment Description',
session_id='Session 1',
var_name_hdf5='TwoPhotonSeries')
#%% First setup some parameters for data and motion correction
# motion correction parameters
dxy = (1., 1., 5.) # spatial resolution in x, y, and z in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (10., 10., 10.) # maximum shift in um
patch_motion_um = (50., 50., 30.
) # patch size for non-rigid correction in um
# pw_rigid = False # flag to select rigid vs pw_rigid motion correction
niter_rig = 1
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a / b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a / b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24, 4)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
is3D = True
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'niter_rig': niter_rig,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy',
'var_name_hdf5': 'acquisition/TwoPhotonSeries',
'is3D': is3D,
'splits_els': 12,
'splits_rig': 12
}
opts = params.CNMFParams(
params_dict=mc_dict
) #NOTE: default adjustments of parameters are not set yet, manually setting them now
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = True
if display_images:
m_orig = cm.load_movie_chain(fnames,
var_name_hdf5=opts.data['var_name_hdf5'],
is3D=True)
T, h, w, z = m_orig.shape # Time, plane, height, weight
m_orig = np.reshape(np.transpose(m_orig, (3, 0, 1, 2)), (T * z, h, w))
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
# NOTE: ignore dview right now for debugging purposes
# c, dview, n_processes = cm.cluster.setup_cluster(
# backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames,
dview=None,
var_name_hdf5=opts.data['var_name_hdf5'],
**opts.get_group('motion'))
# mc = MotionCorrect(fnames, dview=dview, var_name_hdf5=opts.data['var_name_hdf5'], **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# %% compare with original movie
if display_images:
m_orig = cm.load_movie_chain(fnames,
var_name_hdf5=opts.data['var_name_hdf5'],
is3D=True)
T, h, w, z = m_orig.shape # Time, plane, height, weight
m_orig = np.reshape(np.transpose(m_orig, (3, 0, 1, 2)), (T * z, h, w))
m_els = cm.load(mc.mmap_file, is3D=True)
m_els = np.reshape(np.transpose(m_els, (3, 0, 1, 2)), (T * z, h, w))
ds_ratio = 0.2
moviehandle = cm.concatenate([
m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)
],
axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
def main():
pass # For compatibility between running under Spyder and the CLI
#%%
dr = '/home/nel/Code/Voltage_imaging/exampledata/403106_3min_raw/raw_data/'
total = len([f for f in os.listdir(dr) if f.endswith('.tif')])
n_blocks = 1
n_files = np.int(np.floor(total / n_blocks))
for n in range(n_blocks):
file_list = [dr + 'cameraTube051_{:05n}.tif'.format(i + 1) for i in
np.arange(n_files * n, n_files * (n + 1), 1)]
#%% Select file(s) to be processed (download if not present)
fnames = [tuple([ff for ff in file_list[:36000]]),tuple([ff for ff in file_list[36000:72000]])]
fnames = [tuple([ff for ff in file_list[:72000]])]
# fnames = ['/home/nel/Code/Voltage_imaging/exampledata/403106_3min/datasetblock1.hdf5'] # filename to be processed
if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fnames = [download_demo(fnames[0])]
#%%
# file_list = ['/home/nel/Code/Voltage_imaging/exampledata/Un-aligned image files/403106_3min/cameraTube051_{:05n}.tif'.format(i+1) for i in range(1000)]
# mv = cm.load(file_list)
#%% First setup some parameters for data and motion correction
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
dxy = (1., 1.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (10., 10.) # maximum shift in um
patch_motion_um = (64., 64.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = False # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = False
if display_images:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# @params num_splits_to_process_els if none all the splits are processed and the movie is saved
num_splits_to_process_els = params_movie['num_splits_to_process_els']
# @params upsample_factor_grid upsample factor to avoid smearing when merging patches
upsample_factor_grid = params_movie['upsample_factor_grid']
# @params max_deviation_rigid maximum deviation allowed for patch with respect to rigid shift
max_deviation_rigid = params_movie['max_deviation_rigid']
# %% download movie if not there
if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovieJ.tif']:
# TODO: todocument
download_demo(fname[0])
fname = [os.path.join('example_movies', fname[0])]
# TODO: todocument
m_orig = cm.load_movie_chain(fname[:1])
# %% play movie
downsample_ratio = params_display['downsample_ratio']
offset_mov = -np.min(m_orig[:100])
m_orig.resize(1, 1, downsample_ratio).play(
gain=10, offset=offset_mov, fr=30, magnification=2)
# %% RUN ANALYSIS
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %% INITIALIZING
t1 = time.time()
# movie must be mostly positive for this to work
# TODO : document
hdf5File = args.infile[0]
outDir = args.outdir[0].replace('\\', '/')
online = args.online
nSources = args.nsources
maskThresh = args.maskthresh
inBasename, _ = os.path.splitext(os.path.basename(hdf5File))
outArrayFile = os.path.join(outDir, inBasename) + '_caiman_arrays.npz'
outMatlabFile = os.path.join(outDir, inBasename) + '_caiman_arrays.mat'
outImageFolder = os.path.join(outDir, inBasename + '_images')
if (args.orig_mov) is not None:
if type(args.orig_mov) is not list:
args.orig_mov = [args.orig_mov]
orig_mov = cm.load_movie_chain(args.orig_mov)
if not os.path.exists(outDir):
os.mkdir(outDir)
if not os.path.exists(outImageFolder):
os.mkdir(outImageFolder)
print('Input hdf5 file: {}'.format(hdf5File), flush=True)
print('Output directory: {}'.format(outDir), flush=True)
# load in data
print('Loading data', flush=True)
if args.online:
cnm = cnmf.online_cnmf.load_OnlineCNMF(hdf5File)
expt_names = [
'./lateral_videos/PS3_Vid1_conv.tif', './lateral_videos/PS3_Vid14_conv.tif'
]
expt_fold = [os.path.split(nm)[0].split('/')[1] for nm in gt_names]
print(expt_names)
print(expt_fold)
for e, e1 in zip(gt_names, expt_names):
print(e)
pts = scipy.io.loadmat(e)['points'][0][0]
lat_sess = scipy.signal.savgol_filter(
np.sqrt(
np.diff(np.array(pts[0]).T, axis=0)**2 +
np.diff(np.array(pts[8]).T, axis=0)**2), 3, 1)[2:]
mat_files = [e1]
m = cm.load_movie_chain(mat_files[:1], fr=100)[2:]
pl.figure()
pl.imshow(m[10], cmap='gray')
pl.plot(pts[0][10] / 3, pts[8][10] / 3, 'r*')
if mask_all:
mask = np.ones(m.shape[1:])
np.save(e[:-4] + '_mask_all_lat.npy', mask)
else:
mask = behavior.select_roi(np.median(m[::100], 0), 1)[0]
np.save(e[:-4] + '_mask_lat_hl.npy', mask)
#%%
r_values = []
only_magnitude = False
n_components = 6
if save_movie:
out.release()
out = None
cv2.destroyAllWindows()
#%% save results (optional)
save_results = False
if save_results:
np.savez('results_analysis_online_MOT_CORR.npz',
Cn=Cn, Ab=cnm2.Ab, Cf=cnm2.C_on, b=cnm2.b, f=cnm2.f,
dims=cnm2.dims, tottime=tottime, noisyC=cnm2.noisyC, shifts=shifts)
#%% create correlation image or raw data with applied shifts
Y_off = cm.load_movie_chain(fls)[200:].apply_shifts(shifts,interpolation='cubic')
Cn = Y_off.local_correlations(swap_dim=False)
pl.figure();
crd = cm.utils.visualization.plot_contours(cnm2.Ab[:,cnm2.gnb:], Cn, thr=0.95, display_numbers = False)
#%% extract results from the objects and do some plotting
A, b = cnm2.Ab[:, cnm2.gnb:], cnm2.Ab[:, :cnm2.gnb].toarray()
C, f = cnm2.C_on[cnm2.gnb:cnm2.M, t-t//epochs:t], cnm2.C_on[:cnm2.gnb, t-t//epochs:t]
noisyC = cnm2.noisyC[:,t-t//epochs:t]
b_trace = [osi.b for osi in cnm2.OASISinstances] if hasattr(cnm2, 'OASISinstances') else [0]*C.shape[0]
pl.figure()
crd = cm.utils.visualization.plot_contours(A, Cn, thr=0.9)
view_patches_bar(Yr, scipy.sparse.coo_matrix(A.tocsc()[:, :]), C[:, :], b, f,
dims[0], dims[1], YrA=noisyC[cnm2.gnb:cnm2.M] - C, img=Cn)
if save_results:
np.savez('results_analysis_online_MOT_CORR.npz',
Cn=Cn,
Ab=cnm2.Ab,
Cf=cnm2.C_on,
b=cnm2.b,
f=cnm2.f,
dims=cnm2.dims,
tottime=tottime,
noisyC=cnm2.noisyC,
shifts=shifts)
#%% create correlation image or raw data with applied shifts
Y_off = cm.load_movie_chain(fls)[200:].apply_shifts(shifts,
interpolation='cubic')
Cn = Y_off.local_correlations(swap_dim=False)
pl.figure()
crd = cm.utils.visualization.plot_contours(cnm2.Ab[:, cnm2.gnb:],
Cn,
thr=0.95,
display_numbers=False)
#%% extract results from the objects and do some plotting
A, b = cnm2.Ab[:, cnm2.gnb:], cnm2.Ab[:, :cnm2.gnb].toarray()
C, f = cnm2.C_on[cnm2.gnb:cnm2.M,
t - t // epochs:t], cnm2.C_on[:cnm2.gnb, t - t // epochs:t]
noisyC = cnm2.noisyC[:, t - t // epochs:t]
b_trace = [osi.b for osi in cnm2.OASISinstances] if hasattr(
cnm2, 'OASISinstances') else [0] * C.shape[0]
if mot_correct:
# Motion correction setup
#-------------------------------------------------------------------------------
mc = MotionCorrect(tf, dview=None, **opts.get_group('motion'))
# Run piecewise-rugu nituib cirrectuib ysubg NoRMCorre
#-------------------------------------------------------------------------------
mc.motion_correct(save_movie=True)
m_els = cm.load(mc.fname_tot_els)
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0 # maximum shift to be used for trimming against NaNs
# Compare wth original movie - This isn't currently working particularly well
#-------------------------------------------------------------------------------
if display_movie:
m_orig = cm.load_movie_chain(tf)
ds_ratio = 0.2
cm.concatenate([
m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)
],
axis=2).play(fr=60, gain=15, magnification=2,
offset=0) # press q to exit
# Memory Mapping
#-------------------------------------------------------------------------------
mapfile = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C')
Yr, dims, T = cm.load_memmap(mapfile)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# compute some summary images (correlation and peak to noise)
# if ds_factor != 1:
# m=m.resize(ds_factor,ds_factor,1)
# if len(Cn) == 0:
# Cn = np.zeros(m.shape[1:])
#
# Cn = np.maximum(Cn,m.local_correlations(swap_dim=False))
# pl.imshow(Cn,cmap='gray')
# pl.pause(.1)
#%%
# Cn = Y.local_correlations(eight_neighbours=True, swap_dim=False)
# np.save('Cn_90k.npy',Cn)
#%%
if ds > 1:
Y = cm.load_movie_chain(fls[:init_files])[
:initbatch].resize(1. / ds, 1. / ds)
else:
Y = cm.load_movie_chain(fls[:init_files])[:initbatch]
if mot_corr:
mc = Y.motion_correct(max_shift, max_shift)
Y = mc[0].astype(np.float32)
borders = np.max(mc[1])
else:
Y = Y.astype(np.float32)
if ds_factor != 1:
Y = Y.resize(ds_factor, ds_factor, 1)
gSig = 1 + (np.multiply(gSig, ds_factor)).astype(np.int)
img_min = Y.min()
from caiman.source_extraction.cnmf import params as params
from caiman.utils.utils import download_demo
from caiman.summary_images import local_correlations_movie_offline
#%% start by loading basic confiugurations
logging.basicConfig(format=
"%(relativeCreated)12d [%(filename)s:%(funcName)20s():%(lineno)s] [%(process)d] %(message)s",
# filename="/tmp/caiman.log",
level=logging.WARNING)
#%% os.getcwd() to figure out where you are and os.chdir() to your directory with the movies
fnames = ['test.tif'] # filename to be processed
#%%Check your movie first
display_movie = True # False if you dont want to see it. Press q to exit
if display_movie:
m_orig = cm.load_movie_chain(fnames) #this parameters work for a 256X256 for the 2p
ds_ratio = 0.2
m_orig.resize(1, 1, ds_ratio).play(
q_max=85, fr=3, magnification=2) # pick your fr and change your q_max appropiatetly
#%% Start building the parameters (I will build all parameters for future uses like CNMF not only mc)
#%% build parameters
# dataset dependent parameters
fr = 3 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
# parameters for source extraction and deconvolution
p = 0 # order of the autoregressive system (p=0 deconvolution off)
gnb = 2 # number of global background components
merge_thr = 0.95 # merging threshold, max correlation allowed
rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
import os
import caiman as cm
import argparse
parser = argparse.ArgumentParser(
description='save multiple video files as a single tif')
parser.add_argument('infile',
type=str,
nargs=1,
help='.txt file containing chunk file locations')
parser.add_argument('outfile',
type=str,
nargs=1,
help='output file name (.tif file name should be used)')
args = vars(parser.parse_args())
infile = args['infile'][0]
outfile = args['outfile'][0]
if '.txt' in infile:
with open(infile, 'r') as f:
flist = []
for line in f:
flist = flist + [line.replace('\n', '')]
m = cm.load_movie_chain(flist)
m.save(outfile)
# @params fname name of the movie
fname = params_movie['fname']
niter_rig = params_movie['niter_rig']
# @params max_shifts maximum allow rigid shift
max_shifts = params_movie['max_shifts']
# @params splits_rig for parallelization split the movies in num_splits chuncks across time
splits_rig = params_movie['splits_rig']
# @params num_splits_to_process_ri if none all the splits are processed and the movie is saved
num_splits_to_process_rig = params_movie['num_splits_to_process_rig']
# %% download movie if not there
channel = 0
m_orig = cm.load_movie_chain(fname[:1],
channel=(slice(None), channel, slice(None),
slice(None)))
m_orig.save(fname[0][:-4] + '_channel_' + str(channel) + '.tif')
fname[0] = fname[0][:-4] + '_channel_' + str(channel) + '.tif'
# %% play movie
downsample_ratio = params_display['downsample_ratio']
offset_mov = -np.min(m_orig[:100])
m_orig.resize(1, 1, downsample_ratio).play(gain=10,
offset=offset_mov,
fr=30,
magnification=5)
# %% RUN ANALYSIS
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters
# dataset dependent parameters
display_images = False # Set this to true to show movies and plots
fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
# motion correction parameters
niter_rig = 1 # number of iterations for rigid motion correction
max_shifts = (6, 6) # maximum allow rigid shift
# for parallelization split the movies in num_splits chuncks across time
splits_rig = 56
# start a new patch for pw-rigid motion correction every x pixels
strides = (48, 48)
# overlap between pathes (size of patch strides+overlaps)
overlaps = (24, 24)
# for parallelization split the movies in num_splits chuncks across time
splits_els = 56
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
# parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thresh = 0.8 # merging threshold, max correlation allowed
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
rf = 15
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons
# initialization method (if analyzing dendritic data using 'sparse_nmf')
init_method = 'greedy_roi'
is_dendrites = False # flag for analyzing dendritic data
# sparsity penalty for dendritic data analysis through sparse NMF
alpha_snmf = None
# parameters for component evaluation
min_SNR = 2.5 # signal to noise ratio for accepting a component
rval_thr = 0.8 # space correlation threshold for accepting a component
cnn_thr = 0.8 # threshold for CNN based classifier
#%% download the dataset if it's not present in your folder
if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fname = [download_demo(fname[0])]
#%% play the movie
# playing the movie using opencv. It requires loading the movie in memory. To
# close the video press q
m_orig = cm.load_movie_chain(fname[:1])
downsample_ratio = 0.2
offset_mov = -np.min(m_orig[:100])
moviehandle = m_orig.resize(1, 1, downsample_ratio)
if display_images:
moviehandle.play(gain=10, offset=offset_mov, fr=30, magnification=2)
#%% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
#%%% MOTION CORRECTION
# first we create a motion correction object with the parameters specified
min_mov = cm.load(fname[0], subindices=range(200)).min()
# this will be subtracted from the movie to make it non-negative
mc = MotionCorrect(fname[0],
min_mov,
dview=dview,
max_shifts=max_shifts,
niter_rig=niter_rig,
splits_rig=splits_rig,
strides=strides,
overlaps=overlaps,
splits_els=splits_els,
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
shifts_opencv=True,
nonneg_movie=True)
# note that the file is not loaded in memory
#%% Run piecewise-rigid motion correction using NoRMCorre
mc.motion_correct_pwrigid(save_movie=True)
m_els = cm.load(mc.fname_tot_els)
bord_px_els = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
# maximum shift to be used for trimming against NaNs
#%% compare with original movie
moviehandle = cm.concatenate([
m_orig.resize(1, 1, downsample_ratio) + offset_mov,
m_els.resize(1, 1, downsample_ratio)
],
axis=2)
display_images = False
if display_images:
moviehandle.play(fr=60, q_max=99.5, magnification=2,
offset=0) # press q to exit
#%% MEMORY MAPPING
# memory map the file in order 'C'
fnames = mc.fname_tot_els # name of the pw-rigidly corrected file.
border_to_0 = bord_px_els # number of pixels to exclude
fname_new = cm.save_memmap(fnames,
base_name='memmap_',
order='C',
border_to_0=bord_px_els) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
d1, d2 = dims
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
#%% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
#%% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
t1 = time.time()
cnm = cnmf.CNMF(n_processes=1,
k=K,
gSig=gSig,
merge_thresh=merge_thresh,
p=0,
dview=dview,
rf=rf,
stride=stride_cnmf,
memory_fact=1,
method_init=init_method,
alpha_snmf=alpha_snmf,
only_init_patch=False,
gnb=gnb,
border_pix=bord_px_els)
cnm = cnm.fit(images)
#%% plot contours of found components
Cn = cm.local_correlations(images.transpose(1, 2, 0))
Cn[np.isnan(Cn)] = 0
plt.figure()
crd = plot_contours(cnm.A, Cn, thr=0.9)
plt.title('Contour plots of found components')
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
idx_components, idx_components_bad, SNR_comp, r_values, cnn_preds = \
estimate_components_quality_auto(images, cnm.A, cnm.C, cnm.b, cnm.f,
cnm.YrA, fr, decay_time, gSig, dims,
dview=dview, min_SNR=min_SNR,
r_values_min=rval_thr, use_cnn=False,
thresh_cnn_min=cnn_thr)
#%% PLOT COMPONENTS
if display_images:
plt.figure()
plt.subplot(121)
crd_good = cm.utils.visualization.plot_contours(cnm.A[:,
idx_components],
Cn,
thr=.8,
vmax=0.75)
plt.title('Contour plots of accepted components')
plt.subplot(122)
crd_bad = cm.utils.visualization.plot_contours(
cnm.A[:, idx_components_bad], Cn, thr=.8, vmax=0.75)
plt.title('Contour plots of rejected components')
#%% VIEW TRACES (accepted and rejected)
if display_images:
view_patches_bar(Yr,
cnm.A.tocsc()[:, idx_components],
cnm.C[idx_components],
cnm.b,
cnm.f,
dims[0],
dims[1],
YrA=cnm.YrA[idx_components],
img=Cn)
view_patches_bar(Yr,
cnm.A.tocsc()[:, idx_components_bad],
cnm.C[idx_components_bad],
cnm.b,
cnm.f,
dims[0],
dims[1],
YrA=cnm.YrA[idx_components_bad],
img=Cn)
#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
A_in, C_in, b_in, f_in = cnm.A[:, idx_components], cnm.C[
idx_components], cnm.b, cnm.f
cnm2 = cnmf.CNMF(n_processes=1,
k=A_in.shape[-1],
gSig=gSig,
p=p,
dview=dview,
merge_thresh=merge_thresh,
Ain=A_in,
Cin=C_in,
b_in=b_in,
f_in=f_in,
rf=None,
stride=None,
gnb=gnb,
method_deconvolution='oasis',
check_nan=True)
cnm2 = cnm2.fit(images)
#%% Extract DF/F values
F_dff = detrend_df_f(cnm2.A,
cnm2.b,
cnm2.C,
cnm2.f,
YrA=cnm2.YrA,
quantileMin=8,
frames_window=250)
#%% Show final traces
cnm2.view_patches(Yr, dims=dims, img=Cn)
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
#%% reconstruct denoised movie
denoised = cm.movie(cnm2.A.dot(cnm2.C) + cnm2.b.dot(cnm2.f)).reshape(
dims + (-1, ), order='F').transpose([2, 0, 1])
#%% play along side original data
moviehandle = cm.concatenate([
m_els.resize(1, 1, downsample_ratio),
denoised.resize(1, 1, downsample_ratio)
],
axis=2)
if display_images:
moviehandle.play(fr=60, gain=15, magnification=2,
offset=0) # press q to exit
def piecewise_rigid_registration(fnames,
max_shifts=(6, 6),
strides=(48, 48),
overlaps=(24, 24),
num_frames_split=100,
max_deviation_rigid=3,
pw_rigid=False,
shifts_opencv=True,
border_nan='copy',
highPassFilter=False,
sigma=7):
'''Apply CaImAn piecewise rigid registration on raw imaging data
:param fnames: full path filename for motion correction, possible extensions are tif, avi, npy
:param max_shifts: maximum allowed rigid shift in pixels (view the movie to get a sense of motion)
:param strides: create a new patch every x pixels for pw-rigid correction
:param overlaps: overlap between pathes (size of patch strides+overlaps)
:param num_frames_split: length in frames of each chunk of the movie (to be processed in parallel)
:param max_deviation_rigid: maximum deviation allowed for patch with respect to rigid shifts
:param pw_rigid: flag for performing rigid or piecewise rigid motion correction
:param shifts_opencv: flag for correcting motion using bicubic interpolation (otherwise FFT interpolation is used)
:param border_nan: replicate values along the boundary (if True, fill in with NaN)
:param highPassFilter: whether applying high pass filter
:param sigma: lowpass gaussian filter sigma value
:return: motion corrected image in numpy
'''
fnames = [fnames]
m_orig = cm.load_movie_chain(fnames)
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
# create a motion correction object
mc = MotionCorrect(fnames,
dview=dview,
max_shifts=max_shifts,
strides=strides,
overlaps=overlaps,
max_deviation_rigid=max_deviation_rigid,
shifts_opencv=shifts_opencv,
nonneg_movie=True,
border_nan=border_nan)
# correct for rigid motion correction and save the file (in memory mapped form)
mc.motion_correct(save_movie=True)
# load motion corrected movie and apply piece-wise rigid registration
m_rig = cm.load(mc.mmap_file)
mc.pw_rigid = True # turn the flag to True for pw-rigid motion correction
mc.template = mc.mmap_file # use the template obtained before to save in computation (optional)
mc.motion_correct(save_movie=True, template=mc.total_template_rig)
cm.stop_server(dview=dview) # stop the server
m_els = cm.load(mc.fname_tot_els)
Y = np.array(m_els.tolist())
if highPassFilter:
Y = high_pass_filtering(Y, sigma)
return Y
def main():
pass # For compatibility between running under Spyder and the CLI
#%% First setup some parameters for data and motion correction
# dataset dependent parameters
fname = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid motion correction in um
# motion correction parameters
pwrigid_motion_correct = True # flag to select rigid vs pw_rigid motion correction
max_shifts = tuple([int(a/b) for a, b in zip(max_shift_um, dxy)])
# maximum allow rigid shift in pixels
# for parallelization split the movies in num_splits chuncks across time
splits_rig = 56
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch strides+overlaps)
overlaps = (24, 24)
# for parallelization split the movies in num_splits chuncks across time
splits_els = 56
upsample_factor_grid = 4 # upsample factor to avoid smearing when merging patches
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
#%% download the dataset if it's not present in your folder
if fname[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fname = [download_demo(fname[0])]
#%% play the movie
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = False
if display_images:
m_orig = cm.load_movie_chain(fname)
downsample_ratio = 0.2
moviehandle = m_orig.resize(1, 1, downsample_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
#%% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
#%%% MOTION CORRECTION
# first we create a motion correction object with the parameters specified
min_mov = cm.load(fname[0], subindices=range(200)).min()
# this will be subtracted from the movie to make it non-negative
mc = MotionCorrect(fname, min_mov, dview=dview, max_shifts=max_shifts,
splits_rig=splits_rig,
strides=strides, overlaps=overlaps,
splits_els=splits_els, border_nan='copy',
upsample_factor_grid=upsample_factor_grid,
max_deviation_rigid=max_deviation_rigid,
shifts_opencv=True, nonneg_movie=True)
# note that the file is not loaded in memory
#%% Run piecewise-rigid motion correction using NoRMCorre
if pwrigid_motion_correct:
mc.motion_correct_pwrigid(save_movie=True)
m_els = cm.load(mc.fname_tot_els)
bord_px_els = np.ceil(np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
fnames = mc.fname_tot_els # name of the pw-rigidly corrected file.
else:
mc.motion_correct_rigid(save_movie=True)
m_els = cm.load(mc.fname_tot_rig)
bord_px_els = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
fnames = mc.fname_tot_rig # name of the rigidly corrected file.
# maximum shift to be used for trimming against NaNs
#%% compare with original movie
if display_images:
downsample_ratio = 0.2
moviehandle = cm.concatenate([m_orig.resize(1, 1, downsample_ratio) - min_mov,
m_els.resize(1, 1, downsample_ratio)],
axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
#%% MEMORY MAPPING
# memory map the file in order 'C'
border_to_0 = bord_px_els # exclude borders due to motion correction
# border_to_0 = 0 if mc.border_nan is 'copy' else bord_px_els
# you can include boundaries if you used the 'copy' option in the motion
# correction, although be careful abou the components near the boundaries
fname_new = cm.save_memmap(fnames, base_name='memmap_', order='C',
border_to_0=border_to_0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
#%% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
#%% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thresh = 0.8 # merging threshold, max correlation allowed
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
rf = 15
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
# parameters for component evaluation
opts = params.CNMFParams(dims=dims, fr=fr, decay_time=decay_time,
method_init=method_init, gSig=gSig,
merge_thresh=merge_thresh, p=p, gnb=gnb, k=K,
rf=rf, stride=stride_cnmf, rolling_sum=True)
#%% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
opts.set('temporal', {'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
#%% plot contours of found components
Cn = cm.local_correlations(images.transpose(1, 2, 0))
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2.5 # signal to noise ratio for accepting a component
rval_thr = 0.8 # space correlation threshold for accepting a component
cnn_thr = 0.8 # threshold for CNN based classifier
cnm.params.set('quality', {'fr': fr,
'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr})
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
#%% PLOT COMPONENTS
cnm.estimates.plot_contours(img=Cn, idx=cnm.estimates.idx_components)
#%% VIEW TRACES (accepted and rejected)
if display_images:
cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components)
cnm.estimates.view_components(images, img=Cn, idx=cnm.estimates.idx_components_bad)
#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm.dview = None
cnm2 = deepcopy(cnm)
cnm2.dview = dview
cnm2.params.set('patch', {'rf': None})
cnm2.params.set('temporal', {'p': p})
cnm2 = cnm2.fit(images)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(Yr, img=Cn)
#%% reconstruct denoised movie (press q to exit)
if display_images:
cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2,
magnification=2,
bpx=border_to_0,
include_bck=True)
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
import caiman as cm
import os
import numpy as np
os.chdir('C:\\Users\\grossar\\Data\\Caiman\\CaImAn\\')
#source activate CaImAn
### Loading movies
single_movie = cm.load('example_movie\\demoMovie.tif')
print(single_movie.shape)
file_names = ['example_movies/demoMovie.tif', 'example_movies/demoMovie.tif'
] # for the sake of the example we repeat the same movie
movies_chained = cm.load_movie_chain(file_names)
print(movies_chained.shape)
### Generating a random movie
movie_random = cm.movie(np.random.random([1000, 100, 100]))
### Saving Movies
movie_random.save('movie_random-10-29.tif')
np.random.random([2, 3, 3])
### Visualizing Movies
movies_chained.play(magnification=2, fr=30, q_min=0.1, q_max=99.75)
'./gc-AG052014-02/062314_-213 -218 67_WHISK_COND_A__2/points_trial1.mat',
'./AG052014-01/063014_98 -639 144_COND_A__1/points_trial9.mat',
'./AG052014-01/070214_79 -645 131_COND_A_/points_trial99.mat']
expt_fold = [os.path.split(nm)[0].split('/')[1] for nm in gt_names]
expt_names = [os.path.split(nm)[0].split('/')[2] for nm in gt_names]
print(expt_names)
print(expt_fold)
#%%
for e in gt_names:
print(e)
pts = scipy.io.loadmat(e)['points'][0][0]
whisk_sess = scipy.signal.savgol_filter(
np.abs(np.diff(np.array(pts[8]).T, axis=0)), 3, 1)[2:]
num_tr = str(re.findall('\d+', e[-8:-4])[0])
mat_files = [os.path.join(os.path.split(e)[0], 'trial' + num_tr + '.mat')]
m = cm.load_movie_chain(mat_files[:1], fr=100)[2:]
pl.figure()
pl.imshow(m[0], cmap='gray')
pl.plot(pts[0][10] / 3, pts[8][10] / 3, 'r*')
mask = behavior.select_roi(np.median(m[::100], 0), 1)[0]
if mask_all:
np.save(e[:-4] + '_mask_all.npy', mask)
else:
np.save(e[:-4] + '_mask_whisk.npy', mask)
# if ds_factor != 1:
# m=m.resize(ds_factor,ds_factor,1)
# if len(Cn) == 0:
# Cn = np.zeros(m.shape[1:])
#
# Cn = np.maximum(Cn,m.local_correlations(swap_dim=False))
# pl.imshow(Cn,cmap='gray')
# pl.pause(.1)
#%%
# Cn = Y.local_correlations(eight_neighbours=True, swap_dim=False)
# np.save('Cn_90k.npy',Cn)
#%%
if ds > 1:
Y = cm.load_movie_chain(fls[:init_files])[:initbatch].resize(
1. / ds, 1. / ds)
else:
Y = cm.load_movie_chain(fls[:init_files])[:initbatch]
if mot_corr:
mc = Y.motion_correct(max_shift, max_shift)
Y = mc[0].astype(np.float32)
borders = np.max(mc[1])
else:
Y = Y.astype(np.float32)
if ds_factor != 1:
Y = Y.resize(ds_factor, ds_factor, 1)
gSig = 1 + (np.multiply(gSig, ds_factor)).astype(np.int)
img_min = Y.min()
Y -= img_min
# process input file
if '.txt' in infile[0]:
with open(infile[0], 'r') as f:
flist = []
for line in f:
flist = flist + [line.replace('\n','')]
fname = flist
elif os.path.isdir(infile[0]):
tmpMovPath = os.path.join(infile[0], 'tmp_mov.tif')
if not os.path.isfile(tmpMovPath):
tfiles = glob.glob( os.path.join(infile[0], '*.tiff')) + glob.glob( os.path.join(infile[0], '*.tif'))
tfiles = sorted(list(set(tfiles)))
with warnings.catch_warnings():
tmpMov = cm.load_movie_chain(tfiles)
tmpMov.save(tmpMovPath)
del tmpMov
fname = [tmpMovPath]
else:
fname = infile
print('Using file(s): {}'.format(fname))
# get file extension (of first file)
_, fext = os.path.splitext(fname[0])
## process template into ndarray
if mc_temp is not None:
mc_temp = tif.imread(mc_temp)
if len(mc_temp.shape)>2:
def main():
pass # For compatibility between running under Spyder and the CLI
#%%
"""
General parameters
"""
play_movie = 1
plot_extras = 1
plot_extras_cell = 1
compute_mc_metrics = 1
#%% Select file(s) to be processed (download if not present)
"""
Load file
"""
#fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
#if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
# fnames = [download_demo(fnames[0])]
#fnames = ['/home/yuriy/Desktop/Data/rest1_5_9_19_cut.tif']
#f_dir = 'C:\\Users\\rylab_dataPC\\Desktop\\Yuriy\\caiman_data\\short\\'
f_dir = 'G:\\analysis\\190828-calcium_voltage\\soma_dendrites\\pCAG_jREGECO1a_ASAP3_anesth_001\\'
f_name = 'Ch1'
f_ext = 'tif'
fnames = [f_dir + f_name + '.' + f_ext]
#fnames = ['C:/Users/rylab_dataPC/Desktop/Yuriy/caiman_data/rest1_5_9_19_2_cut_ca.hdf5']
#%% First setup some parameters for data and motion correction
"""
Parameters
"""
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 1
#0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
#max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
max_shifts = [6, 6]
# start a new patch for pw-rigid motion correction every x pixels
#strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
strides = [48, 48]
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
if play_movie:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# type "mc."and press TAB to see all interesting associated variables and self. outputs
# interesting outputs
# saved file is mc.fname_tot_els / mc.fname_tot_rig
# mc.x_shifts_els / mc.y_shifts_els: shifts in x/y per frame per patch
# mc.coord_shifts_els: coordinates associated to patches with shifts
# mc.total_template_els: updated template for pw
# mc.total_template_rig: updated template for rigid
# mc.templates_rig: templates for each iteration in rig
#%% # compute metrics for the results (TAKES TIME!!)
if compute_mc_metrics:
# not finished
bord_px_els = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(np.int)
final_size = np.subtract(
mc.total_template_els.shape,
2 * bord_px_els) # remove pixels in the boundaries
winsize = 100
swap_dim = False
resize_fact_flow = .2 # downsample for computing ROF
tmpl_rig, correlations_orig, flows_orig, norms_orig, crispness_orig = cm.motion_correction.compute_metrics_motion_correction(
fnames[0],
final_size[0],
final_size[1],
swap_dim,
winsize=winsize,
play_flow=False,
resize_fact_flow=resize_fact_flow)
plt.figure()
plt.plot(correlations_orig)
# %% compare with original movie
if play_movie:
m_orig = cm.load_movie_chain(fnames)
m_els = cm.load(mc.mmap_file)
ds_ratio = 0.2
moviehandle = cm.concatenate([
m_orig.resize(1, 1, ds_ratio) - mc.min_mov * mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)
],
axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
del m_orig
del m_els
if plot_extras:
# plot total template
plt.figure()
plt.imshow(mc.total_template_els)
plt.title('Template after iteration')
# plot x and y corrections
plt.figure()
plt.plot(mc.shifts_rig)
plt.title('Rigid motion correction xy movement')
plt.legend(['x shift', 'y shift'])
plt.xlabel('frames')
# %% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file,
base_name='memmap_',
order='C',
border_to_0=border_to_0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
# %% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
# %% parameters for source extraction and deconvolution
p = 2 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 2 # number of components per patch
gSig = [15, 15] # expected half size of neurons in pixels
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
ssub = 1 # spatial subsampling during initialization
tsub = 1 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {
'fnames': fnames,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub
}
opts.change_params(params_dict=opts_dict)
# %% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
if plot_extras_cell:
num_cell_plot = 51
plt.figure()
plt.plot(cnm.estimates.C[num_cell_plot, :])
plt.title('Temporal component')
plt.legend(['Cell ' + str(num_cell_plot)])
# plot component sptial profile A
# first convert back to dense components
plot_spat_A = cnm.estimates.A[:, num_cell_plot].toarray().reshape(
list(dims))
plt.figure()
plt.imshow(plot_spat_A)
plt.title('Spatial component cell ' + str(num_cell_plot))
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% plot contours of found components
Cn = cm.local_correlations(images, swap_dim=False)
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
if plot_extras:
plt.figure()
plt.imshow(Cn)
plt.title('Local correlations')
# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm.params.change_params({'p': p})
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # signal to noise ratio for accepting a component
rval_thr = 0.90 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set(
'quality', {
'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest
})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% PLOT COMPONENTS
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
plt.suptitle('Component selection: min_SNR=' + str(min_SNR) +
'; rval_thr=' + str(rval_thr) + '; cnn prob range=[' +
str(cnn_lowest) + ' ' + str(cnn_thr) + ']')
# %% VIEW TRACES (accepted and rejected)
if plot_extras:
cnm2.estimates.view_components(images,
img=Cn,
idx=cnm2.estimates.idx_components)
plt.suptitle('Accepted')
cnm2.estimates.view_components(images,
img=Cn,
idx=cnm2.estimates.idx_components_bad)
plt.suptitle('Rejected')
#plt.figure();
#plt.plot(cnm2.estimates.YrA[0,:]+cnm2.estimates.C[0,:])
#
#
#
#
#plt.figure();
#plt.plot(cnm2.estimates.R[0,:]-cnm2.estimates.YrA[0,:]);
#plt.plot();
#plt.show();
#
#
#plt.figure();
#plt.plot(cnm2.estimates.detrend_df_f[1,:])
# these store the good and bad components, and next step sorts them
# cnm2.estimates.idx_components
# cnm2.estimates.idx_components_bad
#%% update object with selected components
#cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(img=Cn)
plt.suptitle("Final results")
#%% Save the mc data as in cmn struct as well
##
#mc_out = dict(
# pw_rigid = mc.pw_rigid,
# fname = mc.fname,
# mmap_file = mc.mmap_file,
# total_template_els = mc.total_template_els,
# total_template_rig = mc.total_template_rig,
# border_nan = mc.border_nan,
# border_to_0 = mc.border_to_0,
# x_shifts_els = mc.x_shifts_els,
# y_shifts_els = mc.y_shifts_els,
# Cn = Cn
# )
#
#
#deepdish.io.save(fnames[0] + '_mc_data.hdf5', mc_out)
#%% reconstruct denoised movie (press q to exit)
if play_movie:
cnm2.estimates.play_movie(images,
q_max=99.9,
gain_res=2,
magnification=2,
bpx=border_to_0,
include_bck=False) # background not shown
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
save_results = True
if save_results:
cnm2.save(fnames[0][:-4] + '_results.hdf5')
def main():
pass # For compatibility between running under Spyder and the CLI
#%% Select file(s) to be processed (download if not present)
root = '/Users/hheiser/Desktop/testing data/chronic_M2N3/0d_baseline/channel1'
fnames = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\1\file_00003.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\2\file_00004.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\3\file_00005.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\4\file_00006.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\5\file_00007.tif',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M22\20191208\N1\6\file_00008.tif']
fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fnames = [download_demo(fnames[0])]
#%% First setup some parameters for data and motion correction
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
#%% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = True
if display_images:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=30, magnification=1, do_loop=False)
#%% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
#%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
#%% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
#%% compare with original movie
if display_images:
m_orig = cm.load_movie_chain(fnames[:3])
m_els = cm.load(mmap_file[:3])
ds_ratio = 0.2
moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)], axis=2)
moviehandle.play(fr=15, q_max=99.5, magnification=2) # press q to exit
#%% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan == 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
mmap_file = [r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\1\file_00001_els__d1_512_d2_512_d3_1_order_F_frames_1147_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\2\file_00002_els__d1_512_d2_512_d3_1_order_F_frames_2520_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\3\file_00003_els__d1_512_d2_512_d3_1_order_F_frames_3814_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\4\file_00004_els__d1_512_d2_512_d3_1_order_F_frames_5154_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\5\file_00005_els__d1_512_d2_512_d3_1_order_F_frames_2677_.mmap',
r'W:\Neurophysiology-Storage1\Wahl\Hendrik\PhD\Data\Batch2\M19\20191219\N2\6\file_00006_els__d1_512_d2_512_d3_1_order_F_frames_3685_.mmap']
fname_new = r'E:\PhD\Data\DG\M14_20191014\N1\memmap__d1_512_d2_512_d3_1_order_C_frames_34939_.mmap'
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C',
border_to_0=0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
#%% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
#%% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons in pixels
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
ssub = 2 # spatial subsampling during initialization
tsub = 2 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {'fnames': fnames,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub}
opts.change_params(params_dict=opts_dict)
#%% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
#opts.change_params({'p': 1,'rf':None, 'only_init':False})
opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
#%% RUN CNMF SEEDED WITH MANUAL MASK
# load mask
mask = np.asarray(imageio.imread('/Users/hheiser/Desktop/testing data/file_00020_no_motion/avg_mask_fixed.png'), dtype=bool)
# get component ROIs from the mask and plot them
Ain, labels, mR = cm.base.rois.extract_binary_masks(mask)
# plot original mask and extracted labels to check mask
fig, ax = plt.subplots(1,2)
ax[0].imshow(-mR,cmap='binary')
ax[0].set_title('Original mask')
ax[1].imshow(labels)
ax[1].set_title('Extracted labelled ROIs')
""""
plt.figure()
crd = cm.utils.visualization.plot_contours(
Ain.astype('float32'), mR, thr=0.99, display_numbers=True) # todo check if this is important for the pipeline
plt.title('Contour plots of detected ROIs in the structural channel')
"""
opts.change_params({'rf': None, 'only_init': False})
# run CNMF seeded with this mask
cnm_corr = cnmf.CNMF(n_processes, params=opts, dview=dview, Ain=Ain)
cnm_corr_seed = cnm_corr_seed.fit(images)
#cnm_seed = cnm_seed.fit_file(motion_correct=False)
#%% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
#%% plot contours of found components
Cn = cm.local_correlations(images, swap_dim=False)
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
#%% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm.params.change_params({'p': p})
cnm2 = cnm.refit(images, dview=dview)
#%% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # signal to noise ratio for accepting a component
rval_thr = 0.85 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set('quality', {'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest})
cnm2.estimates.evaluate_components(images, params=cnm2.params, dview=dview)
#%% PLOT COMPONENTS
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
#%% VIEW TRACES (accepted and rejected)
if display_images:
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components)
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components_bad)
#%% update object with selected components
#### -> will delete rejected components!
cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(img=Cn)
#%% reconstruct denoised movie (press q to exit)
if display_images:
cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2,
magnification=1,
bpx=border_to_0,
include_bck=True) # background not shown
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
#%% save results
dirname = fnames[0][:-4] + "_results.hdf5"
cnm2.estimates.Cn = Cn
cnm2.save(dirname)
#load results
cnm2 = cnmf.load_CNMF(dirname)
mov_name = fnames[0][:-4] + "_movie_restored_2_gain.tif"
helper.save_movie(cnm2.estimates,images,mov_name,frame_range=range(200),include_bck=True)
def main():
pass # For compatibility between running under Spyder and the CLI
#%% Select file(s) to be processed (download if not present)
fnames = ['Sue_2x_3000_40_-46.tif'] # filename to be processed
if fnames[0] in ['Sue_2x_3000_40_-46.tif', 'demoMovie.tif']:
fnames = [download_demo(fnames[0])]
#%% First setup some parameters for data and motion correction
# dataset dependent parameters
fr = 30 # imaging rate in frames per second
decay_time = 0.4 # length of a typical transient in seconds
dxy = (2., 2.) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% play the movie (optional)
# playing the movie using opencv. It requires loading the movie in memory.
# To close the video press q
display_images = True
if display_images:
m_orig = cm.load_movie_chain(fnames)
ds_ratio = 0.2
moviehandle = m_orig.resize(1, 1, ds_ratio)
moviehandle.play(q_max=99.5, fr=60, magnification=2)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# %% compare with original movie
if display_images:
m_orig = cm.load_movie_chain(fnames)
m_els = cm.load(mc.mmap_file)
ds_ratio = 0.2
moviehandle = cm.concatenate([m_orig.resize(1, 1, ds_ratio) - mc.min_mov*mc.nonneg_movie,
m_els.resize(1, 1, ds_ratio)], axis=2)
moviehandle.play(fr=60, q_max=99.5, magnification=2) # press q to exit
# %% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file, base_name='memmap_', order='C',
border_to_0=border_to_0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
# %% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 2 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 15
# half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 4 # number of components per patch
gSig = [4, 4] # expected half size of neurons in pixels
# initialization method (if analyzing dendritic data using 'sparse_nmf')
method_init = 'greedy_roi'
ssub = 2 # spatial subsampling during initialization
tsub = 2 # temporal subsampling during intialization
# parameters for component evaluation
opts_dict = {'fnames': fnames,
'p': p,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub}
opts.change_params(params_dict=opts_dict);
# %% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0). If you want to have
# deconvolution within each patch change params.patch['p_patch'] to a
# nonzero value
#opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% plot contours of found components
Cns = local_correlations_movie_offline(mc.mmap_file[0],
remove_baseline=True, window=1000, stride=1000,
winSize_baseline=100, quantil_min_baseline=10,
dview=dview)
Cn = Cns.max(axis=0)
Cn[np.isnan(Cn)] = 0
cnm.estimates.plot_contours(img=Cn)
plt.title('Contour plots of found components')
#%% save results
cnm.estimates.Cn = Cn
cnm.save(fname_new[:-5]+'_init.hdf5')
# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # signal to noise ratio for accepting a component
rval_thr = 0.85 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.1 # neurons with cnn probability lower than this value are rejected
cnm2.params.set('quality', {'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
# %% PLOT COMPONENTS
cnm2.estimates.plot_contours(img=Cn, idx=cnm2.estimates.idx_components)
# %% VIEW TRACES (accepted and rejected)
if display_images:
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components)
cnm2.estimates.view_components(images, img=Cn,
idx=cnm2.estimates.idx_components_bad)
#%% update object with selected components
cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#%% Show final traces
cnm2.estimates.view_components(img=Cn)
#%%
cnm2.estimates.Cn = Cn
cnm2.save(cnm2.mmap_file[:-4] + 'hdf5')
#%% reconstruct denoised movie (press q to exit)
if display_images:
cnm2.estimates.play_movie(images, q_max=99.9, gain_res=2,
magnification=2,
bpx=border_to_0,
include_bck=False) # background not shown
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
log_files = glob.glob('*_LOG_*')
for log_file in log_files:
os.remove(log_file)
# @params fname name of the movie
fname = params_movie['fname']
niter_rig = params_movie['niter_rig']
# @params max_shifts maximum allow rigid shift
max_shifts = params_movie['max_shifts']
# @params splits_rig for parallelization split the movies in num_splits chuncks across time
splits_rig = params_movie['splits_rig']
# @params num_splits_to_process_ri if none all the splits are processed and the movie is saved
num_splits_to_process_rig = params_movie['num_splits_to_process_rig']
# %% download movie if not there
channel = 0
m_orig = cm.load_movie_chain(fname[:1], channel=(
slice(None), channel, slice(None), slice(None)))
m_orig.save(fname[0][:-4] + '_channel_' + str(channel) + '.tif')
fname[0] = fname[0][:-4] + '_channel_' + str(channel) + '.tif'
# %% play movie
downsample_ratio = params_display['downsample_ratio']
offset_mov = -np.min(m_orig[:100])
m_orig.resize(1, 1, downsample_ratio).play(
gain=10, offset=offset_mov, fr=30, magnification=5)
# %% RUN ANALYSIS
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %% INITIALIZING
t1 = time.time()
# movie must be mostly positive for this to work
import sys,os
import pickle
import pandas as pd
#%% motion correction with concatenated videos
videoconcat = sys.argv[1]
newpath = os.path.dirname(videoconcat)
ms_ts_name = os.path.join(newpath,"ms_ts.pkl")
if os.path.exists(ms_ts_name):
with open(ms_ts_name, "rb") as f:
ms_ts= pickle.load(f)
else:
print("there is no mt_ts.pkl existed")
fnames=[videoconcat]
m_orig = cm.load_movie_chain(fnames)
# start a cluster for parallel processing (if a cluster already exists it will be closed and a new session will be opened)
if 'dview' in locals():
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# dataset dependent parameters
fr = 30 # movie frame rate
decay_time = 0.4 # length of a typical transient in seconds
motion_correct = True # flag for motion correction
# motion correction parameters
pw_rigid = False # flag for pw-rigid motion correction
gSig_filt = (8,8) # size of filter, in general gSig (see below),
def get_mc_movie(self):
path = self.get_mc_movie_path()
return cm.load_movie_chain([str(path)])
