Cn = np.array(
cm.load(('/'.join(params_movie['gtname'].split('/')[:-2] +
['projections', 'correlation_image.tif'])
))).squeeze()
check_nan = False
# %% start cluster
# TODO: show screenshot 10
try:
cm.stop_server()
dview.terminate()
except:
print('No clusters to stop')
c, dview, n_processes = setup_cluster(backend=backend_patch,
n_processes=n_processes,
single_thread=False)
# %%
params_dict = {
'fnames': [fname_new],
'fr': params_movie['fr'],
'decay_time': params_movie['decay_time'],
'rf': params_movie['rf'],
'stride': params_movie['stride_cnmf'],
'K': params_movie['K'],
'gSig': params_movie['gSig'],
'merge_thr': params_movie['merge_thresh'],
'p': global_params['p'],
'nb': global_params['gnb'],
'only_init': global_params['only_init_patch'],
'dview': dview,
from ipyparallel import Client
# mpl.use('Qt5Agg')
import pylab as pl
pl.ion()
#%%
import caiman as cm
from caiman.source_extraction.cnmf import cnmf as cnmf
from caiman.source_extraction.cnmf.utilities import extract_DF_F
from caiman.components_evaluation import evaluate_components
from caiman.utils.visualization import plot_contours, view_patches_bar
from caiman.utils.utils import download_demo
from caiman.cluster import setup_cluster
# %% RUN ANALYSIS
c, dview, n_processes = setup_cluster(backend='local',
n_processes=None,
single_thread=False)
#%%
is_patches = True
is_dendrites = False
if is_dendrites == True:
# THIS METHOd CAN GIVE POSSIBLY INCONSISTENT RESULTS ON SOMAS WHEN NOT USED WITH PATCHES
init_method = 'sparse_nmf'
alpha_snmf = 10e1 # this controls sparsity
else:
init_method = 'greedy_roi'
alpha_snmf = None #10e2 # this controls sparsity
#%% FOR LOADING ALL TIFF FILES IN A FILE AND SAVING THEM ON A SINGLE MEMORY MAPPABLE FILE
fnames = ['demoMovieJ.tif']
results = []
# %%
for params_movie in np.array(params_movies)[ID]:
# %% start cluster
#
# TODO: show screenshot 10
fname_new = os.path.join(base_folder, params_movie['fname'])
if generate_data:
try:
cm.stop_server()
dview.terminate()
except:
print('No clusters to stop')
c, dview, n_processes = setup_cluster(backend='multiprocessing',
n_processes=20,
single_thread=False)
# %% prepare ground truth masks
gt_file = os.path.join(
os.path.split(fname_new)[0],
os.path.split(fname_new)[1][:-4] + 'match_masks.npz')
gSig = params_movie['gSig']
with np.load(gt_file, encoding='latin1') as ld:
Cn_orig = ld['Cn']
dims = (ld['d1'][()], ld['d2'][()])
gt_estimate = Estimates(A=scipy.sparse.csc_matrix(ld['A_gt'][()]),
b=ld['b_gt'],
C=ld['C_gt'],
f=ld['f_gt'],
def run(work_dir: str, UUID: str, save_temp_files: str):
logging.basicConfig(
stream=sys.stdout,
level=logging.DEBUG,
format=
"%(relativeCreated)12d [%(filename)s:%(funcName)20s():%(lineno)s] [%(process)d] %(message)s"
)
start_time = time()
batch_dir = os.environ['CURR_BATCH_DIR']
save_temp_files = bool(int(save_temp_files))
output = {'status': 0, 'output_info': ''}
n_processes = int(os.environ['_MESMERIZE_N_THREADS'])
filepath = os.path.join(work_dir, UUID)
imgpath = f'{filepath}_input.tiff'
input_params = pickle.load(open(f'{filepath}.params', 'rb'))
print('******** Creating process pool *********')
c, dview, n_processes = setup_cluster(backend='local',
n_processes=n_processes,
single_thread=False,
ignore_preexisting=True)
try:
if input_params['use_memmap']:
memmap_uuid = input_params['memmap_uuid']
memmap_batchdir = glob(
os.path.join(batch_dir, f'memmap-{memmap_uuid}*.mmap'))
# Check batch dir
if len(memmap_batchdir) > 0:
memmap_path = memmap_batchdir[0]
print(
f'********** Found existing memmap in batch dir: {memmap_path} ********** '
)
# copy to work dir
if not os.path.samefile(batch_dir, work_dir):
print('**** Copying memmap to work dir ****')
shutil.copy(memmap_path, work_dir)
memmap_path = glob(
os.path.join(work_dir,
f'memmap-{memmap_uuid}*.mmap'))[0]
else:
# remake the memmap with the same UUID so that future batch items can rely on it
print(
'********** Memmap not found, re-making memmap with the same UUID **********'
)
memmap_path = cm.save_memmap([imgpath],
base_name=f'memmap-{memmap_uuid}',
is_3D=True,
order='C',
dview=dview)
else:
print('********** Making memmap **********')
memmap_path = cm.save_memmap([imgpath],
base_name=f'memmap-{UUID}',
is_3D=True,
order='C',
dview=dview)
print(f'Using memmap:\n{memmap_path}')
print('********** Loading memmap **********')
Yr, dims, T = cm.load_memmap(memmap_path)
Y = np.reshape(Yr, dims + (T, ), order='F')
images = np.reshape(Yr.T, [T] + list(dims), order='F')
if input_params['use_patches']:
cnm = cnmf.CNMF(n_processes=n_processes,
dview=dview,
only_init_patch=True,
**input_params['cnmf_kwargs'])
else:
cnm = cnmf.CNMF(n_processes,
dview=dview,
**input_params['cnmf_kwargs'])
cnm.fit(images)
print('Number of components:' + str(cnm.estimates.A.shape[-1]))
cnm.params.change_params(
params_dict={
**input_params['eval_kwargs'], 'use_cnn': False
})
cnm.estimates.evaluate_components(images, cnm.params, dview=dview)
if input_params['refit']:
cnm.params.set('temporal', {'p': input_params['cnmf_kwargs']['p']})
cnm_ = cnm.refit(images)
else:
cnm_ = cnm
out_filename = f'{UUID}_results.hdf5'
cnm_.save(out_filename)
output_files = [out_filename]
# Save the memmap
if save_temp_files:
print("***** Keeping memmap file *****")
# copy to batch dir if batch_dir != work_dir
if not os.path.samefile(batch_dir, work_dir):
print("***** Copying memmap file to batch dir *****")
shutil.copy(memmap_path, batch_dir)
# Delete the memmap from the work dir
if not os.path.samefile(batch_dir, work_dir):
print("***** Deleting memmap files from work dir *****")
try:
os.remove(memmap_path)
except:
pass
output.update({
'output': UUID,
'status': 1,
'output_files': output_files,
'saved_memmap': save_temp_files
})
except Exception as e:
output.update({'status': 0, 'output_info': traceback.format_exc()})
cm.stop_server(dview=dview)
end_time = time()
processing_time = (end_time - start_time) / 60
output.update({'processing_time': processing_time})
json.dump(output, open(filepath + '.out', 'w'))
import scipy
from ipyparallel import Client
# mpl.use('Qt5Agg')
import pylab as pl
pl.ion()
#%%
import caiman as cm
from caiman.source_extraction.cnmf import cnmf as cnmf
from caiman.source_extraction.cnmf.utilities import extract_DF_F
from caiman.components_evaluation import evaluate_components
from caiman.utils.visualization import plot_contours, view_patches_bar
from caiman.utils.utils import download_demo
from caiman.cluster import setup_cluster
# %% RUN ANALYSIS
c, dview, n_processes = setup_cluster(
backend='local', n_processes=None, single_thread=False)
#%%
is_patches = True
is_dendrites = False
if is_dendrites == True:
# THIS METHOd CAN GIVE POSSIBLY INCONSISTENT RESULTS ON SOMAS WHEN NOT USED WITH PATCHES
init_method = 'sparse_nmf'
alpha_snmf = 10e1 # this controls sparsity
else:
init_method = 'greedy_roi'
alpha_snmf = None # 10e2 # this controls sparsity
#%% FOR LOADING ALL TIFF FILES IN A FILE AND SAVING THEM ON A SINGLE MEMORY MAPPABLE FILE
fnames = ['demoMovieJ.tif']
base_folder = './example_movies/' # folder containing the demo files
strides = parameters[
'strides'] # start a new patch for pw-rigid motion correction every x pixels
overlaps = parameters[
'overlaps'] # overlap between patches. Patch size = strides + overlaps
splits_els = parameters[
'splits_els'] # for parallelisation split movies in num_splits chunks across time.
# Tip: make sure len(movie)/num_splits > 100
upsample_factor_grid = parameters[
'upsample_factor_grid'] # upsample factor to avoid smearing when merging patches
max_deviation_rigid = parameters[
'max_deviation_rigid'] # maximum deviation allowed for patch w.r.t. rigid shifts.
c, dview, n_processes = setup_cluster(
backend=args.backend, # use this one
n_processes=args.
num_processes, # number of processes to use, if you go out of memory try to reduce this one
)
if path.exists(path.splitext(filename)[0] + '_mc_template.npy'):
mc_template = np.load(
path.splitext(filename)[0] + '_mc_template.npy')
bord_px = cell_info[animal][session]['motion_correction'][
'bord_moco']
else:
# Motion correction rigid to obtain template
mc = motion_correct_oneP_rigid(fnames,
gSig_filt=gSig_filt,
max_shifts=max_shifts,
splits_rig=splits_rig,
