def preprocess_data(base,
image_folder='z0',
scale=0.5,
name='movie.tif',
edge=5,
corrected_name='movie_corrected.tif'):
for root, dirs, files in os.walk(base):
offset = -1 * len(image_folder)
if image_folder == root[offset:]:
print('Preprocessing ', root)
fname = root
oname = root[:(-1 - len(image_folder))]
# Convert and crop stack
try:
saved_path = convert_and_crop_stack(fname, oname, scale, name)
except (FileExistsError, FileNotFoundError) as e:
print(oname, ' cannot be preprocessed, skipping...')
continue
# Motion correction, cut off edges
mc = MotionCorrect(saved_path, 0)
mc.motion_correct_rigid()
with tifffile.TiffWriter(oname + os.sep + corrected_name,
imagej=True) as tif:
shifted = mc.apply_shifts_movie(saved_path)
shifted = shifted[:, edge:-edge, edge:-edge]
converted_stack = np.array(shifted * (2**16), dtype='uint16')
tif.save(converted_stack)
def motion_correct_file(args, n_iter=0, max_iter=10):
try:
in_fn, file_index, template, mc_args, \
subidx, apply_subidx, apply_ofn = args
max_shifts, strides, overlaps, \
max_deviation_rigid, shifts_opencv, border_nan = mc_args
mc = MotionCorrect(in_fn,
dview=None,
max_shifts=max_shifts,
strides=strides,
overlaps=overlaps,
max_deviation_rigid=max_deviation_rigid,
shifts_opencv=shifts_opencv,
nonneg_movie=True,
splits_els=1,
splits_rig=1,
border_nan=border_nan,
subidx=subidx)
mc.motion_correct(template=template, save_movie=False)
if apply_subidx is not None:
applied_mov = mc.apply_shifts_movie(in_fn, apply_subidx)
applied_mov.save(apply_ofn)
return (file_index, np.mean(mc.templates_rig, axis=0))
except Exception as e:
if n_iter < max_iter:
print("Retrying %s due to %s" % (in_fn, e))
return motion_correct_file(args, n_iter=n_iter + 1)
else:
return Exception("Failed max_iter: %s times" % max_iter)
def motion_correct(
self, fname: str,
co_fname: str) -> Tuple[np.ndarray, dict, Optional[np.ndarray]]:
"""
Uses caiman non-rigid motion correction to remove/reduce motion artefacts
Note: ALways saves an intermediate mem-map representation in order C of the corrected 32-bit stack
:param fname: The filename of the source file
:param co_fname: Filename of a stack that should be co-aligned or None
:return:
[0]: Corrected stack as a memmap
[1]: Wrapped CaImAn parameter dictionary
[2]: Corrected co-stack as a memmap or None
"""
cont_folder = path.dirname(fname) # the containing folder
stack_name = path.split(fname)[1]
save_dir = cont_folder + f"/{self.ana_dir}"
if not path.exists(save_dir):
makedirs(save_dir)
print("Created analysis directory", flush=True)
out_name = save_dir + '/' + stack_name
if co_fname is not None:
co_stack_name = path.split(co_fname)[1]
co_out_name = save_dir + '/' + co_stack_name
else:
co_out_name = None
test_image = imread(
fname, key=0) # load first frame of stack to compute resolution
assert test_image.shape[0] == test_image.shape[1]
resolution = self.fov_um / test_image.shape[0]
fr = 1 / self.time_per_frame # frame-rate
dxy = (resolution, resolution) # spatial resolution in um/pixel
max_shift_um = (self.neuron_radius * 4, self.neuron_radius * 4
) # maximally allow shift by ~2 cell diameters
# maximum allowed rigid shift in pixels
max_shifts = [int(a / b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple(
[int(a / b) for a, b in zip(self.patch_motion_um, dxy)])
# overlap between patches (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts (unit unclear - likely pixels as it is int)
max_deviation_rigid = 3
# create parameter dictionary
mc_dict = {
'fnames': [fname],
'fr': fr,
'decay_time': self.decay_time,
'dxy': dxy,
'pw_rigid': self.pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(backend='local',
n_processes=None,
single_thread=False)
try:
# motion correction
mc = MotionCorrect(fname, dview=dview, **opts.get_group('motion'))
# Run (piecewise-rigid motion) correction using NoRMCorre
# if we don't save the movie into a memmap, there doesn't seem to be
# any possibility to get at the corrected data later???
mc.motion_correct(save_movie=True)
# memory mapping
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file,
base_name=out_name,
order='C',
border_to_0=border_to_0)
# delete the original mem-map
if mc.fname_tot_els is None:
[remove(fn) for fn in mc.fname_tot_rig]
else:
[remove(fn) for fn in mc.fname_tot_els]
# now load the new file and transform output into [z,y,x] stack
yr, dims, n_t = cm.load_memmap(fname_new)
images = np.reshape(yr.T, [n_t] + list(dims), order='F')
if self.save_projection:
# save anatomical projection as 16bit tif
anat_projection = images.copy()
anat_projection = np.sum(anat_projection, 0)
anat_projection -= np.min(anat_projection)
anat_projection /= np.max(anat_projection)
anat_projection *= (2**16 - 1)
anat_projection[anat_projection < 0] = 0
anat_projection[anat_projection > (2**16 - 1)] = (2**16 - 1)
anat_projection = anat_projection.astype(np.uint16)
imsave(out_name,
anat_projection,
imagej=True,
resolution=(1 / dxy[0], 1 / dxy[1]),
metadata={
'axes': 'YX',
'unit': 'um'
})
# Repeat for the co-stack if present
if co_fname is not None:
co_aligned_images = np.array(mc.apply_shifts_movie(co_fname))
if self.save_projection:
# save anatomical projection as 16bit tif
anat_projection = np.sum(co_aligned_images, 0)
anat_projection -= np.min(anat_projection)
anat_projection /= np.max(anat_projection)
anat_projection *= (2**16 - 1)
anat_projection[anat_projection < 0] = 0
anat_projection[anat_projection > (2**16 - 1)] = (2**16 -
1)
anat_projection = anat_projection.astype(np.uint16)
imsave(co_out_name,
anat_projection,
imagej=True,
resolution=(1 / dxy[0], 1 / dxy[1]),
metadata={
'axes': 'YX',
'unit': 'um'
})
else:
co_aligned_images = None
finally:
cm.stop_server(dview=dview)
return images, {"Motion Correction": mc_dict}, co_aligned_images
