def save_tif_to_mmap_online(movie_iterable,
save_base_name='YrOL_',
order='C',
add_to_movie=0,
border_to_0=0) -> str:
# todo: todocument
if isinstance(movie_iterable, basestring
): # Allow specifying a filename rather than its data rep
with tifffile.TiffFile(
movie_iterable) as tf: # And load it if that happens
movie_iterable = cm.movie(tf)
count = 0
new_mov = []
dims = (len(movie_iterable),
) + movie_iterable[0].shape # TODO: Don't pack length into dims
fname_tot = (memmap_frames_filename(save_base_name, dims[1:], dims[0],
order))
big_mov = np.memmap(fname_tot,
mode='w+',
dtype=np.float32,
shape=prepare_shape((np.prod(dims[1:]), dims[0])),
order=order)
for page in movie_iterable:
if count % 100 == 0:
logging.debug(count)
if 'tifffile' in str(type(movie_iterable[0])):
page = page.asarray()
img = np.array(page, dtype=np.float32)
new_img = img
if save_base_name is not None:
big_mov[:, count] = np.reshape(new_img,
np.prod(dims[1:]),
order='F')
else:
new_mov.append(new_img)
if border_to_0 > 0:
img[:border_to_0, :] = 0
img[:, :border_to_0] = 0
img[:, -border_to_0:] = 0
img[-border_to_0:, :] = 0
big_mov[:, count] = np.reshape(img + add_to_movie,
np.prod(dims[1:]),
order='F')
count += 1
big_mov.flush()
del big_mov
return fname_tot
def save(self,
file_name,
to32=True,
order='F',
imagej=False,
bigtiff=True,
excitation_lambda=488.0,
compress=0,
q_max=99.75,
q_min=1,
var_name_hdf5='mov',
sess_desc='some_description',
identifier='some identifier',
imaging_plane_description='some imaging plane description',
emission_lambda=520.0,
indicator='OGB-1',
location='brain',
starting_time=0.,
experimenter='Dr Who',
lab_name=None,
institution=None,
experiment_description='Experiment Description',
session_id='Session ID'):
"""
Save the timeseries in single precision. Supported formats include
TIFF, NPZ, AVI, MAT, HDF5/H5, MMAP, and NWB
Args:
file_name: str
name of file. Possible formats are tif, avi, npz, mmap and hdf5
to32: Bool
whether to transform to 32 bits
order: 'F' or 'C'
C or Fortran order
var_name_hdf5: str
Name of hdf5 file subdirectory
q_max, q_min: float in [0, 100]
percentile for maximum/minimum clipping value if saving as avi
(If set to None, no automatic scaling to the dynamic range [0, 255] is performed)
Raises:
Exception 'Extension Unknown'
"""
name, extension = os.path.splitext(file_name)[:2]
extension = extension.lower()
logging.debug("Parsing extension " + str(extension))
if extension in ['.tif', '.tiff', '.btf']:
with tifffile.TiffWriter(file_name, bigtiff=bigtiff,
imagej=imagej) as tif:
for i in range(self.shape[0]):
if i % 200 == 0:
logging.debug(str(i) + ' frames saved')
curfr = self[i].copy()
if to32 and not ('float32' in str(self.dtype)):
curfr = curfr.astype(np.float32)
tif.save(curfr, compress=compress)
elif extension == '.npz':
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
np.savez(file_name,
input_arr=input_arr,
start_time=self.start_time,
fr=self.fr,
meta_data=self.meta_data,
file_name=self.file_name)
elif extension == '.avi':
codec = None
try:
codec = cv2.FOURCC('I', 'Y', 'U', 'V')
except AttributeError:
codec = cv2.VideoWriter_fourcc(*'IYUV')
if q_max is None or q_min is None:
data = self.astype(np.uint8)
else:
if q_max < 100:
maxmov = np.nanpercentile(self[::max(1,
len(self) // 100)],
q_max)
else:
maxmov = np.nanmax(self)
if q_min > 0:
minmov = np.nanpercentile(self[::max(1,
len(self) // 100)],
q_min)
else:
minmov = np.nanmin(self)
data = 255 * (self - minmov) / (maxmov - minmov)
np.clip(data, 0, 255, data)
data = data.astype(np.uint8)
y, x = data[0].shape
vw = cv2.VideoWriter(file_name,
codec,
self.fr, (x, y),
isColor=True)
for d in data:
vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR))
vw.release()
elif extension == '.mat':
if self.file_name[0] is not None:
f_name = self.file_name
else:
f_name = ''
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
if self.meta_data[0] is None:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': [],
'file_name': f_name
})
else:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': self.meta_data,
'file_name': f_name
})
elif extension in ('.hdf5', '.h5'):
with h5py.File(file_name, "w") as f:
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
dset = f.create_dataset(var_name_hdf5, data=input_arr)
dset.attrs["fr"] = self.fr
dset.attrs["start_time"] = self.start_time
try:
dset.attrs["file_name"] = [
a.encode('utf8') for a in self.file_name
]
except:
logging.warning('No file saved')
if self.meta_data[0] is not None:
logging.debug("Metadata for saved file: " +
str(self.meta_data))
dset.attrs["meta_data"] = cpk.dumps(self.meta_data)
elif extension == '.mmap':
base_name = name
T = self.shape[0]
dims = self.shape[1:]
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
input_arr = np.transpose(input_arr,
list(range(1,
len(dims) + 1)) + [0])
input_arr = np.reshape(input_arr, (np.prod(dims), T), order='F')
fname_tot = memmap_frames_filename(base_name, dims, T, order)
fname_tot = os.path.join(os.path.split(file_name)[0], fname_tot)
big_mov = np.memmap(fname_tot,
mode='w+',
dtype=np.float32,
shape=(np.uint64(np.prod(dims)), np.uint64(T)),
order=order)
big_mov[:] = np.asarray(input_arr, dtype=np.float32)
big_mov.flush()
del big_mov, input_arr
return fname_tot
elif extension == '.nwb':
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
# Create NWB file
nwbfile = NWBFile(sess_desc,
identifier,
datetime.now(tzlocal()),
experimenter=experimenter,
lab=lab_name,
institution=institution,
experiment_description=experiment_description,
session_id=session_id)
# Get the device
device = Device('imaging_device')
nwbfile.add_device(device)
# OpticalChannel
optical_channel = OpticalChannel('OpticalChannel',
'main optical channel',
emission_lambda=emission_lambda)
imaging_plane = nwbfile.create_imaging_plane(
name='ImagingPlane',
optical_channel=optical_channel,
description=imaging_plane_description,
device=device,
excitation_lambda=excitation_lambda,
imaging_rate=self.fr,
indicator=indicator,
location=location)
# Images
image_series = TwoPhotonSeries(name=var_name_hdf5,
dimension=self.shape[1:],
data=input_arr,
imaging_plane=imaging_plane,
starting_frame=[0],
starting_time=starting_time,
rate=self.fr)
nwbfile.add_acquisition(image_series)
with NWBHDF5IO(file_name, 'w') as io:
io.write(nwbfile)
return file_name
else:
logging.error("Extension " + str(extension) + " unknown")
raise Exception('Extension Unknown')
def save_memmap_join(mmap_fnames: List[str],
base_name: str = None,
n_chunks: int = 20,
dview=None,
add_to_mov=0) -> str:
"""
Makes a large file memmap from a number of smaller files
Args:
mmap_fnames: list of memory mapped files
base_name: string, will be the first portion of name to be solved
n_chunks: number of chunks in which to subdivide when saving, smaller requires more memory
dview: cluster handle
add_to_mov: (undocumented)
"""
tot_frames = 0
order = 'C'
for f in mmap_fnames:
Yr, dims, T = load_memmap(f)
logging.debug(
(f, T)
) # TODO: Add a text header so this isn't just numeric output, but what to say?
tot_frames += T
del Yr
d = np.prod(dims)
if base_name is None:
base_name = mmap_fnames[0]
base_name = base_name[:base_name.find('_d1_')] + '-#-' + str(
len(mmap_fnames))
fname_tot = memmap_frames_filename(base_name, dims, tot_frames, order)
fname_tot = os.path.join(os.path.split(mmap_fnames[0])[0], fname_tot)
logging.info("Memmap file for fname_tot: " + str(fname_tot))
big_mov = np.memmap(fname_tot,
mode='w+',
dtype=np.float32,
shape=prepare_shape((d, tot_frames)),
order='C')
step = np.int(old_div(d, n_chunks))
pars = []
for ref in range(0, d - step + 1, step):
pars.append([
fname_tot, d, tot_frames, mmap_fnames, ref, ref + step, add_to_mov
])
if len(pars[-1]) != 7:
raise Exception(
'You cannot change the number of element in list without changing the statement below (pars[]..)'
)
else:
# last batch should include the leftover pixels
pars[-1][-2] = d
if dview is not None:
if 'multiprocessing' in str(type(dview)):
dview.map_async(save_portion, pars).get(4294967)
else:
my_map(dview, save_portion, pars)
else:
list(map(save_portion, pars))
np.savez(base_name + '.npz', mmap_fnames=mmap_fnames, fname_tot=fname_tot)
logging.info('Deleting big mov')
del big_mov
sys.stdout.flush()
return fname_tot
def save(self,
file_name,
to32=True,
order='F',
imagej=False,
bigtiff=True,
software='CaImAn',
compress=0,
var_name_hdf5='mov'):
"""
Save the timeseries in various formats
Args:
file_name: str
name of file. Possible formats are tif, avi, npz, mmap and hdf5
to32: Bool
whether to transform to 32 bits
order: 'F' or 'C'
C or Fortran order
var_name_hdf5: str
Name of hdf5 file subdirectory
Raises:
Exception 'Extension Unknown'
"""
name, extension = os.path.splitext(file_name)[:2]
extension = extension.lower()
logging.debug("Parsing extension " + str(extension))
if extension == '.tif': # load avi file
with tifffile.TiffWriter(file_name, bigtiff=bigtiff,
imagej=imagej) as tif:
for i in range(self.shape[0]):
if i % 200 == 0:
logging.debug(str(i) + ' frames saved')
curfr = self[i].copy()
if to32 and not ('float32' in str(self.dtype)):
curfr = curfr.astype(np.float32)
tif.save(curfr, compress=compress)
elif extension == '.npz':
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
np.savez(file_name,
input_arr=input_arr,
start_time=self.start_time,
fr=self.fr,
meta_data=self.meta_data,
file_name=self.file_name)
elif extension == '.avi':
codec = None
try:
codec = cv2.FOURCC('I', 'Y', 'U', 'V')
except AttributeError:
codec = cv2.VideoWriter_fourcc(*'IYUV')
np.clip(self, np.percentile(self, 1), np.percentile(self, 99),
self)
minn, maxx = np.min(self), np.max(self)
data = 255 * (self - minn) / (maxx - minn)
data = data.astype(np.uint8)
y, x = data[0].shape
vw = cv2.VideoWriter(file_name,
codec,
self.fr, (x, y),
isColor=True)
for d in data:
vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR))
vw.release()
elif extension == '.mat':
if self.file_name[0] is not None:
f_name = self.file_name
else:
f_name = ''
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
if self.meta_data[0] is None:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': [],
'file_name': f_name
})
else:
savemat(
file_name, {
'input_arr': np.rollaxis(input_arr, axis=0, start=3),
'start_time': self.start_time,
'fr': self.fr,
'meta_data': self.meta_data,
'file_name': f_name
})
elif extension in ('.hdf5', '.h5'):
with h5py.File(file_name, "w") as f:
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
dset = f.create_dataset(var_name_hdf5, data=input_arr)
dset.attrs["fr"] = self.fr
dset.attrs["start_time"] = self.start_time
try:
dset.attrs["file_name"] = [
a.encode('utf8') for a in self.file_name
]
except:
logging.warning('No file saved')
if self.meta_data[0] is not None:
logging.debug("Metadata for saved file: " +
str(self.meta_data))
dset.attrs["meta_data"] = cpk.dumps(self.meta_data)
elif extension == '.mmap':
base_name = name
T = self.shape[0]
dims = self.shape[1:]
if to32 and not ('float32' in str(self.dtype)):
input_arr = self.astype(np.float32)
else:
input_arr = np.array(self)
input_arr = np.transpose(input_arr,
list(range(1,
len(dims) + 1)) + [0])
input_arr = np.reshape(input_arr, (np.prod(dims), T), order='F')
fname_tot = memmap_frames_filename(base_name, dims, T, order)
fname_tot = os.path.join(os.path.split(file_name)[0], fname_tot)
big_mov = np.memmap(fname_tot,
mode='w+',
dtype=np.float32,
shape=(np.uint64(np.prod(dims)), np.uint64(T)),
order=order)
big_mov[:] = np.asarray(input_arr, dtype=np.float32)
big_mov.flush()
del big_mov, input_arr
return fname_tot
else:
logging.error("Extension " + str(extension) + " unknown")
raise Exception('Extension Unknown')
