def save_tif_to_mmap_online(movie_iterable, save_base_name='YrOL_', order='C', add_to_movie=0, border_to_0=0) -> str: # todo: todocument if isinstance(movie_iterable, basestring ): # Allow specifying a filename rather than its data rep with tifffile.TiffFile( movie_iterable) as tf: # And load it if that happens movie_iterable = cm.movie(tf) count = 0 new_mov = [] dims = (len(movie_iterable), ) + movie_iterable[0].shape # TODO: Don't pack length into dims fname_tot = (memmap_frames_filename(save_base_name, dims[1:], dims[0], order)) big_mov = np.memmap(fname_tot, mode='w+', dtype=np.float32, shape=prepare_shape((np.prod(dims[1:]), dims[0])), order=order) for page in movie_iterable: if count % 100 == 0: logging.debug(count) if 'tifffile' in str(type(movie_iterable[0])): page = page.asarray() img = np.array(page, dtype=np.float32) new_img = img if save_base_name is not None: big_mov[:, count] = np.reshape(new_img, np.prod(dims[1:]), order='F') else: new_mov.append(new_img) if border_to_0 > 0: img[:border_to_0, :] = 0 img[:, :border_to_0] = 0 img[:, -border_to_0:] = 0 img[-border_to_0:, :] = 0 big_mov[:, count] = np.reshape(img + add_to_movie, np.prod(dims[1:]), order='F') count += 1 big_mov.flush() del big_mov return fname_tot
def save(self, file_name, to32=True, order='F', imagej=False, bigtiff=True, excitation_lambda=488.0, compress=0, q_max=99.75, q_min=1, var_name_hdf5='mov', sess_desc='some_description', identifier='some identifier', imaging_plane_description='some imaging plane description', emission_lambda=520.0, indicator='OGB-1', location='brain', starting_time=0., experimenter='Dr Who', lab_name=None, institution=None, experiment_description='Experiment Description', session_id='Session ID'): """ Save the timeseries in single precision. Supported formats include TIFF, NPZ, AVI, MAT, HDF5/H5, MMAP, and NWB Args: file_name: str name of file. Possible formats are tif, avi, npz, mmap and hdf5 to32: Bool whether to transform to 32 bits order: 'F' or 'C' C or Fortran order var_name_hdf5: str Name of hdf5 file subdirectory q_max, q_min: float in [0, 100] percentile for maximum/minimum clipping value if saving as avi (If set to None, no automatic scaling to the dynamic range [0, 255] is performed) Raises: Exception 'Extension Unknown' """ name, extension = os.path.splitext(file_name)[:2] extension = extension.lower() logging.debug("Parsing extension " + str(extension)) if extension in ['.tif', '.tiff', '.btf']: with tifffile.TiffWriter(file_name, bigtiff=bigtiff, imagej=imagej) as tif: for i in range(self.shape[0]): if i % 200 == 0: logging.debug(str(i) + ' frames saved') curfr = self[i].copy() if to32 and not ('float32' in str(self.dtype)): curfr = curfr.astype(np.float32) tif.save(curfr, compress=compress) elif extension == '.npz': if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) np.savez(file_name, input_arr=input_arr, start_time=self.start_time, fr=self.fr, meta_data=self.meta_data, file_name=self.file_name) elif extension == '.avi': codec = None try: codec = cv2.FOURCC('I', 'Y', 'U', 'V') except AttributeError: codec = cv2.VideoWriter_fourcc(*'IYUV') if q_max is None or q_min is None: data = self.astype(np.uint8) else: if q_max < 100: maxmov = np.nanpercentile(self[::max(1, len(self) // 100)], q_max) else: maxmov = np.nanmax(self) if q_min > 0: minmov = np.nanpercentile(self[::max(1, len(self) // 100)], q_min) else: minmov = np.nanmin(self) data = 255 * (self - minmov) / (maxmov - minmov) np.clip(data, 0, 255, data) data = data.astype(np.uint8) y, x = data[0].shape vw = cv2.VideoWriter(file_name, codec, self.fr, (x, y), isColor=True) for d in data: vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR)) vw.release() elif extension == '.mat': if self.file_name[0] is not None: f_name = self.file_name else: f_name = '' if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) if self.meta_data[0] is None: savemat( file_name, { 'input_arr': np.rollaxis(input_arr, axis=0, start=3), 'start_time': self.start_time, 'fr': self.fr, 'meta_data': [], 'file_name': f_name }) else: savemat( file_name, { 'input_arr': np.rollaxis(input_arr, axis=0, start=3), 'start_time': self.start_time, 'fr': self.fr, 'meta_data': self.meta_data, 'file_name': f_name }) elif extension in ('.hdf5', '.h5'): with h5py.File(file_name, "w") as f: if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) dset = f.create_dataset(var_name_hdf5, data=input_arr) dset.attrs["fr"] = self.fr dset.attrs["start_time"] = self.start_time try: dset.attrs["file_name"] = [ a.encode('utf8') for a in self.file_name ] except: logging.warning('No file saved') if self.meta_data[0] is not None: logging.debug("Metadata for saved file: " + str(self.meta_data)) dset.attrs["meta_data"] = cpk.dumps(self.meta_data) elif extension == '.mmap': base_name = name T = self.shape[0] dims = self.shape[1:] if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) input_arr = np.transpose(input_arr, list(range(1, len(dims) + 1)) + [0]) input_arr = np.reshape(input_arr, (np.prod(dims), T), order='F') fname_tot = memmap_frames_filename(base_name, dims, T, order) fname_tot = os.path.join(os.path.split(file_name)[0], fname_tot) big_mov = np.memmap(fname_tot, mode='w+', dtype=np.float32, shape=(np.uint64(np.prod(dims)), np.uint64(T)), order=order) big_mov[:] = np.asarray(input_arr, dtype=np.float32) big_mov.flush() del big_mov, input_arr return fname_tot elif extension == '.nwb': if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) # Create NWB file nwbfile = NWBFile(sess_desc, identifier, datetime.now(tzlocal()), experimenter=experimenter, lab=lab_name, institution=institution, experiment_description=experiment_description, session_id=session_id) # Get the device device = Device('imaging_device') nwbfile.add_device(device) # OpticalChannel optical_channel = OpticalChannel('OpticalChannel', 'main optical channel', emission_lambda=emission_lambda) imaging_plane = nwbfile.create_imaging_plane( name='ImagingPlane', optical_channel=optical_channel, description=imaging_plane_description, device=device, excitation_lambda=excitation_lambda, imaging_rate=self.fr, indicator=indicator, location=location) # Images image_series = TwoPhotonSeries(name=var_name_hdf5, dimension=self.shape[1:], data=input_arr, imaging_plane=imaging_plane, starting_frame=[0], starting_time=starting_time, rate=self.fr) nwbfile.add_acquisition(image_series) with NWBHDF5IO(file_name, 'w') as io: io.write(nwbfile) return file_name else: logging.error("Extension " + str(extension) + " unknown") raise Exception('Extension Unknown')
def save_memmap_join(mmap_fnames: List[str], base_name: str = None, n_chunks: int = 20, dview=None, add_to_mov=0) -> str: """ Makes a large file memmap from a number of smaller files Args: mmap_fnames: list of memory mapped files base_name: string, will be the first portion of name to be solved n_chunks: number of chunks in which to subdivide when saving, smaller requires more memory dview: cluster handle add_to_mov: (undocumented) """ tot_frames = 0 order = 'C' for f in mmap_fnames: Yr, dims, T = load_memmap(f) logging.debug( (f, T) ) # TODO: Add a text header so this isn't just numeric output, but what to say? tot_frames += T del Yr d = np.prod(dims) if base_name is None: base_name = mmap_fnames[0] base_name = base_name[:base_name.find('_d1_')] + '-#-' + str( len(mmap_fnames)) fname_tot = memmap_frames_filename(base_name, dims, tot_frames, order) fname_tot = os.path.join(os.path.split(mmap_fnames[0])[0], fname_tot) logging.info("Memmap file for fname_tot: " + str(fname_tot)) big_mov = np.memmap(fname_tot, mode='w+', dtype=np.float32, shape=prepare_shape((d, tot_frames)), order='C') step = np.int(old_div(d, n_chunks)) pars = [] for ref in range(0, d - step + 1, step): pars.append([ fname_tot, d, tot_frames, mmap_fnames, ref, ref + step, add_to_mov ]) if len(pars[-1]) != 7: raise Exception( 'You cannot change the number of element in list without changing the statement below (pars[]..)' ) else: # last batch should include the leftover pixels pars[-1][-2] = d if dview is not None: if 'multiprocessing' in str(type(dview)): dview.map_async(save_portion, pars).get(4294967) else: my_map(dview, save_portion, pars) else: list(map(save_portion, pars)) np.savez(base_name + '.npz', mmap_fnames=mmap_fnames, fname_tot=fname_tot) logging.info('Deleting big mov') del big_mov sys.stdout.flush() return fname_tot
def save(self, file_name, to32=True, order='F', imagej=False, bigtiff=True, software='CaImAn', compress=0, var_name_hdf5='mov'): """ Save the timeseries in various formats Args: file_name: str name of file. Possible formats are tif, avi, npz, mmap and hdf5 to32: Bool whether to transform to 32 bits order: 'F' or 'C' C or Fortran order var_name_hdf5: str Name of hdf5 file subdirectory Raises: Exception 'Extension Unknown' """ name, extension = os.path.splitext(file_name)[:2] extension = extension.lower() logging.debug("Parsing extension " + str(extension)) if extension == '.tif': # load avi file with tifffile.TiffWriter(file_name, bigtiff=bigtiff, imagej=imagej) as tif: for i in range(self.shape[0]): if i % 200 == 0: logging.debug(str(i) + ' frames saved') curfr = self[i].copy() if to32 and not ('float32' in str(self.dtype)): curfr = curfr.astype(np.float32) tif.save(curfr, compress=compress) elif extension == '.npz': if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) np.savez(file_name, input_arr=input_arr, start_time=self.start_time, fr=self.fr, meta_data=self.meta_data, file_name=self.file_name) elif extension == '.avi': codec = None try: codec = cv2.FOURCC('I', 'Y', 'U', 'V') except AttributeError: codec = cv2.VideoWriter_fourcc(*'IYUV') np.clip(self, np.percentile(self, 1), np.percentile(self, 99), self) minn, maxx = np.min(self), np.max(self) data = 255 * (self - minn) / (maxx - minn) data = data.astype(np.uint8) y, x = data[0].shape vw = cv2.VideoWriter(file_name, codec, self.fr, (x, y), isColor=True) for d in data: vw.write(cv2.cvtColor(d, cv2.COLOR_GRAY2BGR)) vw.release() elif extension == '.mat': if self.file_name[0] is not None: f_name = self.file_name else: f_name = '' if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) if self.meta_data[0] is None: savemat( file_name, { 'input_arr': np.rollaxis(input_arr, axis=0, start=3), 'start_time': self.start_time, 'fr': self.fr, 'meta_data': [], 'file_name': f_name }) else: savemat( file_name, { 'input_arr': np.rollaxis(input_arr, axis=0, start=3), 'start_time': self.start_time, 'fr': self.fr, 'meta_data': self.meta_data, 'file_name': f_name }) elif extension in ('.hdf5', '.h5'): with h5py.File(file_name, "w") as f: if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) dset = f.create_dataset(var_name_hdf5, data=input_arr) dset.attrs["fr"] = self.fr dset.attrs["start_time"] = self.start_time try: dset.attrs["file_name"] = [ a.encode('utf8') for a in self.file_name ] except: logging.warning('No file saved') if self.meta_data[0] is not None: logging.debug("Metadata for saved file: " + str(self.meta_data)) dset.attrs["meta_data"] = cpk.dumps(self.meta_data) elif extension == '.mmap': base_name = name T = self.shape[0] dims = self.shape[1:] if to32 and not ('float32' in str(self.dtype)): input_arr = self.astype(np.float32) else: input_arr = np.array(self) input_arr = np.transpose(input_arr, list(range(1, len(dims) + 1)) + [0]) input_arr = np.reshape(input_arr, (np.prod(dims), T), order='F') fname_tot = memmap_frames_filename(base_name, dims, T, order) fname_tot = os.path.join(os.path.split(file_name)[0], fname_tot) big_mov = np.memmap(fname_tot, mode='w+', dtype=np.float32, shape=(np.uint64(np.prod(dims)), np.uint64(T)), order=order) big_mov[:] = np.asarray(input_arr, dtype=np.float32) big_mov.flush() del big_mov, input_arr return fname_tot else: logging.error("Extension " + str(extension) + " unknown") raise Exception('Extension Unknown')