def test_missing_frame_overlapping_rois(self): data = np.ones((10, 2, 6, 8, 2)) data[:, 0, 1:4, 2:6, 1] = 1000 data[3:5, ...] = np.nan seq = sima.Sequence.create('ndarray', data) dataset = sima.ImagingDataset([seq], savedir=None) mask1 = np.array([[[False, False, False, False, False, False], [False, True, True, False, False, False], [False, True, True, False, False, False], [False, False, False, False, False, False]]]) roi1 = sima.ROI.ROI(mask=mask1) mask2 = np.array([[[False, False, False, False, False, False], [False, False, True, True, True, True], [False, False, True, True, True, True], [False, False, True, True, True, True]]]) roi2 = sima.ROI.ROI(mask=mask2) rois = sima.ROI.ROIList([roi1, roi2]) signals = dataset.extract(rois=rois, signal_channel=1, remove_overlap=False, n_processes=1, demix_channel=None) assert_array_equal(signals['raw'][0][:, 3:5], np.nan) assert_(np.all(np.isfinite(signals['raw'][0][:, :3]))) assert_(np.all(np.isfinite(signals['raw'][0][:, 5:])))
def check_create_sima_dataset(fpath): """ If motion_correct flag set to false and there's no .sima folder, bypass sima MC but still make a .sima folder with sequences to allow for downstream processing :param fpath: :return: """ fdir = os.path.split(fpath)[0] fname = os.path.split(fpath)[1] fbasename = os.path.splitext(fname)[0] fext = os.path.splitext(fname)[1] sima_folder_path = os.path.join(fdir, fbasename + '_mc.sima') if not os.path.exists(sima_folder_path): # create a sima sequence with the data if fext == '.tif' or fext == '.tiff': sequences = [sima.Sequence.create('TIFF', fpath)] elif fext == '.h5': sequences = [sima.Sequence.create('HDF5', fpath, 'tyx')] # creates sima imaging dataset, but more importantly saves a .sima folder required for downstream processing sima.ImagingDataset(sequences, sima_folder_path)
def extract(self, block): import sima # reshape the block to (t, z, y, x, c) dims = block.shape if len(dims) == 3: # (t, x, y) reshapedBlock = block.reshape(dims[0], 1, dims[2], dims[1], 1) else: # (t, x, y, z) reshapedBlock = block.reshape(dims[0], dims[3], dims[2], dims[1], 1) # create SIMA dataset from block dataset = sima.ImagingDataset( [sima.Sequence.create('ndarray', reshapedBlock)], None) # apply the sima strategy to the dataset rois = self.strategy.segment(dataset) # convert the coordinates between the SIMA and thunder conventions coords = [asarray(where(array(roi))).T for roi in rois] if len(dims) == 3: coords = [c[:, 1:] for c in coords] coords = [c[:, ::-1] for c in coords] # format the sources sources = [Source(c) for c in coords] return sources
def setup(self): pattern = np.tile([0.4, 0.8, 1.2, 1.6], (3, 1)) data = np.ones((20, 2, 6, 8, 2)) for t in range(20): data[t, 1, 1:4, 2:6, 0] = np.roll(pattern, t, 1) * 1000 seq = sima.Sequence.create('ndarray', data) self.dataset = sima.ImagingDataset([seq], savedir=None)
def tosima(self, startIdx=0, stopIdx=None): """ Convert the experiment to a sima.ImagingDataset Parameters ---------- startIdx : int, optional Starting index (inclusive) for conversion. stopIdx : int Last index (exclusive) for conversion. Returns ------- dataset : sima.ImagingDataset A sima.ImagingDataset """ if os.path.exists(self.sima_dir): try: return sima.ImagingDataset.load(self.sima_dir) except EOFError as err: sys.stderr.write("Could not read from " + self.sima_dir + "regenerating sima files: " + err + "\n") except IndexError as err: sys.stderr.write("Could not read from " + self.sima_dir + "regenerating sima files: " + err + "\n") if self.mptifs is None and (not os.path.exists(self.rawfile) or self.rawfile is None): # The string paths[i][j] is a unix style expression for the # filenames for plane i and channel j sequences = [ sima.Sequence.create( 'TIFFs', [[self.filetrunk + self.format_index("?") + ".tif"]]) ] else: rawarray = self.read_raw() if rawarray.ndim == 3: sequences = [ sima.Sequence.create( 'ndarray', rawarray[:, np.newaxis, :, :, np.newaxis]) ] else: sequences = [sima.Sequence.create('ndarray', rawarray)] if stopIdx is None: stopIdx = self.nframes else: sequences = [seq[:stopIdx, :, :, :, :] for seq in sequences] if startIdx != 0: sequences = [seq[startIdx:, :, :, :, :] for seq in sequences] return sima.ImagingDataset(sequences, self.sima_dir, channel_names=[ self.chan, ])
def setup(self): self.tmp_dir = os.path.join(os.path.dirname(__file__), 'tmp') try: os.mkdir(self.tmp_dir) except: pass pattern = np.tile([0.4, 0.8, 1.2, 1.6], (3, 1)) data = np.ones((20, 2, 6, 8, 2)) for t in range(20): data[t, 1, 1:4, 2:6, 0] = np.roll(pattern, t, 1) * 1000 path = os.path.join(self.tmp_dir, "test_extract.sima") seq = sima.Sequence.create('ndarray', data) self.dataset = sima.ImagingDataset([seq], savedir=path)
def setup(self): for frame in Sequence.create('HDF5', example_hdf5(), 'yxt'): break frame_shifts = [np.array([[[0, 0]], [[-5, -10]]])] self.frame_shifts = [np.array([[[5, 10]], [[0, 0]]])] self.correlations = [np.array([[1], [0.9301478]])] shifted = frame.copy() shifted = np.roll(shifted, -frame_shifts[0][1, 0, 1], axis=2) shifted = np.roll(shifted, -frame_shifts[0][1, 0, 0], axis=1) frames = np.array([frame, shifted]) self.hm2d = hmm.HiddenMarkov2D(n_processes=1, verbose=False) self.dataset = sima.ImagingDataset( [Sequence.create('ndarray', frames)], None)
def check_create_sima_dataset(fpath): fdir = os.path.split(fpath)[0] fname = os.path.split(fpath)[1] fbasename = os.path.splitext(fname)[0] fext = os.path.splitext(fname)[1] sima_folder_path = os.path.join(fdir, fbasename + '_mc.sima') if not os.path.exists(sima_folder_path): # create a sima sequence with the data if fext == '.tif' or fext == '.tiff': sequences = [sima.Sequence.create('TIFF', fpath)] elif fext == '.h5': sequences = [sima.Sequence.create('HDF5', fpath, 'tyx')] # creates sima imaging dataset, but more importantly saves a .sima folder required for downstream processing sima.ImagingDataset(sequences, sima_folder_path)
def initialize_sbx_datasets(path, calc_time_averages=False): """Locate and initialize a SIMA dataset for all Scanbox sbx files.""" for directory, folders, files in os.walk(path): for sbx_file in fnmatch.filter(files, '*.sbx'): info_file = os.path.splitext(sbx_file)[0] + '.mat' sima_dir = os.path.splitext(sbx_file)[0] + '.sima' if info_file in files and sima_dir not in folders: print("Initializing SIMA dataset: {}".format( os.path.join(directory, sima_dir))) seq = sbxread(os.path.join(directory, sbx_file), os.path.join(directory, info_file)) dset = sima.ImagingDataset([seq], savedir=os.path.join( directory, sima_dir)) if calc_time_averages: print("Calculating time averages: {}".format( os.path.join(directory, sima_dir))) dset.time_averages
def _estimate(self, dataset): if not next(iter(dataset)).shape[2] % 2 == 0: raise ValueError( 'Resonant motion correction requires an even number of rows') downsampled_dataset = sima.ImagingDataset([ sima.Sequence.join(*it.chain.from_iterable( (seq[:, :, ::2, :, c], seq[:, :, 1::2, :, c]) for c in range(seq.shape[4]))) for seq in dataset ], None) downsampled_displacements = self._base_strategy.estimate( downsampled_dataset) displacements = [] for d_disps in downsampled_displacements: disps = np.repeat(d_disps, 2, axis=2) # Repeat the displacements disps[:, :, :, 0] *= 2 # multiply y-shifts by 2 disps[:, :, 1::2, -1] += self._offset # shift even rows by offset displacements.append(disps) return displacements
new_templ[None, None, :, :, None], newim[None, None, :, :, None], new_templ[None, None, :, :, None], newim[None, None, :, :, None], new_templ[None, None, :, :, None], newim[None, None, :, :, None], new_templ[None, None, :, :, None], newim[None, None, :, :, None], new_templ[None, None, :, :, None], newim[None, None, :, :, None], new_templ[None, None, :, :, None], newim[None, None, :, :, None] ], axis=0)) ] dataset = sima.ImagingDataset(sequences, None) mc_approach = sima.motion.HiddenMarkov2D(granularity=(granularity, gran_n), max_displacement=[max_shift] * 2, verbose=True, n_processes=14) shifts_sima = mc_approach.estimate(dataset) vx = cv2.resize(shifts_sima[0][0][0][:, 0].astype(np.float32), newim.shape) vy = cv2.resize(shifts_sima[0][0][0][:, 1].astype(np.float32), newim.shape) pl.imshow(apply_field(newim, vx, vy)) # mov_sima = mc_approach.correct(dataset,None,correction_channels = [1]) print(np.sum(shifts_sima)) break #%% pl.subplot(1, 2, 1) pl.imshow(shifts_sima[0][:, 0, :, 0].squeeze(), aspect='auto') pl.colorbar()
cluster_movie = np.zeros(shape=(int(epoch_frames.sum()),1,mov_xDim,mov_yDim,1)) original_movie = np.zeros(shape=(np.shape(time_series)[0],1,mov_xDim,mov_yDim,1)) original_movie[:,0,:,:,0]= time_series startFrame = 0 for index, epoch in enumerate(epochs_to_use): if index>0: startFrame = endFrame endFrame = startFrame + epoch_frames[index] cluster_movie[int(startFrame):int(endFrame),0,:,:,0] = wholeTraces_allTrials[epoch].mean(axis=3) # Create a sima dataset and export the cluster movie b = sima.Sequence.create('ndarray',cluster_movie) cluster_dataset = sima.ImagingDataset([b],None) cluster_dataset.export_frames([[[os.path.join(selected_movie_dir,'cluster_vid.tif')]]], fill_gaps=True,scale_values=True) #%% nCpu=(multiprocessing.cpu_count() - 1) cluster_1d_max_size_pixel = cluster_max_1d_size_micron/x_size cluster_1d_min_size_pixel = cluster_min_1d_size_micron/x_size area_max = int(math.pow(2/x_size, 2)) # 4um square max area #area_min = int(math.pow(cluster_1d_min_size_pixel, 2)) # 1um square min area area_min = int(math.pow(1.1/x_size, 2)) # 1um square min area #if area_min < 10:
def setup(self): data = np.ones((10, 2, 6, 8, 2)) data[:, 0, 1:4, 2:6, 1] = 1000 seq = sima.Sequence.create('ndarray', data) self.dataset = sima.ImagingDataset([seq], savedir=None)
def setup(self): self.strategy = sima.motion.ResonantCorrection( sima.motion.HiddenMarkov2D(n_processes=1, verbose=False), 5) self.dataset = sima.ImagingDataset( [Sequence.create('HDF5', example_volume(), 'tzyxc')], None)
def create_sequence(full_path_to_file, individualframename): # Retrieve S3 filename without path temp = os.path.splitext(os.path.basename(full_path_to_file)) filename = temp[0] if filename[-4:] == '_CH1': dual_channel = True else: dual_channel = False if is_single_file: file_format = temp[-1] if dual_channel: # Identify file format to create SIMA Sequence object if file_format == '.tif': import_format = 'TIFF' sequences1 = sima.Sequence.create(import_format, filename + file_format) sequences2 = sima.Sequence.create( import_format, filename[:-1] + '2' + file_format) sequences = [sima.Sequence.join(sequences1, sequences2)] else: # i.e., if file_format == '.h5': import_format = 'HDF5' sequences1 = sima.Sequence.create(import_format, filename + file_format, dim_order_for_hdf5) sequences2 = sima.Sequence.create( import_format, filename[:-1] + '2' + file_format, dim_order_for_hdf5) sequences = [sima.Sequence.join(sequences1, sequences2)] else: # Identify file format to create SIMA Sequence object if file_format == '.tif': import_format = 'TIFF' sequences = [ sima.Sequence.create(import_format, os.path.basename(full_path_to_file)) ] else: # i.e., if file_format == '.h5': import_format = 'HDF5' sequences = [ sima.Sequence.create(import_format, os.path.basename(full_path_to_file), dim_order_for_hdf5) ] else: # file format is tiff import_format = 'TIFFs' if dual_channel: # 'filename' is really the directory. An individual frame's # file name prefix is defined by 'individualframename' sequences1 = sima.Sequence.create(import_format, [[ base_path_on_ec2 + filename + '/' + individualframename + '_*.tif' ]]) sequences2 = sima.Sequence.create(import_format, [[ base_path_on_ec2 + filename[:-1] + '2' + '/' + individualframename + '_*.tif' ]]) sequences = [sima.Sequence.join(sequences1, sequences2)] else: # 'filename' is really the directory. An individual frame's # file name prefix is defined by 'individualframename' sequences = [ sima.Sequence.create(import_format, [[ base_path_on_ec2 + filename + '/' + individualframename + '_*.tif' ]]) ] sima.ImagingDataset(sequences, filename + '.sima') return filename, dual_channel
def correct(self, dataset, savedir, channel_names=None, info=None, correction_channels=None, trim_criterion=None): """Create a motion-corrected dataset. Parameters ---------- dataset : sima.ImagingDataset or list of sima.Sequence Dataset or sequences to be motion corrected. savedir : str The directory used to store the dataset. If the directory name does not end with .sima, then this extension will be appended. channel_names : list of str, optional Names for the channels. Defaults to ['0', '1', '2', ...]. info : dict Data for the order and timing of the data acquisition. See sima.ImagingDataset for details. correction_channels : list of int, optional Information from the channels corresponding to these indices will be used for motion correction. By default, all channels will be used. trim_criterion : float, optional The required fraction of frames during which a location must be within the field of view for it to be included in the motion-corrected imaging frames. By default, only locations that are always within the field of view are retained. Returns ------- dataset : sima.ImagingDataset The motion-corrected dataset. """ sequences = [s for s in dataset] if correction_channels: correction_channels = [ sima.misc.resolve_channels(c, channel_names, len(sequences[0])) for c in correction_channels ] mc_sequences = [ s[:, :, :, :, correction_channels] for s in sequences ] else: mc_sequences = sequences displacements = self.estimate(sima.ImagingDataset(mc_sequences, None)) disp_dim = displacements[0].shape[-1] max_disp = np.ceil( np.max(list( it.chain.from_iterable( d.reshape(-1, disp_dim) for d in displacements)), axis=0)).astype(np.int64) frame_shape = np.array(sequences[0].shape)[1:-1] # (z, y, x) if len(max_disp) == 2: # if 2D displacements frame_shape[1:3] += max_disp else: # if 3D displacements frame_shape += max_disp corrected_sequences = [ s.apply_displacements(d, frame_shape) for s, d in zip(sequences, displacements) ] planes, rows, columns = _trim_coords(trim_criterion, displacements, sequences[0].shape[1:4], frame_shape) corrected_sequences = [ s[:, planes, rows, columns] for s in corrected_sequences ] return [ sima.ImagingDataset(corrected_sequences, savedir, channel_names=channel_names), rows, columns ] # CZ added rows and columns (trimming edges)
fig = plt.figure() axes = [fig.add_subplot(311), fig.add_subplot(312), fig.add_subplot(313)] for chan in zip(errors, corrs, fold_corrs): for i in range(3): axes[i].plot(range(1, 17), chan[i]) axes[0].set_ylabel('error') axes[0].legend([str(c+1) for c in range(len(errors))]) axes[1].set_ylabel('corr') axes[2].set_ylabel('fold corr') for i in range(1, 3): axes[i].plot([1, 16], [0, 0], 'k') plt.show() if __name__ == '__main__': path = '/data/Joseph/Imaging/JT39/2014-09-13/TSeries-FOV1-000/TSeries-FOV1-000_Cycle00001_Element00001.h5' seq = sima.Sequence.create('HDF5', path, 'tzyxc')[5:10, ::20] ds = sima.ImagingDataset([seq], savedir=None) # path = '/data/Jeff/running/jz107/TSeries-12112014-Day1-Session1-000/TSeries-12112014-Day1-Session1-000_Cycle00001_Element00001.h5' # path = '/data/Nathan/2photon/acuteRemapping/nd99/10282014/ctxA-002/ctxA-002_Cycle00001_Element00001.h5' # path = '/data/Nathan/2photon/acuteRemapping/nd113/12102014/ctxA-002/ctxA-002_Cycle00001_Element00001.h5' # path = '/data/Nathan/2photon/acuteRemapping/nd101/10292014/ctxB-001/ctxB-001_Cycle00001_Element00001.h5' seq = sima.Sequence.create('HDF5', path, 'tzyxc')[0:20] ds = sima.ImagingDataset([seq], savedir=None) print "Analyze:" analyze(ds)
# 'weight' : 'regular', # 'size' : 15} # #pl.rc('font', **font) pl.rcParams['pdf.fonttype'] = 42 #%% test against SIMA import sima import sima.motion from sima.motion import HiddenMarkov2D #fname_gr = 'M_FLUO_t.tif' #fname_gr = 'Sue_1000.tif' #fname_gr = 'Sue_2000.tif' fname_gr = 'Sue_1000_T.tif' fname_gr = 'Sue_1000_T.tifrow1_example_sima_T.tif' sequences = [sima.Sequence.create('TIFF', fname_gr)] dataset = sima.ImagingDataset(sequences, fname_gr) #%% import time t1 = time.time() granularity = 'row' gran_n = 1 mc_approach = sima.motion.HiddenMarkov2D(granularity=(granularity, gran_n), max_displacement=max_shifts, verbose=True, n_processes=14) new_dataset = mc_approach.correct(dataset, None) t2 = time.time() - t1 print(t2) #% new_dataset.export_frames(
data_dir="./real_data/" ds_name='example_data.sima' # Create filenames tiff_filenames = [ [data_dir + '20150303_AF{n1:02d}_{n2:03d}.tif'.format(n1=animal_nr, n2=session_nr) for animal_nr in range(3,4)] for session_nr in range(2, 3)] # The resulting filenames are printed for clarification. print "TIFF filenames:\n", tiff_filenames # Finally, we construct a MultiPageTIFF iterable using each of the filenames. sequences = [ sima.Sequence.join(*[sima.Sequence.create('TIFF', chan) for chan in cycle]) for cycle in tiff_filenames] dataset = sima.ImagingDataset(sequences, ds_name) # load using : #dataset = sima.ImagingDataset.load(ds_name) # Set motion correction parameters mc_approach = sima.motion.PlaneTranslation2D(max_displacement=[15, 30]) # Apply motion correction # Needs dataset or sequences and save dir/dataset name dataset = mc_approach.correct(dataset, ds_name) # Output corrected frames output_filenames = [ [[channel.replace('.tif', '_corrected.tif') for channel in cycle]] for cycle in tiff_filenames