def test_missing_frame_overlapping_rois(self):
data = np.ones((10, 2, 6, 8, 2))
data[:, 0, 1:4, 2:6, 1] = 1000
data[3:5, ...] = np.nan
seq = sima.Sequence.create('ndarray', data)
dataset = sima.ImagingDataset([seq], savedir=None)
mask1 = np.array([[[False, False, False, False, False, False],
[False, True, True, False, False, False],
[False, True, True, False, False, False],
[False, False, False, False, False, False]]])
roi1 = sima.ROI.ROI(mask=mask1)
mask2 = np.array([[[False, False, False, False, False, False],
[False, False, True, True, True, True],
[False, False, True, True, True, True],
[False, False, True, True, True, True]]])
roi2 = sima.ROI.ROI(mask=mask2)
rois = sima.ROI.ROIList([roi1, roi2])
signals = dataset.extract(rois=rois,
signal_channel=1,
remove_overlap=False,
n_processes=1,
demix_channel=None)
assert_array_equal(signals['raw'][0][:, 3:5], np.nan)
assert_(np.all(np.isfinite(signals['raw'][0][:, :3])))
assert_(np.all(np.isfinite(signals['raw'][0][:, 5:])))
def check_create_sima_dataset(fpath):
"""
If motion_correct flag set to false and there's no .sima folder, bypass sima MC but still make a
.sima folder with sequences to allow for downstream processing
:param fpath:
:return:
"""
fdir = os.path.split(fpath)[0]
fname = os.path.split(fpath)[1]
fbasename = os.path.splitext(fname)[0]
fext = os.path.splitext(fname)[1]
sima_folder_path = os.path.join(fdir, fbasename + '_mc.sima')
if not os.path.exists(sima_folder_path):
# create a sima sequence with the data
if fext == '.tif' or fext == '.tiff':
sequences = [sima.Sequence.create('TIFF', fpath)]
elif fext == '.h5':
sequences = [sima.Sequence.create('HDF5', fpath, 'tyx')]
# creates sima imaging dataset, but more importantly saves a .sima folder required for downstream processing
sima.ImagingDataset(sequences, sima_folder_path)
def extract(self, block):
import sima
# reshape the block to (t, z, y, x, c)
dims = block.shape
if len(dims) == 3: # (t, x, y)
reshapedBlock = block.reshape(dims[0], 1, dims[2], dims[1], 1)
else: # (t, x, y, z)
reshapedBlock = block.reshape(dims[0], dims[3], dims[2], dims[1],
1)
# create SIMA dataset from block
dataset = sima.ImagingDataset(
[sima.Sequence.create('ndarray', reshapedBlock)], None)
# apply the sima strategy to the dataset
rois = self.strategy.segment(dataset)
# convert the coordinates between the SIMA and thunder conventions
coords = [asarray(where(array(roi))).T for roi in rois]
if len(dims) == 3:
coords = [c[:, 1:] for c in coords]
coords = [c[:, ::-1] for c in coords]
# format the sources
sources = [Source(c) for c in coords]
return sources
def setup(self):
pattern = np.tile([0.4, 0.8, 1.2, 1.6], (3, 1))
data = np.ones((20, 2, 6, 8, 2))
for t in range(20):
data[t, 1, 1:4, 2:6, 0] = np.roll(pattern, t, 1) * 1000
seq = sima.Sequence.create('ndarray', data)
self.dataset = sima.ImagingDataset([seq], savedir=None)
def tosima(self, startIdx=0, stopIdx=None):
"""
Convert the experiment to a sima.ImagingDataset
Parameters
----------
startIdx : int, optional
Starting index (inclusive) for conversion.
stopIdx : int
Last index (exclusive) for conversion.
Returns
-------
dataset : sima.ImagingDataset
A sima.ImagingDataset
"""
if os.path.exists(self.sima_dir):
try:
return sima.ImagingDataset.load(self.sima_dir)
except EOFError as err:
sys.stderr.write("Could not read from " + self.sima_dir +
"regenerating sima files: " + err + "\n")
except IndexError as err:
sys.stderr.write("Could not read from " + self.sima_dir +
"regenerating sima files: " + err + "\n")
if self.mptifs is None and (not os.path.exists(self.rawfile)
or self.rawfile is None):
# The string paths[i][j] is a unix style expression for the
# filenames for plane i and channel j
sequences = [
sima.Sequence.create(
'TIFFs',
[[self.filetrunk + self.format_index("?") + ".tif"]])
]
else:
rawarray = self.read_raw()
if rawarray.ndim == 3:
sequences = [
sima.Sequence.create(
'ndarray', rawarray[:, np.newaxis, :, :, np.newaxis])
]
else:
sequences = [sima.Sequence.create('ndarray', rawarray)]
if stopIdx is None:
stopIdx = self.nframes
else:
sequences = [seq[:stopIdx, :, :, :, :] for seq in sequences]
if startIdx != 0:
sequences = [seq[startIdx:, :, :, :, :] for seq in sequences]
return sima.ImagingDataset(sequences,
self.sima_dir,
channel_names=[
self.chan,
])
def setup(self):
self.tmp_dir = os.path.join(os.path.dirname(__file__), 'tmp')
try:
os.mkdir(self.tmp_dir)
except:
pass
pattern = np.tile([0.4, 0.8, 1.2, 1.6], (3, 1))
data = np.ones((20, 2, 6, 8, 2))
for t in range(20):
data[t, 1, 1:4, 2:6, 0] = np.roll(pattern, t, 1) * 1000
path = os.path.join(self.tmp_dir, "test_extract.sima")
seq = sima.Sequence.create('ndarray', data)
self.dataset = sima.ImagingDataset([seq], savedir=path)
def setup(self):
for frame in Sequence.create('HDF5', example_hdf5(), 'yxt'):
break
frame_shifts = [np.array([[[0, 0]], [[-5, -10]]])]
self.frame_shifts = [np.array([[[5, 10]], [[0, 0]]])]
self.correlations = [np.array([[1], [0.9301478]])]
shifted = frame.copy()
shifted = np.roll(shifted, -frame_shifts[0][1, 0, 1], axis=2)
shifted = np.roll(shifted, -frame_shifts[0][1, 0, 0], axis=1)
frames = np.array([frame, shifted])
self.hm2d = hmm.HiddenMarkov2D(n_processes=1, verbose=False)
self.dataset = sima.ImagingDataset(
[Sequence.create('ndarray', frames)], None)
def check_create_sima_dataset(fpath):
fdir = os.path.split(fpath)[0]
fname = os.path.split(fpath)[1]
fbasename = os.path.splitext(fname)[0]
fext = os.path.splitext(fname)[1]
sima_folder_path = os.path.join(fdir, fbasename + '_mc.sima')
if not os.path.exists(sima_folder_path):
# create a sima sequence with the data
if fext == '.tif' or fext == '.tiff':
sequences = [sima.Sequence.create('TIFF', fpath)]
elif fext == '.h5':
sequences = [sima.Sequence.create('HDF5', fpath, 'tyx')]
# creates sima imaging dataset, but more importantly saves a .sima folder required for downstream processing
sima.ImagingDataset(sequences, sima_folder_path)
def initialize_sbx_datasets(path, calc_time_averages=False):
"""Locate and initialize a SIMA dataset for all Scanbox sbx files."""
for directory, folders, files in os.walk(path):
for sbx_file in fnmatch.filter(files, '*.sbx'):
info_file = os.path.splitext(sbx_file)[0] + '.mat'
sima_dir = os.path.splitext(sbx_file)[0] + '.sima'
if info_file in files and sima_dir not in folders:
print("Initializing SIMA dataset: {}".format(
os.path.join(directory, sima_dir)))
seq = sbxread(os.path.join(directory, sbx_file),
os.path.join(directory, info_file))
dset = sima.ImagingDataset([seq],
savedir=os.path.join(
directory, sima_dir))
if calc_time_averages:
print("Calculating time averages: {}".format(
os.path.join(directory, sima_dir)))
dset.time_averages
def _estimate(self, dataset):
if not next(iter(dataset)).shape[2] % 2 == 0:
raise ValueError(
'Resonant motion correction requires an even number of rows')
downsampled_dataset = sima.ImagingDataset([
sima.Sequence.join(*it.chain.from_iterable(
(seq[:, :, ::2, :, c], seq[:, :, 1::2, :, c])
for c in range(seq.shape[4]))) for seq in dataset
], None)
downsampled_displacements = self._base_strategy.estimate(
downsampled_dataset)
displacements = []
for d_disps in downsampled_displacements:
disps = np.repeat(d_disps, 2, axis=2) # Repeat the displacements
disps[:, :, :, 0] *= 2 # multiply y-shifts by 2
disps[:, :, 1::2, -1] += self._offset # shift even rows by offset
displacements.append(disps)
return displacements
new_templ[None, None, :, :, None], newim[None, None, :, :,
None],
new_templ[None, None, :, :, None], newim[None, None, :, :,
None],
new_templ[None, None, :, :, None], newim[None, None, :, :,
None],
new_templ[None, None, :, :, None], newim[None, None, :, :,
None],
new_templ[None, None, :, :, None], newim[None, None, :, :,
None],
new_templ[None, None, :, :, None], newim[None, None, :, :,
None]
],
axis=0))
]
dataset = sima.ImagingDataset(sequences, None)
mc_approach = sima.motion.HiddenMarkov2D(granularity=(granularity, gran_n),
max_displacement=[max_shift] * 2,
verbose=True,
n_processes=14)
shifts_sima = mc_approach.estimate(dataset)
vx = cv2.resize(shifts_sima[0][0][0][:, 0].astype(np.float32), newim.shape)
vy = cv2.resize(shifts_sima[0][0][0][:, 1].astype(np.float32), newim.shape)
pl.imshow(apply_field(newim, vx, vy))
# mov_sima = mc_approach.correct(dataset,None,correction_channels = [1])
print(np.sum(shifts_sima))
break
#%%
pl.subplot(1, 2, 1)
pl.imshow(shifts_sima[0][:, 0, :, 0].squeeze(), aspect='auto')
pl.colorbar()
cluster_movie = np.zeros(shape=(int(epoch_frames.sum()),1,mov_xDim,mov_yDim,1))
original_movie = np.zeros(shape=(np.shape(time_series)[0],1,mov_xDim,mov_yDim,1))
original_movie[:,0,:,:,0]= time_series
startFrame = 0
for index, epoch in enumerate(epochs_to_use):
if index>0:
startFrame = endFrame
endFrame = startFrame + epoch_frames[index]
cluster_movie[int(startFrame):int(endFrame),0,:,:,0] = wholeTraces_allTrials[epoch].mean(axis=3)
# Create a sima dataset and export the cluster movie
b = sima.Sequence.create('ndarray',cluster_movie)
cluster_dataset = sima.ImagingDataset([b],None)
cluster_dataset.export_frames([[[os.path.join(selected_movie_dir,'cluster_vid.tif')]]],
fill_gaps=True,scale_values=True)
#%%
nCpu=(multiprocessing.cpu_count() - 1)
cluster_1d_max_size_pixel = cluster_max_1d_size_micron/x_size
cluster_1d_min_size_pixel = cluster_min_1d_size_micron/x_size
area_max = int(math.pow(2/x_size, 2)) # 4um square max area
#area_min = int(math.pow(cluster_1d_min_size_pixel, 2)) # 1um square min area
area_min = int(math.pow(1.1/x_size, 2)) # 1um square min area
#if area_min < 10:
def setup(self):
data = np.ones((10, 2, 6, 8, 2))
data[:, 0, 1:4, 2:6, 1] = 1000
seq = sima.Sequence.create('ndarray', data)
self.dataset = sima.ImagingDataset([seq], savedir=None)
def setup(self):
self.strategy = sima.motion.ResonantCorrection(
sima.motion.HiddenMarkov2D(n_processes=1, verbose=False), 5)
self.dataset = sima.ImagingDataset(
[Sequence.create('HDF5', example_volume(), 'tzyxc')], None)
def create_sequence(full_path_to_file, individualframename):
# Retrieve S3 filename without path
temp = os.path.splitext(os.path.basename(full_path_to_file))
filename = temp[0]
if filename[-4:] == '_CH1':
dual_channel = True
else:
dual_channel = False
if is_single_file:
file_format = temp[-1]
if dual_channel:
# Identify file format to create SIMA Sequence object
if file_format == '.tif':
import_format = 'TIFF'
sequences1 = sima.Sequence.create(import_format,
filename + file_format)
sequences2 = sima.Sequence.create(
import_format, filename[:-1] + '2' + file_format)
sequences = [sima.Sequence.join(sequences1, sequences2)]
else: # i.e., if file_format == '.h5':
import_format = 'HDF5'
sequences1 = sima.Sequence.create(import_format,
filename + file_format,
dim_order_for_hdf5)
sequences2 = sima.Sequence.create(
import_format, filename[:-1] + '2' + file_format,
dim_order_for_hdf5)
sequences = [sima.Sequence.join(sequences1, sequences2)]
else:
# Identify file format to create SIMA Sequence object
if file_format == '.tif':
import_format = 'TIFF'
sequences = [
sima.Sequence.create(import_format,
os.path.basename(full_path_to_file))
]
else: # i.e., if file_format == '.h5':
import_format = 'HDF5'
sequences = [
sima.Sequence.create(import_format,
os.path.basename(full_path_to_file),
dim_order_for_hdf5)
]
else: # file format is tiff
import_format = 'TIFFs'
if dual_channel:
# 'filename' is really the directory. An individual frame's
# file name prefix is defined by 'individualframename'
sequences1 = sima.Sequence.create(import_format, [[
base_path_on_ec2 + filename + '/' + individualframename +
'_*.tif'
]])
sequences2 = sima.Sequence.create(import_format, [[
base_path_on_ec2 + filename[:-1] + '2' + '/' +
individualframename + '_*.tif'
]])
sequences = [sima.Sequence.join(sequences1, sequences2)]
else:
# 'filename' is really the directory. An individual frame's
# file name prefix is defined by 'individualframename'
sequences = [
sima.Sequence.create(import_format, [[
base_path_on_ec2 + filename + '/' + individualframename +
'_*.tif'
]])
]
sima.ImagingDataset(sequences, filename + '.sima')
return filename, dual_channel
def correct(self,
dataset,
savedir,
channel_names=None,
info=None,
correction_channels=None,
trim_criterion=None):
"""Create a motion-corrected dataset.
Parameters
----------
dataset : sima.ImagingDataset or list of sima.Sequence
Dataset or sequences to be motion corrected.
savedir : str
The directory used to store the dataset. If the directory
name does not end with .sima, then this extension will
be appended.
channel_names : list of str, optional
Names for the channels. Defaults to ['0', '1', '2', ...].
info : dict
Data for the order and timing of the data acquisition.
See sima.ImagingDataset for details.
correction_channels : list of int, optional
Information from the channels corresponding to these indices
will be used for motion correction. By default, all channels
will be used.
trim_criterion : float, optional
The required fraction of frames during which a location must
be within the field of view for it to be included in the
motion-corrected imaging frames. By default, only locations
that are always within the field of view are retained.
Returns
-------
dataset : sima.ImagingDataset
The motion-corrected dataset.
"""
sequences = [s for s in dataset]
if correction_channels:
correction_channels = [
sima.misc.resolve_channels(c, channel_names, len(sequences[0]))
for c in correction_channels
]
mc_sequences = [
s[:, :, :, :, correction_channels] for s in sequences
]
else:
mc_sequences = sequences
displacements = self.estimate(sima.ImagingDataset(mc_sequences, None))
disp_dim = displacements[0].shape[-1]
max_disp = np.ceil(
np.max(list(
it.chain.from_iterable(
d.reshape(-1, disp_dim) for d in displacements)),
axis=0)).astype(np.int64)
frame_shape = np.array(sequences[0].shape)[1:-1] # (z, y, x)
if len(max_disp) == 2: # if 2D displacements
frame_shape[1:3] += max_disp
else: # if 3D displacements
frame_shape += max_disp
corrected_sequences = [
s.apply_displacements(d, frame_shape)
for s, d in zip(sequences, displacements)
]
planes, rows, columns = _trim_coords(trim_criterion, displacements,
sequences[0].shape[1:4],
frame_shape)
corrected_sequences = [
s[:, planes, rows, columns] for s in corrected_sequences
]
return [
sima.ImagingDataset(corrected_sequences,
savedir,
channel_names=channel_names), rows, columns
] # CZ added rows and columns (trimming edges)
fig = plt.figure()
axes = [fig.add_subplot(311),
fig.add_subplot(312),
fig.add_subplot(313)]
for chan in zip(errors, corrs, fold_corrs):
for i in range(3):
axes[i].plot(range(1, 17), chan[i])
axes[0].set_ylabel('error')
axes[0].legend([str(c+1) for c in range(len(errors))])
axes[1].set_ylabel('corr')
axes[2].set_ylabel('fold corr')
for i in range(1, 3):
axes[i].plot([1, 16], [0, 0], 'k')
plt.show()
if __name__ == '__main__':
path = '/data/Joseph/Imaging/JT39/2014-09-13/TSeries-FOV1-000/TSeries-FOV1-000_Cycle00001_Element00001.h5'
seq = sima.Sequence.create('HDF5', path, 'tzyxc')[5:10, ::20]
ds = sima.ImagingDataset([seq], savedir=None)
# path = '/data/Jeff/running/jz107/TSeries-12112014-Day1-Session1-000/TSeries-12112014-Day1-Session1-000_Cycle00001_Element00001.h5'
# path = '/data/Nathan/2photon/acuteRemapping/nd99/10282014/ctxA-002/ctxA-002_Cycle00001_Element00001.h5'
# path = '/data/Nathan/2photon/acuteRemapping/nd113/12102014/ctxA-002/ctxA-002_Cycle00001_Element00001.h5'
# path = '/data/Nathan/2photon/acuteRemapping/nd101/10292014/ctxB-001/ctxB-001_Cycle00001_Element00001.h5'
seq = sima.Sequence.create('HDF5', path, 'tzyxc')[0:20]
ds = sima.ImagingDataset([seq], savedir=None)
print "Analyze:"
analyze(ds)
# 'weight' : 'regular',
# 'size' : 15}
#
#pl.rc('font', **font)
pl.rcParams['pdf.fonttype'] = 42
#%% test against SIMA
import sima
import sima.motion
from sima.motion import HiddenMarkov2D
#fname_gr = 'M_FLUO_t.tif'
#fname_gr = 'Sue_1000.tif'
#fname_gr = 'Sue_2000.tif'
fname_gr = 'Sue_1000_T.tif'
fname_gr = 'Sue_1000_T.tifrow1_example_sima_T.tif'
sequences = [sima.Sequence.create('TIFF', fname_gr)]
dataset = sima.ImagingDataset(sequences, fname_gr)
#%%
import time
t1 = time.time()
granularity = 'row'
gran_n = 1
mc_approach = sima.motion.HiddenMarkov2D(granularity=(granularity, gran_n),
max_displacement=max_shifts,
verbose=True,
n_processes=14)
new_dataset = mc_approach.correct(dataset, None)
t2 = time.time() - t1
print(t2)
#%
new_dataset.export_frames(
data_dir="./real_data/"
ds_name='example_data.sima'
# Create filenames
tiff_filenames = [ [data_dir + '20150303_AF{n1:02d}_{n2:03d}.tif'.format(n1=animal_nr, n2=session_nr) for animal_nr in range(3,4)] for session_nr in range(2, 3)]
# The resulting filenames are printed for clarification.
print "TIFF filenames:\n", tiff_filenames
# Finally, we construct a MultiPageTIFF iterable using each of the filenames.
sequences = [
sima.Sequence.join(*[sima.Sequence.create('TIFF', chan) for chan in cycle])
for cycle in tiff_filenames]
dataset = sima.ImagingDataset(sequences, ds_name)
# load using :
#dataset = sima.ImagingDataset.load(ds_name)
# Set motion correction parameters
mc_approach = sima.motion.PlaneTranslation2D(max_displacement=[15, 30])
# Apply motion correction
# Needs dataset or sequences and save dir/dataset name
dataset = mc_approach.correct(dataset, ds_name)
# Output corrected frames
output_filenames = [
[[channel.replace('.tif', '_corrected.tif') for channel in cycle]]
for cycle in tiff_filenames
