def _extract_st_rois(input_rois, min_area):
""" Extract ROIs from the spatio-temporal components
Parameters
----------
input_rois : ROIList
list of ROIs
min_area : int
The minimum size in number of pixels that an ROI can be.
Returns
-------
rois : list
A list of sima.ROI ROI objects
"""
rois = []
for frame in input_rois:
img = np.array(frame)
img[np.where(img > 0)] = 1
img, seg_count = ndimage.measurements.label(img)
for i in range(seg_count):
segment = np.where(img == i + 1)
if segment[0].size >= min_area:
thisroi = np.zeros(img.shape, 'bool')
thisroi[segment] = True
rois.append(ROI(mask=thisroi, im_shape=thisroi.shape))
return rois
def apply(self, rois, dataset=None):
""" Remove overlapping ROIs
Parameters
----------
rois : list
list of sima.ROI ROIs
percent_overlap : float
percent of the smaller ROIs total area which must be covered in
order for the ROIs to be evaluated as overlapping
Returns
-------
rois : list
A list of sima.ROI ROI objects with the overlapping ROIs combined
"""
for roi in rois:
roi.mask = roi.mask
for i in range(len(rois)): # TODO: more efficient strategy
for j in [j for j in range(len(rois)) if j != i]:
if rois[i] is not None and rois[j] is not None:
overlap = np.logical_and(rois[i], rois[j])
small_area = min(np.size(rois[i]), np.size(rois[j]))
if len(np.where(overlap)[0]) > \
self.percent_overlap * small_area:
new_shape = np.logical_or(rois[i], rois[j])
rois[i] = ROI(mask=new_shape.astype('bool'))
rois[j] = None
return ROIList(roi for roi in rois if roi is not None)
def simplifyRoi():
roi_id = request.form.get('roiId')
frame_shape = json.loads(request.form.get('frame_shape'))
points = json.loads(request.form.get('points'))
roi_data = []
for i, plane in enumerate(points):
if plane is None or not len(plane):
continue
array_dat = np.array(plane)
z_dims = i * np.ones((array_dat.shape[:2] + (1, )))
plane_data = np.concatenate((array_dat, z_dims), axis=2)
roi_data.extend(list(plane_data))
try:
roi = ROI(polygons=roi_data, im_shape=frame_shape[:3])
except:
return jsonify(result='failed to create ROI')
smoother = SmoothROIBoundaries()
roi = smoother.apply([roi])[0]
convertedRoi = []
try:
for i in xrange(roi.im_shape[0]):
convertedRoi.append([])
except:
for i in xrange(np.max(np.array(roi.coords)[:, :, 2])):
convertedRoi.append([])
for poly in roi.polygons:
coords = np.array(poly.exterior.coords)
plane = int(coords[0, -1])
coords = coords[:, :2].astype(int).tolist()
convertedRoi[plane].append(coords)
return jsonify({roi_id: {'points': convertedRoi}})
def _segment(self, dataset):
channel = sima.misc.resolve_channels(self._params['channel'],
dataset.channel_names)
if dataset.savedir is not None:
pca_path = os.path.join(dataset.savedir,
'opca_' + str(channel) + '.npz')
else:
pca_path = None
if dataset.savedir is not None:
ica_path = os.path.join(dataset.savedir,
'ica_' + str(channel) + '.npz')
else:
ica_path = None
if self._params['verbose']:
print('performing PCA...')
components = self._params['components']
if isinstance(components, int):
components = list(range(components))
_, space_pcs, time_pcs = oPCA.dataset_opca(
dataset, channel, components[-1] + 1, path=pca_path)
space_pcs = np.real(space_pcs)
if self._params['verbose']:
print('performing ICA...')
st_components = _stica(
space_pcs, time_pcs, mu=self._params['mu'], path=ica_path,
n_components=space_pcs.shape[-1])
return ROIList([ROI(st_components[..., i]) for i in
range(st_components.shape[-1])])
def _extract_st_rois(frames, min_area=50, spatial_sep=True):
""" Extract ROIs from the spatio-temporal components
Parameters
----------
frames : list
list of arrays containing stICA components
min_area : int
The minimum size in number of pixels that an ROI can be. Default: 50
spatial_sep : bool
If True, the stICA components will be segmented spatially and
non-contiguous poitns will be made into sparate ROIs. Default: True
Returns
-------
rois : list
A list of sima.ROI ROI objects
"""
rois = []
for frame_no in range(len(frames)):
img = np.array(frames[frame_no])
img[np.where(img > 0)] = 1
img, seg_count = measurements.label(img)
component_mask = np.zeros(img.shape, 'bool')
for i in xrange(seg_count):
segment = np.where(img == i + 1)
if segment[0].size >= min_area:
if spatial_sep:
thisroi = np.zeros(img.shape, 'bool')
thisroi[segment] = True
rois.append(ROI(mask=thisroi, im_shape=thisroi.shape))
else:
component_mask[segment] = True
if not spatial_sep and np.any(component_mask):
rois.append(ROI(mask=component_mask, im_shape=thisroi.shape))
frame_no = frame_no + 1
return rois
def __call__(self, roi):
roi_static = []
roi_cpy = np.array(roi)
for frame in roi_cpy:
# copy the component, remove pixels with low weights
frame[frame < 2 * np.std(frame)] = 0
# smooth the component via static removal and gaussian blur
for _ in range(self.x_smoothing):
check = frame[1:-1, :-2] + frame[1:-1, 2:] + \
frame[:-2, 1:-1] + frame[2, 1:-1]
z = np.zeros(frame.shape)
z[1:-1, 1:-1] = check
frame[np.logical_not(z)] = 0
blurred = ndimage.gaussian_filter(frame, sigma=1)
frame = blurred + frame
frame = frame / np.max(frame)
frame[frame < 2 * np.std(frame)] = 0
# calculate the remaining static in the component
static = np.sum(np.abs(frame[1:-1, 1:-1] - frame[:-2, 1:-1])) + \
np.sum(np.abs(frame[1:-1, 1:-1] - frame[2:, 1:-1])) + \
np.sum(np.abs(frame[1:-1, 1:-1] - frame[1:-1, :-2])) + \
np.sum(np.abs(frame[1:-1, 1:-1] - frame[1:-1, 2:])) + \
np.sum(np.abs(frame[1:-1, 1:-1] - frame[2:, 2:])) + \
np.sum(np.abs(frame[1:-1, 1:-1] - frame[:-2, 2:])) + \
np.sum(np.abs(frame[1:-1, 1:-1] - frame[2:, :-2])) + \
np.sum(np.abs(frame[1:-1, 1:-1] - frame[:-2, :-2]))
static = np.sum(static) * 2.0 / (frame.shape[0] * frame.shape[1])
roi_static.append(static)
# decide if the component should be accepted or rejected
if np.mean(static) < self.threshold:
return ROI(mask=roi_cpy), ROI(mask=roi), None
else:
return None, None, ROI(mask=roi)
def updateRoi():
ds_path = request.form.get('path')
label = request.form.get('label')
points = json.loads(request.form.get('points'))
roi_label = request.form.get('roiLabel')
roi_id = request.form.get('roiId')
dataset = ImagingDataset.load(ds_path)
roi_data = []
for i, plane in enumerate(points):
if plane is None or not len(plane):
continue
array_dat = np.array(plane)
z_dims = i * np.ones((array_dat.shape[:2] + (1, )))
plane_data = np.concatenate((array_dat, z_dims), axis=2)
roi_data.extend(list(plane_data))
if len(roi_data) == 0:
return jsonify(result="no polygons to save")
for poly in roi_data:
if poly.shape[0] < 3:
raise Exception("unable to store polygon with less then 3 points")
roi = ROI(polygons=roi_data, im_shape=dataset.frame_shape[:3])
roi.label = roi_label
roi.id = roi_id
try:
rois = dataset.ROIs[label]
except KeyError:
rois = []
rois = filter(lambda r: r.id != roi_id, rois)
rois.append(roi)
dataset.add_ROIs(ROIList(rois), label=label)
return jsonify(result='success')
def __call__(self, roi):
smoothed_polygons = []
coords = roi.coords
for polygon in coords:
if polygon.shape[0] > self.min_verts:
plane = polygon[0, -1]
smoothed_coords = approximate_polygon(polygon[:, :2],
self.tolerance)
smoothed_coords = np.hstack((smoothed_coords, plane * np.ones(
(smoothed_coords.shape[0], 1))))
else:
smoothed_coords = polygon
smoothed_polygons += [smoothed_coords]
return ROI(polygons=smoothed_polygons, im_shape=roi.im_shape)
def apply(self, rois, dataset):
channel = sima.misc.resolve_channels(self._channel,
dataset.channel_names)
processed_im = _processed_image_ca1pc(dataset, channel,
self._x_diameter,
self._y_diameter)[0]
shape = processed_im.shape[:2]
ROIs = ROIList([])
for roi in rois:
roi_indices = np.nonzero(roi.mask[0])
roi_indices = np.ravel_multi_index(roi_indices, shape)
# pixel values in the cut
vals = processed_im.flat[roi_indices]
# indices of those values below the otsu threshold
# if all values are identical, continue without adding an ROI
try:
roi_indices = roi_indices[vals < threshold_otsu(vals)]
except ValueError:
continue
# apply binary opening and closing to the surviving pixels
# expand the shape by 1 in all directions to correct for edge
# effects of binary opening/closing
twoD_indices = [np.unravel_index(x, shape) for x in roi_indices]
mask = np.zeros([x + 2 for x in shape])
for indices in twoD_indices:
mask[indices[0] + 1, indices[1] + 1] = 1
mask = ndimage.binary_closing(ndimage.binary_opening(mask))
mask = mask[1:-1, 1:-1]
roi_indices = np.where(mask.flat)[0]
# label blobs in each cut
labeled_array, num_features = measurements.label(mask)
for feat in range(num_features):
blob_inds = np.where(labeled_array.flat == feat + 1)[0]
twoD_indices = [np.unravel_index(x, shape) for x in blob_inds]
mask = np.zeros(shape)
for x in twoD_indices:
mask[x] = 1
ROIs.append(ROI(mask=mask))
return ROIs
def _rois_from_cuts(cls, cuts):
"""Return ROI structures each containing the full extent of a cut.
Parameters
----------
cuts : list of sima.normcut.CutRegion
The segmented regions identified by normalized cuts.
Returns
-------
sima.ROI.ROIList
ROI structures corresponding to each cut.
"""
ROIs = ROIList([])
for cut in cuts:
if len(cut.indices):
mask = np.zeros(cut.shape)
for x in cut.indices:
mask[np.unravel_index(x, cut.shape)] = 1
ROIs.append(ROI(mask=mask))
return ROIs
def _segment(self, dataset):
channel = sima.misc.resolve_channels(self._params['channel'],
dataset.channel_names)
if dataset.savedir is not None:
pca_path = os.path.join(dataset.savedir,
'opca_' + str(channel) + '.npz')
else:
pca_path = None
if dataset.savedir is not None:
ica_path = os.path.join(dataset.savedir,
'ica_' + str(channel) + '.npz')
else:
ica_path = None
if self._params['verbose']:
print('performing PCA...')
components = self._params['components']
if isinstance(components, int):
components = list(range(components))
_, space_pcs, time_pcs = oPCA.dataset_opca(
dataset, channel, components[-1] + 1, path=pca_path)
space_pcs = np.real(space_pcs)
# Remove components greater than the number of PCs returned
# in case more components were asked for than the number of
# independent dimensions in the dataset.
components = [c for c in components if c < time_pcs.shape[1]]
if self._params['verbose']:
print('performing ICA...')
st_components = _stica(
space_pcs, time_pcs, mu=self._params['mu'], path=ica_path,
n_components=space_pcs.shape[-1])
return ROIList([ROI(st_components[..., i]) for i in
range(st_components.shape[-1])])
def _remove_overlapping(rois, percent_overlap=0.9):
""" Remove overlapping ROIs
Parameters
----------
rois : list
list of sima.ROI ROIs
percent_overlap : float
percent of the smaller ROIs total area which must be covered in order
for the ROIs to be evaluated as overlapping
Returns
-------
rois : list
A list of sima.ROI ROI objects with the overlapping ROIs combined
"""
if percent_overlap > 0 and percent_overlap <= 1:
for roi in rois:
roi.mask = roi.mask
for i in xrange(len(rois)):
for j in [j for j in xrange(len(rois)) if j != i]:
if rois[i] is not None and rois[j] is not None:
overlap = np.logical_and(rois[i].mask.toarray(),
rois[j].mask.toarray())
small_area = np.min((rois[i].mask.size, rois[j].mask.size))
if len(np.where(overlap)[0]) > \
percent_overlap * small_area:
new_shape = np.logical_or(rois[i].mask.toarray(),
rois[j].mask.toarray())
rois[i] = ROI(mask=new_shape.astype('bool'),
im_shape=rois[i].mask.shape)
rois[j] = None
return ROIList(roi for roi in rois if roi is not None)
def set_z(roi, z):
old_mask = roi.mask
return ROI(mask=[
sparse.lil_matrix(old_mask[0].shape, old_mask[0].dtype)
for _ in range(z - 1)
] + [old_mask[0]])
def __call__(self, roi):
frame = roi.mask[0].todense().copy()
frame[frame > 0] = 1
check = frame[:-2, :-2] + frame[1:-1, :-2] + frame[2:, :-2] + \
frame[:-2, 1:-1] + frame[2:, 1:-1] + frame[:-2:, 2:] + \
frame[1:-1, 2:] + frame[2:, 2:]
z = np.zeros(frame.shape)
z[1:-1, 1:-1] = check
# initialize and array to hold the new polygon and find the first point
b = []
rows, cols = np.where(z > 0)
p = [cols[0], rows[0]]
base = p
# establish an iteration limit to ensue the loop terminates if
# smoothing is unsuccessful
limit = 1500
radius = self.radius
# store whether the radius of search is increased above the initial
# value
tmp_rad = False
for _ in range(limit - 1):
b.append(p)
# find the list of all points at the given radius and adjust to be
# lined up for clockwise traversal
x = np.roll(
np.array(
list(p[0] + list(range(-radius, radius))) +
[p[0] + radius] * (2 * radius + 1) +
list(p[0] + list(range(-radius, radius))[::-1]) +
[p[0] - (radius + 1)] * (2 * radius + 1)), -2)
y = np.roll(
np.array([p[1] - radius] * (2 * radius) +
list(p[1] + list(range(-radius, radius))) +
[p[1] + radius] * (2 * radius + 1) +
list(p[1] +
list(range(-radius, (radius + 1)))[::-1])),
-radius)
# insure that the x and y points are within the image
x[x < 0] = 0
y[y < 0] = 0
x[x >= z.shape[1]] = z.shape[1] - 1
y[y >= z.shape[0]] = z.shape[0] - 1
vals = z[y, x]
# ensure that the vals array has a valid transition from 0 to 1
# otherwise the algorithm has failed
if len(np.where(np.roll(vals, 1) == 0)[0]) == 0 or \
len(np.where(vals > 0)[0]) == 0:
return roi, False
idx = np.intersect1d(
np.where(vals > 0)[0],
np.where(np.roll(vals, 1) == 0)[0])[0]
p = [x[idx], y[idx]]
# check if the traversal is near to the starting point indicating
# that the algorithm has completed. If less then 3 points are found
# this is not yet a valid ROI
if ((p[0] - base[0]) ** 2 + (p[1] - base[1]) ** 2) ** 0.5 < \
1.5 * radius and len(b) > 3:
new_roi = ROI(polygons=[b], im_shape=roi.im_shape)
if new_roi.mask[0].size != 0:
# "well formed ROI"
return new_roi, True
# if p is already in the list of polygon points, increase the
# radius of search. if radius is already larger then 6, blur the
# mask and try again
if p in b:
if radius > 6:
radius = 3
z = ndimage.gaussian_filter(z, sigma=1)
b = []
rows, cols = np.where(z > 0)
p = [cols[0], rows[0]]
base = p
tmp_rad = False
else:
radius = radius + 1
tmp_rad = True
if len(b) > 3:
p = b[-3]
del b[-3:]
elif tmp_rad:
tmp_rad = False
radius = 3
# The maximum number of cycles has completed and no suitable smoothed
# ROI has been determined
return roi, False
def _segment(self, dataset):
channel = sima.misc.resolve_channels(self._params.channel,
dataset.channel_names)
if dataset.savedir is not None:
pca_path = os.path.join(dataset.savedir,
'opca_' + str(channel) + '.npz')
else:
pca_path = None
if dataset.savedir is not None:
ica_path = os.path.join(dataset.savedir,
'ica_' + str(channel) + '.npz')
else:
ica_path = None
if self._params.verbose:
print 'performing PCA...'
if isinstance(self._params.components, int):
self._params.components = range(self._params.components)
_, space_pcs, time_pcs = _OPCA(dataset,
channel,
self._params.components[-1] + 1,
path=pca_path)
space_pcs = np.real(
space_pcs.reshape(dataset.frame_shape[1:3] +
(space_pcs.shape[2], )))
space_pcs = np.array(
[space_pcs[:, :, i] for i in self._params.components]).transpose(
(1, 2, 0))
time_pcs = np.array([time_pcs[:, i]
for i in self._params.components]).transpose(
(1, 0))
if self._params.verbose:
print 'performing ICA...'
st_components = _stica(space_pcs,
time_pcs,
mu=self._params.mu,
path=ica_path,
n_components=space_pcs.shape[2])
if self._params.x_smoothing > 0 or self._params.static_threshold > 0:
accepted, _, _ = _find_useful_components(
st_components,
self._params.static_threshold,
x_smoothing=self._params.x_smoothing)
if self._params.min_area > 0 or self._params.spatial_sep:
rois = _extract_st_rois(accepted,
min_area=self._params.min_area,
spatial_sep=self._params.spatial_sep)
if self._params.smooth_rois:
if self._params.verbose:
print 'smoothing ROIs...'
rois = [_smooth_roi(roi)[0] for roi in rois]
if self._params.verbose:
print 'removing overlapping ROIs...'
rois = _remove_overlapping(
rois, percent_overlap=self._params.overlap_per)
else:
rois = [
ROI(st_components[:, :, i])
for i in xrange(st_components.shape[2])
]
return ROIList(rois)
def _load_version0(path):
"""Load a SIMA 0.x dataset
Parameters
----------
path : str
The path to the original saved dataset, ending in .sima
Examples
--------
>>> from sima.misc import example_data
>>> from sima.misc.convert import _load_version0
>>> ds = _load_version0(example_data())
"""
def parse_channel(channel):
"""Parse an old format channel stored a dictionary
Parameters
----------
channel : dict
Returns
-------
result : sima.Sequence
A sequence equivalent to the old format channel.
"""
_resolve_paths(channel, path)
klass = channel.pop('__class__')
if klass == 'sima.iterables.MultiPageTIFF':
result = Sequence.create('TIFF', channel['path'])
try:
clip = channel['clip']
except KeyError:
pass
else:
if clip is not None:
s = (slice(None), slice(None)) + tuple(
slice(*[None if x is 0 else x for x in dim])
for dim in clip)
result = result[s]
return result
elif klass == 'sima.iterables.HDF5':
raise Exception('TODO')
else:
raise Exception('Format not recognized.')
def parse_sequence(sequence):
channels = [parse_channel(c) for c in sequence]
return Sequence.join(channels)
with open(os.path.join(path, 'dataset.pkl'), 'rb') as f:
unpickler = Unpickler(f)
dataset_dict = unpickler.load()
iterables = dataset_dict.pop('iterables')
sequences = [parse_sequence(seq) for seq in iterables]
# Apply displacements if they exist
try:
with open(os.path.join(path, 'displacements.pkl'), 'rb') as f:
displacements = pkl.load(f)
except IOError:
pass
else:
assert all(np.all(d >= 0) for d in displacements)
max_disp = np.max(list(chain(*displacements)), axis=0)
frame_shape = np.array(sequences[0].shape)[1:]
frame_shape[1:3] += max_disp
sequences = [
s.apply_displacements(d.reshape(s.shape[:3] + (2,)), frame_shape)
for s, d in zip(sequences, displacements)]
try:
trim_coords = dataset_dict.pop('_lazy__trim_coords')
except KeyError:
try:
trim_criterion = dataset_dict.pop('trim_criterion')
except KeyError:
pass
else:
raise Exception(
'Parsing of trim_criterion ' + str(trim_criterion) +
' not yet implemented')
else:
sequences = [s[:, :, trim_coords[0][0]:trim_coords[1][0],
trim_coords[0][1]:trim_coords[1][1]]
for s in sequences]
ds = ImagingDataset(sequences, None)
ds.savedir = path
# Add ROIs if they exist
try:
with open(os.path.join(path, 'rois.pkl'), 'rb') as f:
rois = pkl.load(f)
except IOError:
pass
else:
roi_lists = {}
for label, roi_list_dict in rois.iteritems():
roi_list = []
for roi in roi_list_dict['rois']:
mask = roi['mask']
polygons = roi['polygons']
if mask is not None:
new_roi = ROI(mask=mask)
else:
new_roi = ROI(polygons=polygons)
new_roi.id = roi['id']
new_roi.label = roi['label']
new_roi.tags = roi['tags']
new_roi.im_shape = roi['im_shape']
roi_list.append(new_roi)
roi_lists[label] = ROIList(roi_list)
roi_lists[label].timestamp = roi_list_dict['timestamp']
for label, roi_list in roi_lists.iteritems():
ds.add_ROIs(roi_list, label=label)
return ds
