def measure_expression(self, im, labels, segment_ids):
"""
Measure expression levels.
Args:
im (np.ndarray[float]) - 3D array of pixel values
labels (np.ndarray[int]) - cell segment labels
segment_ids (np.ndarray[int]) - ordered segment IDs
"""
# split R/G/B image channels
drop = lambda x: x.reshape(*x.shape[:2])
channels = [drop(x) for x in np.split(im, self.colordepth, axis=-1)]
# compute means
means = [mean(channel, labels, segment_ids) for channel in channels]
# compute std
with catch_warnings():
filterwarnings('ignore')
evaluate_std = lambda x: standard_deviation(x, labels, segment_ids)
stds = [evaluate_std(channel) for channel in channels]
# compile dictionaries
self.levels = dict(enumerate(means))
self.std = dict(enumerate(stds))
def interpolate_curvature(curvatures, times, kind='cubic'):
""" Interpolate the curvatures.
A curvature is a parametrisation `s`, d(angle)/ ds and a parameter between [0,1].
Return a surface f(s,t).
"""
import numpy as np
from scipy import interpolate
from scipy.ndimage import measurements
if len(curvatures) == 3 and not isinstance(curvatures[2], tuple):
curvatures= [curvatures]
curv_abs = [c[0] for c in curvatures]
curves = [c[1] for c in curvatures]
params = [c[2] for c in curvatures]
n = len(params)
if n==1:
curv_abs*=2
curves*=2
curves[0]=np.zeros(len(curves[0]))
params = [0.,1.]
elif False:
if params[0] != 0.:
params.insert(0,0.)
curves.insert(0,curves[0])
curv_abs.insert(0, curv_abs[0])
if params[-1] != 0:
params.append(1.)
curves.append(curves[-1])
curv_abs.append(curv_abs[-1])
# compute a common parametrisation s
# We conserve the last parametrisation because the result is very sensitive
if True:
min_s = min(np.diff(s).min() for s in curv_abs)
s_new = np.unique(np.array(curv_abs).flatten())
ds = np.diff(s_new)
k = np.cumsum(ds >= min_s)
labels = range(k.min(), k.max()+1)
s = np.zeros(len(labels)+1)
s[1:] = measurements.mean(s_new[1:], k, labels)
s[-1]=1.
s = s_new
else:
s = np.array(curv_abs[-1])
# renormalise all the curves
curves = [np.interp(s[1:-1], old_s[1:-1], old_crv) for old_s, old_crv in zip(curv_abs,curves)]
# interpolate correctly the curvatures
x = s[1:-1]
y = np.array(params)
z = np.array(curves)
#f = interpolate.interp2d(y, x, z, kind=kind)
f = interpolate.RectBivariateSpline(x, y, z.T, kx=1, ky=1)
return s, f(x,times)
def _showMeanStd(self, busy=True):
if busy:
busy = BusyIndicator()
dmin, dmax = self._subset_range
subset, mask = self.image.optimalRavel(self._subset)
dprint(5, "computing mean")
mean = measurements.mean(subset, labels=mask, index=None if mask is None else False)
dprint(5, "computing std")
std = measurements.standard_deviation(subset, labels=mask, index=None if mask is None else False)
dprint(5, "done")
text = " ".join([("%s: " + DataValueFormat) % (name, value) for name, value in
("min", dmin), ("max", dmax), ("mean", mean), ("std", std)] + ["np: %d" % self._subset.size])
self._wlab_stats.setText(text)
self._wmore_stats.hide()
# update markers
ypos = 0.3
self._line_mean.line.setData([mean, mean], [0, 1])
self._line_mean.marker.setValue(mean, ypos)
self._line_mean.setText(("\u03BC=" + DataValueFormat) % mean)
self._line_mean.show()
self._line_std.line.setData([mean - std, mean + std], [ypos, ypos])
self._line_std.marker.setValue(mean, ypos)
self._line_std.setText(("\u03C3=" + DataValueFormat) % std)
self._line_std.show()
self._histplot.replot()
def _threshold_and_mask_single_frame(im, threshold=None, mask=None):
image = copy.deepcopy(im)
if mask is not None:
image *= mask
if threshold is not None:
mean_value = measurements.mean(image, mask) * threshold
image[image <= mean_value] = 0
image[image > mean_value] = 1
return image
def measure(im, labels, num_objs, measurement_type='mean'):
if measurement_type == 'mean':
res = spm.mean(im, labels, range(1, num_objs))
elif measurement_type == 'max':
res = spm.maximum(im, labels, range(1, num_objs))
else:
raise ValueError('Unsporrted measurement type: {}'.format(measurement_type))
return res
def _get_prev_lab(prevlabelIm, labelIm, label, center):
if int(
labelIm[int(round(center[0])),
int(round(center[1]))]
) == label: # in case center is inluced in object -> simply return value @ center
prev_lab = int(prevlabelIm[int(round(center[0])),
int(round(center[1]))])
else:
prev_lab = int(round(mean(prevlabelIm, labelIm, label)))
return prev_lab
def get_superpixel_means_band(label_array: np.ndarray,
band: np.ndarray) -> np.ndarray:
"""Assume labels in label_array are 0, 1, 2, ..., n
Returns array of shape = (len(np.unique(labels)) x 1)
"""
labels_ = label_array + 1 # scipy wants labels to begin at 1 and transforms to 1, 2, ..., n+1
labels_unique = np.unique(labels_)
means = measurements.mean(band, labels=labels_, index=labels_unique)
return means.reshape((-1, 1))
def getLMRectStats(self, rect):
xx1, xx2, yy1, yy2 = self._lmRectToPix(rect)
if xx1 is not None:
subset = self.image.image()[xx1:xx2, yy1:yy2]
subset, mask = self.image.optimalRavel(subset)
mmin, mmax = measurements.extrema(subset, labels=mask, index=None if mask is None else False)[:2]
mean = measurements.mean(subset, labels=mask, index=None if mask is None else False)
std = measurements.standard_deviation(subset, labels=mask, index=None if mask is None else False)
ssum = measurements.sum(subset, labels=mask, index=None if mask is None else False)
return xx1, xx2, yy1, yy2, mmin, mmax, mean, std, ssum, subset.size
return None
def relabelise(filled, A , nofroi):
labels, nlabs = meas.label(filled)
aves = meas.mean(A ,labels=labels, index = range(1,nlabs+1) )
avlabs = zip(aves, np.arange(1,nlabs+1))
avlabs.sort()
intes = [ a for a,l in avlabs[-nofroi:]]
amed = np.median(intes)
labs = [ l for a,l in avlabs[-nofroi:] if a>amed/100]
newmask = np.zeros(filled.shape,"i")
i=1
for l in labs:
newmask[np.equal(labels,l)]=i
i+=1
return newmask
def getPeaks(im, nstd=6, maxsize=None):
"""
Detects the peaks using edge detection
Parameters
----------
im: 2d array
The image to fit
nstd: integer, default 6
Number of STD to use for theshold
maxsize: integer
Maximim size of the peak
Returns
-------
peaks: 2d array
mask of the peaks location
Notes
-----
Position of high slope and intern parts are marked
"""
im = np.asarray(im, dtype='float')
imblur = np.empty(im.shape)
edge = cr.Scharr_edge(im, imblur=imblur)
threshold = np.nanmean(edge) + 6 * np.nanstd(edge)
peaks = edge > threshold
labels, n = msr.label(peaks)
intensity_inclusions = msr.mean(imblur, labels, np.arange(n) + 1)
for i in np.arange(n) + 1:
if intensity_inclusions[i - 1] > np.nanmean(imblur):
high, m = msr.label(imblur > intensity_inclusions[i - 1])
for j in np.unique(high[np.logical_and(labels == i, high > 0)]):
labels[high == j] = i
if maxsize is not None and np.sum(labels == i) > maxsize:
labels[labels == i] = 0
else:
labels[labels == i] = 0
"""
from matplotlib.pyplot import figure, hist, plot, title, ylim
figure()
hist(edge[np.isfinite(edge)],100)
plot(threshold*np.array([1,1]),[0,ylim()[1]])
title(str(threshold))
#"""
return labels > 0
def mean_intensity(images, labeled_array, index=None):
"""Compute the mean intensity for each ROI in the image list
Parameters
----------
images : list
List of images
labeled_array : array
labeled array; 0 is background.
Each ROI is represented by a nonzero integer. It is not required that
the ROI labels are contiguous
index : int, list, optional
The ROI's to use. If None, this function will extract averages for all
ROIs
Returns
-------
mean_intensity : array
The mean intensity of each ROI for all `images`
Dimensions:
len(mean_intensity) == len(index)
len(mean_intensity[0]) == len(images)
index : list
The labels for each element of the `mean_intensity` list
"""
if labeled_array.shape != images[0].shape[0:]:
raise ValueError(
"`images` shape (%s) needs to be equal to the labeled_array shape"
"(%s)" % (images[0].shape, labeled_array.shape)
)
# handle various input for `index`
if index is None:
index = list(np.unique(labeled_array))
index.remove(0)
try:
len(index)
except TypeError:
index = [index]
# pre-allocate an array for performance
# might be able to use list comprehension to make this faster
mean_intensity = np.zeros((images.shape[0], len(index)))
for n, img in enumerate(images):
# use a mean that is mask-aware
mean_intensity[n] = ndim.mean(img, labeled_array, index=index)
return mean_intensity, index
def mean_intensity(images, labeled_array, index=None):
"""Compute the mean intensity for each ROI in the image list
Parameters
----------
images : list
List of images
labeled_array : array
labeled array; 0 is background.
Each ROI is represented by a nonzero integer. It is not required that
the ROI labels are contiguous
index : int, list, optional
The ROI's to use. If None, this function will extract averages for all
ROIs
Returns
-------
mean_intensity : array
The mean intensity of each ROI for all `images`
Dimensions:
- len(mean_intensity) == len(index)
- len(mean_intensity[0]) == len(images)
index : list
The labels for each element of the `mean_intensity` list
"""
if labeled_array.shape != images[0].shape[0:]:
raise ValueError(
"`images` shape (%s) needs to be equal to the labeled_array shape"
"(%s)" % (images[0].shape, labeled_array.shape))
# handle various input for `index`
if index is None:
index = list(np.unique(labeled_array))
index.remove(0)
try:
len(index)
except TypeError:
index = [index]
# pre-allocate an array for performance
# might be able to use list comprehension to make this faster
mean_intensity = np.zeros((images.shape[0], len(index)))
for n, img in enumerate(images):
# use a mean that is mask-aware
mean_intensity[n] = ndim.mean(img, labeled_array, index=index)
return mean_intensity, index
def sptr_mean(path, fname, boolean_mask):
'''
sptr_mean takes an input of the hyperspectral image and the boolean array
from hyper_pixcorr function to give an output image with mean spectral intensities
across the sources in each spectral channel
Input Parameters:
------------
path = str
Path to the directory where hyperspectral images are located
fname = str
File name of raw hyperspectral image
boolean_mask = np.array
Output boolean mask of sources from hyper_pixcorr function
Output:
------------
src_sptr_mean = np.array
Mean Spectrum of sources across corresponsing source pixels
'''
# Reading the Raw hyperspectral image
cube = utils.read_hyper(path, fname)
# Storing the hyperspectral image as a memmap for future computations
img = 1.0 * cube.data
#Labeling the sources in Boolean Mask
mask = boolean_mask
labels, count = mm.label(mask)
index = np.arange(count + 1)
sptr_stack = []
for i in range(0, img.shape[0]):
channel = img[i, :-1, :-1]
src_mean = mm.mean(channel, labels, index)
sptr_stack.append(src_mean) #redund
#sptr_stack = np.array(sptr_stack) #redund
#sources = np.array([sptr_stack[:,i] for i in range(count)]) #redund
return np.array(sptr_stack)
def img_to_signals(in_file,
atlas_file,
mask_file=None,
background_label=0,
min_n_voxels=50):
in_img = nib.load(in_file)
atlas_img = nib.load(atlas_file)
assert nvol(atlas_img) == 1
assert atlas_img.shape[:3] == in_img.shape[:3]
assert np.allclose(atlas_img.affine, in_img.affine)
labels = np.asanyarray(atlas_img.dataobj).astype(np.int32)
nlabel = labels.max()
if mask_file is not None:
mask_img = nib.load(mask_file)
assert nvol(mask_img) == 1
assert mask_img.shape[:3] == in_img.shape[:3]
assert np.allclose(mask_img.affine, in_img.affine)
mask_data = np.asanyarray(mask_img.dataobj).astype(np.bool)
labels[np.logical_not(mask_data)] = background_label
assert np.all(labels >= 0)
indices, counts = np.unique(labels, return_counts=True)
indices = indices[counts >= min_n_voxels]
indices = np.setdiff1d(indices, [background_label])
in_data = in_img.get_fdata()
if in_data.ndim == 3:
in_data = in_data[:, :, :, np.newaxis]
assert in_data.ndim == 4
result = np.full((in_data.shape[3], nlabel), np.nan)
for i, img in enumerate(np.moveaxis(in_data, 3, 0)):
result[i, indices - 1] = mean(img, labels=labels, index=indices)
return result
def mean_intensity(images, labels, index=None):
"""
Mean intensities for ROIS' of the labeled array for set of images
Parameters
----------
images : array
Intensity array of the images
dimensions are: (num_img, num_rows, num_cols)
labels : array
labeled array; 0 is background.
Each ROI is represented by a distinct label (i.e., integer).
index : list
labels list
eg: 5 ROI's
index = [1, 2, 3, 4, 5]
Returns
-------
mean_intensity : array
mean intensity of each ROI for the set of images as an array
shape (len(images), number of labels)
"""
if labels.shape != images[0].shape[0:]:
raise ValueError("Shape of the images should be equal to"
" shape of the label array")
if index is None:
index = np.arange(1, np.max(labels) + 1)
mean_intensity = np.zeros((images.shape[0], index.shape[0]))
for n, img in enumerate(images):
mean_intensity[n] = ndim.mean(img, labels, index=index)
return mean_intensity, index
def saveInclusions(im, labels, maskWorm, imagesFolder, fn, i, um2px2ratio=1,
im_number_per_worm_pixel=[], im_intensity=[], im_size=[],
im_index=[], j=0):
if not os.path.isdir(imagesFolder):
os.mkdir(imagesFolder)
imagesFolder = imagesFolder + str(i)
if j > 0:
imagesFolder = imagesFolder + '_{}'.format(j)
# get worm coverage
prct = maskWorm.sum() / np.product(im.shape)
nl = labels.max()
# extract normalized intensity
intensity_inclusions = msr.mean(im, labels, np.arange(nl) + 1)
# extract size
size_inclusions = np.bincount(labels.flat)[1:] * um2px2ratio
# plot image
f = figure()
imshow(im)
colorbar()
# if no inclusions
if nl < 1:
plt.title(fn + ', no Inclusions, %.1f %% worm' % (prct * 100))
plt.savefig(imagesFolder + '_image')
f.close()
else:
# Fill array
im_number_per_worm_pixel.append(nl / maskWorm.sum())
im_intensity.append(intensity_inclusions.mean())
im_size.append(size_inclusions.mean())
im_index.append((i, j))
plt.title(fn + ', %.1f %% worm' % (prct * 100))
plt.savefig(imagesFolder + '_image')
plt.close()
# plot detected inclusions
# coordinates = peak_local_max(im, min_distance=20)
f = figure()
imshow(labels)
imshow(maskWorm, alpha=.5)
# plot(coordinates[:, 1], coordinates[:, 0], 'w.')
plt.title(
"%d inclusions, mean = %.2f" %
(nl, intensity_inclusions.mean()))
plt.savefig(imagesFolder + '_inclusions')
plt.close()
# plot intensity
f = figure()
plt.hist(intensity_inclusions, 20)
plt.xlabel('Normalized intensity')
plt.ylabel('Number of inclusions')
plt.savefig(imagesFolder + '_intensityHistogram')
plt.close()
f = figure()
plt.hist(size_inclusions, 20)
plt.xlabel('Size')
plt.ylabel('Number of inclusions')
plt.savefig(imagesFolder + '_sizeHistogram')
plt.close()
writeFileInfos(os.path.splitext(fn)[0] + "_{}.txt".format(j),
intensity_inclusions, size_inclusions, prct)
def _measure(self, channel):
""" Returns measured <channel> level in for each contour. """
labels = self.segmentation
index = np.arange(self.num_nuclei)
return measurements.mean(self.im[channel], labels=labels, index=index)
def getMACDSignal(self, t): """t時点におけるMACDシグナルを返します.""" dat = [self.getMACD(t-i) for i in xrange(self.__z)] return mean(dat)
def interpolate_curvature(curvatures, times, kind='cubic'):
""" Interpolate the curvatures.
A curvature is a parametrisation `s`, d(angle)/ ds and a parameter between [0,1].
Return a surface f(s,t).
"""
import numpy as np
from scipy import interpolate
from scipy.ndimage import measurements
if len(curvatures) == 3 and not isinstance(curvatures[2], tuple):
curvatures = [curvatures]
curv_abs = [c[0] for c in curvatures]
curves = [c[1] for c in curvatures]
params = [c[2] for c in curvatures]
n = len(params)
if n == 1:
curv_abs *= 2
curves *= 2
curves[0] = np.zeros(len(curves[0]))
params = [0., 1.]
elif False:
if params[0] != 0.:
params.insert(0, 0.)
curves.insert(0, curves[0])
curv_abs.insert(0, curv_abs[0])
if params[-1] != 0:
params.append(1.)
curves.append(curves[-1])
curv_abs.append(curv_abs[-1])
# compute a common parametrisation s
# We conserve the last parametrisation because the result is very sensitive
if True:
min_s = min(np.diff(s).min() for s in curv_abs)
s_new = np.unique(np.array(curv_abs).flatten())
ds = np.diff(s_new)
k = np.cumsum(ds >= min_s)
labels = range(k.min(), k.max() + 1)
s = np.zeros(len(labels) + 1)
s[1:] = measurements.mean(s_new[1:], k, labels)
s[-1] = 1.
s = s_new
else:
s = np.array(curv_abs[-1])
# renormalise all the curves
curves = [
np.interp(s[1:-1], old_s[1:-1], old_crv)
for old_s, old_crv in zip(curv_abs, curves)
]
# interpolate correctly the curvatures
x = s[1:-1]
y = np.array(params)
z = np.array(curves)
#f = interpolate.interp2d(y, x, z, kind=kind)
f = interpolate.RectBivariateSpline(x, y, z.T, kx=1, ky=1)
return s, f(x, times)
def meansignals(in_file,
atlas_file,
mask_file=None,
background_label=0,
min_region_coverage=0.5,
output_coverage=False):
in_img = nib.load(in_file)
atlas_img = nib.load(atlas_file)
assert nvol(atlas_img) == 1
assert atlas_img.shape[:3] == in_img.shape[:3]
assert np.allclose(atlas_img.affine, in_img.affine)
labels = np.asanyarray(atlas_img.dataobj).astype(np.int32)
nlabel = labels.max()
assert background_label <= nlabel
indices = np.arange(0, nlabel + 1, dtype=np.int32)
out_region_coverage = None
if mask_file is not None:
mask_img = nib.load(mask_file)
assert nvol(mask_img) == 1
assert mask_img.shape[:3] == in_img.shape[:3]
assert np.allclose(mask_img.affine, in_img.affine)
mask_data = np.asanyarray(mask_img.dataobj).astype(np.bool)
pre_counts = np.bincount(np.ravel(labels), minlength=nlabel + 1)
pre_counts = pre_counts[:nlabel + 1].astype(np.float64)
labels[np.logical_not(mask_data)] = background_label
post_counts = np.bincount(np.ravel(labels), minlength=nlabel + 1)
post_counts = post_counts[:nlabel + 1].astype(np.float64)
region_coverage = post_counts / pre_counts
region_coverage[np.isclose(pre_counts, 0)] = 0
out_region_coverage = list(
region_coverage[indices != background_label])
assert len(out_region_coverage) == nlabel
indices = indices[region_coverage >= min_region_coverage]
indices = np.setdiff1d(indices, [background_label])
assert np.all(labels >= 0)
in_data = in_img.get_fdata()
if in_data.ndim == 3:
in_data = in_data[:, :, :, np.newaxis]
assert in_data.ndim == 4
result = np.full((in_data.shape[3], nlabel), np.nan)
for i, img in enumerate(np.moveaxis(in_data, 3, 0)):
result[i, indices - 1] = mean(img,
labels=labels.reshape(img.shape),
index=indices)
if output_coverage is True:
return result, out_region_coverage
else:
return result
def label_objects(dataset=None, labels=None, out=None, out_features=None,
source=None, return_labels=False):
DM = DataModel.instance()
log.info('+ Loading data into memory')
data = DM.load_slices(dataset)
if labels is None:
data += 1
labels = set(np.unique(data)) - set([0])
else:
data += 1
labels = np.asarray(labels) + 1
obj_labels = []
log.info('+ Extracting individual objects')
new_labels = np.zeros(data.shape, np.int32)
total_labels = 0
num = 0
for label in labels:
mask = (data == label)
tmp_data = data.copy()
tmp_data[~mask] = 0
tmp_labels, num = splabel(tmp_data, structure=octahedron(1))
mask = (tmp_labels > 0)
new_labels[mask] = tmp_labels[mask] + total_labels
total_labels += num
obj_labels += [label] * num
log.info('+ {} Objects found'.format(total_labels))
log.info('+ Saving results')
DM.create_empty_dataset(out, DM.data_shape, new_labels.dtype)
DM.write_slices(out, new_labels, params=dict(active=True, num_objects=total_labels))
log.info('+ Loading source to memory')
data = DM.load_slices(source)
objs = new_labels
objects = new_labels
num_objects = total_labels
objlabels = np.arange(1, num_objects+1)
log.info('+ Computing Average intensity')
feature = measure.mean(data, objs, index=objlabels)
DM.create_empty_dataset(out_features[0], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[0], feature, params=dict(active=True))
"""log.info('+ Computing Median intensity')
objs.shape = -1
data.shape = -1
feature = binned_statistic(objs, data, statistic='median',
bins=num_objects+1)[0]
feature = feature[objlabels]
out_features[1].write_direct(feature)
out_features[1].attrs['active'] = True
objs.shape = dataset.shape
data.shape = dataset.shape"""
log.info('+ Computing Sum of intensity')
feature = measure.sum(data, objs, index=objlabels)
DM.create_empty_dataset(out_features[1], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[1], feature, params=dict(active=True))
log.info('+ Computing Standard Deviation of intensity')
feature = measure.standard_deviation(data, objs, index=objlabels)
DM.create_empty_dataset(out_features[2], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[2], feature, params=dict(active=True))
log.info('+ Computing Variance of intensity')
feature = measure.variance(data, objs, index=objlabels)
DM.create_empty_dataset(out_features[3], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[3], feature, params=dict(active=True))
log.info('+ Computing Area')
objs.shape = -1
feature = np.bincount(objs, minlength=num_objects+1)[1:]
DM.create_empty_dataset(out_features[4], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[4], feature, params=dict(active=True))
DM.create_empty_dataset(out_features[5], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[5], np.log10(feature), params=dict(active=True))
objs.shape = data.shape
log.info('+ Computing Bounding Box')
obj_windows = measure.find_objects(objs)
feature = []; depth = []; height = []; width = [];
for w in obj_windows:
feature.append((w[0].stop - w[0].start) *
(w[1].stop - w[1].start) *
(w[2].stop - w[2].start))
depth.append(w[0].stop - w[0].start)
height.append(w[1].stop - w[1].start)
width.append(w[2].stop - w[2].start)
feature = np.asarray(feature, np.float32)
DM.create_empty_dataset(out_features[6], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[6], feature, params=dict(active=True))
#depth
depth = np.asarray(depth, np.float32)
DM.create_empty_dataset(out_features[7], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[7], depth, params=dict(active=True))
# height
height = np.asarray(height, np.float32)
DM.create_empty_dataset(out_features[8], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[8], height, params=dict(active=True))
# width
width = np.asarray(width, np.float32)
DM.create_empty_dataset(out_features[9], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[9], width, params=dict(active=True))
# log10
DM.create_empty_dataset(out_features[10], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[10], np.log10(feature), params=dict(active=True))
log.info('+ Computing Oriented Bounding Box')
ori_feature = []; ori_depth = []; ori_height = []; ori_width = [];
for i, w in enumerate(obj_windows):
z, y, x = np.where(objs[w] == i+1)
coords = np.c_[z, y, x]
if coords.shape[0] >= 3:
coords = PCA(n_components=3).fit_transform(coords)
cmin, cmax = coords.min(0), coords.max(0)
zz, yy, xx = (cmax[0] - cmin[0] + 1,
cmax[1] - cmin[1] + 1,
cmax[2] - cmin[2] + 1)
ori_feature.append(zz * yy * xx)
ori_depth.append(zz)
ori_height.append(yy)
ori_width.append(xx)
ori_feature = np.asarray(ori_feature, np.float32)
DM.create_empty_dataset(out_features[11], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[11], ori_feature, params=dict(active=True))
#depth
ori_depth = np.asarray(ori_depth, np.float32)
DM.create_empty_dataset(out_features[12], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[12], ori_depth, params=dict(active=True))
# height
ori_height = np.asarray(ori_height, np.float32)
DM.create_empty_dataset(out_features[13], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[13], ori_height, params=dict(active=True))
# width
ori_width = np.asarray(ori_width, np.float32)
DM.create_empty_dataset(out_features[14], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[14], ori_width, params=dict(active=True))
# log10
DM.create_empty_dataset(out_features[15], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[15], np.log10(ori_feature), params=dict(active=True))
log.info('+ Computing Positions')
pos = measure.center_of_mass(objs, labels=objs, index=objlabels)
pos = np.asarray(pos, dtype=np.float32)
DM.create_empty_dataset(out_features[16], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[16], pos[:, 2].copy(), params=dict(active=True))
DM.create_empty_dataset(out_features[17], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[17], pos[:, 1].copy(), params=dict(active=True))
DM.create_empty_dataset(out_features[18], (num_objects,), np.float32, check=False)
DM.write_dataset(out_features[18], pos[:, 0].copy(), params=dict(active=True))
if return_labels:
return out, total_labels, np.asarray(obj_labels)
return out, total_labels
def commit(cutout,
low_threshold=LOW_THRESHOLD,
high_threshold=HIGH_THRESHOLD,
cost_benefit_ratio=COST_BENEFIT_RATIO,
close=lambda x, y: True,
force=False):
V = cutout.parent
unique_list = cutout.central_unique_list
traced_list = measurements.mean(cutout.traced, cutout.raw_labels,
unique_list)
current_list = measurements.mean(cutout.current_object, cutout.raw_labels,
unique_list)
volumes = measurements.sum(np.ones_like(cutout.current_object),
cutout.raw_labels, unique_list)
positive_indices = [
i for i in xrange(len(unique_list)) if high_threshold < traced_list[i]
]
uncertain_indices = [
i for i in xrange(len(unique_list))
if low_threshold <= traced_list[i] <= high_threshold
]
negative_indices = [
i for i in xrange(len(unique_list)) if traced_list[i] < low_threshold
]
cost = sum([volumes[i]*max(traced_list[i],1-traced_list[i]) for i in uncertain_indices]) + \
sum([volumes[i]*traced_list[i] for i in negative_indices]) + \
sum([volumes[i]*(1-traced_list[i]) for i in positive_indices])
benefit = sum([
abs(volumes[i] * (traced_list[i] - current_list[i]))
for i in positive_indices + negative_indices
])
"""
cost = sum([volumes[i] * traced_list[i]*(1-traced_list[i]) for i in xrange(len(unique_list))])
benefit = sum([abs(volumes[i]*(round(traced_list[i])-current_list[i])) for i in xrange(len(unique_list))])
"""
cost2 = sum([volumes[i] for i in uncertain_indices]) + \
sum([volumes[i]*traced_list[i] for i in negative_indices]) + \
sum([volumes[i]*(1-traced_list[i]) for i in positive_indices])
benefit2 = sum([
abs(volumes[i] * (round(traced_list[i]) - current_list[i]))
for i in positive_indices + negative_indices
])
if not (len(uncertain_indices) == 0 or
(cost_benefit_ratio is not None
and benefit2 > cost_benefit_ratio * cost2) or force):
raise ReconstructionException("not confident")
split_point = (high_threshold + low_threshold) / 2
rounded_positive = [
unique_list[i] for i in xrange(len(unique_list))
if traced_list[i] > split_point
]
rounded_negative = [
unique_list[i] for i in xrange(len(unique_list))
if traced_list[i] <= split_point
]
if not V.valid.isdisjoint(rounded_positive):
raise ReconstructionException("blocking merge to valid segment")
full_segment = bfs(V.G, rounded_positive)
if not V.glial.isdisjoint(full_segment):
raise ReconstructionException("blocking merge to glial cell")
if len(V.dendrite & full_segment) > 2:
raise ReconstructionException("blocking merge of two dendritic trunks")
original_components = list(
nx.connected_components(V.G.subgraph(cutout.unique_list)))
regiongraphs.add_clique(V.G, rounded_positive, guard=close)
regiongraphs.delete_bipartite(V.G, rounded_positive, rounded_negative)
new_components = list(
nx.connected_components(V.G.subgraph(cutout.unique_list)))
changed_list = set(cutout.unique_list) - set.union(
*([set([])] + [s for s in original_components if s in new_components]))
changed_cutout = indicator(cutout.raw_labels, changed_list)
V.changed[cutout.region] = np.maximum(V.changed[cutout.region],
changed_cutout)
if not GLOBAL_EXPAND:
cutout.G = V.G.subgraph(cutout.unique_list)
return len(changed_list) > 0
lopen = open(lname, "w")
lopen.write("Extracting lightcurves for {0} observations...\n========\n" \
.format(nobs))
# -- initialize the lightcurve array
lcs = np.zeros((nobs, nbbl), dtype=float) - 9999
# -- loop over observations
lopen.write(" FIX THE DIMENSIONS!!!\n")
for ii, fname in enumerate(flist):
if ii % 10 == 0:
lopen.write(" obs {0} of {1}\n".format(ii, nobs))
lopen.flush()
# - get luminosities
img = imread(fname)[5::2, 1::2]
lcs[ii] = ndm.mean(img, bbls, blist)
# - periodically write to file
if (ii + 1) % 100 == 0:
np.save(oname, lcs[:ii])
# -- write to file
lopen.write("\nWriting to npy...\n========\n")
lopen.flush()
np.save(oname, lcs)
lopen.write("FINISHED in {0}s\n".format(time.time() - t0))
lopen.flush()
lopen.close()
def watershed_in_block(
affs,
block,
context,
rag_provider,
fragments_out,
num_voxels_in_block,
mask=None,
fragments_in_xy=False,
epsilon_agglomerate=0.0,
filter_fragments=0.0,
min_seed_distance=10,
replace_sections=None):
'''
Args:
filter_fragments (float):
Filter fragments that have an average affinity lower than this
value.
min_seed_distance (int):
Controls distance between seeds in the initial watershed. Reducing
this value improves downsampled segmentation.
'''
total_roi = affs.roi
logger.debug("reading affs from %s", block.read_roi)
affs = affs.intersect(block.read_roi)
affs.materialize()
if affs.dtype == np.uint8:
logger.info("Assuming affinities are in [0,255]")
max_affinity_value = 255.0
affs.data = affs.data.astype(np.float32)
else:
max_affinity_value = 1.0
if mask is not None:
logger.debug("reading mask from %s", block.read_roi)
mask_data = get_mask_data_in_roi(mask, affs.roi, affs.voxel_size)
logger.debug("masking affinities")
affs.data *= mask_data
# extract fragments
fragments_data, _ = watershed_from_affinities(
affs.data,
max_affinity_value,
fragments_in_xy=fragments_in_xy,
min_seed_distance=min_seed_distance)
if mask is not None:
fragments_data *= mask_data.astype(np.uint64)
if filter_fragments > 0:
if fragments_in_xy:
average_affs = np.mean(affs.data[0:2]/max_affinity_value, axis=0)
else:
average_affs = np.mean(affs.data/max_affinity_value, axis=0)
filtered_fragments = []
fragment_ids = np.unique(fragments_data)
for fragment, mean in zip(
fragment_ids,
measurements.mean(
average_affs,
fragments_data,
fragment_ids)):
if mean < filter_fragments:
filtered_fragments.append(fragment)
filtered_fragments = np.array(
filtered_fragments,
dtype=fragments_data.dtype)
replace = np.zeros_like(filtered_fragments)
replace_values(fragments_data, filtered_fragments, replace, inplace=True)
if epsilon_agglomerate > 0:
logger.info(
"Performing initial fragment agglomeration until %f",
epsilon_agglomerate)
generator = waterz.agglomerate(
affs=affs.data/max_affinity_value,
thresholds=[epsilon_agglomerate],
fragments=fragments_data,
scoring_function='OneMinus<HistogramQuantileAffinity<RegionGraphType, 25, ScoreValue, 256, false>>',
discretize_queue=256,
return_merge_history=False,
return_region_graph=False)
fragments_data[:] = next(generator)
# cleanup generator
for _ in generator:
pass
if replace_sections:
logger.info("Replacing sections...")
block_begin = block.write_roi.get_begin()
shape = block.write_roi.get_shape()
z_context = context[0]/affs.voxel_size[0]
logger.info("Z context: %i",z_context)
mapping = {}
voxel_offset = block_begin[0]/affs.voxel_size[0]
for i,j in zip(
range(fragments_data.shape[0]),
range(shape[0])):
mapping[i] = i
mapping[j] = int(voxel_offset + i) \
if block_begin[0] == total_roi.get_begin()[0] \
else int(voxel_offset + (i - z_context))
logging.info('Mapping: %s', mapping)
replace = [k for k,v in mapping.items() if v in replace_sections]
for r in replace:
logger.info("Replacing mapped section %i with zero", r)
fragments_data[r] = 0
#todo add key value replacement option
fragments = daisy.Array(fragments_data, affs.roi, affs.voxel_size)
# crop fragments to write_roi
fragments = fragments[block.write_roi]
fragments.materialize()
max_id = fragments.data.max()
# ensure we don't have IDs larger than the number of voxels (that would
# break uniqueness of IDs below)
if max_id > num_voxels_in_block:
logger.warning(
"fragments in %s have max ID %d, relabelling...",
block.write_roi, max_id)
fragments.data, max_id = relabel(fragments.data)
assert max_id < num_voxels_in_block
# ensure unique IDs
id_bump = block.block_id[1]*num_voxels_in_block
logger.debug("bumping fragment IDs by %i", id_bump)
fragments.data[fragments.data>0] += id_bump
fragment_ids = range(id_bump + 1, id_bump + 1 + int(max_id))
# store fragments
logger.debug("writing fragments to %s", block.write_roi)
fragments_out[block.write_roi] = fragments
# following only makes a difference if fragments were found
if max_id == 0:
return
# get fragment centers
fragment_centers = {
fragment: block.write_roi.get_offset() + affs.voxel_size*daisy.Coordinate(center)
for fragment, center in zip(
fragment_ids,
measurements.center_of_mass(fragments.data, fragments.data, fragment_ids))
if not np.isnan(center[0])
}
# store nodes
rag = rag_provider[block.write_roi]
rag.add_nodes_from([
(node, {
'center_z': c[0],
'center_y': c[1],
'center_x': c[2]
}
)
for node, c in fragment_centers.items()
])
rag.write_nodes(block.write_roi)
def evaluate(self, t): self.asset.lock(t) dat = self.asset.getPreviousData(t, self.length) self.asset.unlock() return mean(dat)
def evaluate(self, t):
self.asset.lock(t)
dat = self.asset.getPreviousData(t, self.length)
self.asset.unlock()
return mean(dat)
def analyze(cutout, example_id):
V = cutout.parent
unique_list = cutout.central_unique_list
args = [cutout.raw_labels, unique_list]
tic()
guess = measurements.mean(cutout.traced, *args)
truth = measurements.mean(
cutout.local_human_labels[np.unravel_index(
np.argmax(cutout.traced),
cutout.raw_labels.shape)] == cutout.local_human_labels, *args)
volumes = measurements.sum(np.ones_like(cutout.raw_labels), *args)
histogram_list = list(ndimage.histogram(cutout.traced, 0, 1, 10, *args))
histogram = np.histogram(crop(cutout.traced, CENTRAL_CROP), bins=10)
toc("compute statistics")
tic()
high_threshold = HIGH_THRESHOLD
low_threshold = LOW_THRESHOLD
traced_list = measurements.mean(cutout.traced, cutout.raw_labels,
unique_list)
proofread_list = measurements.mean(cutout.proofread_object,
cutout.raw_labels, unique_list)
current_list = measurements.mean(cutout.current_object, cutout.raw_labels,
unique_list)
volumes = measurements.sum(np.ones_like(cutout.current_object),
cutout.raw_labels, unique_list)
positive_indices = [
i for i in xrange(len(unique_list)) if high_threshold < traced_list[i]
]
uncertain_indices = [
i for i in xrange(len(unique_list))
if low_threshold <= traced_list[i] <= high_threshold
]
negative_indices = [
i for i in xrange(len(unique_list)) if traced_list[i] < low_threshold
]
cost = sum([volumes[i]*max(traced_list[i],1-traced_list[i]) for i in uncertain_indices]) + \
sum([volumes[i]*traced_list[i] for i in negative_indices]) + \
sum([volumes[i]*(1-traced_list[i]) for i in positive_indices])
benefit = sum([
abs(volumes[i] * (traced_list[i] - current_list[i]))
for i in positive_indices + negative_indices
])
cost2 = sum([volumes[i] for i in uncertain_indices]) + \
sum([volumes[i]*traced_list[i] for i in negative_indices]) + \
sum([volumes[i]*(1-traced_list[i]) for i in positive_indices])
benefit2 = sum([
abs(volumes[i] * (round(traced_list[i]) - current_list[i]))
for i in positive_indices + negative_indices
])
true_cost = sum([
volumes[i] * abs(round(traced_list[i]) - proofread_list[i])
for i in xrange(len(unique_list))
])
true_benefit = sum([
volumes[i] * (abs(proofread_list[i] - round(current_list[i])) -
abs(proofread_list[i] - traced_list[i]))
for i in xrange(len(unique_list))
])
toc("cost benefit analysis")
tic()
positive = [
unique_list[i] for i in xrange(len(unique_list)) if guess[i] > 0.5
]
negative = [
unique_list[i] for i in xrange(len(unique_list)) if guess[i] <= 0.5
]
new_graph = V.G.subgraph(cutout.unique_list).copy()
regiongraphs.add_clique(new_graph, positive)
regiongraphs.delete_bipartite(new_graph, positive, negative)
new_obj = indicator(cutout.raw_labels, bfs(new_graph, positive))
new_errors_cutout = crop(
reconstruct_utils.discrim_online_daemon(cutout.image, new_obj),
ERRORS_CROP)
old_errors_cutout = crop(cutout.errors * new_obj, ERRORS_CROP)
#d_error = crop(new_errors_cutout,ERRORS_CROP) - crop(old_errors_cutout,ERRORS_CROP)
#print(np.histogram(d_error, bins=20, range=(-1.0,1.0)))
satisfaction = np.sum(
crop(np.abs(cutout.current_object - cutout.traced), CENTRAL_CROP))
toc("computing change in error")
guess_margin = np.min(np.append(guess[guess > 0.5], 1)) - np.max(
np.append(guess[guess <= 0.5], 0))
true_margin = np.min(np.append(guess[truth > 0.5], 1)) - np.max(
np.append(guess[truth <= 0.5], 0))
df1 = pd.DataFrame.from_dict({
"guess":
guess,
"truth":
truth,
"volume":
volumes,
"seg_id":
unique_list,
"example_id": [example_id for i in unique_list],
"histogram":
histogram_list
})
df2 = pd.DataFrame.from_dict({
"guess_margin": [guess_margin],
"true_margin": [true_margin],
"err_max": [np.max(new_errors_cutout)],
"err_min": [np.min(new_errors_cutout)],
"err_mean": [np.mean(new_errors_cutout)],
"satisfaction": [satisfaction],
"histogram": [histogram],
"example_id": [example_id],
"cost": [cost],
"benefit": [benefit],
"cost2": [cost2],
"benefit2": [benefit2],
"true_cost": [true_cost],
"true_benefit": [true_benefit],
})
return df1, df2