def _msge_with_gradient_underdetermined(data, delta, xvschema, skipstep, p):
""" Calculate the mean squared generalization error and it's gradient for
underdetermined equation system.
"""
(l, m, t) = data.shape
d = None
j, k = 0, 0
nt = sp.ceil(t / skipstep)
for trainset, testset in xvschema(t, skipstep):
(a, b) = _construct_var_eqns(sp.atleast_3d(data[:, :, trainset]), p)
(c, d) = _construct_var_eqns(sp.atleast_3d(data[:, :, testset]), p)
e = sp.linalg.inv(sp.eye(a.shape[0]) * delta**2 + a.dot(a.transpose()))
cc = c.transpose().dot(c)
be = b.transpose().dot(e)
bee = be.dot(e)
bea = be.dot(a)
beea = bee.dot(a)
beacc = bea.dot(cc)
dc = d.transpose().dot(c)
j += sp.sum(beacc * bea - 2 * bea * dc) + sp.sum(d**2)
k += sp.sum(beea * dc - beacc * beea) * 4 * delta
return j / (nt * d.size), k / (nt * d.size)
def _msge_with_gradient_underdetermined(self, data, delta, xvschema, skipstep):
""" Calculate the mean squared generalization error and it's gradient for underdetermined equation system.
"""
(l, m, t) = data.shape
d = None
j, k = 0, 0
nt = sp.ceil(t / skipstep)
for s in range(0, t, skipstep):
trainset, testset = xvschema(s, t)
(a, b) = self._construct_eqns(sp.atleast_3d(data[:, :, trainset]))
(c, d) = self._construct_eqns(sp.atleast_3d(data[:, :, testset]))
e = sp.linalg.inv(sp.eye(a.shape[0]) * delta ** 2 + a.dot(a.transpose()))
cc = c.transpose().dot(c)
be = b.transpose().dot(e)
bee = be.dot(e)
bea = be.dot(a)
beea = bee.dot(a)
beacc = bea.dot(cc)
dc = d.transpose().dot(c)
j += sp.sum(beacc * bea - 2 * bea * dc) + sp.sum(d ** 2)
k += sp.sum(beea * dc - beacc * beea) * 4 * delta
return j / (nt * d.size), k / (nt * d.size)
def _msge_with_gradient_overdetermined(data, delta, xvschema, skipstep, p):
""" Calculate the mean squared generalization error and it's gradient for
overdetermined equation system.
"""
(l, m, t) = data.shape
d = None
l, k = 0, 0
nt = sp.ceil(t / skipstep)
for trainset, testset in xvschema(t, skipstep):
(a, b) = _construct_var_eqns(sp.atleast_3d(data[:, :, trainset]), p)
(c, d) = _construct_var_eqns(sp.atleast_3d(data[:, :, testset]), p)
e = sp.linalg.inv(sp.eye(a.shape[1]) * delta ** 2 +
a.transpose().dot(a))
ba = b.transpose().dot(a)
dc = d.transpose().dot(c)
bae = ba.dot(e)
baee = bae.dot(e)
baecc = bae.dot(c.transpose().dot(c))
l += sp.sum(baecc * bae - 2 * bae * dc) + sp.sum(d ** 2)
k += sp.sum(baee * dc - baecc * baee) * 4 * delta
return l / (nt * d.size), k / (nt * d.size)
def _msge_with_gradient_overdetermined(data, delta, xvschema, skipstep, p):
""" Calculate the mean squared generalization error and it's gradient for
overdetermined equation system.
"""
(l, m, t) = data.shape
d = None
l, k = 0, 0
nt = sp.ceil(t / skipstep)
for trainset, testset in xvschema(t, skipstep):
(a, b) = _construct_var_eqns(sp.atleast_3d(data[:, :, trainset]), p)
(c, d) = _construct_var_eqns(sp.atleast_3d(data[:, :, testset]), p)
e = sp.linalg.inv(sp.eye(a.shape[1]) * delta**2 + a.transpose().dot(a))
ba = b.transpose().dot(a)
dc = d.transpose().dot(c)
bae = ba.dot(e)
baee = bae.dot(e)
baecc = bae.dot(c.transpose().dot(c))
l += sp.sum(baecc * bae - 2 * bae * dc) + sp.sum(d**2)
k += sp.sum(baee * dc - baecc * baee) * 4 * delta
return l / (nt * d.size), k / (nt * d.size)
def _msge_with_gradient_overdetermined(self, data, delta, xvschema, skipstep):
""" Calculate the mean squared generalization error and it's gradient for overdetermined equation system.
"""
(l, m, t) = data.shape
d = None
l, k = 0, 0
nt = sp.ceil(t / skipstep)
for s in range(0, t, skipstep):
#print(s,drange)
trainset, testset = xvschema(s, t)
(a, b) = self._construct_eqns(sp.atleast_3d(data[:, :, trainset]))
(c, d) = self._construct_eqns(sp.atleast_3d(data[:, :, testset]))
#e = sp.linalg.inv(np.eye(a.shape[1])*delta**2 + a.transpose().dot(a), overwrite_a=True, check_finite=False)
e = sp.linalg.inv(sp.eye(a.shape[1]) * delta ** 2 + a.transpose().dot(a))
ba = b.transpose().dot(a)
dc = d.transpose().dot(c)
bae = ba.dot(e)
baee = bae.dot(e)
baecc = bae.dot(c.transpose().dot(c))
l += sp.sum(baecc * bae - 2 * bae * dc) + sp.sum(d ** 2)
k += sp.sum(baee * dc - baecc * baee) * 4 * delta
return l / (nt * d.size), k / (nt * d.size)
def fit(self, data):
""" Fit VAR model to data.
Parameters
----------
data : array-like
shape = [n_samples, n_channels, n_trials] or
[n_samples, n_channels]
Continuous or segmented data set.
Returns
-------
self : :class:`VAR`
The :class:`VAR` object to facilitate method chaining (see usage
example)
"""
data = sp.atleast_3d(data)
if self.delta == 0 or self.delta is None:
# ordinary least squares
(x, y) = self._construct_eqns(data)
else:
# regularized least squares (ridge regression)
(x, y) = self._construct_eqns_rls(data)
(b, res, rank, s) = sp.linalg.lstsq(x, y)
self.coef = b.transpose()
self.residuals = data - self.predict(data)
self.rescov = sp.cov(cat_trials(self.residuals[self.p:, :, :]),
rowvar=False)
return self
def fit(self, data):
""" Fit VAR model to data.
Parameters
----------
data : array-like, shape = [n_samples, n_channels, n_trials] or [n_samples, n_channels]
Continuous or segmented data set.
Returns
-------
self : :class:`VAR`
The :class:`VAR` object to facilitate method chaining (see usage example)
"""
data = sp.atleast_3d(data)
if self.delta == 0 or self.delta is None:
# ordinary least squares
(x, y) = self._construct_eqns(data)
else:
# regularized least squares (ridge regression)
(x, y) = self._construct_eqns_rls(data)
(b, res, rank, s) = sp.linalg.lstsq(x, y)
self.coef = b.transpose()
self.residuals = data - self.predict(data)
self.rescov = sp.cov(cat_trials(self.residuals), rowvar=False)
return self
def __init__(self, template, spacing=[1, 1, 1], **kwargs):
template = sp.atleast_3d(template)
super().__init__(shape=template.shape, **kwargs)
coords = sp.unravel_index(range(template.size), template.shape)
self['pore.template_coords'] = sp.vstack(coords).T
self['pore.template_indices'] = self.Ps
self['pore.drop'] = template.flatten() == 0
topotools.trim(network=self, pores=self.pores('drop'))
del self['pore.drop']
def apply_chords(im, spacing=0, axis=0, trim_edges=True):
r"""
Adds chords to the void space in the specified direction. The chords are
separated by 1 voxel plus the provided spacing.
Parameters
----------
im : ND-array
An image of the porous material with void marked as True.
spacing : int (default = 0)
Chords are automatically separated by 1 voxel and this argument
increases the separation.
axis : int (default = 0)
The axis along which the chords are drawn.
trim_edges : bool (default = True)
Whether or not to remove chords that touch the edges of the image.
These chords are artifically shortened, so skew the chord length
distribution
Returns
-------
An ND-array of the same size as ```im``` with True values indicating the
chords.
See Also
--------
apply_chords_3D
"""
if spacing < 0:
raise Exception('Spacing cannot be less than 0')
dims1 = sp.arange(0, im.ndim)
dims2 = sp.copy(dims1)
dims2[axis] = 0
dims2[0] = axis
im = sp.moveaxis(a=im, source=dims1, destination=dims2)
im = sp.atleast_3d(im)
ch = sp.zeros_like(im, dtype=bool)
if im.ndim == 2:
ch[:, ::2+spacing, ::2+spacing] = 1
if im.ndim == 3:
ch[:, ::4+2*spacing, ::4+2*spacing] = 1
chords = im*ch
chords = sp.squeeze(chords)
if trim_edges:
temp = clear_border(spim.label(chords == 1)[0]) > 0
chords = temp*chords
chords = sp.moveaxis(a=chords, source=dims1, destination=dims2)
return chords
def bundle_of_tubes(shape: List[int], spacing: int):
r"""
Create a 3D image of a bundle of tubes, in the form of a rectangular
plate with randomly sized holes through it.
Parameters
----------
shape : list
The size the image, with the 3rd dimension indicating the plate
thickness. If the 3rd dimension is not given then a thickness of
1 voxel is assumed.
spacing : scalar
The center to center distance of the holes. The hole sizes will be
randomly distributed between this values down to 3 voxels.
Returns
-------
image : ND-array
A boolean array with ``True`` values denoting the pore space
"""
shape = sp.array(shape)
if sp.size(shape) == 1:
shape = sp.full((3, ), int(shape))
if sp.size(shape) == 2:
shape = sp.hstack((shape, [1]))
temp = sp.zeros(shape=shape[:2])
Xi = sp.ceil(sp.linspace(spacing/2,
shape[0]-(spacing/2)-1,
int(shape[0]/spacing)))
Xi = sp.array(Xi, dtype=int)
Yi = sp.ceil(sp.linspace(spacing/2,
shape[1]-(spacing/2)-1,
int(shape[1]/spacing)))
Yi = sp.array(Yi, dtype=int)
temp[tuple(sp.meshgrid(Xi, Yi))] = 1
inds = sp.where(temp)
for i in range(len(inds[0])):
r = sp.random.randint(1, (spacing/2))
try:
s1 = slice(inds[0][i]-r, inds[0][i]+r+1)
s2 = slice(inds[1][i]-r, inds[1][i]+r+1)
temp[s1, s2] = ps_disk(r)
except ValueError:
odd_shape = sp.shape(temp[s1, s2])
temp[s1, s2] = ps_disk(r)[:odd_shape[0], :odd_shape[1]]
im = sp.broadcast_to(array=sp.atleast_3d(temp), shape=shape)
return im
def __init__(self, template, spacing=[1, 1, 1], **kwargs):
template = sp.atleast_3d(template)
if 'shape' in kwargs:
del kwargs['shape']
logger.warning('shape argument ignored, inferred from template')
super().__init__(shape=template.shape, spacing=spacing, **kwargs)
coords = sp.unravel_index(range(template.size), template.shape)
self['pore.template_coords'] = sp.vstack(coords).T
self['pore.template_indices'] = self.Ps
self['pore.drop'] = template.flatten() == 0
topotools.trim(network=self, pores=self.pores('drop'))
del self['pore.drop']
# remove labels pertaining to surface pores, then redo post-trim
self.clear(mode='labels')
self['pore.internal'] = True
self['throat.internal'] = True
topotools.find_surface_pores(self)
def predict(self, data):
""" Predict samples on actual data.
The result of this function is used for calculating the residuals.
Parameters
----------
data : array-like
shape = [n_samples, n_channels, n_trials] or
[n_samples, n_channels]
Continuous or segmented data set.
Returns
-------
predicted : shape = `data`.shape
Data as predicted by the VAR model.
Notes
-----
Residuals are obtained by r = x - var.predict(x)
"""
data = sp.atleast_3d(data)
(l, m, t) = data.shape
p = int(sp.shape(self.coef)[1] / m)
y = sp.zeros(data.shape)
if t > l-p: # which takes less loop iterations
for k in range(1, p + 1):
bp = self.coef[:, (k - 1)::p]
for n in range(p, l):
y[n, :, :] += bp.dot(data[n - k, :, :])
else:
for k in range(1, p + 1):
bp = self.coef[:, (k - 1)::p]
for s in range(t):
y[p:, :, s] += data[(p - k):(l - k), :, s].dot(bp.T)
return y
def get_subscripts(network, shape, **kwargs):
r'''
Return the 3D subscripts (i,j,k) into the cubic network
Parameters
----------
shape : list
The (i,j,k) shape of the network in number of pores in each direction
'''
if network.num_pores('internal') != _sp.prod(shape):
print('Supplied shape does not match Network size, cannot proceed')
else:
template = _sp.atleast_3d(_sp.empty(shape))
a = _sp.indices(_sp.shape(template))
i = a[0].flatten()
j = a[1].flatten()
k = a[2].flatten()
ind = _sp.vstack((i, j, k)).T
vals = _sp.ones((network.Np, 3)) * _sp.nan
vals[network.pores('internal')] = ind
return vals
def predict(self, data):
""" Predict samples on actual data.
The result of this function is used for calculating the residuals.
Parameters
----------
data : array-like
shape = [n_samples, n_channels, n_trials] or
[n_samples, n_channels]
Continuous or segmented data set.
Returns
-------
predicted : shape = `data`.shape
Data as predicted by the VAR model.
Notes
-----
Residuals are obtained by r = x - var.predict(x)
"""
data = sp.atleast_3d(data)
(l, m, t) = data.shape
p = int(sp.shape(self.coef)[1] / m)
y = sp.zeros(data.shape)
if t > l-p: # which takes less loop iterations
for k in range(1, p + 1):
bp = self.coef[:, (k - 1)::p]
for n in range(p, l):
y[n, :, :] += bp.dot(data[n - k, :, :])
else:
for k in range(1, p + 1):
bp = self.coef[:, (k - 1)::p]
for s in range(t):
y[p:, :, s] += data[(p - k):(l - k), :, s].dot(bp.T)
return y
def get_subscripts(network, shape, **kwargs):
r"""
Return the 3D subscripts (i,j,k) into the cubic network
Parameters
----------
shape : list
The (i,j,k) shape of the network in number of pores in each direction
"""
if network.num_pores('internal') != _sp.prod(shape):
print('Supplied shape does not match Network size, cannot proceed')
else:
template = _sp.atleast_3d(_sp.empty(shape))
a = _sp.indices(_sp.shape(template))
i = a[0].flatten()
j = a[1].flatten()
k = a[2].flatten()
ind = _sp.vstack((i, j, k)).T
vals = _sp.ones((network.Np, 3))*_sp.nan
vals[network.pores('internal')] = ind
return vals
def fromarray(self, array, propname):
r"""
Apply data to the network based on a rectangular array filled with
values. Each array location corresponds to a pore in the network.
Parameters
----------
array : array_like
The rectangular array containing the values to be added to the
network. This array must be the same shape as the original network.
propname : string
The name of the pore property being added.
"""
array = sp.atleast_3d(array)
if sp.shape(array) != self._shape:
raise Exception('The received array does not match the original network')
temp = array.flatten()
Ps = sp.array(self['pore.index'][self.pores('internal')], dtype=int)
propname = 'pore.' + propname.split('.')[-1]
self[propname] = sp.nan
self[propname][self.pores('internal')] = temp[Ps]
def fit(self, data):
""" Fit VAR model to data.
Parameters
----------
data : array-like, shape = [n_samples, n_channels, n_trials] or [n_samples, n_channels]
Continuous or segmented data set.
Returns
-------
self : :class:`VAR`
The :class:`VAR` object.
"""
data = sp.atleast_3d(data)
(x, y) = self._construct_eqns(data)
self.fitting_model.fit(x, y)
self.coef = self.fitting_model.coef_
self.residuals = data - self.predict(data)
self.rescov = sp.cov(datatools.cat_trials(self.residuals[self.p :, :, :]), rowvar=False)
return self
def fit(self, data):
""" Fit VAR model to data.
Parameters
----------
data : array-like, shape = [n_samples, n_channels, n_trials] or [n_samples, n_channels]
Continuous or segmented data set.
Returns
-------
self : :class:`VAR`
The :class:`VAR` object.
"""
data = sp.atleast_3d(data)
(x, y) = self._construct_eqns(data)
self.fitting_model.fit(x, y)
self.coef = self.fitting_model.coef_
self.residuals = data - self.predict(data)
self.rescov = sp.cov(datatools.cat_trials(self.residuals[self.p:, :, :]), rowvar=False)
return self
def fromarray(self, array, propname):
r'''
Apply data to the network based on a rectangular array filled with
values. Each array location corresponds to a pore in the network.
Parameters
----------
array : array_like
The rectangular array containing the values to be added to the
network. This array must be the same shape as the original network.
propname : string
The name of the pore property being added.
'''
array = sp.atleast_3d(array)
if sp.shape(array) != self._shape:
raise Exception(
'The received array does not match the original network')
temp = array.flatten()
Ps = sp.array(self['pore.index'][self.pores('internal')], dtype=int)
propname = 'pore.' + propname.split('.')[-1]
self[propname] = sp.nan
self[propname][self.pores('internal')] = temp[Ps]
def optimize_delta_bisection(self, data, skipstep=1):
""" Find optimal ridge penalty with bisection search.
Parameters
----------
data : array-like
shape = [n_samples, n_channels, n_trials] or
[n_samples, n_channels]
Continuous or segmented data set.
skipstep : int, optional
Speed up calculation by skipping samples during cost function
calculation
Returns
-------
self : :class:`VAR`
The :class:`VAR` object to facilitate method chaining (see usage
example)
"""
data = sp.atleast_3d(data)
(l, m, t) = data.shape
assert (t > 1)
maxsteps = 10
maxdelta = 1e50
a = -10
b = 10
trform = lambda x: sp.sqrt(sp.exp(x))
msge = _get_msge_with_gradient_func(data.shape, self.p)
(ja, ka) = msge(data, trform(a), self.xvschema, skipstep, self.p)
(jb, kb) = msge(data, trform(b), self.xvschema, skipstep, self.p)
# before starting the real bisection, assure the interval contains 0
while sp.sign(ka) == sp.sign(kb):
print('Bisection initial interval (%f,%f) does not contain zero. '
'New interval: (%f,%f)' % (a, b, a * 2, b * 2))
a *= 2
b *= 2
(ja, ka) = msge(data, trform(a), self.xvschema, skipstep, self.p)
(jb, kb) = msge(data, trform(b), self.xvschema, skipstep, self.p)
if trform(b) >= maxdelta:
print('Bisection: could not find initial interval.')
print(' ********* Delta set to zero! ************ ')
return 0
nsteps = 0
while nsteps < maxsteps:
# point where the line between a and b crosses zero
# this is not very stable!
#c = a + (b-a) * np.abs(ka) / np.abs(kb-ka)
c = (a + b) / 2
(j, k) = msge(data, trform(c), self.xvschema, skipstep, self.p)
if sp.sign(k) == sp.sign(ka):
a, ka = c, k
else:
b, kb = c, k
nsteps += 1
tmp = trform([a, b, a + (b - a) * np.abs(ka) / np.abs(kb - ka)])
print('%d Bisection Interval: %f - %f, (projected: %f)' %
(nsteps, tmp[0], tmp[1], tmp[2]))
self.delta = trform(a + (b - a) * np.abs(ka) / np.abs(kb - ka))
print('Final point: %f' % self.delta)
return self
def image_preprocessing(arr,params):
arr = sp.atleast_3d(arr)
smallest_edge = min(arr.shape[:2])
rep = params
preproc_lsum = rep['preproc']['lsum_ksize']
if preproc_lsum is None:
preproc_lsum = 1
smallest_edge -= (preproc_lsum-1)
normin_kshape = rep['normin']['kshape']
smallest_edge -= (normin_kshape[0]-1)
filter_kshape = rep['filter']['kshape']
smallest_edge -= (filter_kshape[0]-1)
normout_kshape = rep['normout']['kshape']
smallest_edge -= (normout_kshape[0]-1)
pool_lsum = rep['pool']['lsum_ksize']
smallest_edge -= (pool_lsum-1)
arrh, arrw, _ = arr.shape
if smallest_edge <= 0 and rep['conv_mode'] == 'valid':
if arrh > arrw:
new_w = arrw - smallest_edge + 1
new_h = int(np.round(1.*new_w * arrh/arrw))
print new_w, new_h
raise
elif arrh < arrw:
new_h = arrh - smallest_edge + 1
new_w = int(np.round(1.*new_h * arrw/arrh))
print new_w, new_h
raise
else:
pass
# TODO: finish image size adjustment
assert min(arr.shape[:2]) > 0
# use the first 3 channels only
orig_imga = arr.astype("float32")[:,:,:3]
# make sure that we don't have a 3-channel (pseudo) gray image
if orig_imga.shape[2] == 3 \
and (orig_imga[:,:,0]-orig_imga.mean(2) < 0.1*orig_imga.max()).all() \
and (orig_imga[:,:,1]-orig_imga.mean(2) < 0.1*orig_imga.max()).all() \
and (orig_imga[:,:,2]-orig_imga.mean(2) < 0.1*orig_imga.max()).all():
orig_imga = sp.atleast_3d(orig_imga[:,:,0])
# rescale to [0,1]
#print orig_imga.min(), orig_imga.max()
if orig_imga.min() == orig_imga.max():
raise MinMaxError("[ERROR] orig_imga.min() == orig_imga.max() "
"orig_imga.min() = %f, orig_imga.max() = %f"
% (orig_imga.min(), orig_imga.max())
)
orig_imga -= orig_imga.min()
orig_imga /= orig_imga.max()
# -- color conversion
# insure 3 dims
#print orig_imga.shape
if orig_imga.ndim == 2 or orig_imga.shape[2] == 1:
orig_imga_new = sp.empty(orig_imga.shape[:2] + (3,), dtype="float32")
orig_imga.shape = orig_imga_new[:,:,0].shape
orig_imga_new[:,:,0] = 0.2989*orig_imga
orig_imga_new[:,:,1] = 0.5870*orig_imga
orig_imga_new[:,:,2] = 0.1141*orig_imga
orig_imga = orig_imga_new
if params['color_space'] == 'rgb':
orig_imga_conv = orig_imga
# elif params['color_space'] == 'rg':
# orig_imga_conv = colorconv.rg_convert(orig_imga)
elif params['color_space'] == 'rg2':
orig_imga_conv = colorconv.rg2_convert(orig_imga)
elif params['color_space'] == 'gray':
orig_imga_conv = colorconv.gray_convert(orig_imga)
orig_imga_conv.shape = orig_imga_conv.shape + (1,)
elif params['color_space'] == 'opp':
orig_imga_conv = colorconv.opp_convert(orig_imga)
elif params['color_space'] == 'oppnorm':
orig_imga_conv = colorconv.oppnorm_convert(orig_imga)
elif params['color_space'] == 'chrom':
orig_imga_conv = colorconv.chrom_convert(orig_imga)
# elif params['color_space'] == 'opponent':
# orig_imga_conv = colorconv.opponent_convert(orig_imga)
# elif params['color_space'] == 'W':
# orig_imga_conv = colorconv.W_convert(orig_imga)
elif params['color_space'] == 'hsv':
orig_imga_conv = colorconv.hsv_convert(orig_imga)
else:
raise ValueError, "params['color_space'] not understood"
return orig_imga,orig_imga_conv
def regionprops_3D(im):
r"""
Calculates various metrics for each labeled region in a 3D image.
The ``regionsprops`` method in **skimage** is very thorough for 2D images,
but is a bit limited when it comes to 3D images, so this function aims
to fill this gap.
Parameters
----------
im : array_like
An imaging containing at least one labeled region. If a boolean image
is received than the ``True`` voxels are treated as a single region
labeled ``1``. Regions labeled 0 are ignored in all cases.
Returns
-------
An augmented version of the list returned by skimage's ``regionprops``.
Information, such as ``volume``, can be found for region A using the
following syntax: ``result[A-1].volume``.
Notes
-----
This function may seem slow compared to the skimage version, but that is
because they defer calculation of certain properties until they are
accessed while this one evalulates everything (inlcuding the deferred
properties from skimage's ``regionprops``)
Regions can be identified using a watershed algorithm, which can be a bit
tricky to obtain desired results. *PoreSpy* includes the SNOW algorithm,
which may be helpful.
"""
print('_' * 60)
print('Calculating regionprops')
results = regionprops(im, coordinates='xy')
for i in tqdm(range(len(results))):
mask = results[i].image
mask_padded = sp.pad(mask, pad_width=1, mode='constant')
temp = spim.distance_transform_edt(mask_padded)
dt = extract_subsection(temp, shape=mask.shape)
# ---------------------------------------------------------------------
# Slice indices
results[i].slice = results[i]._slice
# ---------------------------------------------------------------------
# Volume of regions in voxels
results[i].volume = results[i].area
# ---------------------------------------------------------------------
# Volume of bounding box, in voxels
results[i].bbox_volume = sp.prod(mask.shape)
# ---------------------------------------------------------------------
# Create an image of the border
results[i].border = dt == 1
# ---------------------------------------------------------------------
# Create an image of the maximal inscribed sphere
r = dt.max()
inv_dt = spim.distance_transform_edt(dt < r)
results[i].inscribed_sphere = inv_dt < r
# ---------------------------------------------------------------------
# Find surface area using marching cubes and analyze the mesh
tmp = sp.pad(sp.atleast_3d(mask), pad_width=1, mode='constant')
tmp = spim.convolve(tmp, weights=ball(1)) / 5
verts, faces, norms, vals = marching_cubes_lewiner(volume=tmp, level=0)
results[i].surface_mesh_vertices = verts
results[i].surface_mesh_simplices = faces
area = mesh_surface_area(verts, faces)
results[i].surface_area = area
# ---------------------------------------------------------------------
# Find sphericity
vol = results[i].volume
r = (3 / 4 / sp.pi * vol)**(1 / 3)
a_equiv = 4 * sp.pi * (r)**2
a_region = results[i].surface_area
results[i].sphericity = a_equiv / a_region
# ---------------------------------------------------------------------
# Find skeleton of region
results[i].skeleton = skeletonize_3d(mask)
# ---------------------------------------------------------------------
# Volume of convex image, equal to area in 2D, so just translating
results[i].convex_volume = results[i].convex_area
# ---------------------------------------------------------------------
# Convert region grid to a graph
am = grid_to_graph(*mask.shape, mask=mask)
results[i].graph = am
return results
def optimize_delta_bisection(self, data, skipstep=1):
""" Find optimal ridge penalty with bisection search.
Parameters
----------
data : array-like, shape = [n_samples, n_channels, n_trials] or [n_samples, n_channels]
Continuous or segmented data set.
skipstep : int, optional
Speed up calculation by skipping samples during cost function calculation
Returns
-------
self : :class:`VAR`
The :class:`VAR` object to facilitate method chaining (see usage example)
"""
data = sp.atleast_3d(data)
(l, m, t) = data.shape
assert (t > 1)
maxsteps = 10
maxdelta = 1e50
a = -10
b = 10
transform = lambda x: sp.sqrt(sp.exp(x))
msge = self._get_msge_with_gradient_func(data.shape)
(ja, ka) = msge(data, transform(a), self.xvschema, skipstep)
(jb, kb) = msge(data, transform(b), self.xvschema, skipstep)
# before starting the real bisection, make sure the interval actually contains 0
while sp.sign(ka) == sp.sign(kb):
print('Bisection initial interval (%f,%f) does not contain zero. New interval: (%f,%f)' % (a, b, a * 2, b * 2))
a *= 2
b *= 2
(jb, kb) = msge(data, transform(b), self.xvschema, skipstep)
if transform(b) >= maxdelta:
print('Bisection: could not find initial interval.')
print(' ********* Delta set to zero! ************ ')
return 0
nsteps = 0
while nsteps < maxsteps:
# point where the line between a and b crosses zero
# this is not very stable!
#c = a + (b-a) * np.abs(ka) / np.abs(kb-ka)
c = (a + b) / 2
(j, k) = msge(data, transform(c), self.xvschema, skipstep)
if sp.sign(k) == sp.sign(ka):
a, ka = c, k
else:
b, kb = c, k
nsteps += 1
tmp = transform([a, b, a + (b - a) * np.abs(ka) / np.abs(kb - ka)])
print('%d Bisection Interval: %f - %f, (projected: %f)' % (nsteps, tmp[0], tmp[1], tmp[2]))
self.delta = transform(a + (b - a) * np.abs(ka) / np.abs(kb - ka))
print('Final point: %f' % self.delta)
return self
def regionprops_3D(im):
r"""
Calculates various metrics for each labeled region in a 3D image.
The ``regionsprops`` method in **skimage** is very thorough for 2D images,
but is a bit limited when it comes to 3D images, so this function aims
to fill this gap.
Parameters
----------
im : array_like
An imaging containing at least one labeled region. If a boolean image
is received than the ``True`` voxels are treated as a single region
labeled ``1``. Regions labeled 0 are ignored in all cases.
Returns
-------
props : list
An augmented version of the list returned by skimage's ``regionprops``.
Information, such as ``volume``, can be found for region A using the
following syntax: ``result[A-1].volume``.
The returned list contains all the metrics normally returned by
**skimage.measure.regionprops** plus the following:
'slice': Slice indices into the image that can be used to extract the
region
'volume': Volume of the region in number of voxels.
'bbox_volume': Volume of the bounding box that contains the region.
'border': The edges of the region, found as the locations where
the distance transform is 1.
'inscribed_sphere': An image containing the largest sphere can can
fit entirely inside the region.
'surface_mesh_vertices': Obtained by applying the marching cubes
algorithm on the region, AFTER first blurring the voxel image. This
allows marching cubes more freedom to fit the surface contours. See
also ``surface_mesh_simplices``
'surface_mesh_simplices': This accompanies ``surface_mesh_vertices``
and together they can be used to define the region as a mesh.
'surface_area': Calculated using the mesh obtained as described above,
using the ``porespy.metrics.mesh_surface_area`` method.
'sphericity': Defined as the ratio of the area of a sphere with the
same volume as the region to the actual surface area of the region.
'skeleton': The medial axis of the region obtained using the
``skeletonize_3D`` method from **skimage**.
'convex_volume': Same as convex_area, but translated to a more
meaningful name.
See Also
--------
snow_partitioning
Notes
-----
This function may seem slow compared to the skimage version, but that is
because they defer calculation of certain properties until they are
accessed, while this one evalulates everything (inlcuding the deferred
properties from skimage's ``regionprops``)
Regions can be identified using a watershed algorithm, which can be a bit
tricky to obtain desired results. *PoreSpy* includes the SNOW algorithm,
which may be helpful.
"""
print('_' * 60)
print('Calculating regionprops')
results = regionprops(im, coordinates='xy')
for i in tqdm(range(len(results))):
mask = results[i].image
mask_padded = sp.pad(mask, pad_width=1, mode='constant')
temp = spim.distance_transform_edt(mask_padded)
dt = extract_subsection(temp, shape=mask.shape)
# ---------------------------------------------------------------------
# Slice indices
results[i].slice = results[i]._slice
# ---------------------------------------------------------------------
# Volume of regions in voxels
results[i].volume = results[i].area
# ---------------------------------------------------------------------
# Volume of bounding box, in voxels
results[i].bbox_volume = sp.prod(mask.shape)
# ---------------------------------------------------------------------
# Create an image of the border
results[i].border = dt == 1
# ---------------------------------------------------------------------
# Create an image of the maximal inscribed sphere
r = dt.max()
inv_dt = spim.distance_transform_edt(dt < r)
results[i].inscribed_sphere = inv_dt < r
# ---------------------------------------------------------------------
# Find surface area using marching cubes and analyze the mesh
tmp = sp.pad(sp.atleast_3d(mask), pad_width=1, mode='constant')
tmp = spim.convolve(tmp, weights=ball(1)) / 5
verts, faces, norms, vals = marching_cubes_lewiner(volume=tmp, level=0)
results[i].surface_mesh_vertices = verts
results[i].surface_mesh_simplices = faces
area = mesh_surface_area(verts, faces)
results[i].surface_area = area
# ---------------------------------------------------------------------
# Find sphericity
vol = results[i].volume
r = (3 / 4 / sp.pi * vol)**(1 / 3)
a_equiv = 4 * sp.pi * (r)**2
a_region = results[i].surface_area
results[i].sphericity = a_equiv / a_region
# ---------------------------------------------------------------------
# Find skeleton of region
results[i].skeleton = skeletonize_3d(mask)
# ---------------------------------------------------------------------
# Volume of convex image, equal to area in 2D, so just translating
results[i].convex_volume = results[i].convex_area
return results
idx, x = d1[xitem]
idy, y = d2[yitem]
assert (idx==idy).all()
sel = sp.bitwise_and(x, y)
mfnames = fnames[idx][sel]
nmfnames = len(mfnames)
if nmfnames <= 0:
continue
mpath = path.join(output_path, xitem+'_'+yitem)
os.makedirs(mpath)
for n, (fname1, fname2) in enumerate(mfnames):
arr1 = sp.atleast_3d(sp.misc.imread(fname1).astype('float32'))
arr2 = sp.atleast_3d(sp.misc.imread(fname2).astype('float32'))
# grayscale?
if arr1.shape[2] == 1:
arr1 = sp.concatenate([arr1,arr1,arr1], 2)
if arr2.shape[2] == 1:
arr2 = sp.concatenate([arr2,arr2,arr2], 2)
fullarr = sp.concatenate([arr1, arr2], axis=1)
basename1 = path.basename(fname1)
basename2 = path.basename(fname2)
out_fname = path.join(mpath, "%s-%s" % (basename1, basename2))
sp.misc.imsave(out_fname, fullarr)
sys.stdout.write("\rProcessing '%s': "
"%6.2f%%" % (mpath, (100.*(n+1.)/nmfnames)))
sys.stdout.flush()
def v1like_fromarray(arr, params, featsel):
""" Applies a simple V1-like model and generates a feature vector from
its outputs.
Inputs:
arr -- image's array
params -- representation parameters (dict)
featsel -- features to include to the vector (dict)
Outputs:
fvector -- corresponding feature vector
"""
if 'conv_mode' not in params:
params['conv_mode'] = 'same'
if 'color_space' not in params:
params['color_space'] = 'gray'
arr = sp.atleast_3d(arr)
smallest_edge = min(arr.shape[:2])
rep = params
preproc_lsum = rep['preproc']['lsum_ksize']
if preproc_lsum is None:
preproc_lsum = 1
smallest_edge -= (preproc_lsum-1)
normin_kshape = rep['normin']['kshape']
smallest_edge -= (normin_kshape[0]-1)
filter_kshape = rep['filter']['kshape']
smallest_edge -= (filter_kshape[0]-1)
normout_kshape = rep['normout']['kshape']
smallest_edge -= (normout_kshape[0]-1)
pool_lsum = rep['pool']['lsum_ksize']
smallest_edge -= (pool_lsum-1)
arrh, arrw, _ = arr.shape
if smallest_edge <= 0 and rep['conv_mode'] == 'valid':
if arrh > arrw:
new_w = arrw - smallest_edge + 1
new_h = int(np.round(1.*new_w * arrh/arrw))
print new_w, new_h
raise
elif arrh < arrw:
new_h = arrh - smallest_edge + 1
new_w = int(np.round(1.*new_h * arrw/arrh))
print new_w, new_h
raise
else:
pass
# TODO: finish image size adjustment
assert min(arr.shape[:2]) > 0
# use the first 3 channels only
orig_imga = arr.astype("float32")[:,:,:3]
# make sure that we don't have a 3-channel (pseudo) gray image
if orig_imga.shape[2] == 3 \
and (orig_imga[:,:,0]-orig_imga.mean(2) < 0.1*orig_imga.max()).all() \
and (orig_imga[:,:,1]-orig_imga.mean(2) < 0.1*orig_imga.max()).all() \
and (orig_imga[:,:,2]-orig_imga.mean(2) < 0.1*orig_imga.max()).all():
orig_imga = sp.atleast_3d(orig_imga[:,:,0])
# rescale to [0,1]
#print orig_imga.min(), orig_imga.max()
if orig_imga.min() == orig_imga.max():
raise MinMaxError("[ERROR] orig_imga.min() == orig_imga.max() "
"orig_imga.min() = %f, orig_imga.max() = %f"
% (orig_imga.min(), orig_imga.max())
)
orig_imga -= orig_imga.min()
orig_imga /= orig_imga.max()
# -- color conversion
# insure 3 dims
#print orig_imga.shape
if orig_imga.ndim == 2 or orig_imga.shape[2] == 1:
orig_imga_new = sp.empty(orig_imga.shape[:2] + (3,), dtype="float32")
orig_imga.shape = orig_imga_new[:,:,0].shape
orig_imga_new[:,:,0] = 0.2989*orig_imga
orig_imga_new[:,:,1] = 0.5870*orig_imga
orig_imga_new[:,:,2] = 0.1141*orig_imga
orig_imga = orig_imga_new
# -
if params['color_space'] == 'rgb':
orig_imga_conv = orig_imga
# elif params['color_space'] == 'rg':
# orig_imga_conv = colorconv.rg_convert(orig_imga)
elif params['color_space'] == 'rg2':
orig_imga_conv = colorconv.rg2_convert(orig_imga)
elif params['color_space'] == 'gray':
orig_imga_conv = colorconv.gray_convert(orig_imga)
orig_imga_conv.shape = orig_imga_conv.shape + (1,)
elif params['color_space'] == 'opp':
orig_imga_conv = colorconv.opp_convert(orig_imga)
elif params['color_space'] == 'oppnorm':
orig_imga_conv = colorconv.oppnorm_convert(orig_imga)
elif params['color_space'] == 'chrom':
orig_imga_conv = colorconv.chrom_convert(orig_imga)
# elif params['color_space'] == 'opponent':
# orig_imga_conv = colorconv.opponent_convert(orig_imga)
# elif params['color_space'] == 'W':
# orig_imga_conv = colorconv.W_convert(orig_imga)
elif params['color_space'] == 'hsv':
orig_imga_conv = colorconv.hsv_convert(orig_imga)
else:
raise ValueError, "params['color_space'] not understood"
# -- process each map
fvector_l = []
for cidx in xrange(orig_imga_conv.shape[2]):
imga0 = orig_imga_conv[:,:,cidx]
assert(imga0.min() != imga0.max())
# -- 0. preprocessing
#imga0 = imga0 / 255.0
# flip image ?
if 'flip_lr' in params['preproc'] and params['preproc']['flip_lr']:
imga0 = imga0[:,::-1]
if 'flip_ud' in params['preproc'] and params['preproc']['flip_ud']:
imga0 = imga0[::-1,:]
# smoothing
lsum_ksize = params['preproc']['lsum_ksize']
conv_mode = params['conv_mode']
if lsum_ksize is not None:
k = sp.ones((lsum_ksize), 'f') / lsum_ksize
imga0 = conv(conv(imga0, k[sp.newaxis,:], conv_mode),
k[:,sp.newaxis], conv_mode)
# whiten full image (assume True)
if 'whiten' not in params['preproc'] or params['preproc']['whiten']:
imga0 -= imga0.mean()
if imga0.std() != 0:
imga0 /= imga0.std()
# -- 1. input normalization
imga1 = v1like_norm(imga0[:,:,sp.newaxis], conv_mode, **params['normin'])
#print imga1.shape
# -- 2. linear filtering
filt_l = get_gabor_filters(params['filter'])
imga2 = v1like_filter(imga1[:,:,0], conv_mode, filt_l)
#print imga2.shape
#raise
# -- 3. simple non-linear activation (clamping)
minout = params['activ']['minout'] # sustain activity
maxout = params['activ']['maxout'] # saturation
imga3 = imga2.clip(minout, maxout)
#print imga3.shape
# -- 4. output normalization
imga4 = v1like_norm(imga3, conv_mode, **params['normout'])
#print imga4.shape
# -- 5. sparsify ?
if "sparsify" in params and params["sparsify"]:
imga4 = (imga4.max(2)[:,:,None] == imga4)
#print imga4.shape
#raise
# -- 6. volume dimension reduction
imga5 = v1like_pool(imga4, conv_mode, **params['pool'])
output = imga5
#print imga5.shape
# -- 7. handle features to include
feat_l = []
# include input norm histograms ?
f_normin_hists = featsel['normin_hists']
if f_normin_hists is not None:
division, nfeatures = f_norminhists
feat_l += [rephists(imga1, division, nfeatures)]
# include filter output histograms ?
f_filter_hists = featsel['filter_hists']
if f_filter_hists is not None:
division, nfeatures = f_filter_hists
feat_l += [rephists(imga2, division, nfeatures)]
# include activation output histograms ?
f_activ_hists = featsel['activ_hists']
if f_activ_hists is not None:
division, nfeatures = f_activ_hists
feat_l += [rephists(imga3, division, nfeatures)]
# include output norm histograms ?
f_normout_hists = featsel['normout_hists']
if f_normout_hists is not None:
division, nfeatures = f_normout_hists
feat_l += [rephists(imga4, division, nfeatures)]
# include representation output histograms ?
f_pool_hists = featsel['pool_hists']
if f_pool_hists is not None:
division, nfeatures = f_pool_hists
feat_l += [rephists(imga5, division, nfeatures)]
# include representation output ?
f_output = featsel['output']
if f_output and len(feat_l) != 0:
fvector = sp.concatenate([output.ravel()]+feat_l)
else:
fvector = output
fvector_l += [fvector]
# --
# include grayscale values ?
f_input_gray = featsel['input_gray']
if f_input_gray is not None:
shape = f_input_gray
#print orig_imga.shape
fvector_l += [sp.misc.imresize(colorconv.gray_convert(orig_imga), shape).ravel()]
# include color histograms ?
f_input_colorhists = featsel['input_colorhists']
if f_input_colorhists is not None:
nbins = f_input_colorhists
colorhists = sp.empty((3,nbins), 'f')
if orig_imga.ndim == 3:
for d in xrange(3):
h = sp.histogram(orig_imga[:,:,d].ravel(),
bins=nbins,
range=[0,255])
binvals = h[0].astype('f')
colorhists[d] = binvals
else:
raise ValueError, "orig_imga.ndim == 3"
#h = sp.histogram(orig_imga[:,:].ravel(),
# bins=nbins,
# range=[0,255])
#binvals = h[0].astype('f')
#colorhists[:] = binvals
#feat_l += [colorhists.ravel()]
fvector_l += [colorhists.ravel()]
# -- done !
fvector_l = [fvector.ravel() for fvector in fvector_l]
out = sp.concatenate(fvector_l).ravel()
return out
def extract_backgrounds(
input_filenames,
# --
output_dir = DEFAULT_OUTPUT_DIR,
# --
extract_width = DEFAULT_EXTRACT_WIDTH,
extract_height = DEFAULT_EXTRACT_HEIGHT,
output_width = DEFAULT_OUTPUT_WIDTH,
output_height = DEFAULT_OUTPUT_HEIGHT,
# --
#rot_range = DEFAULT_ROT_RANGE,
# --
nimages = DEFAULT_NIMAGES,
# color = DEFAULT_COLOR,
):
assert os.path.isdir(output_dir)
print "Number of input files:", len(input_filenames)
print "Number of output files:", nimages
pbar = ProgressBar(widgets=widgets, maxval=nimages)
pbar.start()
for n in xrange(nimages):
# 1. select a random (valid) image
done = False
while not done:
fname = input_filenames[sp.random.randint(len(input_filenames))]
arr_in = sp.atleast_3d(imread(fname))
arr_h, arr_w = shap = arr_in.shape[:2]
if (sp.array([extract_height, extract_width])*(1.+SIZE_EPSILON)
<= shap).all():
done = True
arr_in = arr_in.mean(2)
# 2. select a random (valid) position
posx = sp.random.randint(arr_w-extract_width)
posy = sp.random.randint(arr_h-extract_height)
# 3. extract the patch
arr_out = arr_in[posy:posy+extract_height, posx:posx+extract_width]
assert arr_out.shape == (extract_height, extract_width)
# 4. resize the image
arr_out = imresize(arr_out, (output_height, output_width))
assert arr_out.shape == (output_height, output_width)
# 5. normalize
arr_out = arr_out.astype('float32')
arr_out -= arr_out.mean()
arr_out_std = arr_out.std()
assert arr_out_std != 0
arr_out /= arr_out_std
# 6. save output
sha = hashlib.sha1(arr_out.tostring()).hexdigest()
output_filename = os.path.join(output_dir, "%s.png" % sha)
imsave(output_filename, arr_out)
pbar.update(n+1)
pbar.finish()
def apply_chords(im, spacing=1, axis=0, trim_edges=True, label=False):
r"""
Adds chords to the void space in the specified direction. The chords are
separated by 1 voxel plus the provided spacing.
Parameters
----------
im : ND-array
An image of the porous material with void marked as ``True``.
spacing : int
Separation between chords. The default is 1 voxel. This can be
decreased to 0, meaning that the chords all touch each other, which
automatically sets to the ``label`` argument to ``True``.
axis : int (default = 0)
The axis along which the chords are drawn.
trim_edges : bool (default = ``True``)
Whether or not to remove chords that touch the edges of the image.
These chords are artifically shortened, so skew the chord length
distribution.
label : bool (default is ``False``)
If ``True`` the chords in the returned image are each given a unique
label, such that all voxels lying on the same chord have the same
value. This is automatically set to ``True`` if spacing is 0, but is
``False`` otherwise.
Returns
-------
image : ND-array
A copy of ``im`` with non-zero values indicating the chords.
See Also
--------
apply_chords_3D
"""
if spacing < 0:
raise Exception('Spacing cannot be less than 0')
if spacing == 0:
label = True
result = sp.zeros(im.shape, dtype=int) # Will receive chords at end
slxyz = [slice(None, None, spacing * (axis != i) + 1) for i in [0, 1, 2]]
slices = tuple(slxyz[:im.ndim])
s = [[0, 1, 0], [0, 1, 0], [0, 1, 0]] # Straight-line structuring element
if im.ndim == 3: # Make structuring element 3D if necessary
s = sp.pad(sp.atleast_3d(s),
pad_width=((0, 0), (0, 0), (1, 1)),
mode='constant',
constant_values=0)
im = im[slices]
s = sp.swapaxes(s, 0, axis)
chords = spim.label(im, structure=s)[0]
if trim_edges: # Label on border chords will be set to 0
chords = clear_border(chords)
result[slices] = chords # Place chords into empty image created at top
if label is False: # Remove label if not requested
result = result > 0
return result
def __init__(self, image):
super().__init__()
image = sp.atleast_3d(image)
self.image = sp.array(image, dtype=bool)
def __init__(self, image):
super().__init__()
image = sp.atleast_3d(image)
self.image = sp.array(image, dtype=bool)