def circ_mask(arr, axis, ratio=1, val=0., ncore=None):
"""
Apply circular mask to a 3D array.
Parameters
----------
arr : ndarray
Arbitrary 3D array.
axis : int
Axis along which mask will be performed.
ratio : int, optional
Ratio of the mask's diameter in pixels to
the smallest edge size along given axis.
val : int, optional
Value for the masked region.
Returns
-------
ndarray
Masked array.
"""
arr = dtype.as_float32(arr)
val = np.float32(val)
_arr = arr.swapaxes(0, axis)
dx, dy, dz = _arr.shape
mask = _get_mask(dy, dz, ratio)
with mproc.set_numexpr_threads(ncore):
ne.evaluate('where(mask, _arr, val)', out=_arr)
return _arr.swapaxes(0, axis)
def remove_nan(arr, val=0., ncore=None):
"""
Replace NaN values in array with a given value.
Parameters
----------
arr : ndarray
Input array.
val : float, optional
Values to be replaced with NaN values in array.
ncore : int, optional
Number of cores that will be assigned to jobs.
Returns
-------
ndarray
Corrected array.
"""
arr = dtype.as_float32(arr)
val = np.float32(val)
with mproc.set_numexpr_threads(ncore):
ne.evaluate('where(arr!=arr, val, arr)', out=arr)
return arr
def remove_nan(arr, val=0., ncore=None):
"""
Replace NaN values in array with a given value.
Parameters
----------
arr : ndarray
Input array.
val : float, optional
Values to be replaced with NaN values in array.
ncore : int, optional
Number of cores that will be assigned to jobs.
Returns
-------
ndarray
Corrected array.
"""
arr = dtype.as_float32(arr)
val = np.float32(val)
with mproc.set_numexpr_threads(ncore):
ne.evaluate('where(arr!=arr, val, arr)', out=arr)
return arr
def circ_mask(arr, axis, ratio=1, val=0., ncore=None):
"""
Apply circular mask to a 3D array.
Parameters
----------
arr : ndarray
Arbitrary 3D array.
axis : int
Axis along which mask will be performed.
ratio : int, optional
Ratio of the mask's diameter in pixels to
the smallest edge size along given axis.
val : int, optional
Value for the masked region.
Returns
-------
ndarray
Masked array.
"""
arr = dtype.as_float32(arr)
val = np.float32(val)
_arr = arr.swapaxes(0, axis)
dx, dy, dz = _arr.shape
mask = _get_mask(dy, dz, ratio)
with mproc.set_numexpr_threads(ncore):
ne.evaluate('where(mask, _arr, val)', out=_arr)
return _arr.swapaxes(0, axis)
def remove_outlier1d(arr, dif, size=3, axis=0, ncore=None, out=None):
"""
Remove high intensity bright spots from an array, using a one-dimensional
median filter along the specified axis.
Dula: also removes dark spots
Parameters
----------
arr : ndarray
Input array.
dif : float
Expected difference value between outlier value and
the median value of the array.
size : int
Size of the median filter.
axis : int, optional
Axis along which median filtering is performed.
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process will be done in-place.
Returns
-------
ndarray
Corrected array.
"""
arr = arr.astype(np.float32, copy=False)
dif = np.float32(dif)
tmp = np.empty_like(arr)
other_axes = [i for i in range(arr.ndim) if i != axis]
largest = np.argmax([arr.shape[i] for i in other_axes])
lar_axis = other_axes[largest]
ncore, chnk_slices = mproc.get_ncore_slices(arr.shape[lar_axis],
ncore=ncore)
filt_size = [1] * arr.ndim
filt_size[axis] = size
with cf.ThreadPoolExecutor(ncore) as e:
slc = [slice(None)] * arr.ndim
for i in range(ncore):
slc[lar_axis] = chnk_slices[i]
e.submit(snf.median_filter,
arr[slc],
size=filt_size,
output=tmp[slc],
mode='mirror')
with mproc.set_numexpr_threads(ncore):
out = ne.evaluate('where(abs(arr-tmp)>=dif,tmp,arr)', out=out)
return out
def remove_outlier(arr, dif, size=3, axis=0, ncore=None, out=None):
"""
Remove high intensity bright spots from a N-dimensional array by chunking
along the specified dimension, and performing (N-1)-dimensional median
filtering along the other dimensions.
Parameters
----------
arr : ndarray
Input array.
dif : float
Expected difference value between outlier value and
the median value of the array.
size : int
Size of the median filter.
axis : int, optional
Axis along which to chunk.
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process
will be done in-place.
Returns
-------
ndarray
Corrected array.
"""
tmp = np.empty_like(arr)
ncore, chnk_slices = mproc.get_ncore_slices(arr.shape[axis], ncore=ncore)
filt_size = [size] * arr.ndim
filt_size[axis] = 1
with cf.ThreadPoolExecutor(ncore) as e:
slc = [slice(None)] * arr.ndim
for i in range(ncore):
slc[axis] = chnk_slices[i]
e.submit(scipy.ndimage.median_filter,
arr[tuple(slc)],
size=filt_size,
output=tmp[tuple(slc)])
arr = dtype.as_float32(arr)
tmp = dtype.as_float32(tmp)
dif = np.float32(dif)
with mproc.set_numexpr_threads(ncore):
out = ne.evaluate('where(arr-tmp>=dif,tmp,arr)', out=out)
return out
def remove_outlier1d(arr, dif, size=3, axis=0, ncore=None, out=None):
"""
Remove high intensity bright spots from an array, using a one-dimensional
median filter along the specified axis.
Parameters
----------
arr : ndarray
Input array.
dif : float
Expected difference value between outlier value and
the median value of the array.
size : int
Size of the median filter.
axis : int, optional
Axis along which median filtering is performed.
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process
will be done in-place.
Returns
-------
ndarray
Corrected array.
"""
arr = dtype.as_float32(arr)
dif = np.float32(dif)
tmp = np.empty_like(arr)
other_axes = [i for i in range(arr.ndim) if i != axis]
largest = np.argmax([arr.shape[i] for i in other_axes])
lar_axis = other_axes[largest]
ncore, chnk_slices = mproc.get_ncore_slices(
arr.shape[lar_axis], ncore=ncore)
filt_size = [1]*arr.ndim
filt_size[axis] = size
with cf.ThreadPoolExecutor(ncore) as e:
slc = [slice(None)]*arr.ndim
for i in range(ncore):
slc[lar_axis] = chnk_slices[i]
e.submit(filters.median_filter, arr[slc], size=filt_size,
output=tmp[slc], mode='mirror')
with mproc.set_numexpr_threads(ncore):
out = ne.evaluate('where(arr-tmp>=dif,tmp,arr)', out=out)
return out
def remove_outlier(arr, dif, size=3, axis=0, ncore=None, out=None):
"""
Remove high intensity bright spots from a N-dimensional array by chunking
along the specified dimension, and performing (N-1)-dimensional median
filtering along the other dimensions.
Parameters
----------
arr : ndarray
Input array.
dif : float
Expected difference value between outlier value and
the median value of the array.
size : int
Size of the median filter.
axis : int, optional
Axis along which to chunk.
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process
will be done in-place.
Returns
-------
ndarray
Corrected array.
"""
arr = dtype.as_float32(arr)
dif = np.float32(dif)
tmp = np.empty_like(arr)
ncore, chnk_slices = mproc.get_ncore_slices(arr.shape[axis], ncore=ncore)
filt_size = [size]*arr.ndim
filt_size[axis] = 1
with cf.ThreadPoolExecutor(ncore) as e:
slc = [slice(None)]*arr.ndim
for i in range(ncore):
slc[axis] = chnk_slices[i]
e.submit(filters.median_filter, arr[tuple(slc)], size=filt_size,
output=tmp[tuple(slc)])
with mproc.set_numexpr_threads(ncore):
out = ne.evaluate('where(arr-tmp>=dif,tmp,arr)', out=out)
return out
def remove_outlier(arr, dif, size=3, axis=0, ncore=None, out=None):
"""
Remove high intensity bright spots from a 3D array along specified
dimension.
Parameters
----------
arr : ndarray
Input array.
dif : float
Expected difference value between outlier value and
the median value of the array.
size : int
Size of the median filter.
axis : int, optional
Axis along which median filtering is performed.
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process will be done in-place.
Returns
-------
ndarray
Corrected array.
"""
arr = dtype.as_float32(arr)
dif = np.float32(dif)
tmp = np.empty_like(arr)
if ncore is None:
ncore = mproc.mp.cpu_count()
e = cf.ThreadPoolExecutor(ncore)
slc = [slice(None)]*len(arr.shape)
for i in range(arr.shape[axis]):
slc[axis] = i
e.submit(filters.median_filter, arr[slc], size=(size, size),
output=tmp[slc])
e.shutdown()
with mproc.set_numexpr_threads(ncore):
out = ne.evaluate('where(arr-tmp>=dif,tmp,arr)', out=out)
return out
def normalize(arr, flat, dark, cutoff=None, ncore=None, out=None):
"""
Normalize raw projection data using the flat and dark field projections.
Parameters
----------
arr : ndarray
3D stack of projections.
flat : ndarray
3D flat field data.
dark : ndarray
3D dark field data.
cutoff : float, optional
Permitted maximum vaue for the normalized data.
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr,
process will be done in-place.
Returns
-------
ndarray
Normalized 3D tomographic data.
"""
arr = dtype.as_float32(arr)
l = np.float32(1e-6)
flat = np.mean(flat, axis=0, dtype=np.float32)
dark = np.mean(dark, axis=0, dtype=np.float32)
with mproc.set_numexpr_threads(ncore):
#denom = ne.evaluate('flat-dark')
#ne.evaluate('where(denom<l,l,denom)', out=denom)
#out = ne.evaluate('arr-dark', out=out)
denom = flat - dark
denom[denom < l] = l
out = arr - dark
out[out < l] = l
out[:] /= denom
#ne.evaluate('out/denom', out=out, truediv=True)
if cutoff is not None:
cutoff = np.float32(cutoff)
out[out > cutoff] = cutoff
#ne.evaluate('where(out>cutoff,cutoff,out)', out=out)
return out
def minus_log(arr, ncore=None, out=None):
"""
Computation of the minus log of a given array.
Parameters
----------
arr : ndarray
3D stack of projections.
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process will be done in-place.
Returns
-------
ndarray
Minus-log of the input data.
"""
arr = dtype.as_float32(arr)
with mproc.set_numexpr_threads(ncore):
out = ne.evaluate('-log(arr)', out=out)
return out
def minus_log(arr, ncore=None, out=None):
"""
Computation of the minus log of a given array.
Parameters
----------
arr : ndarray
3D stack of projections.
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process will be done in-place.
Returns
-------
ndarray
Minus-log of the input data.
"""
arr = dtype.as_float32(arr)
with mproc.set_numexpr_threads(ncore):
out = ne.evaluate('-log(arr)', out=out)
return out
def normalize(arr, flat, dark, cutoff=None, ncore=None, out=None):
"""
Normalize raw projection data using the flat and dark field projections.
Parameters
----------
arr : ndarray
3D stack of projections.
flat : ndarray
3D flat field data.
dark : ndarray
3D dark field data.
cutoff : float, optional
Permitted maximum vaue for the normalized data.
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process will be done in-place.
Returns
-------
ndarray
Normalized 3D tomographic data.
"""
arr = dtype.as_float32(arr)
l = np.float32(1e-6)
flat = np.mean(flat, axis=0, dtype=np.float32)
dark = np.mean(dark, axis=0, dtype=np.float32)
with mproc.set_numexpr_threads(ncore):
denom = ne.evaluate('flat-dark')
ne.evaluate('where(denom<l,l,denom)', out=denom)
out = ne.evaluate('arr-dark', out=out)
ne.evaluate('out/denom', out=out, truediv=True)
if cutoff is not None:
cutoff = np.float32(cutoff)
ne.evaluate('where(out>cutoff,cutoff,out)', out=out)
return out
def normalize_nf(tomo,
flats,
dark,
flat_loc,
cutoff=None,
ncore=None,
out=None):
"""
Normalize raw 3D projection data with flats taken more than once during
tomography. Normalization for each projection is done with the mean of the
nearest set of flat fields (nearest flat fields).
Parameters
----------
tomo : ndarray
3D tomographic data.
flats : ndarray
3D flat field data.
dark : ndarray
3D dark field data.
flat_loc : list of int
Indices of flat field data within tomography
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process
will be done in-place.
Returns
-------
ndarray
Normalized 3D tomographic data.
"""
tomo = dtype.as_float32(tomo)
flats = dtype.as_float32(flats)
dark = dtype.as_float32(dark)
l = np.float32(1e-6)
if cutoff is not None:
cutoff = np.float32(cutoff)
if out is None:
out = np.empty_like(tomo)
dark = np.median(dark, axis=0)
denom = np.empty_like(dark)
num_flats = len(flat_loc)
total_flats = flats.shape[0]
total_tomo = tomo.shape[0]
num_per_flat = total_flats // num_flats
tend = 0
for m, loc in enumerate(flat_loc):
fstart = m * num_per_flat
fend = (m + 1) * num_per_flat
flat = np.median(flats[fstart:fend], axis=0)
# Normalization can be parallelized much more efficiently outside this
# for loop accounting for the nested parallelism arising from
# chunking the total normalization and each chunked normalization
tstart = 0 if m == 0 else tend
tend = total_tomo if m >= num_flats-1 \
else int(np.round((flat_loc[m+1]-loc)/2)) + loc
tomo_l = tomo[tstart:tend]
out_l = out[tstart:tend]
with mproc.set_numexpr_threads(ncore):
ne.evaluate('flat-dark', out=denom)
ne.evaluate('where(denom<l,l,denom)', out=denom)
ne.evaluate('(tomo_l-dark)/denom', out=out_l, truediv=True)
if cutoff is not None:
ne.evaluate('where(out_l>cutoff,cutoff,out_l)', out=out_l)
return out
def pad(arr, axis, npad=None, mode='constant', ncore=None, **kwargs):
"""
Pad an array along specified axis.
Parameters
----------
arr : ndarray
Input array.
npad : int, optional
New dimension after padding.
axis : int, optional
Axis along which padding will be performed.
mode : str or function
One of the following string values or a user supplied function.
'constant'
Pads with a constant value.
'edge'
Pads with the edge values of array.
constant_values : float, optional
Used in 'constant'. Pad value
ncore : int, optional
Number of cores that will be assigned to jobs.
Returns
-------
ndarray
Padded 3D array.
"""
allowedkwargs = {'constant': ['constant_values'],
'edge': [], }
kwdefaults = {'constant_values': 0, }
if isinstance(mode, str):
if mode not in allowedkwargs:
raise ValueError("'mode' keyword value '{}' not in allowed values {}".format(mode, list(allowedkwargs.keys())))
for key in kwargs:
if key not in allowedkwargs[mode]:
raise ValueError('%s keyword not in allowed keywords %s' %
(key, allowedkwargs[mode]))
for kw in allowedkwargs[mode]:
kwargs.setdefault(kw, kwdefaults[kw])
else:
raise ValueError('mode keyword value must be string, got %s: ' %
type(mode))
if npad is None:
npad = _get_npad(arr.shape[axis])
newshape = list(arr.shape)
newshape[axis] += 2*npad
slc_in, slc_l, slc_r, slc_l_v, slc_r_v = _get_slices(arr.shape, axis, npad)
out = np.empty(newshape, dtype=arr.dtype)
if arr.dtype in [np.float32, np.float64, np.bool,
np.int32, np.int64, np.complex128]:
# Datatype supported by numexpr
with mproc.set_numexpr_threads(ncore):
ne.evaluate("arr", out=out[slc_in])
if mode == 'constant':
np_cast = getattr(np, str(arr.dtype))
cval = np_cast(kwargs['constant_values'])
ne.evaluate("cval", out=out[slc_l])
ne.evaluate("cval", out=out[slc_r])
elif mode == 'edge':
ne.evaluate("vec", local_dict={'vec': arr[slc_l_v]},
out=out[slc_l])
ne.evaluate("vec", local_dict={'vec': arr[slc_r_v]},
out=out[slc_r])
else:
# Datatype not supported by numexpr, use numpy instead
out[slc_in] = arr
if mode == 'constant':
out[slc_l] = kwargs['constant_values']
out[slc_r] = kwargs['constant_values']
elif mode == 'edge':
out[slc_l] = arr[slc_l_v]
out[slc_r] = arr[slc_r_v]
return out
def normalize_nf(tomo, flats, dark, flat_loc,
cutoff=None, ncore=None, out=None):
"""
Normalize raw 3D projection data with flats taken more than once during
tomography. Normalization for each projection is done with the mean of the
nearest set of flat fields (nearest flat fields).
Parameters
----------
tomo : ndarray
3D tomographic data.
flats : ndarray
3D flat field data.
dark : ndarray
3D dark field data.
flat_loc : list of int
Indices of flat field data within tomography
ncore : int, optional
Number of cores that will be assigned to jobs.
out : ndarray, optional
Output array for result. If same as arr, process will be done in-place.
Returns
-------
ndarray
Normalized 3D tomographic data.
"""
tomo = dtype.as_float32(tomo)
flats = dtype.as_float32(flats)
dark = dtype.as_float32(dark)
l = np.float32(1e-6)
if cutoff is not None:
cutoff = np.float32(cutoff)
if out is None:
out = np.empty_like(tomo)
dark = np.median(dark, axis=0)
denom = np.empty_like(dark)
num_flats = len(flat_loc)
total_flats = flats.shape[0]
total_tomo = tomo.shape[0]
num_per_flat = total_flats//num_flats
tend = 0
for m, loc in enumerate(flat_loc):
fstart = m*num_per_flat
fend = (m + 1)*num_per_flat
flat = np.median(flats[fstart:fend], axis=0)
# Normalization can be parallelized much more efficiently outside this
# for loop accounting for the nested parallelism arising from
# chunking the total normalization and each chunked normalization
tstart = 0 if m == 0 else tend
tend = total_tomo if m >= num_flats-1 \
else int(np.round((flat_loc[m+1]-loc)/2)) + loc
tomo_l = tomo[tstart:tend]
out_l = out[tstart:tend]
with mproc.set_numexpr_threads(ncore):
ne.evaluate('flat-dark', out=denom)
ne.evaluate('where(denom<l,l,denom)', out=denom)
ne.evaluate('(tomo_l-dark)/denom', out=out_l, truediv=True)
if cutoff is not None:
ne.evaluate('where(out_l>cutoff,cutoff,out_l)', out=out_l)
return out
def pad(arr, axis, npad=None, mode='constant', ncore=None, **kwargs):
"""
Pad an array along specified axis.
Parameters
----------
arr : ndarray
Input array.
npad : int, optional
New dimension after padding.
axis : int, optional
Axis along which padding will be performed.
mode : str or function
One of the following string values or a user supplied function.
'constant'
Pads with a constant value.
'edge'
Pads with the edge values of array.
constant_values : float, optional
Used in 'constant'. Pad value
ncore : int, optional
Number of cores that will be assigned to jobs.
Returns
-------
ndarray
Padded 3D array.
"""
allowedkwargs = {
'constant': ['constant_values'],
'edge': [],
}
kwdefaults = {
'constant_values': 0,
}
if isinstance(mode, str):
if mode not in allowedkwargs:
raise ValueError(
"'mode' keyword value '{}' not in allowed values {}".format(
mode, list(allowedkwargs.keys())))
for key in kwargs:
if key not in allowedkwargs[mode]:
raise ValueError('%s keyword not in allowed keywords %s' %
(key, allowedkwargs[mode]))
for kw in allowedkwargs[mode]:
kwargs.setdefault(kw, kwdefaults[kw])
else:
raise ValueError('mode keyword value must be string, got %s: ' %
type(mode))
if npad is None:
npad = _get_npad(arr.shape[axis])
newshape = list(arr.shape)
newshape[axis] += 2 * npad
slc_in, slc_l, slc_r, slc_l_v, slc_r_v = _get_slices(arr.shape, axis, npad)
# convert to tuples
slc_in = tuple(slc_in)
slc_l = tuple(slc_l)
slc_r = tuple(slc_r)
slc_l_v = tuple(slc_l_v)
slc_r_v = tuple(slc_r_v)
out = np.empty(newshape, dtype=arr.dtype)
if arr.dtype in [
np.float32, np.float64, np.bool, np.int32, np.int64, np.complex128
]:
# Datatype supported by numexpr
with mproc.set_numexpr_threads(ncore):
ne.evaluate("arr", out=out[slc_in])
if mode == 'constant':
np_cast = getattr(np, str(arr.dtype))
cval = np_cast(kwargs['constant_values'])
ne.evaluate("cval", out=out[slc_l])
ne.evaluate("cval", out=out[slc_r])
elif mode == 'edge':
ne.evaluate("vec",
local_dict={'vec': arr[slc_l_v]},
out=out[slc_l])
ne.evaluate("vec",
local_dict={'vec': arr[slc_r_v]},
out=out[slc_r])
else:
# Datatype not supported by numexpr, use numpy instead
out[slc_in] = arr
if mode == 'constant':
out[slc_l] = kwargs['constant_values']
out[slc_r] = kwargs['constant_values']
elif mode == 'edge':
out[slc_l] = arr[slc_l_v]
out[slc_r] = arr[slc_r_v]
return out