def plot_cub_as_curve(c, colors=None, plot_kwargs=None, legend_prefix='',
show_axis_labels=True, show_legend=False, axes=None):
"""
Plot a cuboid (ndims <= 2) as curve(s).
If the input is 1D: one single curve.
If the input is 2D:
* multiple curves are plotted: one for each domain value on the 1st axis.
* legends are shown to display which domain value is associated
to which curve.
Args:
- colors (dict <domain value>: <matplotlib color>):
associate domain values of the 1st axis to color curves
- plot_kwargs (dict <arg name>:<arg value>):
dictionary of named argument passed to the plot function
- legend_prefix (str): prefix to prepend to legend labels.
Return:
None
"""
import matplotlib.pyplot as plt
def protect_latex_str(s):
return s.replace('_','\_')
axes = axes or plt.gca()
colors = colors or {}
plot_kwargs = plot_kwargs or {}
if c.get_ndims() == 1:
dom = c.axes_domains[c.axes_names[0]]
if np.issubsctype(dom.dtype, str):
dom = np.arange(len(dom))
axes.plot(dom, c.data, **plot_kwargs)
if np.issubsctype(c.axes_domains[c.axes_names[0]], str):
set_int_tick_labels(axes.xaxis, c.axes_domains[c.axes_names[0]],
rotation=30)
elif c.get_ndims() == 2:
for val, sub_c in c.split(c.axes_names[0]).iteritems():
pkwargs = plot_kwargs.copy()
col = colors.get(val, None)
if col is not None:
pkwargs['color'] = col
pkwargs['label'] = protect_latex_str(legend_prefix + \
c.axes_names[0] + \
'=' + str(val))
plot_cub_as_curve(sub_c, plot_kwargs=pkwargs, axes=axes,
show_axis_labels=False)
if show_legend:
axes.legend()
else:
raise Exception('xndarray has too many dims (%d), expected at most 2' \
%c.get_ndims())
if show_axis_labels:
if c.get_ndims() == 1:
axes.set_xlabel(protect_latex_str(c.axes_names[0]))
else:
axes.set_xlabel(protect_latex_str(c.axes_names[1]))
axes.set_ylabel(protect_latex_str(c.value_label))
def test_float32_input_inv():
# Check that an float32 input is correctly output as float32
Yl, Yh = dtwavexfm(np.array([1, 2, 3, 4]).astype(np.float32))
assert np.issubsctype(Yl.dtype, np.float32)
assert np.all(list(np.issubsctype(x.dtype, np.complex64) for x in Yh))
recon = dtwaveifm(Yl, Yh)
assert np.issubsctype(recon.dtype, np.float32)
def test_float32_recon():
# Check that an float32 input is correctly output as float32
Yl, Yh = dtwavexfm3(ellipsoid.astype(np.float32))
assert np.issubsctype(Yl.dtype, np.float32)
assert np.all(list(np.issubsctype(x.dtype, np.complex64) for x in Yh))
recon = dtwaveifm3(Yl, Yh)
assert np.issubsctype(recon.dtype, np.float32)
def is_range_domain(d):
#print 'd:', d.dtype
if not (np.issubsctype(d.dtype, np.unicode) or \
np.issubsctype(d.dtype, np.str) or (d.size==1)):
delta = np.diff(d)
return (delta == delta[0]).all()
else:
return False
def test_float32_input():
# Check that an float32 input is correctly output as float32
Yl, Yh = dtwavexfm2(lena.astype(np.float32))
assert np.issubsctype(Yl.dtype, np.float32)
assert np.all(list(np.issubsctype(x.dtype, np.complex64) for x in Yh))
lena_recon = dtwaveifm2(Yl, Yh)
assert np.issubsctype(lena_recon.dtype, np.float32)
def _encode_sql_value(val):
if np.issubsctype(type(val), np.str_):
return sqlite3.Binary(val)
elif np.issubsctype(type(val), np.bool_):
return bool(val)
elif np.issubsctype(type(val), np.integer):
return int(val)
elif np.issubsctype(type(val), np.floating):
return float(val)
else:
return val
def _encode_sql_value(val):
# This is a non-trivial speedup for bulk inserts (e.g. creating thousands
# of events while loading a file):
if type(val) in ok_types:
return val
if np.issubsctype(type(val), np.str_):
return sqlite3.Binary(val)
elif np.issubsctype(type(val), np.bool_):
return bool(val)
elif np.issubsctype(type(val), np.integer):
return int(val)
elif np.issubsctype(type(val), np.floating):
return float(val)
else:
return val
def convert_value_read(value):
"""Convert attribute value from bytes to string."""
if isinstance(value, bytes):
return value.decode()
elif not np.isscalar(value) and np.issubsctype(value, np.bytes_):
return value.astype(np.str_)
return value
def quantize(img, levels=32, dtype=np.uint8):
"""Uniform quantization from float [0, 1] to int [0, levels-1]."""
img = np.asarray(img)
assert np.issubsctype(img, np.floating), img.dtype
assert np.all(img >= 0) and np.all(img <= 1), (img.min(), img.max())
# img = skimage.img_as_ubyte(img)
# img //= int(round(256 / levels))
return (img * levels).clip(0, levels-1).astype(dtype)
def appropriate_complex_type_for(X):
"""Return an appropriate complex data type depending on the type of X. If X
is already complex, return that, if it is floating point return a complex
type of the appropriate size and if it is integer, choose an complex
floating point type depending on the result of :py:func:`numpy.asfarray`.
"""
X = asfarray(X)
if np.issubsctype(X.dtype, np.complex64) or np.issubsctype(X.dtype, np.complex128):
return X.dtype
elif np.issubsctype(X.dtype, np.float32):
return np.complex64
elif np.issubsctype(X.dtype, np.float64):
return np.complex128
# God knows, err on the side of caution
return np.complex128
def isnan(a):
"""
isnan is equivalent to np.isnan, except that it returns False instead of
raising a TypeError if the argument is an array of non-numeric.
"""
if isinstance(a, np.ndarray):
return np.issubsctype(a, np.floating) and np.isnan(a)
else:
return np.isnan(a)
def _copy_array_if_base_present(a):
"""
Copies the array if its base points to a parent array.
"""
if a.base is not None:
return a.copy()
elif np.issubsctype(a, np.float32):
return np.array(a, dtype=np.double)
else:
return a
def test_ufuncs():
# Cannot use fixture due to bug in pytest
fn = Rn(3)
for name, n_args, n_out, _ in UFUNCS:
if (np.issubsctype(fn.dtype, np.floating) and
name in ['bitwise_and', 'bitwise_or', 'bitwise_xor', 'invert',
'left_shift', 'right_shift']):
# Skip integer only methods if floating point type
continue
yield _impl_test_ufuncs, fn, name, n_args, n_out
def _update_colors(self, numcolors=None):
""" Update the colors cache using our color mapper and based
on our number of levels. The **mode** parameter accounts for fenceposting:
- If **mode** is "poly", then the number of colors to generate is 1
less than the number of levels
- If **mode** is "line", then the number of colors to generate is
equal to the number of levels
"""
if numcolors is None:
numcolors = len(self._levels)
colors = self.colors
# If we are given no colors, set a default for all levels
if colors is None:
self._color_map_trait = "black"
self._colors = [self._color_map_trait_] * numcolors
# If we are given a single color, apply it to all levels
elif isinstance(colors, basestring):
self._color_map_trait = colors
self._colors = [self._color_map_trait_] * numcolors
# If we are given a colormap, use it to map all the levels to colors
elif isinstance(colors, ColorMapper):
self._colors = []
mapped_colors = self.color_mapper.map_screen(array(self._levels))
for i in range(numcolors):
self._color_map_trait = tuple(mapped_colors[i])
self._colors.append(self._color_map_trait_)
# A list or tuple
# This could be a length 3 or 4 sequence of scalars, which indicates
# a color; otherwise, this is interpreted as a list of items to
# be converted via self._color_map_trait.
else:
if len(colors) in (3,4) and \
(isscalar(colors[0]) and issubsctype(type(colors[0]), number)):
self._color_map_trait = colors
self._colors = [self._color_map_trait_] * numcolors
else:
# if the list of colors is shorter than the list of levels, simply
# repeat colors from the beginning of the list as needed
self._colors = []
for i in range(len(self._levels)):
self._color_map_trait = colors[i%len(colors)]
self._colors.append(self._color_map_trait_)
self._colors_cache_valid = True
return
def test_ufuncs(fn, ufunc):
name, n_args, n_out, _ = ufunc
if (np.issubsctype(fn.dtype, np.floating) and
name in ['bitwise_and',
'bitwise_or',
'bitwise_xor',
'invert',
'left_shift',
'right_shift']):
# Skip integer only methods if floating point type
return
# Get the ufunc from numpy as reference
ufunc = getattr(np, name)
# Create some data
arrays, vectors = example_vectors(fn, n_args + n_out)
in_arrays = arrays[:n_args]
out_arrays = arrays[n_args:]
data_vector = vectors[0]
in_vectors = vectors[1:n_args]
out_vectors = vectors[n_args:]
# Out of place:
np_result = ufunc(*in_arrays)
vec_fun = getattr(data_vector.ufunc, name)
odl_result = vec_fun(*in_vectors)
assert all_almost_equal(np_result, odl_result)
# Test type of output
if n_out == 1:
assert isinstance(odl_result, fn.element_type)
elif n_out > 1:
for i in range(n_out):
assert isinstance(odl_result[i], fn.element_type)
# In place:
np_result = ufunc(*(in_arrays + out_arrays))
vec_fun = getattr(data_vector.ufunc, name)
odl_result = vec_fun(*(in_vectors + out_vectors))
assert all_almost_equal(np_result, odl_result)
# Test inplace actually holds:
if n_out == 1:
assert odl_result is out_vectors[0]
elif n_out > 1:
for i in range(n_out):
assert odl_result[i] is out_vectors[i]
def slice(self, columns_specifier):
"""Locate a subset of design matrix columns, specified symbolically.
A patsy design matrix has two levels of structure: the individual
columns (which are named), and the :ref:`terms <formulas>` in
the formula that generated those columns. This is a one-to-many
relationship: a single term may span several columns. This method
provides a user-friendly API for locating those columns.
(While we talk about columns here, this is probably most useful for
indexing into other arrays that are derived from the design matrix,
such as regression coefficients or covariance matrices.)
The `columns_specifier` argument can take a number of forms:
* A term name
* A column name
* A :class:`Term` object
* An integer giving a raw index
* A raw slice object
In all cases, a Python :func:`slice` object is returned, which can be
used directly for indexing.
Example::
y, X = dmatrices("y ~ a", demo_data("y", "a", nlevels=3))
betas = np.linalg.lstsq(X, y)[0]
a_betas = betas[X.design_info.slice("a")]
(If you want to look up a single individual column by name, use
``design_info.column_name_indexes[name]``.)
"""
if isinstance(columns_specifier, slice):
return columns_specifier
if np.issubsctype(type(columns_specifier), np.integer):
return slice(columns_specifier, columns_specifier + 1)
if (self.term_slices is not None
and columns_specifier in self.term_slices):
return self.term_slices[columns_specifier]
if columns_specifier in self.term_name_slices:
return self.term_name_slices[columns_specifier]
if columns_specifier in self.column_name_indexes:
idx = self.column_name_indexes[columns_specifier]
return slice(idx, idx + 1)
raise PatsyError("unknown column specified '%s'"
% (columns_specifier,))
def slice(self, columns_specifier):
"""Locate a subset of design matrix columns, specified symbolically.
A patsy design matrix has two levels of structure: the individual
columns (which are named), and the :ref:`terms <formulas>` in
the formula that generated those columns. This is a one-to-many
relationship: a single term may span several columns. This method
provides a user-friendly API for locating those columns.
(While we talk about columns here, this is probably most useful for
indexing into other arrays that are derived from the design matrix,
such as regression coefficients or covariance matrices.)
The `columns_specifier` argument can take a number of forms:
* A term name
* A column name
* A :class:`Term` object
* An integer giving a raw index
* A raw slice object
In all cases, a Python :func:`slice` object is returned, which can be
used directly for indexing.
Example::
y, X = dmatrices("y ~ a", demo_data("y", "a", nlevels=3))
betas = np.linalg.lstsq(X, y)[0]
a_betas = betas[X.design_info.slice("a")]
(If you want to look up a single individual column by name, use
``design_info.column_name_indexes[name]``.)
"""
if isinstance(columns_specifier, slice):
return columns_specifier
if np.issubsctype(type(columns_specifier), np.integer):
return slice(columns_specifier, columns_specifier + 1)
if (self.term_slices is not None
and columns_specifier in self.term_slices):
return self.term_slices[columns_specifier]
if columns_specifier in self.term_name_slices:
return self.term_name_slices[columns_specifier]
if columns_specifier in self.column_name_indexes:
idx = self.column_name_indexes[columns_specifier]
return slice(idx, idx + 1)
raise PatsyError("unknown column specified '%s'" %
(columns_specifier, ))
def test_ufuncs(fn, ufunc):
name, n_args, n_out, _ = ufunc
if (np.issubsctype(fn.dtype, np.floating) and name in [
'bitwise_and', 'bitwise_or', 'bitwise_xor', 'invert', 'left_shift',
'right_shift'
]):
# Skip integer only methods if floating point type
return
# Get the ufunc from numpy as reference
ufunc = getattr(np, name)
# Create some data
arrays, vectors = noise_elements(fn, n_args + n_out)
in_arrays = arrays[:n_args]
out_arrays = arrays[n_args:]
data_vector = vectors[0]
in_vectors = vectors[1:n_args]
out_vectors = vectors[n_args:]
# Out of place:
np_result = ufunc(*in_arrays)
vec_fun = getattr(data_vector.ufuncs, name)
odl_result = vec_fun(*in_vectors)
assert all_almost_equal(np_result, odl_result)
# Test type of output
if n_out == 1:
assert isinstance(odl_result, fn.element_type)
elif n_out > 1:
for i in range(n_out):
assert isinstance(odl_result[i], fn.element_type)
# In place:
np_result = ufunc(*(in_arrays + out_arrays))
vec_fun = getattr(data_vector.ufuncs, name)
odl_result = vec_fun(*(in_vectors + out_vectors))
assert all_almost_equal(np_result, odl_result)
# Test inplace actually holds:
if n_out == 1:
assert odl_result is out_vectors[0]
elif n_out > 1:
for i in range(n_out):
assert odl_result[i] is out_vectors[i]
def test_ufuncs():
# Cannot use fixture due to bug in pytest
spaces = [odl.uniform_discr([0, 0], [1, 1], [2, 2])]
if odl.CUDA_AVAILABLE:
spaces += [odl.uniform_discr([0, 0], [1, 1], [2, 2], impl='cuda')]
for fn in spaces:
for name, n_args, n_out, _ in odl.util.ufuncs.UFUNCS:
if (np.issubsctype(fn.dtype, np.floating) and
name in ['bitwise_and',
'bitwise_or',
'bitwise_xor',
'invert',
'left_shift',
'right_shift']):
# Skip integer only methods if floating point type
continue
yield _impl_test_ufuncs, fn, name, n_args, n_out
def __eq__(self, other):
"""
Two nodes are equal if all attributes from the list are equal (ignoring gurobi_stats)
"""
for attr in set(self.__slots__) - {'_invar', 'gurobi_stats'}:
if self.__getattribute__(attr) != other.__getattribute__(attr):
return False
# invar is dictionary of numpy arrays
if self.invar.keys() != other.invar.keys():
return False
# If a single invariant is different, nodes are not equal
for inv_name in self.invar.keys():
array_comp_func = np.allclose if np.issubsctype(
self.invar[inv_name], float) else np.array_equal
if not array_comp_func(self.invar[inv_name],
other.invar[inv_name]):
return False
return True
def mask_source_lonlats(source_def, mask):
"""Mask source longitudes and latitudes to match data mask."""
source_geo_def = source_def
# the data may have additional masked pixels
# let's compare them to see if we can use the same area
# assume lons and lats mask are the same
if mask is not None and mask is not False and isinstance(source_geo_def, SwathDefinition):
if np.issubsctype(mask.dtype, np.bool):
# copy the source area and use it for the rest of the calculations
LOG.debug("Copying source area to mask invalid dataset points")
if mask.ndim != source_geo_def.lons.ndim:
raise ValueError("Can't mask area, mask has different number "
"of dimensions.")
return SwathDefinition(source_geo_def.lons.where(~mask),
source_geo_def.lats.where(~mask))
else:
return SwathDefinition(source_geo_def.lons.where(~xu.isnan(mask)),
source_geo_def.lats.where(~xu.isnan(mask)))
return source_geo_def
def mask_source_lonlats(source_def, mask):
"""Mask source longitudes and latitudes to match data mask."""
source_geo_def = source_def
# the data may have additional masked pixels
# let's compare them to see if we can use the same area
# assume lons and lats mask are the same
if mask is not None and mask is not False and isinstance(
source_geo_def, SwathDefinition):
if np.issubsctype(mask.dtype, np.bool):
# copy the source area and use it for the rest of the calculations
LOG.debug("Copying source area to mask invalid dataset points")
if mask.ndim != source_geo_def.lons.ndim:
raise ValueError("Can't mask area, mask has different number "
"of dimensions.")
return SwathDefinition(source_geo_def.lons.where(~mask),
source_geo_def.lats.where(~mask))
else:
return SwathDefinition(source_geo_def.lons.where(~xu.isnan(mask)),
source_geo_def.lats.where(~xu.isnan(mask)))
return source_geo_def
def __eq__(self, other):
"""
Two nodes are equal if all attributes from the list are equal
"""
for attr in set(
self.__slots__) - {'_invar', SYN, RAW, RAW_HOUSING, UNIT_SYN}:
if self.__getattribute__(attr) != other.__getattribute__(attr):
return False
# invar is dictionary of numpy arrays
if self.invar.keys() != other.invar.keys():
return False
# If a single invariant is different, nodes are not equal
for inv_name in self.invar.keys():
array_comp_func = np.allclose if np.issubsctype(
self.invar[inv_name], float) else np.array_equal
if not array_comp_func(self.invar[inv_name],
other.invar[inv_name]):
return False
# When raw/syn is np.array have to call np.array_equal
if not np.array_equal(self.getDenseSyn(), other.getDenseSyn()):
return False
if not np.array_equal(self.getDenseRaw(), other.getDenseRaw()):
return False
if not np.array_equal(self.getDenseRawHousing(),
other.getDenseRawHousing()):
return False
if not np.array_equal(self.getDenseSynHousing(),
other.getDenseSynHousing()):
return False
return True
def validate(self, value: np.ndarray, context: str = '') -> None:
if not isinstance(value, np.ndarray):
raise TypeError(
'{} is not a numpy array; {}'.format(repr(value), context))
if not any(
np.issubsctype(value.dtype.type, valid_type) for valid_type in
self.valid_types):
raise TypeError(
f'type of {value} is not any of {self.valid_types}'
f' it is {value.dtype}; {context}')
if self.shape is not None:
shape = self.shape
if np.shape(value) != shape:
raise ValueError(
f'{repr(value)} does not have expected shape {shape},'
f' it has shape {np.shape(value)}; {context}')
# Only check if max is not inf as it can be expensive for large arrays
if self._max_value != (float("inf")) and self._max_value is not None:
if not (np.max(value) <= self._max_value):
raise ValueError(
'{} is invalid: all values must be between '
'{} and {} inclusive; {}'.format(
repr(value), self._min_value,
self._max_value, context))
# Only check if min is not -inf as it can be expensive for large arrays
if self._min_value != (-float("inf")) and self._min_value is not None:
if not (self._min_value <= np.min(value)):
raise ValueError(
'{} is invalid: all values must be between '
'{} and {} inclusive; {}'.format(
repr(value), self._min_value,
self._max_value, context))
def is_real_floating_dtype(dtype):
"""Return ``True`` if ``dtype`` is a real floating point type."""
return np.issubsctype(dtype, np.floating)
def test_ufunc(fn_impl, ufunc):
space = odl.uniform_discr([0, 0], [1, 1], (2, 2), impl=fn_impl)
name, n_args, n_out, _ = ufunc
if (np.issubsctype(space.dtype, np.floating) and
name in ['bitwise_and',
'bitwise_or',
'bitwise_xor',
'invert',
'left_shift',
'right_shift']):
# Skip integer only methods if floating point type
return
# Get the ufunc from numpy as reference
ufunc = getattr(np, name)
# Create some data
arrays, vectors = noise_elements(space, n_args + n_out)
in_arrays = arrays[:n_args]
out_arrays = arrays[n_args:]
data_vector = vectors[0]
in_vectors = vectors[1:n_args]
out_vectors = vectors[n_args:]
# Verify type
assert isinstance(data_vector.ufuncs,
odl.util.ufuncs.DiscreteLpUfuncs)
# Out-of-place:
np_result = ufunc(*in_arrays)
vec_fun = getattr(data_vector.ufuncs, name)
odl_result = vec_fun(*in_vectors)
assert all_almost_equal(np_result, odl_result)
# Test type of output
if n_out == 1:
assert isinstance(odl_result, space.element_type)
elif n_out > 1:
for i in range(n_out):
assert isinstance(odl_result[i], space.element_type)
# In-place:
np_result = ufunc(*(in_arrays + out_arrays))
vec_fun = getattr(data_vector.ufuncs, name)
odl_result = vec_fun(*(in_vectors + out_vectors))
assert all_almost_equal(np_result, odl_result)
# Test in-place actually holds:
if n_out == 1:
assert odl_result is out_vectors[0]
elif n_out > 1:
for i in range(n_out):
assert odl_result[i] is out_vectors[i]
# Test out-of-place with np data
np_result = ufunc(*in_arrays)
vec_fun = getattr(data_vector.ufuncs, name)
odl_result = vec_fun(*in_arrays[1:])
assert all_almost_equal(np_result, odl_result)
# Test type of output
if n_out == 1:
assert isinstance(odl_result, space.element_type)
elif n_out > 1:
for i in range(n_out):
assert isinstance(odl_result[i], space.element_type)
def filter_image_sep2d(image, fh, fv, impl='numpy', padding=None):
"""Filter an image with a separable filter.
Parameters
----------
image : 2D array-like
The image to be filtered. It must have a real (vs. complex) dtype.
fh, fv : 1D array-like
Horizontal (axis 0) and vertical (axis 1) filters. Their sizes
can be at most the image sizes in the respective axes.
impl : {'numpy', 'pyfftw'}, optional
FFT backend to use. The ``pyfftw`` backend requires the
``pyfftw`` package to be installed. It is usually significantly
faster than the NumPy backend.
padding : positive int, optional
Amount of zeros added to the left and right of the image in all
axes before FFT. This helps avoiding wraparound artifacts due to
large boundary values.
For ``None``, the padding is computed as ::
padding = min(max(len(fh), len(fv)) - 1, 64)
A padding of ``len(filt) - 1`` ensures that errors in FFT-based
convolutions are small. At the same time, the padding should not
be excessive to retain efficiency.
Returns
-------
filtered : 2D `numpy.ndarray`
The image filtered horizontally by ``fh`` and vertically by ``fv``.
It has the same shape as ``image``, and its dtype is
``np.result_type(image, fh, fv)``.
"""
# TODO: generalize for nD
impl, impl_in = str(impl).lower(), impl
image = np.asarray(image)
if image.ndim != 2:
raise ValueError('`image` must be 2-dimensional, got image with '
'ndim={}'.format(image.ndim))
if image.size == 0:
raise ValueError('`image` cannot have size 0')
if not np.issubsctype(image.dtype, np.floating):
image = image.astype(float)
fh = np.asarray(fh).astype(image.dtype)
if fh.ndim != 1:
raise ValueError('`fh` must be one-dimensional')
elif fh.size == 0:
raise ValueError('`fh` cannot have size 0')
elif fh.size > image.shape[0]:
raise ValueError('`fh` can be at most `image.shape[0]`, got '
'{} > {}'.format(fh.size, image.shape[0]))
fv = np.asarray(fv).astype(image.dtype)
if fv.ndim != 1:
raise ValueError('`fv` must be one-dimensional')
elif fv.size == 0:
raise ValueError('`fv` cannot have size 0')
elif fv.size > image.shape[0]:
raise ValueError('`fv` can be at most `image.shape[1]`, got '
'{} > {}'.format(fv.size, image.shape[1]))
# Pad image with zeros
if padding is None:
padding = min(max(len(fh), len(fv)) - 1, 64)
if padding != 0:
image_padded = np.pad(image, padding, mode='constant')
else:
image_padded = image.copy() if impl == 'pyfftw' else image
# Prepare filters for the convolution
def prepare_for_fft(filt, n_new):
"""Return padded and shifted filter ready for FFT.
The filter is padded with zeros to the new size, and then shifted
such that such that the middle element of old filter, i.e., the
one at index ``(len(filt) - 1) // 2`` ends up at index 0.
"""
mid = (len(filt) - 1) // 2
padded = np.zeros(n_new, dtype=filt.dtype)
padded[:len(filt) - mid] = filt[mid:]
padded[len(padded) - mid:] = filt[:mid]
return padded
fh = prepare_for_fft(fh, image_padded.shape[0])
fv = prepare_for_fft(fv, image_padded.shape[1])
# Perform the multiplication in Fourier space and apply inverse FFT
if impl == 'numpy':
image_ft = np.fft.rfftn(image_padded)
fh_ft = np.fft.fft(fh)
fv_ft = np.fft.rfft(fv)
image_ft *= fh_ft[:, None]
image_ft *= fv_ft[None, :]
# Important to specify the shape since `irfftn` cannot know the
# original shape
conv = np.fft.irfftn(image_ft, s=image_padded.shape)
if conv.dtype != image.dtype:
conv = conv.astype(image.dtype)
elif impl == 'pyfftw':
if not PYFFTW_AVAILABLE:
raise ValueError(
'`pyfftw` package is not available; you need to install it '
'to use the pyfftw backend')
import pyfftw
import multiprocessing
# Generate output arrays, for half-complex transform of image and
# vertical filter, and full FT of the horizontal filter
out_img_shape = (image_padded.shape[0], image_padded.shape[1] // 2 + 1)
out_img_dtype = np.result_type(image_padded, 1j)
out_img = np.empty(out_img_shape, out_img_dtype)
out_fh_shape = out_img_shape[0]
out_fh_dtype = np.result_type(fh, 1j)
fh_c = fh.astype(out_fh_dtype) # need to make this a C2C trafo
out_fh = np.empty(out_fh_shape, out_fh_dtype)
out_fv_shape = out_img_shape[1]
out_fv_dtype = np.result_type(fv, 1j)
out_fv = np.empty(out_fv_shape, out_fv_dtype)
# Perform the forward transforms of image and filters. We use
# the `FFTW_ESTIMATE` flag to not allow the planner to destroy
# the input.
plan = pyfftw.FFTW(image_padded,
out_img,
axes=(0, 1),
direction='FFTW_FORWARD',
flags=['FFTW_ESTIMATE'],
threads=multiprocessing.cpu_count())
plan(image_padded, out_img)
plan = pyfftw.FFTW(fh_c,
out_fh,
axes=(0, ),
direction='FFTW_FORWARD',
flags=['FFTW_ESTIMATE'],
threads=multiprocessing.cpu_count())
plan(fh_c, out_fh)
plan = pyfftw.FFTW(fv,
out_fv,
axes=(0, ),
direction='FFTW_FORWARD',
flags=['FFTW_ESTIMATE'],
threads=multiprocessing.cpu_count())
plan(fv, out_fv)
# Fourier space multiplication
out_img *= out_fh[:, None]
out_img *= out_fv[None, :]
# Inverse trafo
conv = image_padded # Overwrite
plan = pyfftw.FFTW(out_img.copy(),
conv,
axes=(0, 1),
direction='FFTW_BACKWARD',
flags=['FFTW_ESTIMATE'],
threads=multiprocessing.cpu_count())
plan(out_img, conv)
else:
raise ValueError('unsupported `impl` {!r}'.format(impl_in))
if padding:
return conv[padding:-padding, padding:-padding]
else:
return conv
def is_scalar_dtype(dtype):
"""Return ``True`` if ``dtype`` is a scalar type."""
return np.issubsctype(dtype, np.number)
def is_real_floating_dtype(dtype):
"""Return ``True`` if ``dtype`` is a real floating point type."""
return np.issubsctype(dtype, np.floating)
def is_scalar_dtype(dtype):
"""`True` if ``dtype`` is scalar, else `False`."""
return np.issubsctype(dtype, np.number)
def is_real_floating_dtype(dtype):
"""`True` if ``dtype`` is real floating-point, else `False`."""
return np.issubsctype(dtype, np.floating)
import numpy as np
# Techincally this works, but probably shouldn't. See
#
# https://github.com/numpy/numpy/issues/16366
#
np.maximum_sctype(1) # E: incompatible type "int"
np.issubsctype(1, np.int64) # E: incompatible type "int"
np.issubdtype(1, np.int64) # E: incompatible type "int"
np.find_common_type(
np.int64, np.int64) # E: incompatible type "Type[signedinteger[Any]]"
def __getitem__(self, key):
"""Returns values based on `key`.
All the functionality of ``ndarray.__getitem__()`` is supported
(including fancy indexing), plus a special support for expressions:
Parameters
----------
key : string
The corresponding ctable column name will be returned. If
not a column name, it will be interpret as a boolean
expression (computed via `ctable.eval`) and the rows where
these values are true will be returned as a NumPy
structured array.
See Also
--------
ctable.eval
"""
# First, check for integer
if isinstance(key, _inttypes):
# Get a copy of the len-1 array
ra = self._arr1.copy()
# Fill it
ra[0] = tuple([self.cols[name][key] for name in self.names])
return ra[0]
# Slices
elif type(key) == slice:
(start, stop, step) = key.start, key.stop, key.step
if step and step <= 0:
raise NotImplementedError("step in slice can only be positive")
# Multidimensional keys
elif isinstance(key, tuple):
if len(key) != 1:
raise IndexError("multidimensional keys are not supported")
return self[key[0]]
# List of integers (case of fancy indexing), or list of column names
elif type(key) is list:
if len(key) == 0:
return np.empty(0, self.dtype)
strlist = all(isinstance(v, _strtypes) for v in key)
# Range of column names
if strlist:
cols = [self.cols[name] for name in key]
return ctable(cols, key)
# Try to convert to a integer array
try:
key = np.array(key, dtype=np.int_)
except:
raise IndexError(
"key cannot be converted to an array of indices")
return np.fromiter((self[i] for i in key),
dtype=self.dtype,
count=len(key))
# A boolean array (case of fancy indexing)
elif hasattr(key, "dtype"):
if key.dtype.type == np.bool_:
return self._where(key)
elif np.issubsctype(key, np.int_):
# An integer array
return np.array([tuple(self[i]) for i in key],
dtype=self.dtype)
else:
raise IndexError(
"arrays used as indices must be integer (or boolean)")
# Column name or expression
elif isinstance(key, _strtypes):
if key not in self.names:
# key is not a column name, try to evaluate
arr = self.eval(key, depth=4)
if arr.dtype.type != np.bool_:
raise IndexError("`key` %s does not represent a boolean "
"expression" % key)
return self._where(arr)
return self.cols[key]
# All the rest not implemented
else:
raise NotImplementedError("key not supported: %s" % repr(key))
# From now on, will only deal with [start:stop:step] slices
# Get the corrected values for start, stop, step
(start, stop, step) = slice(start, stop, step).indices(self.len)
# Build a numpy container
n = utils.get_len_of_range(start, stop, step)
ra = np.empty(shape=(n, ), dtype=self.dtype)
# Fill it
for name in self.names:
ra[name][:] = self.cols[name][start:stop:step]
return ra
def sampleK(d, poiMu, Knew, N, D, K, X, F, Z, A, E, FF, prec_x, prec_a,
prec_aa, prec_ab, prec_a_iso):
"""Propose new feature count K using M-H step."""
if prec_a_iso and Knew > 0:
prec_a_new = np.ones(Knew) * prec_a[1]
if not prec_a_iso and Knew > 0:
# sample from Gamma prior
prec_a_new = nr.gamma(prec_aa, 1. / prec_ab, Knew)
# Find singleton clusters
m_neg_d = np.sum(Z[d, :], axis=0) - Z[d, :]
singletons = [kd for kd in range(K) if m_neg_d[kd] == 0]
current_kappa = len(singletons) # current number of singletons
assert (isinstance(current_kappa, int))
# Calculate error E_dn assuming all singletons for row d switched off
Ad = np.copy(A[d, :])
Ad[singletons] = 0
Ed = X[[d], :] - np.dot(Ad, F)
current_a = A[d, singletons].reshape(current_kappa, 1)
if current_kappa > 0:
M = prec_x[d] * np.dot(current_a, current_a.T) + np.eye(current_kappa)
M_inv = np.linalg.inv(M)
m_n = prec_x[d] * np.dot(np.dot(M_inv, current_a), Ed)
log_prop_de = -0.5 * N * np.log(np.linalg.det(M_inv))
log_prop_de += 0.5 * np.trace(np.dot(m_n.T, np.dot(M, m_n)))
log_prop_de += (Knew) * np.log(poiMu)
if Knew > 0:
a = (nr.standard_normal(Knew) * np.sqrt(1. / prec_a_new)).reshape(
Knew, 1)
M_star = prec_x[d] * np.dot(a, a.T) + np.eye(Knew)
M_star_inv = np.linalg.inv(M_star)
m_star_n = prec_x[d] * np.dot(np.dot(M_star_inv, a), Ed)
log_prop_nu = -0.5 * N * np.log(np.linalg.det(M_star_inv))
log_prop_nu += 0.5 * np.trace(
np.dot(m_star_n.T, np.dot(M_star, m_star_n)))
log_prop_nu += (current_kappa) * np.log(poiMu)
if current_kappa > 0 and Knew > 0.5:
log_prop = log_prop_nu - log_prop_de
prop = np.exp(log_prop)
if current_kappa > 0 and Knew == 0:
prop = 0
if current_kappa == 0 and Knew >= 0:
prop = 1
if nr.uniform() < prop:
if (current_kappa > 0):
# Update error term
# We are deleting singleton features, so add beck regarding these.
E[d, :] = E[d, :] + np.dot(A[d, singletons], F[singletons, :])
# Delete redundant features
Z[d, singletons] = 0
m = np.sum(Z, axis=0)
idx = [kd for kd in range(K) if m[kd] == 0] # just refresh
Z = np.delete(Z, idx, axis=1)
A = np.delete(A, idx, axis=1)
F = np.delete(F, idx, axis=0)
FF = np.delete(FF, idx)
prec_a = np.delete(prec_a, idx)
K = len(prec_a)
assert (Z.shape[1] == A.shape[1] == F.shape[0] == K)
if Knew > 0:
Z = np.hstack((Z, np.zeros((D, Knew)))).astype(np.int)
Z[d, range(-Knew, 0)] = 1
M_star_inv_L = np.linalg.cholesky(M_star_inv)
normal_sample = nr.standard_normal((Knew, N))
Fnew = m_star_n + np.dot(M_star_inv_L, normal_sample)
F = np.vstack((F, Fnew))
FFnew = np.dot(Fnew, Fnew.T).diagonal()
FF = np.append(FF, FFnew)
A = np.hstack((A, np.zeros((D, Knew))))
A[d, range(-Knew, 0)] = a.T
prec_a = np.append(prec_a, prec_a_new)
Kold = K
K += Knew
assert (Z.shape[1] == A.shape[1] == F.shape[0] == len(prec_a) == K)
# Update error term based on new values
# We are adding new features, so subtract regarding these.
E[d, :] = E[d, :] - np.dot(a.T, Fnew)
for k in range(Kold, K):
ek = E[d, :] + A[d, k] * F[k, :] # assume A_dk = 0
precision = prec_x[d] * FF[k] + prec_a[k]
mu = (prec_x[d] * np.dot(F[k, :], ek.T)) / precision
sd = np.sqrt(1. / precision)
A[d, k] = nr.normal(mu, sd)
E = X - np.dot(A, F)
assert (np.issubsctype(Z, np.int))
return (Z, A, F, FF, prec_a, E, K)
def is_complex_floating_dtype(dtype):
"""Return ``True`` if ``dtype`` is a complex floating point type."""
return np.issubsctype(dtype, np.complexfloating)
def __setattr__(self, key, value):
# attribute access is switched off until this attribute is created by
# _enable_group_attributes
if not hasattr(self, '_group_attribute_access_active') or key in self.__dict__:
object.__setattr__(self, key, value)
elif key in self._linked_variables:
if not isinstance(value, LinkedVariable):
raise ValueError(('Cannot set a linked variable directly, link '
'it to another variable using "linked_var".'))
linked_var = value.variable
if isinstance(linked_var, DynamicArrayVariable):
raise NotImplementedError(('Linking to variable %s is not '
'supported, can only link to '
'state variables of fixed '
'size.') % linked_var.name)
eq = self.equations[key]
if eq.unit != linked_var.unit:
raise DimensionMismatchError(('Unit of variable %s does not '
'match its link target %s') % (key,
linked_var.name))
if not isinstance(linked_var, Subexpression):
var_length = len(linked_var)
else:
var_length = len(linked_var.owner)
if value.index is not None:
try:
index_array = np.asarray(value.index)
if not np.issubsctype(index_array.dtype, np.int):
raise TypeError()
except TypeError:
raise TypeError(('The index for a linked variable has '
'to be an integer array'))
size = len(index_array)
source_index = value.group.variables.indices[value.name]
if source_index not in ('_idx', '0'):
# we are indexing into an already indexed variable,
# calculate the indexing into the target variable
index_array = value.group.variables[source_index].get_value()[index_array]
if not index_array.ndim == 1 or size != len(self):
raise TypeError(('Index array for linked variable %s '
'has to be a one-dimensional array of '
'length %d, but has shape '
'%s') % (key,
len(self),
str(index_array.shape)))
if min(index_array) < 0 or max(index_array) >= var_length:
raise ValueError('Index array for linked variable %s '
'contains values outside of the valid '
'range [0, %d[' % (key,
var_length))
self.variables.add_array('_%s_indices' % key, unit=Unit(1),
size=size, dtype=index_array.dtype,
constant=True, read_only=True,
values=index_array)
index = '_%s_indices' % key
else:
if linked_var.scalar or (var_length == 1 and self._N != 1):
index = '0'
else:
index = value.group.variables.indices[value.name]
if index == '_idx':
target_length = var_length
else:
target_length = len(value.group.variables[index])
# we need a name for the index that does not clash with
# other names and a reference to the index
new_index = '_' + value.name + '_index_' + index
self.variables.add_reference(new_index,
value.group,
index)
index = new_index
if len(self) != target_length:
raise ValueError(('Cannot link variable %s to %s, the size of '
'the target group does not match '
'(%d != %d). You can provide an indexing '
'scheme with the "index" keyword to link '
'groups with different sizes') % (key,
linked_var.name,
len(self),
target_length))
self.variables.add_reference(key,
value.group,
value.name,
index=index)
log_msg = ('Setting {target}.{targetvar} as a link to '
'{source}.{sourcevar}').format(target=self.name,
targetvar=key,
source=value.variable.owner.name,
sourcevar=value.variable.name)
if index is not None:
log_msg += '(using "{index}" as index variable)'.format(index=index)
logger.debug(log_msg)
else:
if isinstance(value, LinkedVariable):
raise TypeError(('Cannot link variable %s, it has to be marked '
'as a linked variable with "(linked)" in the '
'model equations.') % key)
else:
Group.__setattr__(self, key, value, level=1)
def plot_cub_as_curve(c,
colors=None,
plot_kwargs=None,
legend_prefix='',
show_axis_labels=True,
show_legend=False,
axes=None,
axis_label_fontsize=12):
"""Plot a cuboid (ndims <= 2) as curve(s).
- If the input is 1D: one single curve.
- If the input is 2D:
* multiple curves are plotted: one for each domain value on the 1st axis.
* legends are shown to display which domain value is associated
to which curve.
Parameters
----------
colors : dict <domain value>: <matplotlib color>
associate domain values of the 1st axis to color curves
plot_kwargs : dict <arg name>:<arg value>
dictionary of named argument passed to the plot function
legend_prefix : str
prefix to prepend to legend labels.
Returns
-------
None
"""
import matplotlib.pyplot as plt
def protect_latex_str(s):
return s.replace('_', '\_')
axes = axes or plt.gca()
colors = colors or {}
plot_kwargs = plot_kwargs or {}
if c.get_ndims() == 1:
dom = c.axes_domains[c.axes_names[0]]
if np.issubsctype(dom.dtype, str):
dom = np.arange(len(dom))
axes.plot(dom, c.data, **plot_kwargs)
if np.issubsctype(c.axes_domains[c.axes_names[0]], str):
set_int_tick_labels(axes.xaxis,
c.axes_domains[c.axes_names[0]],
rotation=30)
elif c.get_ndims() == 2:
for val, sub_c in c.split(c.axes_names[0]).iteritems():
pkwargs = plot_kwargs.copy()
col = colors.get(val, None)
if col is not None:
pkwargs['color'] = col
pkwargs['label'] = protect_latex_str(legend_prefix + \
c.axes_names[0] + \
'=' + str(val))
plot_cub_as_curve(sub_c,
plot_kwargs=pkwargs,
axes=axes,
show_axis_labels=False)
if show_legend:
axes.legend()
else:
raise Exception('xndarray has too many dims (%d), expected at most 2' \
%c.get_ndims())
if show_axis_labels:
if c.get_ndims() == 1:
axes.set_xlabel(protect_latex_str(c.axes_names[0]),
fontsize=axis_label_fontsize)
else:
axes.set_xlabel(protect_latex_str(c.axes_names[1]),
fontsize=axis_label_fontsize)
axes.set_ylabel(protect_latex_str(c.value_label),
fontsize=axis_label_fontsize)
def is_int_dtype(dtype):
"""`True` if ``dtype`` is integer, else `False`."""
return np.issubsctype(dtype, np.integer)
import numpy as np
np.issubsctype('S8', str)
np.issubsctype(np.array([1]), np.int)
np.issubsctype(np.array([1]), np.float)
def is_complex_floating_dtype(dtype):
"""`True` if ``dtype`` is complex floating-point, else `False`."""
return np.issubsctype(dtype, np.complexfloating)
def is_int_dtype(dtype):
"""`True` if ``dtype`` is integer, else `False`."""
return np.issubsctype(dtype, np.integer)
def is_int_dtype(dtype):
"""Return ``True`` if ``dtype`` is an integer type."""
return np.issubsctype(dtype, np.integer)
def is_complex_floating_dtype(dtype):
"""`True` if ``dtype`` is complex floating-point, else `False`."""
return np.issubsctype(dtype, np.complexfloating)
def test_ufunc(fn_impl, ufunc):
space = odl.uniform_discr([0, 0], [1, 1], (2, 2), impl=fn_impl)
name, n_args, n_out, _ = ufunc
if (np.issubsctype(space.dtype, np.floating) and
name in ['bitwise_and',
'bitwise_or',
'bitwise_xor',
'invert',
'left_shift',
'right_shift']):
# Skip integer only methods if floating point type
return
# Get the ufunc from numpy as reference
ufunc = getattr(np, name)
# Create some data
arrays, vectors = noise_elements(space, n_args + n_out)
in_arrays = arrays[:n_args]
out_arrays = arrays[n_args:]
data_vector = vectors[0]
in_vectors = vectors[1:n_args]
out_vectors = vectors[n_args:]
# Verify type
assert isinstance(data_vector.ufuncs,
odl.util.ufuncs.DiscreteLpUfuncs)
# Out-of-place:
np_result = ufunc(*in_arrays)
vec_fun = getattr(data_vector.ufuncs, name)
odl_result = vec_fun(*in_vectors)
assert all_almost_equal(np_result, odl_result)
# Test type of output
if n_out == 1:
assert isinstance(odl_result, space.element_type)
elif n_out > 1:
for i in range(n_out):
assert isinstance(odl_result[i], space.element_type)
# In-place:
np_result = ufunc(*(in_arrays + out_arrays))
vec_fun = getattr(data_vector.ufuncs, name)
odl_result = vec_fun(*(in_vectors + out_vectors))
assert all_almost_equal(np_result, odl_result)
# Test in-place actually holds:
if n_out == 1:
assert odl_result is out_vectors[0]
elif n_out > 1:
for i in range(n_out):
assert odl_result[i] is out_vectors[i]
# Test out-of-place with np data
np_result = ufunc(*in_arrays)
vec_fun = getattr(data_vector.ufuncs, name)
odl_result = vec_fun(*in_arrays[1:])
assert all_almost_equal(np_result, odl_result)
# Test type of output
if n_out == 1:
assert isinstance(odl_result, space.element_type)
elif n_out > 1:
for i in range(n_out):
assert isinstance(odl_result[i], space.element_type)
def is_int_dtype(dtype):
"""Return ``True`` if ``dtype`` is an integer type."""
return np.issubsctype(dtype, np.integer)
def __init__(self,
min_value: Optional[numbertypes] = None,
max_value: Optional[numbertypes] = None,
shape: Optional[TSequence[shape_type]] = None,
valid_types: Optional[TSequence[type]] = None) -> None:
if valid_types is not None:
for mytype in valid_types:
is_supported = any(
np.issubsctype(mytype, supported_type)
for supported_type in self.__supported_types)
if not is_supported:
raise TypeError(f"Arrays validator only supports numeric "
f"types: {mytype} is not supported.")
self.valid_types = valid_types
else:
self.valid_types = self.__real_types
supports_complex = any(
np.issubsctype(my_type, np.complexfloating)
for my_type in self.valid_types)
limits_given = min_value is not None or max_value is not None
min_real = any(
np.issubsctype(type(min_value), real_type)
for real_type in self.__real_types)
max_real = any(
np.issubsctype(type(max_value), real_type)
for real_type in self.__real_types)
if min_value is not None and not min_real:
raise TypeError(f"min_value must be a real number. It is "
f"{min_value} of type {type(min_value)}")
if max_value is not None and not max_real:
raise TypeError(f"max_value must be a real number. It is "
f"{max_value} of type {type(max_value)}")
if supports_complex and limits_given:
raise TypeError(
"Setting min_value or max_value is not supported for "
"complex validators.")
min_value_is_valid_type = any(
np.issubsctype(type(min_value), valid_type)
for valid_type in self.valid_types)
max_value_is_valid_type = any(
np.issubsctype(type(max_value), valid_type)
for valid_type in self.valid_types)
if min_value_is_valid_type or min_value is None:
self._min_value = min_value
else:
raise TypeError(f'min_value must be an instance of valid_types. '
f'It is {min_value} of '
f'type {type(min_value)}')
if max_value_is_valid_type or max_value is None:
self._max_value = max_value
else:
raise TypeError(f'max_value must be an instance of valid_types. '
f'It is {max_value} of '
f'type {type(max_value)}')
if min_value is not None and max_value is not None:
valuesok = max_value > min_value
if not valuesok:
raise TypeError(f'max_value must be bigger than min_value')
if not isinstance(shape,
collections.abc.Sequence) and shape is not None:
raise ValueError(f"Shape must be a sequence (List, Tuple ...) "
f"got a {type(shape)}")
self._shape: shape_tuple_type = None
if shape is not None:
self._shape = tuple(shape)
def is_complex_floating_dtype(dtype):
"""Return ``True`` if ``dtype`` is a complex floating point type."""
return np.issubsctype(dtype, np.complexfloating)
def test_ufuncs(odl_tspace_impl, odl_ufunc):
"""Test ufuncs in ``x.ufuncs`` against direct Numpy ufuncs."""
impl = odl_tspace_impl
space = odl.uniform_discr([0, 0], [1, 1], (2, 3), impl=impl)
name = odl_ufunc
# Get the ufunc from numpy as reference
npy_ufunc = getattr(np, name)
nin = npy_ufunc.nin
nout = npy_ufunc.nout
if (np.issubsctype(space.dtype, np.floating) and name in [
'bitwise_and', 'bitwise_or', 'bitwise_xor', 'invert', 'left_shift',
'right_shift'
]):
# Skip integer only methods if floating point type
return
# Create some data
arrays, elements = noise_elements(space, nin + nout)
in_arrays = arrays[:nin]
out_arrays = arrays[nin:]
data_elem = elements[0]
out_elems = elements[nin:]
if nout == 1:
out_arr_kwargs = {'out': out_arrays[0]}
out_elem_kwargs = {'out': out_elems[0]}
elif nout > 1:
out_arr_kwargs = {'out': out_arrays[:nout]}
out_elem_kwargs = {'out': out_elems[:nout]}
# Get function to call, using both interfaces:
# - vec.ufunc(other_args)
# - np.ufunc(vec, other_args)
elem_fun_old = getattr(data_elem.ufuncs, name)
in_elems_old = elements[1:nin]
elem_fun_new = npy_ufunc
in_elems_new = elements[:nin]
# Out-of-place
with np.errstate(all='ignore'): # avoid pytest warnings
npy_result = npy_ufunc(*in_arrays)
odl_result_old = elem_fun_old(*in_elems_old)
assert all_almost_equal(npy_result, odl_result_old)
odl_result_new = elem_fun_new(*in_elems_new)
assert all_almost_equal(npy_result, odl_result_new)
# Test type of output
if nout == 1:
assert isinstance(odl_result_old, space.element_type)
assert isinstance(odl_result_new, space.element_type)
elif nout > 1:
for i in range(nout):
assert isinstance(odl_result_old[i], space.element_type)
assert isinstance(odl_result_new[i], space.element_type)
# In-place with ODL objects as `out`
with np.errstate(all='ignore'): # avoid pytest warnings
npy_result = npy_ufunc(*in_arrays, **out_arr_kwargs)
odl_result_old = elem_fun_old(*in_elems_old, **out_elem_kwargs)
assert all_almost_equal(npy_result, odl_result_old)
odl_result_new = elem_fun_new(*in_elems_new, **out_elem_kwargs)
assert all_almost_equal(npy_result, odl_result_new)
# Check that returned stuff refers to given out
if nout == 1:
assert odl_result_old is out_elems[0]
assert odl_result_new is out_elems[0]
elif nout > 1:
for i in range(nout):
assert odl_result_old[i] is out_elems[i]
assert odl_result_new[i] is out_elems[i]
# In-place with Numpy array as `out` for new interface
out_arrays_new = tuple(np.empty_like(arr) for arr in out_arrays)
if nout == 1:
out_arr_kwargs_new = {'out': out_arrays_new[0]}
elif nout > 1:
out_arr_kwargs_new = {'out': out_arrays_new[:nout]}
with np.errstate(all='ignore'): # avoid pytest warnings
odl_result_arr_new = elem_fun_new(*in_elems_new, **out_arr_kwargs_new)
assert all_almost_equal(npy_result, odl_result_arr_new)
if nout == 1:
assert odl_result_arr_new is out_arrays_new[0]
elif nout > 1:
for i in range(nout):
assert odl_result_arr_new[i] is out_arrays_new[i]
# In-place with data container (tensor) as `out` for new interface
out_tensors_new = tuple(
space.tspace.element(np.empty_like(arr)) for arr in out_arrays)
if nout == 1:
out_tens_kwargs_new = {'out': out_tensors_new[0]}
elif nout > 1:
out_tens_kwargs_new = {'out': out_tensors_new[:nout]}
with np.errstate(all='ignore'): # avoid pytest warnings
odl_result_tens_new = elem_fun_new(*in_elems_new,
**out_tens_kwargs_new)
assert all_almost_equal(npy_result, odl_result_tens_new)
if nout == 1:
assert odl_result_tens_new is out_tensors_new[0]
elif nout > 1:
for i in range(nout):
assert odl_result_tens_new[i] is out_tensors_new[i]
# Check `ufunc.at`
indices = ([0, 0, 1], [0, 1, 2])
mod_array = in_arrays[0].copy()
mod_elem = in_elems_new[0].copy()
if nout > 1:
return # currently not supported by Numpy
if nin == 1:
with np.errstate(all='ignore'): # avoid pytest warnings
npy_result = npy_ufunc.at(mod_array, indices)
odl_result = npy_ufunc.at(mod_elem, indices)
elif nin == 2:
other_array = in_arrays[1][indices]
other_elem = in_elems_new[1][indices]
with np.errstate(all='ignore'): # avoid pytest warnings
npy_result = npy_ufunc.at(mod_array, indices, other_array)
odl_result = npy_ufunc.at(mod_elem, indices, other_elem)
assert all_almost_equal(odl_result, npy_result)
# Check `ufunc.reduce`
if nin == 2 and nout == 1:
in_array = in_arrays[0]
in_elem = in_elems_new[0]
# We only test along one axis since some binary ufuncs are not
# re-orderable, in which case Numpy raises a ValueError
with np.errstate(all='ignore'): # avoid pytest warnings
npy_result = npy_ufunc.reduce(in_array)
odl_result = npy_ufunc.reduce(in_elem)
assert all_almost_equal(odl_result, npy_result)
# In-place using `out` (with ODL vector and array)
out_elem = odl_result.space.element()
out_array = np.empty(odl_result.shape, dtype=odl_result.dtype)
npy_ufunc.reduce(in_elem, out=out_elem)
npy_ufunc.reduce(in_elem, out=out_array)
assert all_almost_equal(out_elem, odl_result)
assert all_almost_equal(out_array, odl_result)
# Using a specific dtype
try:
npy_result = npy_ufunc.reduce(in_array, dtype=complex)
except TypeError:
# Numpy finds no matching loop, bail out
return
else:
odl_result = npy_ufunc.reduce(in_elem, dtype=complex)
assert odl_result.dtype == npy_result.dtype
assert all_almost_equal(odl_result, npy_result)
def normQuant(obj, sigfigs=None, full_norm=True):
"""Normalize quantities such that two things that *should* be equal are
returned as identical objects.
Handles floating point numbers, pint quantities, uncertainties, and
combinations thereof as standalone objects or in sequences, dicts, or numpy
ndarrays. Numerical precision issues and equal quantities represented in
differently-scaled or different systems of units come out identically.
Outputs from this function (**not** the inputs) deemed to be equal by the
above logic will compare to be equal (via the `==` operator and via
`pisa.utils.comparisons.recursiveEquality`) and will also hash to equal
values (via `pisa.utils.hash.hash_obj`).
Parameters
----------
obj
Object to be normalized.
sigfigs : None or int > 0
Number of digits to which to round numbers' significands; if None, do
not round numbers.
full_norm : bool
If True, does full translation and normalization which is good across
independent invocations and is careful about normalizing units, etc.
If false, certain assumptions are made that modify the behavior
described below in the Notes section which help speed things up in the
case of e.g. a minimizer run, where we know certain things won't
change:
* Units are not normalized. They are assumed to stay the same from
run to run.
* sigfigs are not respected; full significant figures are returned
(since it takes time to round all values appropriately).
Returns
-------
normed_obj : object roughly of same type as input `obj`
Simple types are returned as the same type as at the input, Numpy
ndarrays are returned in the same shape and representation as the
input, Mappings (dicts) are returned as OrderedDict, and all other
sequences or iterables are returned as (possibly nested) lists.
Notes
-----
Conversion logic by `obj` type or types found within `obj`:
* **Sequences and OrderedDicts** (but not numpy arrays) are iterated
through recursively.
* **Mappings without ordering** (e.g. dicts) are iterated through
recursively after sorting their keys, and are returned as
OrderedDicts (such that the output is always consistent when
serialized).
* **Sequences** (not numpy arrays) are iterated through recursively.
* **Numpy ndarrays** are treated as the below data types (according to the
array's dtype).
* **Simple objects** (non-floating point / non-sequence / non-numpy / etc.)
are returned unaltered (e.g. strings).
* **Pint quantities** (numbers with units): Convert to their base units.
* **Floating-point numbers** (including the converted pint quantities):
Round values to `sigfigs` significant figures.
* **Numbers with uncertainties** (via the `uncertainties` module) have
their nominal values rounded as above but their standard deviations are
rounded to the same order of magnitude (*not* number of significant
figures) as the nominal.
Therefore passing obj=10.23+/-0.25 and sigfigs=2 returns 10+/-0.0.
Note that **correlations are lost** in the outputs of this function, so
equality of the output requires merely having equal nomial values and
equal standard deviations.
The calculations leading to these equal numbers might have used
independent random variables to arrive at them, however, and so the
`uncertainties` module would have evaluated them to be unequal. [1]
To achieve rounding that masks floating point precision issues, set
`sigfigs` to a value *less than* the number of decimal digits used for the
significand of the calculation floating point precision.
For reference, the IEEE 754 floating point standard [2] uses the following:
* FP16 (half precision): **3.31** significand decimal digits (11 bits)
* FP32 (single precision): **7.22** significand decimal digits (24 bits)
* FP64 (double precision): **15.95** significand decimal digits (53 bits)
* FP128 (quad precision): **34.02** significand decimal digits (113 bits)
Logic for rounding the significand for numpy arrays was derived from [3].
References
----------
[1] https://github.com/lebigot/uncertainties/blob/master/uncertainties/test_uncertainties.py#L436
[2] https://en.wikipedia.org/wiki/IEEE_floating_point
[3] http://stackoverflow.com/questions/18915378, answer by user BlackGriffin.
Examples
--------
Pint quantities hash to unequal values if specified in different scales or
different systems of units (even if the underlying physical quantity is
identical).
>>> from pisa import ureg
>>> from pisa.utils.hash import hash_obj
>>> q0 = 1 * ureg.m
>>> q1 = 100 * ureg.cm
>>> q0 == q1
True
>>> hash_obj(q0) == hash_obj(q1)
False
Even the `to_base_units()` method fails for hashing to equal values, as
`q0` is a float and `q1` is an integer.
>>> hash_obj(q0.to_base_units()) == hash_obj(q1.to_base_units())
False
Even if both quantities are floating point numbers, finite precision
effects in the `to_base_units` conversion can still cause two things which
we "know" are equal to evaluate to be unequal.
>>> q2 = 0.1 * ureg.m
>>> q3 = 1e5 * ureg.um
>>> q2 == q3
True
>>> q2.to_base_units() == q3.to_base_units()
False
`normQuant` handles all of these issues given an appropriate `sigfigs`
argument.
>>> q2_normed = normQuant(q2, sigfigs=12)
>>> q3_normed = normQuant(q3, sigfigs=12)
>>> q2_normed == q3_normed
True
>>> hash_obj(q2_normed) == hash_obj(q3_normed)
True
"""
#logging.trace('-'*80)
#logging.trace('obj: %s', obj)
#logging.trace('type(obj): %s', type(obj))
if not full_norm:
return obj
# Nothing to convert for strings, None, ...
if isinstance(obj, str) or obj is None:
return obj
round_result = False
if sigfigs is not None:
if not (int(sigfigs) == float(sigfigs) and sigfigs > 0):
raise ValueError('`sigfigs` must be an integer > 0.')
round_result = True
sigfigs = int(sigfigs)
# Store kwargs for easily passing to recursive calls of this function
kwargs = dict(sigfigs=sigfigs, full_norm=full_norm)
if hasattr(obj, 'normalized_state'):
return obj.normalized_state
# Recurse into dict by its (sorted) keys (or into OrderedDict using keys in
# their defined order) and return an OrderedDict in either case.
if isinstance(obj, Mapping):
#logging.trace('Mapping')
if isinstance(obj, OrderedDict):
keys = obj.keys()
else:
keys = sorted(obj.keys())
normed_obj = OrderedDict()
for key in keys:
normed_obj[key] = normQuant(obj[key], **kwargs)
return normed_obj
# Sequences, etc. but NOT numpy arrays (or pint quantities, which are
# iterable) get their elements normalized and populated to a new list for
# returning.
# NOTE/TODO: allowing access across unit regestries for pragmatic (if
# incorrect) reasons... may want to revisit this decision.
# pylint: disable=protected-access
misbehaving_sequences = (np.ndarray, pint.quantity._Quantity)
if (isinstance(obj, (Iterable, Iterator, Sequence))
and not isinstance(obj, misbehaving_sequences)):
#logging.trace('Iterable, Iterator, or Sequence but not ndarray or'
# ' _Qauantity')
return [normQuant(x, **kwargs) for x in obj]
# Must be a numpy array or scalar if we got here...
# NOTE: the order in which units (Pint module) and uncertainties
# (uncertainties module) are handled is crucial! Essentially, it appears
# that Pint is aware of uncertainties, but not vice versa. Hence the
# ordering and descriptions used below.
# The outermost "wrapper" of a number or numpy array is its Pint units. If
# units are present, convert to base units, record the base units, and
# strip the units off of the quantity by replacing it with its magnitude
# (in the base units).
has_units = False
if isinstance(obj, pint.quantity._Quantity):
#logging.trace('is a Quantity, converting to base units')
has_units = True
if full_norm:
obj = obj.to_base_units()
units = obj.units
obj = obj.magnitude
# The next layer possible for a number or numpy array to have is
# uncertainties. If uncertainties are attached to `obj`, record a
# "snapshot" (losing correlations) of the standard deviations. Then replace
# the number or array solely with its nominal value(s).
# NOTE: uncertainties.core.AffineScalarFunc includes such functions *and*
# uncertainties.core.Variable objects
has_uncertainties = False
if isinstance(obj, AffineScalarFunc):
#logging.trace('type is AffineScalarFunc')
has_uncertainties = True
std_devs = obj.std_dev
obj = obj.nominal_value
elif isinstance(obj, np.ndarray) and np.issubsctype(obj, AffineScalarFunc):
#logging.trace('ndarray with subsctype is AffineScalarFunc')
has_uncertainties = True
std_devs = unp.std_devs(obj)
obj = unp.nominal_values(obj)
# What is done below will convert scalars into arrays, so get this info
# before it is lost.
is_scalar = isscalar(obj)
if round_result:
#logging.trace('rounding result')
# frexp returns *binary* fraction (significand) and *binary* exponent
bin_significand, bin_exponent = np.frexp(obj)
exponent = LOG10_2 * bin_exponent
exponent_integ = np.floor(exponent)
exponent_fract = exponent - exponent_integ
significand = bin_significand * 10**(exponent_fract)
obj = np.around(significand, sigfigs - 1) * 10**exponent_integ
# Now work our way *up* through the hierarchy: First, reintroduce
# uncertainties
if has_uncertainties and round_result:
#logging.trace('uncertainties and rounding')
std_bin_significand, std_bin_exponent = np.frexp(std_devs)
std_exponent = LOG10_2 * std_bin_exponent
std_exponent_integ = np.floor(std_exponent)
std_exponent_fract = std_exponent - std_exponent_integ
# Don't just scale magnitude by the stddev's fractional exponent; also
# shift to be on the same scale (power-of-10) as the nominal value
delta_order_of_mag = std_exponent_integ - exponent_integ
std_significand = (std_bin_significand *
10**(std_exponent_fract + delta_order_of_mag))
# Now rounding on the stddev's significand occurs at the same order of
# magnitude as rounding on the nominal value (and so scaling is done
# with `exponent_integ`, NOT `std_exponent_integ`)
std_devs = (np.around(std_significand, sigfigs - 1) *
10**exponent_integ)
if has_uncertainties:
#logging.trace('recreate uncertainties array')
obj = unp.uarray(obj, std_devs)
# If it was a scalar, it has become a len-1 array; extract the scalar
if is_scalar:
#logging.trace('converting to scalar')
obj = obj[0]
# Finally, attach units if they were present
if has_units:
#logging.trace('reattaching units')
obj = obj * units
return obj
def is_numeric_dtype(dtype):
"""Return ``True`` if ``dtype`` is a numeric type."""
dtype = np.dtype(dtype)
return np.issubsctype(getattr(dtype, 'base', None), np.number)
def __setattr__(self, key, value):
# attribute access is switched off until this attribute is created by
# _enable_group_attributes
if not hasattr(
self,
'_group_attribute_access_active') or key in self.__dict__:
object.__setattr__(self, key, value)
elif key in self._linked_variables:
if not isinstance(value, LinkedVariable):
raise ValueError(
('Cannot set a linked variable directly, link '
'it to another variable using "linked_var".'))
linked_var = value.variable
if isinstance(linked_var, DynamicArrayVariable):
raise NotImplementedError(('Linking to variable %s is not '
'supported, can only link to '
'state variables of fixed '
'size.') % linked_var.name)
eq = self.equations[key]
if eq.dim is not linked_var.dim:
raise DimensionMismatchError(
('Unit of variable %s does not '
'match its link target %s') % (key, linked_var.name))
if not isinstance(linked_var, Subexpression):
var_length = len(linked_var)
else:
var_length = len(linked_var.owner)
if value.index is not None:
try:
index_array = np.asarray(value.index)
if not np.issubsctype(index_array.dtype, np.int):
raise TypeError()
except TypeError:
raise TypeError(('The index for a linked variable has '
'to be an integer array'))
size = len(index_array)
source_index = value.group.variables.indices[value.name]
if source_index not in ('_idx', '0'):
# we are indexing into an already indexed variable,
# calculate the indexing into the target variable
index_array = value.group.variables[
source_index].get_value()[index_array]
if not index_array.ndim == 1 or size != len(self):
raise TypeError(
('Index array for linked variable %s '
'has to be a one-dimensional array of '
'length %d, but has shape '
'%s') % (key, len(self), str(index_array.shape)))
if min(index_array) < 0 or max(index_array) >= var_length:
raise ValueError('Index array for linked variable %s '
'contains values outside of the valid '
'range [0, %d[' % (key, var_length))
self.variables.add_array('_%s_indices' % key,
size=size,
dtype=index_array.dtype,
constant=True,
read_only=True,
values=index_array)
index = '_%s_indices' % key
else:
if linked_var.scalar or (var_length == 1 and self._N != 1):
index = '0'
else:
index = value.group.variables.indices[value.name]
if index == '_idx':
target_length = var_length
else:
target_length = len(value.group.variables[index])
# we need a name for the index that does not clash with
# other names and a reference to the index
new_index = '_' + value.name + '_index_' + index
self.variables.add_reference(new_index, value.group,
index)
index = new_index
if len(self) != target_length:
raise ValueError(
('Cannot link variable %s to %s, the size of '
'the target group does not match '
'(%d != %d). You can provide an indexing '
'scheme with the "index" keyword to link '
'groups with different sizes') %
(key, linked_var.name, len(self), target_length))
self.variables.add_reference(key,
value.group,
value.name,
index=index)
log_msg = ('Setting {target}.{targetvar} as a link to '
'{source}.{sourcevar}').format(
target=self.name,
targetvar=key,
source=value.variable.owner.name,
sourcevar=value.variable.name)
if index is not None:
log_msg += '(using "{index}" as index variable)'.format(
index=index)
logger.diagnostic(log_msg)
else:
if isinstance(value, LinkedVariable):
raise TypeError(
('Cannot link variable %s, it has to be marked '
'as a linked variable with "(linked)" in the '
'model equations.') % key)
else:
Group.__setattr__(self, key, value, level=1)
def is_int_dtype(dtype):
"""Return ``True`` if ``dtype`` is an integer type."""
dtype = np.dtype(dtype)
return np.issubsctype(getattr(dtype, 'base', None), np.integer)
def __getitem__(self, key):
"""
x.__getitem__(key) <==> x[key]
Returns values based on `key`. All the functionality of
``ndarray.__getitem__()`` is supported (including fancy
indexing), plus a special support for expressions:
Parameters
----------
key : string, int, tuple, list
The corresponding btable column name will be returned. If not a
column name, it will be interpret as a boolean expression
(computed via `btable.eval`) and the rows where these values are
true will be returned as a NumPy structured array. If `key` is an
integer, slice or list then the typical NumPy indexing operation
will be performed over the table.
See Also
--------
btable.eval
"""
# First, check for integer
if isinstance(key, _inttypes):
# Get a copy of the len-1 array
ra = self._arr1.copy()
# Fill it
for name in self.names:
ra[0][name] = self.cols[name][key]
# The next gives conversion problems with unicode
# (the responsible being probably numpy)
#ra[0] = tuple([self.cols[name][key] for name in self.names])
return ra[0]
# Slices
elif type(key) == slice:
(start, stop, step) = key.start, key.stop, key.step
if step and step <= 0 :
raise NotImplementedError("step in slice can only be positive")
# Multidimensional keys
elif isinstance(key, tuple):
if len(key) != 1:
raise IndexError("multidimensional keys are not supported")
return self[key[0]]
# List of integers (case of fancy indexing), or list of column names
elif type(key) is list:
if len(key) == 0:
return np.empty(0, self.dtype)
strlist = [type(v) for v in key] == [str for v in key]
# Range of column names
if strlist:
cols = [self.cols[name] for name in key]
return btable(cols, key)
# Try to convert to a integer array
try:
key = np.array(key, dtype=np.int_)
except:
raise IndexError(
"key cannot be converted to an array of indices")
return np.fromiter((self[i] for i in key),
dtype=self.dtype, count=len(key))
# A boolean array (case of fancy indexing)
elif hasattr(key, "dtype"):
if key.dtype.type == np.bool_:
return self._where(key)
elif np.issubsctype(key, np.int_):
# An integer array
return np.array([self[i] for i in key], dtype=self.dtype)
else:
raise IndexError(
"arrays used as indices must be integer (or boolean)")
# Column name or expression
elif type(key) in _strtypes:
if key not in self.names:
# key is not a column name, try to evaluate
arr = self.eval(key, depth=4)
if arr.dtype.type != np.bool_:
raise IndexError(
"`key` %s does not represent a boolean expression" %
key)
return self._where(arr)
return self.cols[key]
# All the rest not implemented
else:
raise NotImplementedError("key not supported: %s" % repr(key))
# From now on, will only deal with [start:stop:step] slices
# Get the corrected values for start, stop, step
(start, stop, step) = slice(start, stop, step).indices(self.len)
# Build a numpy container
n = utils.get_len_of_range(start, stop, step)
ra = np.empty(shape=(n,), dtype=self.dtype)
# Fill it
for name in self.names:
ra[name][:] = self.cols[name][start:stop:step]
return ra
def is_real_floating_dtype(dtype):
"""Return ``True`` if ``dtype`` is a real floating point type."""
dtype = np.dtype(dtype)
return np.issubsctype(getattr(dtype, 'base', None), np.floating)
def is_scalar_dtype(dtype):
"""`True` if ``dtype`` is scalar, else `False`."""
return np.issubsctype(dtype, np.number)
def is_complex_floating_dtype(dtype):
"""Return ``True`` if ``dtype`` is a complex floating point type."""
dtype = np.dtype(dtype)
return np.issubsctype(getattr(dtype, 'base', None), np.complexfloating)
def is_real_floating_dtype(dtype):
"""`True` if ``dtype`` is real floating-point, else `False`."""
return np.issubsctype(dtype, np.floating)
def _check_string(self,
root,
attribute=True,
raiseExtended=True,
useOpaqueDataType=True):
# Test following string literals
sAsciiBytes = b"abc"
sAsciiUnicode = u"abc"
sLatinBytes = b"\xe423"
sLatinUnicode = u"\xe423" # not used
sUTF8Unicode = u"\u0101bc"
sUTF8Bytes = b"\xc4\x81bc"
sUTF8AsciiUnicode = u"abc"
sUTF8AsciiBytes = b"abc"
# Expected conversion after HDF5 write/read
strmap = {}
strmap["ascii(scalar)"] = sAsciiBytes, sAsciiUnicode
strmap["ext(scalar)"] = sLatinBytes, sLatinBytes
strmap["unicode(scalar)"] = sUTF8Unicode, sUTF8Unicode
strmap["unicode2(scalar)"] = sUTF8AsciiUnicode, sUTF8AsciiUnicode
strmap["ascii(list)"] = (
[sAsciiBytes, sAsciiBytes],
[sAsciiUnicode, sAsciiUnicode],
)
strmap["ext(list)"] = [sLatinBytes,
sLatinBytes], [sLatinBytes, sLatinBytes]
strmap["unicode(list)"] = (
[sUTF8Unicode, sUTF8Unicode],
[sUTF8Unicode, sUTF8Unicode],
)
strmap["unicode2(list)"] = (
[sUTF8AsciiUnicode, sUTF8AsciiUnicode],
[sUTF8AsciiUnicode, sUTF8AsciiUnicode],
)
strmap["mixed(list)"] = (
[sUTF8Unicode, sUTF8AsciiUnicode, sAsciiBytes, sLatinBytes],
[sUTF8Bytes, sUTF8AsciiBytes, sAsciiBytes, sLatinBytes],
)
strmap["ascii(0d-array)"] = np.array(sAsciiBytes), sAsciiUnicode
strmap["ext(0d-array)"] = np.array(sLatinBytes), sLatinBytes
strmap["unicode(0d-array)"] = np.array(sUTF8Unicode), sUTF8Unicode
strmap["unicode2(0d-array)"] = np.array(
sUTF8AsciiUnicode), sUTF8AsciiUnicode
strmap["ascii(1d-array)"] = (
np.array([sAsciiBytes, sAsciiBytes]),
[sAsciiUnicode, sAsciiUnicode],
)
strmap["ext(1d-array)"] = (
np.array([sLatinBytes, sLatinBytes]),
[sLatinBytes, sLatinBytes],
)
strmap["unicode(1d-array)"] = (
np.array([sUTF8Unicode, sUTF8Unicode]),
[sUTF8Unicode, sUTF8Unicode],
)
strmap["unicode2(1d-array)"] = (
np.array([sUTF8AsciiUnicode, sUTF8AsciiUnicode]),
[sUTF8AsciiUnicode, sUTF8AsciiUnicode],
)
strmap["mixed(1d-array)"] = (
np.array([sUTF8Unicode, sUTF8AsciiUnicode, sAsciiBytes]),
[sUTF8Unicode, sUTF8AsciiUnicode, sAsciiUnicode],
)
strmap["mixed2(1d-array)"] = (
np.array([sUTF8AsciiUnicode, sAsciiBytes]),
[sUTF8AsciiUnicode, sAsciiUnicode],
)
path = root.mkdir("test")
prepare_kwargs = {
"raiseExtended": raiseExtended,
"useOpaqueDataType": useOpaqueDataType,
}
def write(name, value):
if attribute:
kwargs = {name: value}
path.update_stats(prepare_kwargs=prepare_kwargs, **kwargs)
else:
path[name].mkfile(prepare_kwargs=prepare_kwargs, data=value)
def read(name):
if attribute:
return path.get_stat(name)
else:
return path[name].read()
# as long as vlen_opaque_dtype is not supported by h5py
def remove00(s):
return bytes(bytearray([b for b in bytearray(s) if b]))
subtest_kwargs = {
"attribute": attribute,
"raiseExtended": raiseExtended,
"useOpaqueDataType": useOpaqueDataType,
}
for name, (value, expectedValue) in strmap.items():
subtest_kwargs["data"] = name
with self.subTest(**subtest_kwargs):
# Write/read
decodingError = "ext" in name or name == "mixed(list)"
expectOpaque = decodingError and useOpaqueDataType
if raiseExtended and decodingError:
with self.assertRaises(UnicodeDecodeError):
write(name, value)
continue
else:
write(name, value)
value = read(name)
# Check type and value
if "list" in name or "1d-array" in name:
self.assertTrue(isinstance(value, np.ndarray))
if expectOpaque:
self.assertTrue(np.issubsctype(value.dtype, np.void))
value = value.tolist() # also converts void to bytes
self.assertEqual(list(map(type, value)),
list(map(type, expectedValue)),
msg=name)
if expectOpaque:
value = list(map(remove00, value))
firstValue = value[0]
else:
if expectOpaque:
value = remove00(value.tobytes())
firstValue = value
msg = "{} {} instead of {}".format(name, type(value),
type(expectedValue))
self.assertEqual(type(value), type(expectedValue), msg=msg)
self.assertEqual(value, expectedValue, msg=name)
# Check HDF5 type id
if not attribute:
with path[name].open() as dset:
typeId = dset.id.get_type()
if expectOpaque:
self.assertTrue(
isinstance(typeId, h5py.h5t.TypeOpaqueID))
else:
self.assertTrue(
isinstance(typeId, h5py.h5t.TypeStringID))
charSet = typeId.get_cset()
if isinstance(firstValue, bytes):
# This is why opaque types are used for extended ASCII strings
expectedCharSet = h5py.h5t.CSET_ASCII
else:
expectedCharSet = h5py.h5t.CSET_UTF8
msg = "{} type {} instead of {}".format(
name, charSet, expectedCharSet)
self.assertEqual(charSet, expectedCharSet, msg=msg)
def __ne__(self, other):
if np.issubsctype(type(other), np.number):
return InequalitySubsetState(self, other, operator.ne)
return other is not self
def select(data, trial=None, invert=False, **axes_to_select):
"""Define the selection of trials, using ranges or actual values.
Parameters
----------
data : instance of Data
data to select from.
trial : list of int or ndarray (dtype='i'), optional
index of trials of interest
**axes_to_select, optional
Values need to be tuple or list. If the values in one axis are string,
then you need to specify all the strings that you want. If the values
are numeric, then you should specify the range. To select only up to one
point, you can use (None, value_of_interest). To select multiple
values, you can pass a numpy array with dtype bool
invert : bool
take the opposite selection
Returns
-------
instance, same class as input
data where selection has been applied.
"""
if trial is not None and not isinstance(trial, Iterable):
raise TypeError('Trial needs to be iterable.')
for axis_to_select, values_to_select in axes_to_select.items():
if (not isinstance(values_to_select, Iterable)
or isinstance(values_to_select, str)):
raise TypeError(axis_to_select + ' needs to be iterable.')
if trial is None:
trial = range(data.number_of('trial'))
else:
trial = trial
if invert:
trial = setdiff1d(range(data.number_of('trial')), trial)
# create empty axis
output = data._copy(axis=False)
for one_axis in output.axis:
output.axis[one_axis] = empty(len(trial), dtype='O')
output.data = empty(len(trial), dtype='O')
to_select = {}
for cnt, i in enumerate(trial):
lg.debug('Selection on trial {0: 6}'.format(i))
for one_axis in output.axis:
values = data.axis[one_axis][i]
if one_axis in axes_to_select.keys():
values_to_select = axes_to_select[one_axis]
if len(values_to_select) == 0:
selected_values = ()
elif isinstance(values_to_select[0], str):
selected_values = asarray(values_to_select, dtype='U')
else:
if isinstance(values_to_select, ndarray) and issubsctype(
values_to_select.dtype, bool):
bool_values = values_to_select
elif (values_to_select[0] is None
and values_to_select[1] is None):
bool_values = ones(len(values), dtype=bool)
elif values_to_select[0] is None:
bool_values = values < values_to_select[1]
elif values_to_select[1] is None:
bool_values = values_to_select[0] <= values
else:
bool_values = ((values_to_select[0] <= values) &
(values < values_to_select[1]))
selected_values = values[bool_values]
if invert:
selected_values = setdiff1d(values, selected_values)
lg.debug('In axis {0}, selecting {1: 6} '
'values'.format(one_axis, len(selected_values)))
to_select[one_axis] = selected_values
else:
lg.debug('In axis ' + one_axis + ', selecting all the '
'values')
selected_values = data.axis[one_axis][i]
output.axis[one_axis][cnt] = selected_values
output.data[cnt] = data(trial=i, **to_select)
return output
