def assert_array_compare(comparison,
x,
y,
err_msg='',
verbose=True,
header='',
fill_value=True):
"""Asserts that a comparison relation between two masked arrays is satisfied
elementwise."""
# Fill the data first
# xf = filled(x)
# yf = filled(y)
# Allocate a common mask and refill
m = mask_or(getmask(x), getmask(y))
x = masked_array(x, copy=False, mask=m, keep_mask=False, subok=False)
y = masked_array(y, copy=False, mask=m, keep_mask=False, subok=False)
if ((x is masked) and not (y is masked)) or \
((y is masked) and not (x is masked)):
msg = build_err_msg([x, y],
err_msg=err_msg,
verbose=verbose,
header=header,
names=('x', 'y'))
raise ValueError(msg)
# OK, now run the basic tests on filled versions
return utils.assert_array_compare(comparison,
x.filled(fill_value),
y.filled(fill_value),
err_msg=err_msg,
verbose=verbose,
header=header)
def assert_array_compare(comparison, x, y, err_msg='', header='',
fill_value=True):
"""Asserts that a comparison relation between two masked arrays is satisfied
elementwise."""
xf = filled(x)
yf = filled(y)
m = mask_or(getmask(x), getmask(y))
x = masked_array(xf, copy=False, subok=False, mask=m).filled(fill_value)
y = masked_array(yf, copy=False, subok=False, mask=m).filled(fill_value)
if ((x is masked) and not (y is masked)) or \
((y is masked) and not (x is masked)):
msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y'))
raise ValueError(msg)
if (x.dtype.char != "O") and (x.dtype.char != "S"):
x = x.astype(float_)
if isinstance(x, N.ndarray) and x.size > 1:
x[N.isnan(x)] = 0
elif N.isnan(x):
x = 0
if (y.dtype.char != "O") and (y.dtype.char != "S"):
y = y.astype(float_)
if isinstance(y, N.ndarray) and y.size > 1:
y[N.isnan(y)] = 0
elif N.isnan(y):
y = 0
try:
cond = (x.shape==() or y.shape==()) or x.shape == y.shape
if not cond:
msg = build_err_msg([x, y],
err_msg
+ '\n(shapes %s, %s mismatch)' % (x.shape,
y.shape),
header=header,
names=('x', 'y'))
assert cond, msg
val = comparison(x,y)
if m is not nomask and fill_value:
val = masked_array(val, mask=m, copy=False)
if isinstance(val, bool):
cond = val
reduced = [0]
else:
reduced = val.ravel()
cond = reduced.all()
reduced = reduced.tolist()
if not cond:
match = 100-100.0*reduced.count(1)/len(reduced)
msg = build_err_msg([x, y],
err_msg
+ '\n(mismatch %s%%)' % (match,),
header=header,
names=('x', 'y'))
assert cond, msg
except ValueError:
msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y'))
raise ValueError(msg)
def almost(a, b, decimal=6, fill_value=True):
"""Returns True if a and b are equal up to decimal places.
If fill_value is True, masked values considered equal. Otherwise, masked values
are considered unequal.
"""
m = mask_or(getmask(a), getmask(b))
d1 = filled(a)
d2 = filled(b)
if d1.dtype.char == "O" or d2.dtype.char == "O":
return np.equal(d1, d2).ravel()
x = filled(masked_array(d1, copy=False, mask=m), fill_value).astype(float_)
y = filled(masked_array(d2, copy=False, mask=m), 1).astype(float_)
d = np.around(np.abs(x - y), decimal) <= 10.0**(-decimal)
return d.ravel()
def almost(a, b, decimal=6, fill_value=True):
"""Returns True if a and b are equal up to decimal places.
If fill_value is True, masked values considered equal. Otherwise, masked values
are considered unequal.
"""
m = mask_or(getmask(a), getmask(b))
d1 = filled(a)
d2 = filled(b)
if d1.dtype.char == "O" or d2.dtype.char == "O":
return N.equal(d1,d2).ravel()
x = filled(masked_array(d1, copy=False, mask=m), fill_value).astype(float_)
y = filled(masked_array(d2, copy=False, mask=m), 1).astype(float_)
d = N.around(N.abs(x-y),decimal) <= 10.0**(-decimal)
return d.ravel()
def dot(a,b, strict=False):
"""Return the dot product of two 2D masked arrays a and b.
Like the generic numpy equivalent, the product sum is over the
last dimension of a and the second-to-last dimension of b. If
strict is True, masked values are propagated: if a masked value
appears in a row or column, the whole row or column is considered
masked.
Parameters
----------
strict : {boolean}
Whether masked data are propagated (True) or set to 0 for
the computation.
Notes
-----
The first argument is not conjugated.
"""
#TODO: Works only with 2D arrays. There should be a way to get it to run with higher dimension
if strict and (a.ndim == 2) and (b.ndim == 2):
a = mask_rows(a)
b = mask_cols(b)
#
d = np.dot(filled(a, 0), filled(b, 0))
#
am = (~getmaskarray(a))
bm = (~getmaskarray(b))
m = ~np.dot(am, bm)
return masked_array(d, mask=m)
def masked_all_like(arr):
"""Return an empty masked array of the same shape and dtype as
the array `a`, where all the data are masked.
"""
a = masked_array(np.empty_like(arr), mask=np.ones(arr.shape, bool))
return a
def dot(a, b, strict=False):
"""
Return the dot product of two 2D masked arrays a and b.
Like the generic numpy equivalent, the product sum is over the last
dimension of a and the second-to-last dimension of b. If strict is True,
masked values are propagated: if a masked value appears in a row or column,
the whole row or column is considered masked.
Parameters
----------
strict : {boolean}
Whether masked data are propagated (True) or set to 0 for the computation.
Notes
-----
The first argument is not conjugated.
"""
#!!!: Works only with 2D arrays. There should be a way to get it to run with higher dimension
if strict and (a.ndim == 2) and (b.ndim == 2):
a = mask_rows(a)
b = mask_cols(b)
#
d = np.dot(filled(a, 0), filled(b, 0))
#
am = ~getmaskarray(a)
bm = ~getmaskarray(b)
m = ~np.dot(am, bm)
return masked_array(d, mask=m)
def masked_all_like(arr):
"""Return an empty masked array of the same shape and dtype as
the array `a`, where all the data are masked.
"""
a = masked_array(np.empty_like(arr),
mask=np.ones(arr.shape, bool))
return a
def assert_array_compare(comparison, x, y, err_msg="", verbose=True, header="", fill_value=True):
"""Asserts that a comparison relation between two masked arrays is satisfied
elementwise."""
# Fill the data first
# xf = filled(x)
# yf = filled(y)
# Allocate a common mask and refill
m = mask_or(getmask(x), getmask(y))
x = masked_array(x, copy=False, mask=m, keep_mask=False, subok=False)
y = masked_array(y, copy=False, mask=m, keep_mask=False, subok=False)
if ((x is masked) and not (y is masked)) or ((y is masked) and not (x is masked)):
msg = build_err_msg([x, y], err_msg=err_msg, verbose=verbose, header=header, names=("x", "y"))
raise ValueError(msg)
# OK, now run the basic tests on filled versions
return utils.assert_array_compare(
comparison, x.filled(fill_value), y.filled(fill_value), err_msg=err_msg, verbose=verbose, header=header
)
def approx (a, b, fill_value=1, rtol=1.e-5, atol=1.e-8):
"""Returns true if all components of a and b are equal subject to given tolerances.
If fill_value is 1, masked values considered equal.
If fill_value is 0, masked values considered unequal.
The relative error rtol should be positive and << 1.0
The absolute error atol comes into play for those elements of b that are
very small or zero; it says how small a must be also.
"""
m = mask_or(getmask(a), getmask(b))
d1 = filled(a)
d2 = filled(b)
if d1.dtype.char == "O" or d2.dtype.char == "O":
return N.equal(d1,d2).ravel()
x = filled(masked_array(d1, copy=False, mask=m), fill_value).astype(float_)
y = filled(masked_array(d2, copy=False, mask=m), 1).astype(float_)
d = N.less_equal(umath.absolute(x-y), atol + rtol * umath.absolute(y))
return d.ravel()
def expand_dims(a, axis):
"""Expands the shape of a by including newaxis before axis.
"""
if not isinstance(a, MaskedArray):
return np.expand_dims(a, axis)
elif getmask(a) is nomask:
return np.expand_dims(a, axis).view(MaskedArray)
m = getmaskarray(a)
return masked_array(np.expand_dims(a, axis), mask=np.expand_dims(m, axis))
def approx(a, b, fill_value=True, rtol=1.e-5, atol=1.e-8):
"""Returns true if all components of a and b are equal subject to given tolerances.
If fill_value is True, masked values considered equal. Otherwise, masked values
are considered unequal.
The relative error rtol should be positive and << 1.0
The absolute error atol comes into play for those elements of b that are very
small or zero; it says how small a must be also.
"""
m = mask_or(getmask(a), getmask(b))
d1 = filled(a)
d2 = filled(b)
if d1.dtype.char == "O" or d2.dtype.char == "O":
return np.equal(d1, d2).ravel()
x = filled(masked_array(d1, copy=False, mask=m), fill_value).astype(float_)
y = filled(masked_array(d2, copy=False, mask=m), 1).astype(float_)
d = np.less_equal(umath.absolute(x - y), atol + rtol * umath.absolute(y))
return d.ravel()
def expand_dims(a, axis):
"""Expands the shape of a by including newaxis before axis.
"""
if not isinstance(a, MaskedArray):
return np.expand_dims(a, axis)
elif getmask(a) is nomask:
return np.expand_dims(a, axis).view(MaskedArray)
m = getmaskarray(a)
return masked_array(np.expand_dims(a, axis),
mask=np.expand_dims(m, axis))
def masked_all(shape, dtype=float):
"""Return an empty masked array of the given shape and dtype,
where all the data are masked.
Parameters
----------
dtype : dtype, optional
Data type of the output.
"""
a = masked_array(np.empty(shape, dtype), mask=np.ones(shape, make_mask_descr(dtype)))
return a
def masked_all(shape, dtype=float):
"""Return an empty masked array of the given shape and dtype,
where all the data are masked.
Parameters
----------
dtype : dtype, optional
Data type of the output.
"""
a = masked_array(np.empty(shape, dtype),
mask=np.ones(shape, bool))
return a
def masked_all(shape, dtype=float):
"""
Empty masked array with all elements masked.
Return an empty masked array of the given shape and dtype, where all the
data are masked.
Parameters
----------
shape : tuple
Shape of the required MaskedArray.
dtype : dtype, optional
Data type of the output.
Returns
-------
a : MaskedArray
A masked array with all data masked.
See Also
--------
masked_all_like : Empty masked array modelled on an existing array.
Examples
--------
>>> import numpy.ma as ma
>>> ma.masked_all((3, 3))
masked_array(data =
[[-- -- --]
[-- -- --]
[-- -- --]],
mask =
[[ True True True]
[ True True True]
[ True True True]],
fill_value=1e+20)
The `dtype` parameter defines the underlying data type.
>>> a = ma.masked_all((3, 3))
>>> a.dtype
dtype('float64')
>>> a = ma.masked_all((3, 3), dtype=np.int32)
>>> a.dtype
dtype('int32')
"""
a = masked_array(np.empty(shape, dtype),
mask=np.ones(shape, make_mask_descr(dtype)))
return a
def __call__(self, *args, **params):
func = getattr(np, self._function)
if len(args)==1:
x = args[0]
if isinstance(x, ndarray):
_d = func(np.asarray(x), **params)
_m = func(getmaskarray(x), **params)
return masked_array(_d, mask=_m)
elif isinstance(x, tuple) or isinstance(x, list):
_d = func(tuple([np.asarray(a) for a in x]), **params)
_m = func(tuple([getmaskarray(a) for a in x]), **params)
return masked_array(_d, mask=_m)
else:
arrays = []
args = list(args)
while len(args)>0 and issequence(args[0]):
arrays.append(args.pop(0))
res = []
for x in arrays:
_d = func(np.asarray(x), *args, **params)
_m = func(getmaskarray(x), *args, **params)
res.append(masked_array(_d, mask=_m))
return res
def __call__(self, *args, **params):
func = getattr(np, self.__name__)
if len(args) == 1:
x = args[0]
if isinstance(x, ndarray):
_d = func(np.asarray(x), **params)
_m = func(getmaskarray(x), **params)
return masked_array(_d, mask=_m)
elif isinstance(x, tuple) or isinstance(x, list):
_d = func(tuple([np.asarray(a) for a in x]), **params)
_m = func(tuple([getmaskarray(a) for a in x]), **params)
return masked_array(_d, mask=_m)
else:
arrays = []
args = list(args)
while len(args) > 0 and issequence(args[0]):
arrays.append(args.pop(0))
res = []
for x in arrays:
_d = func(np.asarray(x), *args, **params)
_m = func(getmaskarray(x), *args, **params)
res.append(masked_array(_d, mask=_m))
return res
def assert_array_compare(comparison,
x,
y,
err_msg='',
header='',
fill_value=True):
"""Asserts that a comparison relation between two masked arrays is satisfied
elementwise."""
xf = filled(x)
yf = filled(y)
m = mask_or(getmask(x), getmask(y))
x = masked_array(xf, copy=False, subok=False, mask=m).filled(fill_value)
y = masked_array(yf, copy=False, subok=False, mask=m).filled(fill_value)
if ((x is masked) and not (y is masked)) or \
((y is masked) and not (x is masked)):
msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y'))
raise ValueError(msg)
if (x.dtype.char != "O") and (x.dtype.char != "S"):
x = x.astype(float_)
if isinstance(x, N.ndarray) and x.size > 1:
x[N.isnan(x)] = 0
elif N.isnan(x):
x = 0
if (y.dtype.char != "O") and (y.dtype.char != "S"):
y = y.astype(float_)
if isinstance(y, N.ndarray) and y.size > 1:
y[N.isnan(y)] = 0
elif N.isnan(y):
y = 0
try:
cond = (x.shape == () or y.shape == ()) or x.shape == y.shape
if not cond:
msg = build_err_msg([x, y],
err_msg + '\n(shapes %s, %s mismatch)' %
(x.shape, y.shape),
header=header,
names=('x', 'y'))
assert cond, msg
val = comparison(x, y)
if m is not nomask and fill_value:
val = masked_array(val, mask=m, copy=False)
if isinstance(val, bool):
cond = val
reduced = [0]
else:
reduced = val.ravel()
cond = reduced.all()
reduced = reduced.tolist()
if not cond:
match = 100 - 100.0 * reduced.count(1) / len(reduced)
msg = build_err_msg([x, y],
err_msg + '\n(mismatch %s%%)' % (match, ),
header=header,
names=('x', 'y'))
assert cond, msg
except ValueError:
msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y'))
raise ValueError(msg)
def average(a, axis=None, weights=None, returned=False):
"""
Average the array over the given axis.
Parameters
----------
axis : {None,int}, optional
Axis along which to perform the operation.
If None, applies to a flattened version of the array.
weights : {None, sequence}, optional
Sequence of weights.
The weights must have the shape of a, or be 1D with length
the size of a along the given axis.
If no weights are given, weights are assumed to be 1.
returned : {False, True}, optional
Flag indicating whether a tuple (result, sum of weights/counts)
should be returned as output (True), or just the result (False).
"""
a = asarray(a)
mask = a.mask
ash = a.shape
if ash == ():
ash = (1,)
if axis is None:
if mask is nomask:
if weights is None:
n = a.sum(axis=None)
d = float(a.size)
else:
w = filled(weights, 0.0).ravel()
n = umath.add.reduce(a._data.ravel() * w)
d = umath.add.reduce(w)
del w
else:
if weights is None:
n = a.filled(0).sum(axis=None)
d = umath.add.reduce((-mask).ravel().astype(int))
else:
w = array(filled(weights, 0.0), float, mask=mask).ravel()
n = add.reduce(a.ravel() * w)
d = add.reduce(w)
del w
else:
if mask is nomask:
if weights is None:
d = ash[axis] * 1.0
n = add.reduce(a._data, axis, dtype=float)
else:
w = filled(weights, 0.0)
wsh = w.shape
if wsh == ():
wsh = (1,)
if wsh == ash:
w = np.array(w, float, copy=0)
n = add.reduce(a * w, axis)
d = add.reduce(w, axis)
del w
elif wsh == (ash[axis],):
ni = ash[axis]
r = [None] * len(ash)
r[axis] = slice(None, None, 1)
w = eval("w[" + repr(tuple(r)) + "] * ones(ash, float)")
n = add.reduce(a * w, axis, dtype=float)
d = add.reduce(w, axis, dtype=float)
del w, r
else:
raise ValueError, "average: weights wrong shape."
else:
if weights is None:
n = add.reduce(a, axis, dtype=float)
d = umath.add.reduce((-mask), axis=axis, dtype=float)
else:
w = filled(weights, 0.0)
wsh = w.shape
if wsh == ():
wsh = (1,)
if wsh == ash:
w = array(w, dtype=float, mask=mask, copy=0)
n = add.reduce(a * w, axis, dtype=float)
d = add.reduce(w, axis, dtype=float)
elif wsh == (ash[axis],):
ni = ash[axis]
r = [None] * len(ash)
r[axis] = slice(None, None, 1)
w = eval("w[" + repr(tuple(r)) + "] * masked_array(ones(ash, float), mask)")
n = add.reduce(a * w, axis, dtype=float)
d = add.reduce(w, axis, dtype=float)
else:
raise ValueError, "average: weights wrong shape."
del w
if n is masked or d is masked:
return masked
result = n / d
del n
if isinstance(result, MaskedArray):
if ((axis is None) or (axis == 0 and a.ndim == 1)) and (result.mask is nomask):
result = result._data
if returned:
if not isinstance(d, MaskedArray):
d = masked_array(d)
if isinstance(d, ndarray) and (not d.shape == result.shape):
d = ones(result.shape, dtype=float) * d
if returned:
return result, d
else:
return result
def average(a, axis=None, weights=None, returned=False):
"""Average the array over the given axis.
Parameters
----------
axis : {None,int}, optional
Axis along which to perform the operation.
If None, applies to a flattened version of the array.
weights : {None, sequence}, optional
Sequence of weights.
The weights must have the shape of a, or be 1D with length
the size of a along the given axis.
If no weights are given, weights are assumed to be 1.
returned : {False, True}, optional
Flag indicating whether a tuple (result, sum of weights/counts)
should be returned as output (True), or just the result (False).
"""
a = asarray(a)
mask = a.mask
ash = a.shape
if ash == ():
ash = (1,)
if axis is None:
if mask is nomask:
if weights is None:
n = a.sum(axis=None)
d = float(a.size)
else:
w = filled(weights, 0.0).ravel()
n = umath.add.reduce(a._data.ravel() * w)
d = umath.add.reduce(w)
del w
else:
if weights is None:
n = a.filled(0).sum(axis=None)
d = umath.add.reduce((-mask).ravel().astype(int))
else:
w = array(filled(weights, 0.0), float, mask=mask).ravel()
n = add.reduce(a.ravel() * w)
d = add.reduce(w)
del w
else:
if mask is nomask:
if weights is None:
d = ash[axis] * 1.0
n = add.reduce(a._data, axis, dtype=float)
else:
w = filled(weights, 0.0)
wsh = w.shape
if wsh == ():
wsh = (1,)
if wsh == ash:
w = np.array(w, float, copy=0)
n = add.reduce(a*w, axis)
d = add.reduce(w, axis)
del w
elif wsh == (ash[axis],):
ni = ash[axis]
r = [None]*len(ash)
r[axis] = slice(None, None, 1)
w = eval ("w["+ repr(tuple(r)) + "] * ones(ash, float)")
n = add.reduce(a*w, axis, dtype=float)
d = add.reduce(w, axis, dtype=float)
del w, r
else:
raise ValueError, 'average: weights wrong shape.'
else:
if weights is None:
n = add.reduce(a, axis, dtype=float)
d = umath.add.reduce((-mask), axis=axis, dtype=float)
else:
w = filled(weights, 0.0)
wsh = w.shape
if wsh == ():
wsh = (1,)
if wsh == ash:
w = array(w, dtype=float, mask=mask, copy=0)
n = add.reduce(a*w, axis, dtype=float)
d = add.reduce(w, axis, dtype=float)
elif wsh == (ash[axis],):
ni = ash[axis]
r = [None]*len(ash)
r[axis] = slice(None, None, 1)
w = eval ("w["+ repr(tuple(r)) + \
"] * masked_array(ones(ash, float), mask)")
n = add.reduce(a*w, axis, dtype=float)
d = add.reduce(w, axis, dtype=float)
else:
raise ValueError, 'average: weights wrong shape.'
del w
if n is masked or d is masked:
return masked
result = n/d
del n
if isinstance(result, MaskedArray):
if ((axis is None) or (axis==0 and a.ndim == 1)) and \
(result.mask is nomask):
result = result._data
if returned:
if not isinstance(d, MaskedArray):
d = masked_array(d)
if isinstance(d, ndarray) and (not d.shape == result.shape):
d = ones(result.shape, dtype=float) * d
if returned:
return result, d
else:
return result
def ediff1d(array, to_end=None, to_begin=None):
"""Return the differences between consecutive elements of an
array, possibly with prefixed and/or appended values.
Parameters
----------
array : {array}
Input array, will be flattened before the difference is taken.
to_end : {number}, optional
If provided, this number will be tacked onto the end of the returned
differences.
to_begin : {number}, optional
If provided, this number will be taked onto the beginning of the
returned differences.
Returns
-------
ed : {array}
The differences. Loosely, this will be (ary[1:] - ary[:-1]).
"""
a = masked_array(array, copy=True)
if a.ndim > 1:
a.reshape((a.size,))
(d, m, n) = (a._data, a._mask, a.size-1)
dd = d[1:]-d[:-1]
if m is nomask:
dm = nomask
else:
dm = m[1:]-m[:-1]
#
if to_end is not None:
to_end = asarray(to_end)
nend = to_end.size
if to_begin is not None:
to_begin = asarray(to_begin)
nbegin = to_begin.size
r_data = np.empty((n+nend+nbegin,), dtype=a.dtype)
r_mask = np.zeros((n+nend+nbegin,), dtype=bool)
r_data[:nbegin] = to_begin._data
r_mask[:nbegin] = to_begin._mask
r_data[nbegin:-nend] = dd
r_mask[nbegin:-nend] = dm
else:
r_data = np.empty((n+nend,), dtype=a.dtype)
r_mask = np.zeros((n+nend,), dtype=bool)
r_data[:-nend] = dd
r_mask[:-nend] = dm
r_data[-nend:] = to_end._data
r_mask[-nend:] = to_end._mask
#
elif to_begin is not None:
to_begin = asarray(to_begin)
nbegin = to_begin.size
r_data = np.empty((n+nbegin,), dtype=a.dtype)
r_mask = np.zeros((n+nbegin,), dtype=bool)
r_data[:nbegin] = to_begin._data
r_mask[:nbegin] = to_begin._mask
r_data[nbegin:] = dd
r_mask[nbegin:] = dm
#
else:
r_data = dd
r_mask = dm
return masked_array(r_data, mask=r_mask)
