def generic_interp_hght(h, hght, field, log=False):
'''
Generic interpolation routine
Parameters
----------
h : number, numpy array
Height (m) of the level for which pressure is desired
hght : numpy array
The array of heights
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given height
'''
if ma.isMaskedArray(hght):
not_masked1 = ~hght.mask
else:
not_masked1 = np.ones(hght.shape)
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape)
not_masked = not_masked1 * not_masked2
if log:
return 10**np.interp(h, hght[not_masked], field[not_masked],
left=ma.masked, right=ma.masked)
else:
return np.interp(h, hght[not_masked], field[not_masked],
left=ma.masked, right=ma.masked)
def to_ham6(img, palette, background=None, out=None):
_debug_array(img)
if background is None:
background = ma.masked
elif isinstance(background, numbers.Integral):
background = palette[background]
if not ma.is_masked(background) and ma.isMaskedArray(img):
img = img.filled(background)
if ma.isMaskedArray(img):
ham6 = ma.empty(img.shape[:2], dtype=np.uint8)
else:
ham6 = np.empty(img.shape[:2], dtype=np.uint8)
for y in range(img.shape[0]):
c = background
for x in range(img.shape[1]):
i, c = ham6_nearest(img[y, x], palette, c)
ham6[y, x] = i
if out is not None:
out[y, x] = c
_debug_array(ham6)
return ham6
def test_peak_with_nan_and_mask(self):
# Single nan in column with single value masked.
latitude = iris.coords.DimCoord(np.arange(0, 5, 1),
standard_name='latitude',
units='degrees')
cube = iris.cube.Cube(ma.array([1, 4, 2, 3, 1], dtype=np.float32),
standard_name='air_temperature',
units='kelvin')
cube.add_dim_coord(latitude, 0)
cube.data[3] = np.nan
cube.data[4] = ma.masked
collapsed_cube = cube.collapsed('latitude', iris.analysis.PEAK)
self.assertArrayAlmostEqual(collapsed_cube.data,
np.array([4.024977], dtype=np.float32))
self.assertTrue(ma.isMaskedArray(collapsed_cube.data))
self.assertEqual(collapsed_cube.data.shape, (1,))
# Only nans in column where values not masked.
cube.data[0:3] = np.nan
collapsed_cube = cube.collapsed('latitude', iris.analysis.PEAK)
self.assertTrue(np.isnan(collapsed_cube.data).all())
self.assertTrue(ma.isMaskedArray(collapsed_cube.data))
self.assertEqual(collapsed_cube.data.shape, (1,))
def test_peak_with_mask(self):
# Single value in column masked.
latitude = iris.coords.DimCoord(np.arange(0, 5, 1),
standard_name='latitude',
units='degrees')
cube = iris.cube.Cube(ma.array([1, 4, 2, 3, 2], dtype=np.float32),
standard_name='air_temperature',
units='kelvin')
cube.add_dim_coord(latitude, 0)
cube.data[3] = ma.masked
collapsed_cube = cube.collapsed('latitude', iris.analysis.PEAK)
self.assertArrayAlmostEqual(collapsed_cube.data,
np.array([4.024977], dtype=np.float32))
self.assertTrue(ma.isMaskedArray(collapsed_cube.data))
self.assertEqual(collapsed_cube.data.shape, (1,))
# Whole column masked.
cube.data[:] = ma.masked
collapsed_cube = cube.collapsed('latitude', iris.analysis.PEAK)
masked_array = ma.array(ma.masked)
self.assertTrue(ma.allequal(collapsed_cube.data, masked_array))
self.assertTrue(ma.isMaskedArray(collapsed_cube.data))
self.assertEqual(collapsed_cube.data.shape, (1,))
def test_rotated_to_osgb(self):
# Rotated Pole data with large extent.
x = np.linspace(311.9, 391.1, 10)
y = np.linspace(-23.6, 24.8, 8)
u, v = uv_cubes(x, y)
ut, vt = rotate_winds(u, v, iris.coord_systems.OSGB())
# Ensure cells with discrepancies in magnitude are masked.
self.assertTrue(ma.isMaskedArray(ut.data))
self.assertTrue(ma.isMaskedArray(vt.data))
# Snapshot of mask with fixed tolerance of atol=2e-3
expected_mask = np.array([[1, 1, 1, 0, 0, 0, 0, 0, 0, 1],
[1, 1, 1, 0, 0, 0, 0, 0, 0, 1],
[1, 1, 1, 1, 0, 0, 0, 0, 1, 1],
[1, 1, 1, 1, 0, 0, 0, 0, 1, 1],
[1, 1, 1, 1, 0, 0, 0, 0, 1, 1],
[1, 1, 1, 1, 1, 0, 0, 1, 1, 1],
[1, 1, 1, 1, 1, 0, 0, 1, 1, 1],
[1, 1, 1, 1, 1, 0, 0, 1, 1, 1]], np.bool)
self.assertArrayEqual(expected_mask, ut.data.mask)
self.assertArrayEqual(expected_mask, vt.data.mask)
# Check unmasked values have sufficiently small error in mag.
expected_mag = np.sqrt(u.data**2 + v.data**2)
# Use underlying data to ignore mask in calculation.
res_mag = np.sqrt(ut.data.data**2 + vt.data.data**2)
# Calculate percentage error (note there are no zero magnitudes
# so we can divide safely).
anom = 100.0 * np.abs(res_mag - expected_mag) / expected_mag
self.assertTrue(anom[~ut.data.mask].max() < 0.1)
def set_data(self, *args):
"""
Set the x and y data
ACCEPTS: (np.array xdata, np.array ydata)
"""
if len(args)==1:
x, y = args[0]
else:
x, y = args
not_masked = 0
if not ma.isMaskedArray(x):
x = np.asarray(x)
not_masked += 1
if not ma.isMaskedArray(y):
y = np.asarray(y)
not_masked += 1
if (not_masked < 2 or
(x is not self._xorig and
(x.shape != self._xorig.shape or np.any(x != self._xorig))) or
(y is not self._yorig and
(y.shape != self._yorig.shape or np.any(y != self._yorig)))):
self._xorig = x
self._yorig = y
self._invalid = True
def recache(self, always=False):
if always or self._invalidx:
xconv = self.convert_xunits(self._xorig)
if ma.isMaskedArray(self._xorig):
x = ma.asarray(xconv, np.float_).filled(np.nan)
else:
x = np.asarray(xconv, np.float_)
x = x.ravel()
else:
x = self._x
if always or self._invalidy:
yconv = self.convert_yunits(self._yorig)
if ma.isMaskedArray(self._yorig):
y = ma.asarray(yconv, np.float_).filled(np.nan)
else:
y = np.asarray(yconv, np.float_)
y = y.ravel()
else:
y = self._y
if len(x) == 1 and len(y) > 1:
x = x * np.ones(y.shape, np.float_)
if len(y) == 1 and len(x) > 1:
y = y * np.ones(x.shape, np.float_)
if len(x) != len(y):
raise RuntimeError('xdata and ydata must be the same length')
self._xy = np.empty((len(x), 2), dtype=np.float_)
self._xy[:, 0] = x
self._xy[:, 1] = y
self._x = self._xy[:, 0] # just a view
self._y = self._xy[:, 1] # just a view
self._subslice = False
if (self.axes and len(x) > 1000 and self._is_sorted(x) and
self.axes.name == 'rectilinear' and
self.axes.get_xscale() == 'linear' and
self._markevery is None and
self.get_clip_on() is True):
self._subslice = True
nanmask = np.isnan(x)
if nanmask.any():
self._x_filled = self._x.copy()
indices = np.arange(len(x))
self._x_filled[nanmask] = np.interp(indices[nanmask],
indices[~nanmask], self._x[~nanmask])
else:
self._x_filled = self._x
if self._path is not None:
interpolation_steps = self._path._interpolation_steps
else:
interpolation_steps = 1
xy = STEP_LOOKUP_MAP[self._drawstyle](*self._xy.T)
self._path = Path(np.asarray(xy).T, None, interpolation_steps)
self._transformed_path = None
self._invalidx = False
self._invalidy = False
def generic_interp_pres(p, pres, field):
'''
Generic interpolation routine
Parameters
----------
p : number, numpy array
Pressure (hPa) of the level for which the field variable is desired
pres : numpy array
The array of pressure
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given pressure
'''
if ma.isMaskedArray(pres):
not_masked1 = ~pres.mask
else:
not_masked1 = np.ones(pres.shape, dtype=bool)
not_masked1[:] = True
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape, dtype=bool)
not_masked2[:] = True
not_masked = not_masked1 * not_masked2
return np.interp(p, pres[not_masked], field[not_masked], left=ma.masked,
right=ma.masked)
def recache(self, always=False):
if always or self._invalidx:
xconv = self.convert_xunits(self._xorig)
if ma.isMaskedArray(self._xorig):
x = ma.asarray(xconv, np.float_)
else:
x = np.asarray(xconv, np.float_)
x = x.ravel()
else:
x = self._x
if always or self._invalidy:
yconv = self.convert_yunits(self._yorig)
if ma.isMaskedArray(self._yorig):
y = ma.asarray(yconv, np.float_)
else:
y = np.asarray(yconv, np.float_)
y = y.ravel()
else:
y = self._y
if len(x) == 1 and len(y) > 1:
x = x * np.ones(y.shape, np.float_)
if len(y) == 1 and len(x) > 1:
y = y * np.ones(x.shape, np.float_)
if len(x) != len(y):
raise RuntimeError("xdata and ydata must be the same length")
x = x.reshape((len(x), 1))
y = y.reshape((len(y), 1))
if ma.isMaskedArray(x) or ma.isMaskedArray(y):
self._xy = ma.concatenate((x, y), 1)
else:
self._xy = np.concatenate((x, y), 1)
self._x = self._xy[:, 0] # just a view
self._y = self._xy[:, 1] # just a view
self._subslice = False
if (
self.axes
and len(x) > 100
and self._is_sorted(x)
and self.axes.name == "rectilinear"
and self.axes.get_xscale() == "linear"
and self._markevery is None
):
self._subslice = True
if hasattr(self, "_path"):
interpolation_steps = self._path._interpolation_steps
else:
interpolation_steps = 1
self._path = Path(self._xy, None, interpolation_steps)
self._transformed_path = None
self._invalidx = False
self._invalidy = False
def __init__(self, cpfile, smapfile, areafile, crop, varname):
with nc(cpfile) as f:
self.year = f.variables['year'][:]
self.week = f.variables['week'][:]
self.county = f.variables['county'][:]
self.day = f.variables['day'][:]
self.rawdata = f.variables[varname][:]
varatt = f.variables['var'].ncattrs()
if 'units' in varatt and f.variables['var'].units == 'mapping':
self.var = array(f.variables['var'].long_name.split(', '))
self.varmap = self.varmap_str[crop]
else:
self.var = f.variables['var'][:]
self.varmap = self.varmap_num[crop]
self.crop = crop
nyears, nweeks, ncounties, nvars = len(self.year), len(self.week), len(self.county), len(self.vars)
self.data = masked_array(zeros((nyears, nweeks, ncounties, nvars)), mask = ones((nyears, nweeks, ncounties, nvars)))
for i in range(nvars):
vmap = self.varmap[i]
if isinstance(vmap, list):
for j in range(ncounties):
for k in range(len(vmap)):
if vmap[k] in self.var: # variable in list
varidx = where(self.var == vmap[k])[0][0]
data = self.rawdata[:, :, varidx, j]
if not isMaskedArray(data) or not data.mask.all():
self.data[:, :, j, i] = data
break
elif vmap != '':
if vmap in self.var:
varidx = where(self.var == vmap)[0][0]
self.data[:, :, :, i] = self.rawdata[:, :, varidx, :]
else: # no data
continue
# discard counties with insufficient data
for j in range(ncounties):
for k in range(nyears):
data = self.data[k, :, j, i]
if isMaskedArray(data):
data = data[~data.mask]
if data.size and data[-1] - data[0] < 40:
self.data[k, :, j, i].mask = True # mask
# aggregate to state level
aggregator = StateAggregator(smapfile, areafile)
self.sdata = aggregator.aggregate(self.data, self.county)
self.state = aggregator.states
def _check_fill_value(self, result, fill0='none', fill1='none'):
expected_fill_value = self._expected_fill_value(fill0, fill1)
if expected_fill_value is None:
data = result.data
if ma.isMaskedArray(data):
np_fill_value = ma.masked_array(0,
dtype=result.dtype).fill_value
self.assertEqual(data.fill_value, np_fill_value)
else:
data = result.data
if ma.isMaskedArray(data):
self.assertEqual(data.fill_value, expected_fill_value)
def generic_interp_hght(h, hght, field, log=False):
'''
Generic interpolation routine
Parameters
----------
h : number, numpy array
Height (m) of the level for which pressure is desired
hght : numpy array
The array of heights
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given height
'''
if ma.isMaskedArray(hght):
not_masked1 = ~hght.mask
else:
not_masked1 = np.ones(hght.shape)
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape)
not_masked = not_masked1 * not_masked2
field_intrp = np.interp(h, hght[not_masked], field[not_masked],
left=ma.masked, right=ma.masked)
if hasattr(h, 'shape') and h.shape == tuple():
h = h[()]
if type(h) != type(ma.masked):
# Bug fix for Numpy v1.10: returns nan on the boundary.
field_intrp = np.where(np.isclose(h, hght[not_masked][0]), field[not_masked][0], field_intrp)
field_intrp = np.where(np.isclose(h, hght[not_masked][-1]), field[not_masked][-1], field_intrp)
# Another bug fix: np.interp() returns masked values as nan. We want ma.masked, dangit!
field_intrp = ma.where(np.isnan(field_intrp), ma.masked, field_intrp)
# ma.where() returns a 0-d array when the arguments are floats, which confuses subsequent code.
if hasattr(field_intrp, 'shape') and field_intrp.shape == tuple():
field_intrp = field_intrp[()]
if log:
return 10 ** field_intrp
else:
return field_intrp
def test_testBasic1d(self):
# Test of basic array creation and properties in 1 dimension.
(x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d
assert_(not isMaskedArray(x))
assert_(isMaskedArray(xm))
assert_equal(shape(xm), s)
assert_equal(xm.shape, s)
assert_equal(xm.dtype, x.dtype)
assert_equal(xm.size, reduce(lambda x, y:x * y, s))
assert_equal(count(xm), len(m1) - reduce(lambda x, y:x + y, m1))
assert_(eq(xm, xf))
assert_(eq(filled(xm, 1.e20), xf))
assert_(eq(x, xm))
def generic_interp_pres(p, pres, field):
'''
Generic interpolation routine
Parameters
----------
p : number, numpy array
Pressure (hPa) of the level for which the field variable is desired
pres : numpy array
The array of pressure
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given pressure
'''
if ma.isMaskedArray(pres):
not_masked1 = ~pres.mask
else:
not_masked1 = np.ones(pres.shape, dtype=bool)
not_masked1[:] = True
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape, dtype=bool)
not_masked2[:] = True
not_masked = not_masked1 * not_masked2
field_intrp = np.interp(p, pres[not_masked], field[not_masked], left=ma.masked,
right=ma.masked)
if hasattr(p, 'shape') and p.shape == tuple():
p = p[()]
if type(p) != type(ma.masked):
# Bug fix for Numpy v1.10: returns nan on the boundary.
field_intrp = ma.where(np.isclose(p, pres[not_masked][0]), field[not_masked][0], field_intrp)
field_intrp = ma.where(np.isclose(p, pres[not_masked][-1]), field[not_masked][-1], field_intrp)
# Another bug fix: np.interp() returns masked values as nan. We want ma.masked, dangit!
field_intrp = ma.where(np.isnan(field_intrp), ma.masked, field_intrp)
# ma.where() returns a 0-d array when the arguments are floats, which confuses subsequent code.
if hasattr(field_intrp, 'shape') and field_intrp.shape == tuple():
field_intrp = field_intrp[()]
return field_intrp
def _weighted_mean_with_mdtol(data, weights, axis=None, mdtol=0):
"""
Return the weighted mean of an array over the specified axis
using the provided weights (if any) and a permitted fraction of
masked data.
Args:
* data (array-like):
Data to be averaged.
* weights (array-like):
An array of the same shape as the data that specifies the contribution
of each corresponding data element to the calculated mean.
Kwargs:
* axis (int or tuple of ints):
Axis along which the mean is computed. The default is to compute
the mean of the flattened array.
* mdtol (float):
Tolerance of missing data. The value returned in each element of the
returned array will be masked if the fraction of masked data exceeds
mdtol. This fraction is weighted by the `weights` array if one is
provided. mdtol=0 means no missing data is tolerated
while mdtol=1 will mean the resulting element will be masked if and
only if all the contributing elements of data are masked.
Defaults to 0.
Returns:
Numpy array (possibly masked) or scalar.
"""
res = ma.average(data, weights=weights, axis=axis)
if ma.isMaskedArray(data) and mdtol < 1:
weights_total = weights.sum(axis=axis)
masked_weights = weights.copy()
masked_weights[~ma.getmaskarray(data)] = 0
masked_weights_total = masked_weights.sum(axis=axis)
frac_masked = np.true_divide(masked_weights_total, weights_total)
mask_pt = frac_masked > mdtol
if np.any(mask_pt):
if np.isscalar(res):
res = ma.masked
elif ma.isMaskedArray(res):
res.mask |= mask_pt
else:
res = ma.masked_array(res, mask=mask_pt)
return res
def cf_label_data(self, cf_data_var):
"""
Return the associated CF-netCDF label variable strings.
Args:
* cf_data_var (:class:`iris.fileformats.cf.CFDataVariable`):
The CF-netCDF data variable which the CF-netCDF label variable describes.
Returns:
String labels.
"""
if not isinstance(cf_data_var, CFDataVariable):
raise TypeError('cf_data_var argument should be of type CFDataVariable. Got %r.' % type(cf_data_var))
# Determine the name of the label string (or length) dimension by
# finding the dimension name that doesn't exist within the data dimensions.
str_dim_name = list(set(self.dimensions) - set(cf_data_var.dimensions))
if len(str_dim_name) != 1:
raise ValueError('Invalid string dimensions for CF-netCDF label variable %r' % self.cf_name)
str_dim_name = str_dim_name[0]
label_data = self[:]
if ma.isMaskedArray(label_data):
label_data = label_data.filled()
# Determine whether we have a string-valued scalar label
# i.e. a character variable that only has one dimension (the length of the string).
if self.ndim == 1:
label_string = b''.join(label_data).strip()
if six.PY3:
label_string = label_string.decode('utf8')
data = np.array([label_string])
else:
# Determine the index of the string dimension.
str_dim = self.dimensions.index(str_dim_name)
# Calculate new label data shape (without string dimension) and create payload array.
new_shape = tuple(dim_len for i, dim_len in enumerate(self.shape) if i != str_dim)
string_basetype = '|S%d' if six.PY2 else '|U%d'
string_dtype = string_basetype % self.shape[str_dim]
data = np.empty(new_shape, dtype=string_dtype)
for index in np.ndindex(new_shape):
# Create the slice for the label data.
if str_dim == 0:
label_index = (slice(None, None),) + index
else:
label_index = index + (slice(None, None),)
label_string = b''.join(label_data[label_index]).strip()
if six.PY3:
label_string = label_string.decode('utf8')
data[index] = label_string
return data
def getVar(self, years):
# interpolate to common times and counties
frac = self.census.getFrac(years)
frac = self.interpolateCounty(frac, self.census.counties, self.counties)
# interpolate to common times
yldsum = self.interpolateTime(self.yldsum, self.ysum.years, years)
yldirr = self.interpolateTime(self.yldirr, self.yirr.years, years, fillgaps = False) # no gap filling!
hvtsum = self.interpolateTime(self.hvtsum, self.hsum.years, years)
hvtirr = self.interpolateTime(self.hvtirr, self.hirr.years, years, fillgaps = False)
# extrapolate irrigated yield, sum area, and irrigated area fraction
yldirr, delta = self.extrapolateYield(yldirr, yldsum, years)
hvtsum = self.extrapolateTime(hvtsum, years)
frac = self.extrapolateFrac(frac, hvtsum, hvtirr, years)
ny, nc, ni = len(years), len(self.counties), len(self.irr)
yld = masked_array(zeros((ny, nc, ni)), mask = ones((ny, nc, ni)))
hvt = masked_array(zeros((ny, nc, ni)), mask = ones((ny, nc, ni)))
for i in range(nc):
for j in range(ny):
hvt[j, i] = self.computeArea(hvtsum[j, i], hvtirr[j, i], frac[j, i])
yld[j, i] = self.computeYield(yldsum[j, i], yldirr[j, i], hvt[j, i], delta[j, i])
# extrapolate area in time
for j in range(ni):
area = hvt[:, i, j]
if isMaskedArray(area) and area.mask.any() and not area.mask.all():
hvt[:, i, j] = interp(years, years[~area.mask], area[~area.mask])
# convert
yld *= self.yldconv
hvt *= self.hvtconv
return yld, hvt
def concatenate(cubes, order=None):
"""
Explicitly force the contiguous major order of cube data
alignment to ensure consistent CML crc32 checksums.
Defaults to contiguous 'C' row-major order.
"""
if order is None:
order = 'C'
cubelist = iris.cube.CubeList(cubes)
result = cubelist.concatenate()
for cube in result:
# Setting the cube.data clears the cube.dtype and cube.fill_value.
# We want to maintain the fill_value for testing purposes, even
# though we have lost the lazy data in order to pin down the array
# data order to overcome testing on different architectures.
fill_value = cube.fill_value
if ma.isMaskedArray(cube.data):
data = np.array(cube.data.data, copy=True, order=order)
mask = np.array(cube.data.mask, copy=True, order=order)
cube.data = ma.array(data, mask=mask)
else:
cube.data = np.array(cube.data, copy=True, order=order)
cube.fill_value = fill_value
return result
def _condition_arg(value):
"""
Validate value is acceptable for conversion purposes.
Will convert into an array if not a scalar, and can be converted
into an array
Parameters
----------
value: int or float value, or sequence of such values
Returns
-------
Scalar value or numpy array
Raises
------
ValueError
If value is not as expected
"""
if isinstance(value, (float, int, long)):
return value
else:
try:
avalue = np.array(value)
dt = str(avalue.dtype)
if not (dt.startswith('int') or dt.startswith('float')):
raise ValueError("Must be convertable to int or float array")
if ma.isMaskedArray(value):
return value
return avalue
except ValueError:
raise ValueError(
"Value not scalar compatible or convertable into a float or "
"integer array")
def __init__(self, cube):
"""
Create a new _ProtoCube from the given cube and record the cube
as a source-cube.
Args:
* cube:
Source :class:`iris.cube.Cube` of the :class:`_ProtoCube`.
"""
# Cache the source-cube of this proto-cube.
self._cube = cube
# The cube signature is a combination of cube and coordinate
# metadata that defines this proto-cube.
self._cube_signature = _CubeSignature(cube)
self._data_is_masked = ma.isMaskedArray(cube.data)
# The coordinate signature allows suitable non-overlapping
# source-cubes to be identified.
self._coord_signature = _CoordSignature(self._cube_signature)
# The list of source-cubes relevant to this proto-cube.
self._skeletons = []
self._add_skeleton(self._coord_signature, cube.data)
# The nominated axis of concatenation.
self._axis = None
def concatenate(cubes, order=None):
"""
Explicitly force the contiguous major order of cube data
alignment to ensure consistent CML crc32 checksums.
Defaults to contiguous 'C' row-major order.
"""
if order is None:
order = 'C'
cubelist = iris.cube.CubeList(cubes)
result = cubelist.concatenate()
for cube in result:
if ma.isMaskedArray(cube.data):
# cube.data = ma.copy(cube.data, order=order)
data = np.array(cube.data.data, copy=True, order=order)
mask = np.array(cube.data.mask, copy=True, order=order)
fill_value = cube.data.fill_value
cube.data = ma.array(data, mask=mask, fill_value=fill_value)
else:
# cube.data = np.copy(cube.data, order=order)
cube.data = np.array(cube.data, copy=True, order=order)
return result
def _masked_to_fill_value(self,value):
fill_value = None
add_fill_value_att = False
if ma.isMaskedArray(value):
#
# If the file variable already has a _FillValue or missing_value attribute
# use it for the fill_value when converting the masked array.
# _FillValue is the preferred fill value attribute but if it is not set
# then look for missing_value.
# Note that it is important to generate the fill_value with the correct type.
# If there is an existing fill value in the file, make it conform to that type (??)
# Otherwise use the type of the numpy array value
#
if self.__dict__.has_key('_FillValue'):
fval = self.__dict__['_FillValue']
fill_value = np.array(fval,dtype=fval.dtype)
elif self.__dict__.has_key('missing_value'):
fval = self.__dict__['missing_value']
fill_value = np.array(fval,dtype=fval.dtype)
elif _is_new_ma:
fill_value = np.array(value.fill_value,dtype=value.dtype)
add_fill_value_att = True
else:
fill_value = np.array(value.fill_value(),dtype=value.dtype)
add_fill_value_att = True
value = value.filled(fill_value)
if add_fill_value_att:
setattr(self._obj,'_FillValue',fill_value)
return value
def convert_mesh_to_triangles(self, meshWidth, meshHeight, coordinates):
"""
Converts a given mesh into a sequence of triangles, each point
with its own color. This is useful for experiments using
`draw_qouraud_triangle`.
"""
Path = mpath.Path
if ma.isMaskedArray(coordinates):
p = coordinates.data
else:
p = coordinates
p_a = p[0:-1, 0:-1]
p_b = p[0:-1, 1:]
p_c = p[1:, 1:]
p_d = p[1:, 0:-1]
p_center = (p_a + p_b + p_c + p_d) / 4.0
triangles = np.concatenate(
(p_a, p_b, p_center, p_b, p_c, p_center, p_c, p_d, p_center, p_d, p_a, p_center), axis=2
)
triangles = triangles.reshape((meshWidth * meshHeight * 4, 3, 2))
c = self.get_facecolor().reshape((meshHeight + 1, meshWidth + 1, 4))
c_a = c[0:-1, 0:-1]
c_b = c[0:-1, 1:]
c_c = c[1:, 1:]
c_d = c[1:, 0:-1]
c_center = (c_a + c_b + c_c + c_d) / 4.0
colors = np.concatenate(
(c_a, c_b, c_center, c_b, c_c, c_center, c_c, c_d, c_center, c_d, c_a, c_center), axis=2
)
colors = colors.reshape((meshWidth * meshHeight * 4, 3, 4))
return triangles, colors
def _get_scale_factor_add_offset(cube, datatype):
"""Utility function used by netCDF data packing tests."""
if isinstance(datatype, dict):
dt = np.dtype(datatype['dtype'])
else:
dt = np.dtype(datatype)
cmax = cube.data.max()
cmin = cube.data.min()
n = dt.itemsize * 8
if ma.isMaskedArray(cube.data):
masked = True
else:
masked = False
if masked:
scale_factor = (cmax - cmin)/(2**n-2)
else:
scale_factor = (cmax - cmin)/(2**n-1)
if dt.kind == 'u':
add_offset = cmin
elif dt.kind == 'i':
if masked:
add_offset = (cmax + cmin)/2
else:
add_offset = cmin + 2**(n-1)*scale_factor
return (scale_factor, add_offset)
def as_lazy_data(data, chunks=_MAX_CHUNK_SIZE):
"""
Convert the input array `data` to a dask array.
Args:
* data:
An array. This will be converted to a dask array.
Kwargs:
* chunks:
Describes how the created dask array should be split up. Defaults to a
value first defined in biggus (being `8 * 1024 * 1024 * 2`).
For more information see
http://dask.pydata.org/en/latest/array-creation.html#chunks.
Returns:
The input array converted to a dask array.
"""
if not is_lazy_data(data):
if ma.isMaskedArray(data):
data = array_masked_to_nans(data)
data = da.from_array(data, chunks=chunks)
return data
def test_scalar_no_overlap(self):
# Slice src so result collapses to a scalar.
src_cube = self.src_cube[:, 1, :]
# Regrid to a single cell with no overlap with masked src cells.
grid_cube = self.grid_cube[2, 1, 3]
res = regrid(src_cube, grid_cube, mdtol=0.8)
self.assertFalse(ma.isMaskedArray(res.data))
def band2txt(band, outfile):
"""Accepts numpy rasterand writes to specified text file on disk."""
if ma.isMaskedArray(band) is True:
outraster = ma.compressed(band)
else:
outraster = band
np.savetxt(outfile, outraster, fmt="%f")
def __loadwts(self, var, vals):
if self.wfile is None:
return ones(len(vals))
else:
vars = var.split('/')
nvars = len(vars)
v = [0] * nvars
w = [0] * nvars
with nc(self.wfile) as f:
for i in range(nvars):
if f.variables[vars[i]].units == 'mapping':
v[i] = array(f.variables[vars[i]].long_name.split(', '))
else:
v[i] = f.variables[vars[i]][:]
w[i] = f.variables['weights_' + vars[i]][:]
nvals = len(vals)
wts = masked_array(zeros(nvals), mask = ones(nvals))
for i in range(nvals):
svals = vals[i].split('/')
for j in range(nvars):
if svals[j].isdigit():
svals[j] = double(svals[j]) # convert to number
idx = where(v[j] == svals[j])[0]
if idx.size:
if isMaskedArray(wts[i]):
wts[i] = w[j][idx[0]]
else:
wts[i] *= w[j][idx[0]]
return wts
def array_masked_to_nans(array):
"""
Convert a masked array to a NumPy `ndarray` filled with NaN values. Input
NumPy arrays with no mask are returned unchanged.
This is used for dask integration, as dask does not support masked arrays.
Args:
* array:
A NumPy `ndarray` or masked array.
Returns:
A NumPy `ndarray`. This is the input array if unmasked, or an array
of floating-point values with NaN values where the mask was `True` if
the input array is masked.
.. note::
The fill value and mask of the input masked array will be lost.
.. note::
Integer masked arrays are cast to 8-byte floats because NaN is a
floating-point value.
"""
if not ma.isMaskedArray(array):
result = array
else:
if ma.is_masked(array):
mask = array.mask
new_dtype = nan_array_type(array.data.dtype)
result = array.data.astype(new_dtype)
result[mask] = np.nan
else:
result = array.data
return result
def identify(cls, variables, ignore=None, target=None, warn=True, monotonic=False):
result = {}
ignore, target = cls._identify_common(variables, ignore, target)
# Identify all CF coordinate variables.
for nc_var_name, nc_var in target.iteritems():
if nc_var_name in ignore:
continue
# String variables can't be coordinates
if np.issubdtype(nc_var.dtype, np.str):
continue
# Restrict to one-dimensional with name as dimension OR zero-dimensional scalar
if not ((nc_var.ndim == 1 and nc_var_name in nc_var.dimensions) or (nc_var.ndim == 0)):
continue
# Restrict to monotonic?
if monotonic:
data = nc_var[:]
# Gracefully fill a masked coordinate.
if ma.isMaskedArray(data):
data = ma.filled(data)
if nc_var.shape == () or nc_var.shape == (1,) or iris.util.monotonic(data):
result[nc_var_name] = CFCoordinateVariable(nc_var_name, nc_var)
else:
result[nc_var_name] = CFCoordinateVariable(nc_var_name, nc_var)
return result
def test_testBasic2d(self):
# Test of basic array creation and properties in 2 dimensions.
for s in [(4, 3), (6, 2)]:
(x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d
x.shape = s
y.shape = s
xm.shape = s
ym.shape = s
xf.shape = s
assert_(not isMaskedArray(x))
assert_(isMaskedArray(xm))
assert_equal(shape(xm), s)
assert_equal(xm.shape, s)
assert_equal(xm.size, reduce(lambda x, y: x * y, s))
assert_equal(count(xm), len(m1) - reduce(lambda x, y: x + y, m1))
assert_(eq(xm, xf))
assert_(eq(filled(xm, 1.e20), xf))
assert_(eq(x, xm))
self.setup()
def strip_mask(arr, fill_value=np.NaN):
"""
Take a masked array and return its data(filled) + mask.
"""
if ma.isMaskedArray(arr):
mask = np.ma.getmaskarray(arr)
arr = np.ma.filled(arr, fill_value)
return mask, arr
else:
return arr
def __init__(self,vals,vals_dmin,vals_dmax,mask=ma.nomask):
super(UncertContainer, self).__init__()
# If input data already masked arrays extract unmasked data
if ma.isMaskedArray(vals):
vals = vals.data
if ma.isMaskedArray(vals_dmin):
vals_dmin = vals_dmin.data
if ma.isMaskedArray(vals_dmax):
vals_dmax = vals_dmax.data
# Adjust negative values
ineg = np.where(vals_dmin <= 0.0)
vals_dmin[ineg] = TOL*vals[ineg]
# Calculate weight based on fractional uncertainty
diff = vals_dmax - vals_dmin
diff_m = ma.masked_where(vals_dmax == vals_dmin,diff)
self.vals = ma.masked_where(vals == 0.0,vals)
self.wt = (self.vals/diff_m)**2
self.uncert = diff_m/self.vals
self.wt.fill_value = np.inf
self.uncert.fill_vaule = np.inf
assert np.all(self.wt.mask == self.uncert.mask)
# Mask data if uncertainty is not finite or if any of the inputs were
# already masked
mm = ma.mask_or(self.wt.mask,mask)
self.vals.mask = mm
self.wt.mask = mm
self.uncert.mask = mm
self.dmin = ma.array(vals_dmin,mask=mm,fill_value=np.inf)
self.dmax = ma.array(vals_dmax,mask=mm,fill_value=np.inf)
self.mask = ma.getmaskarray(self.vals)
def generic_interp_hght(h, hght, field, log=False):
'''
Generic interpolation routine
Parameters
----------
h : number, numpy array
Height (m) of the level for which pressure is desired
hght : numpy array
The array of heights
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given height
'''
if ma.isMaskedArray(hght):
not_masked1 = ~hght.mask
else:
not_masked1 = np.ones(hght.shape)
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape)
not_masked = not_masked1 * not_masked2
if log:
return 10**np.interp(h,
hght[not_masked],
field[not_masked],
left=ma.masked,
right=ma.masked)
else:
return np.interp(h,
hght[not_masked],
field[not_masked],
left=ma.masked,
right=ma.masked)
def _weighted_mean_with_mdtol(data, weights, axis=None, mdtol=0):
"""
Return the weighted mean of an array over the specified axis
using the provided weights (if any) and a permitted fraction of
masked data.
Args:
* data (array-like):
Data to be averaged.
* weights (array-like):
An array of the same shape as the data that specifies the contribution
of each corresponding data element to the calculated mean.
Kwargs:
* axis (int or tuple of ints):
Axis along which the mean is computed. The default is to compute
the mean of the flattened array.
* mdtol (float):
Tolerance of missing data. The value returned in each element of the
returned array will be masked if the fraction of masked data exceeds
mdtol. This fraction is weighted by the `weights` array if one is
provided. mdtol=0 means no missing data is tolerated
while mdtol=1 will mean the resulting element will be masked if and
only if all the contributing elements of data are masked.
Defaults to 0.
Returns:
Numpy array (possibly masked) or scalar.
"""
if ma.is_masked(data):
res, unmasked_weights_sum = ma.average(data,
weights=weights,
axis=axis,
returned=True)
if mdtol < 1:
weights_sum = weights.sum(axis=axis)
frac_masked = 1 - np.true_divide(unmasked_weights_sum, weights_sum)
mask_pt = frac_masked > mdtol
if np.any(mask_pt) and not isinstance(res, ma.core.MaskedConstant):
if np.isscalar(res):
res = ma.masked
elif ma.isMaskedArray(res):
res.mask |= mask_pt
else:
res = ma.masked_array(res, mask=mask_pt)
else:
res = np.average(data, weights=weights, axis=axis)
return res
def test_load_global_xyzt_gems(self):
# Test loading single xyzt CF-netCDF file (multi-cube).
cubes = iris.load(
tests.get_data_path(
('NetCDF', 'global', 'xyz_t', 'GEMS_CO2_Apr2006.nc')))
self.assertCML(cubes, ('netcdf', 'netcdf_global_xyzt_gems.cml'))
# Check the masked array fill value is propogated through the data
# manager loading.
lnsp = cubes[1]
self.assertTrue(ma.isMaskedArray(lnsp.data))
self.assertEqual(-32767.0, lnsp.data.fill_value)
def is_string_like(obj):
'Return True if *obj* looks like a string'
if isinstance(obj, (str, unicode)): return True
# numpy strings are subclass of str, ma strings are not
if ma.isMaskedArray(obj):
if obj.ndim == 0 and obj.dtype.kind in 'SU':
return True
else:
return False
try: obj + ''
except: return False
return True
def set_mask(self,nq,mask,msk_value=1.E20):
"""Set a mask on an array.
"""
if ma.isMaskedArray(nq):
nq=npy.array(nq,subok=True)
q = nan_to_zero(nq)
a_msk = abs(mask-1)
a_msk = a_msk * msk_value
mq = q * mask
mq+= a_msk
#
return mq
def register(self, cube, axis=None):
"""
Determine whether the given source-cube is suitable for concatenation
with this :class:`_ProtoCube`.
Args:
* cube:
The :class:`iris.cube.Cube` source-cube candidate for
concatenation.
Kwargs:
* axis:
Seed the dimension of concatenation for the :class:`_ProtoCube`
rather than rely on negotiation with source-cubes.
Returns:
Boolean.
"""
# Verify and assert the nominated axis.
if axis is not None and self.axis is not None and self.axis != axis:
msg = 'Nominated axis [{}] is not equal ' \
'to negotiated axis [{}]'.format(axis, self.axis)
raise ValueError(msg)
# Check for compatible cube signatures.
cube_signature = _CubeSignature(cube)
match = self._cube_signature == cube_signature
# Check for compatible coordinate signatures.
if match:
coord_signature = _CoordSignature(cube_signature)
candidate_axis = self._coord_signature.candidate_axis(
coord_signature)
match = candidate_axis is not None and \
(candidate_axis == axis or axis is None)
# Check for compatible coordinate extents.
if match:
match = self._sequence(coord_signature.dim_extents[candidate_axis],
candidate_axis)
if match:
# Register the cube as a source-cube for this proto-cube.
self._add_skeleton(coord_signature, cube.data)
self._data_is_masked |= ma.isMaskedArray(cube.data)
# Declare the nominated axis of concatenation.
self._axis = candidate_axis
return match
def test_masked_if_fill_att(self):
file = self.f
#if verbose: print file
if verbose: print('testing MaskedArrayMode MaskedIfFillAtt')
v = file.variables['lat']
assert_equal(hasattr(v, '_FillValue'), False)
vm = v[:]
assert_equal(ma.isMaskedArray(vm), False)
print(type(vm), vm[0].__repr__())
v = file.variables['PT']
assert_equal(v._FillValue, 1e20)
vm = v[0, 0]
assert_equal(ma.isMaskedArray(vm), True)
try:
assert_equal(vm.get_fill_value(),
N.array([1e20], dtype='f')) # numpy 1.6.1rc1
except:
assert_equal(N.array([vm._fill_value], dtype='f'),
N.array([1e20], dtype='f')) # numpy 1.4.1
if verbose: print(type(vm), vm[0].__repr__())
file.close()
def test_load_global_xyzt_gems(self):
# Test loading single xyzt CF-netCDF file (multi-cube).
cubes = iris.load(
tests.get_data_path(
("NetCDF", "global", "xyz_t", "GEMS_CO2_Apr2006.nc")))
cubes = sorted(cubes, key=lambda cube: cube.name())
self.assertCML(cubes, ("netcdf", "netcdf_global_xyzt_gems.cml"))
# Check the masked array fill value is propogated through the data
# manager loading.
lnsp = cubes[1]
self.assertTrue(ma.isMaskedArray(lnsp.data))
self.assertEqual(-32767.0, lnsp.data.fill_value)
def r3_to_r2(phUnw):
""" Given a rank3 of ifgs, convert it to rank2 and a mask. Works with either masked arrays or just arrays.
Inputs:
phUnw | rank 3 array | n_ifgs x height x width
returns:
r2_data['ifgs'] | rank 2 array | ifgs as row vectors
r2_data['mask'] | rank 2 array
History:
2020/06/09 | MEG | Written
"""
import numpy as np
import numpy.ma as ma
if ma.isMaskedArray(phUnw):
n_pixels = len(
ma.compressed(phUnw[0, ])
) # if it's a masked array, get the number of non-masked pixels
mask = ma.getmask(phUnw)[
0, ] # get the mask, which is assumed to be constant through time
else:
n_pixels = len(np.ravel(phUnw[
0, ])) # or if a normal numpy array, just get the number of pixels
mask = np.zeros(phUnw[0, ].shape) # or make a blank mask
r2_ifgs = np.zeros(
(phUnw.shape[0], n_pixels)) # initiate to store ifgs as rows in
for ifg_n, ifg in enumerate(phUnw):
if ma.isMaskedArray(phUnw):
r2_ifgs[ifg_n, ] = ma.compressed(
ifg) # non masked pixels into row vectors
else:
r2_ifgs[ifg_n, ] = np.ravel(
ifg) # or all just pixles into row vectors
r2_data = {
'ifgs': r2_ifgs, # make into a dictionary.
'mask': mask
}
return r2_data
def recache(self):
#if self.axes is None: print 'recache no axes'
#else: print 'recache units', self.axes.xaxis.units, self.axes.yaxis.units
if ma.isMaskedArray(self._xorig) or ma.isMaskedArray(self._yorig):
x = ma.asarray(self.convert_xunits(self._xorig), float)
y = ma.asarray(self.convert_yunits(self._yorig), float)
x = ma.ravel(x)
y = ma.ravel(y)
else:
x = np.asarray(self.convert_xunits(self._xorig), float)
y = np.asarray(self.convert_yunits(self._yorig), float)
x = np.ravel(x)
y = np.ravel(y)
if len(x) == 1 and len(y) > 1:
x = x * np.ones(y.shape, float)
if len(y) == 1 and len(x) > 1:
y = y * np.ones(x.shape, float)
if len(x) != len(y):
raise RuntimeError('xdata and ydata must be the same length')
x = x.reshape((len(x), 1))
y = y.reshape((len(y), 1))
if ma.isMaskedArray(x) or ma.isMaskedArray(y):
self._xy = ma.concatenate((x, y), 1)
else:
self._xy = np.concatenate((x, y), 1)
self._x = self._xy[:, 0] # just a view
self._y = self._xy[:, 1] # just a view
# Masked arrays are now handled by the Path class itself
self._path = Path(self._xy)
self._transformed_path = TransformedPath(self._path,
self.get_transform())
self._invalid = False
def monotonic(array, strict=False, return_direction=False):
"""
Return whether the given 1d array is monotonic.
Note that, the array must not contain missing data.
Kwargs:
* strict (boolean)
Flag to enable strict monotonic checking
* return_direction (boolean)
Flag to change return behaviour to return (monotonic_status, direction)
Direction will be 1 for positive or -1 for negative. The direction is meaningless
if the array is not monotonic.
Returns:
* monotonic_status (boolean)
Whether the array was monotonic.
If the return_direction flag was given then the returned value will be:
``(monotonic_status, direction)``
"""
if array.ndim != 1 or len(array) <= 1:
raise ValueError(
'The array to check must be 1 dimensional and have more than 1 element.'
)
if ma.isMaskedArray(array) and ma.count_masked(array) != 0:
raise ValueError('The array to check contains missing data.')
# Identify the directions of the largest/most-positive and
# smallest/most-negative steps.
d = np.diff(array)
sign_max_d = np.sign(d[np.argmax(d)])
sign_min_d = np.sign(d[np.argmin(d)])
if strict:
monotonic = sign_max_d == sign_min_d and sign_max_d != 0
else:
monotonic = (sign_min_d < 0 and sign_max_d <= 0) or \
(sign_max_d > 0 and sign_min_d >= 0) or \
(sign_min_d == sign_max_d == 0)
if return_direction:
direction = sign_max_d or 1
return monotonic, direction
else:
return monotonic
def data(self, data):
"""
Replaces the currently managed data with the specified data, which must
be of an equivalent shape.
Note that, the only shape promotion permitted is for 0-dimensional
scalar data to be replaced with a single item 1-dimensional data.
Args:
* data:
The :class:`~numpy.ndarray` or :class:`~numpy.ma.core.MaskedArray`
real data, or :class:`~dask.array.core.Array` lazy data to be
managed.
"""
# Ensure we have numpy-like data.
if not (hasattr(data, 'shape') and hasattr(data, 'dtype')):
data = np.asanyarray(data)
# Determine whether the class instance has been created,
# as this method is called from within the __init__.
init_done = (self._lazy_array is not None
or self._real_array is not None)
if init_done and self.shape != data.shape:
# The _ONLY_ data reshape permitted is converting a 0-dimensional
# array i.e. self.shape == () into a 1-dimensional array of length
# one i.e. data.shape == (1,)
if self.shape or data.shape != (1, ):
emsg = 'Require data with shape {!r}, got {!r}.'
raise ValueError(emsg.format(self.shape, data.shape))
# Set lazy or real data, and reset the other.
if is_lazy_data(data):
self._lazy_array = data
self._real_array = None
else:
if not ma.isMaskedArray(data):
# Coerce input data to ndarray (including ndarray subclasses).
data = np.asarray(data)
if isinstance(data, ma.core.MaskedConstant):
# Promote to a masked array so that the fill-value is
# writeable to the data owner.
data = ma.array(data.data, mask=data.mask, dtype=data.dtype)
self._lazy_array = None
self._real_array = data
# Check the manager contract, as the managed data has changed.
self._assert_axioms()
def __init__(self, vertices, codes=None, _interpolation_steps=1):
"""
Create a new path with the given vertices and codes.
*vertices* is an Nx2 numpy float array, masked array or Python
sequence.
*codes* is an N-length numpy array or Python sequence of type
:attr:`matplotlib.path.Path.code_type`.
These two arrays must have the same length in the first
dimension.
If *codes* is None, *vertices* will be treated as a series of
line segments.
If *vertices* contains masked values, they will be converted
to NaNs which are then handled correctly by the Agg
PathIterator and other consumers of path data, such as
:meth:`iter_segments`.
*interpolation_steps* is used as a hint to certain projections,
such as Polar, that this path should be linearly interpolated
immediately before drawing. This attribute is primarily an
implementation detail and is not intended for public use.
"""
if ma.isMaskedArray(vertices):
vertices = vertices.astype(np.float_).filled(np.nan)
else:
vertices = np.asarray(vertices, np.float_)
if codes is not None:
codes = np.asarray(codes, self.code_type)
assert codes.ndim == 1
assert len(codes) == len(vertices)
if len(codes):
assert codes[0] == self.MOVETO
assert vertices.ndim == 2
assert vertices.shape[1] == 2
self.should_simplify = (
rcParams['path.simplify']
and (len(vertices) >= 128 and
(codes is None or np.all(codes <= Path.LINETO))))
self.simplify_threshold = rcParams['path.simplify_threshold']
self.has_nonfinite = not np.isfinite(vertices).all()
self.codes = codes
self.vertices = vertices
self._interpolation_steps = _interpolation_steps
def _remove_noise(self, array: np.ndarray, noise: np.ndarray,
keep_negative: bool, snr_limit: float) -> np.ndarray:
snr = array / utils.transpose(noise)
if self.range_corrected is False:
snr_scale_factor = 6
ind = self._get_altitude_ind()
snr[:, ind] *= snr_scale_factor
if ma.isMaskedArray(array) is False:
array = ma.masked_array(array)
if keep_negative is True:
array[np.abs(snr) < snr_limit] = ma.masked
else:
array[snr < snr_limit] = ma.masked
return array
def __init__(self, file, lat, lon, vars=None):
with nc(file) as f:
if vars is None:
vars = setdiff1d(f.variables, ['lat', 'lon', 'time'])
lats, lons = f.variables['lat'][:], f.variables['lon'][:]
if isMaskedArray(f.variables[vars[0]][0]):
mask = f.variables[vars[0]][
0].mask # pull mask from first variable, first time
else:
mask = zeros((len(lats), len(lons)))
latd = resize(lats, (len(lons), len(lats))).T - lat
lond = resize(lons, (len(lats), len(lons))) - lon
latd = masked_where(mask, latd)
lond = masked_where(mask, lond)
totd = latd**2 + lond**2
idx = where(totd == totd.min())
latidx, lonidx = idx[0][0], idx[1][0]
self.time = f.variables['time'][:]
tunits = f.variables['time'].units
ts = tunits.split('months since ')[1].split(' ')
yr0, mth0, day0 = [int(t) for t in ts[0].split('-')[0:3]]
if len(ts) > 1:
hr0, min0, sec0 = [int(t) for t in ts[1].split(':')[0:3]]
else:
hr0 = min0 = sec0 = 0
self.reftime = datetime(yr0, mth0, day0, hr0, min0, sec0)
nv, nt = len(vars), len(self.time)
self.data = zeros((nv, nt))
self.units = zeros(nv, dtype='|S32')
for i in range(nv):
if vars[i] in f.variables:
var = f.variables[vars[i]]
else:
vidx = foundVar(f.variables.keys(), vars[i])
var = f.variables[f.variables.keys()[vidx]]
self.data[i] = var[:, latidx, lonidx]
self.units[i] = var.units
self.vars = vars # store variable names
self.pridx = foundVar(vars, 'pr') # variable indices
self.maidx = foundVar(vars, 'tmax')
self.miidx = foundVar(vars, 'tmin')
def _write_vars2nc(nc: netCDF4.Dataset, cloudnet_variables: dict) -> None:
"""Iterates over Cloudnet instances and write to netCDF file."""
for obj in cloudnet_variables.values():
if ma.isMaskedArray(obj.data):
fill_value = netCDF4.default_fillvals[obj.data_type]
else:
fill_value = False
size = _get_dimensions(nc, obj.data)
nc_variable = nc.createVariable(obj.name, obj.data_type, size, zlib=True,
fill_value=fill_value)
nc_variable[:] = obj.data
for attr in obj.fetch_attributes():
setattr(nc_variable, attr, getattr(obj, attr))
def test_rotated_to_osgb(self):
# Rotated Pole data with large extent.
x = np.linspace(311.9, 391.1, 10)
y = np.linspace(-23.6, 24.8, 8)
u, v = uv_cubes(x, y)
ut, vt = rotate_winds(u, v, iris.coord_systems.OSGB())
# Ensure cells with discrepancies in magnitude are masked.
self.assertTrue(ma.isMaskedArray(ut.data))
self.assertTrue(ma.isMaskedArray(vt.data))
# Snapshot of mask with fixed tolerance of atol=2e-3
expected_mask = np.array(
[
[1, 1, 1, 0, 0, 0, 0, 0, 0, 1],
[1, 1, 1, 0, 0, 0, 0, 0, 0, 1],
[1, 1, 1, 1, 0, 0, 0, 0, 1, 1],
[1, 1, 1, 1, 0, 0, 0, 0, 1, 1],
[1, 1, 1, 1, 0, 0, 0, 0, 1, 1],
[1, 1, 1, 1, 1, 0, 0, 1, 1, 1],
[1, 1, 1, 1, 1, 0, 0, 1, 1, 1],
[1, 1, 1, 1, 1, 0, 0, 1, 1, 1],
],
np.bool_,
)
self.assertArrayEqual(expected_mask, ut.data.mask)
self.assertArrayEqual(expected_mask, vt.data.mask)
# Check unmasked values have sufficiently small error in mag.
expected_mag = np.sqrt(u.data**2 + v.data**2)
# Use underlying data to ignore mask in calculation.
res_mag = np.sqrt(ut.data.data**2 + vt.data.data**2)
# Calculate percentage error (note there are no zero magnitudes
# so we can divide safely).
anom = 100.0 * np.abs(res_mag - expected_mag) / expected_mag
self.assertTrue(anom[~ut.data.mask].max() < 0.1)
def _write_initial_data(f: netCDF4.Dataset, data: dict) -> None:
for key, array in data.items():
if key in SKIP_ME:
continue
fill_value = 0 if array.ndim > 1 and not ma.isMaskedArray(
array) else None
var = f.createVariable(METADATA[key].name,
_get_dtype(array),
_get_dim(f, array),
zlib=True,
complevel=3,
shuffle=False,
fill_value=fill_value)
var[:] = array
_set_attributes(var, key)
def generic_interp_pres(p, pres, field):
'''
Generic interpolation routine
Parameters
----------
p : number, numpy array
Pressure (hPa) of the level for which the field variable is desired
pres : numpy array
The array of pressure
field : numpy array
The variable which is being interpolated
log : bool
Flag to determine whether the 'field' variable is in log10 space
Returns
-------
Value of the 'field' variable at the given pressure
'''
if ma.isMaskedArray(pres):
not_masked1 = ~pres.mask
else:
not_masked1 = np.ones(pres.shape, dtype=bool)
not_masked1[:] = True
if ma.isMaskedArray(field):
not_masked2 = ~field.mask
else:
not_masked2 = np.ones(field.shape, dtype=bool)
not_masked2[:] = True
not_masked = not_masked1 * not_masked2
return np.interp(p,
pres[not_masked],
field[not_masked],
left=ma.masked,
right=ma.masked)
def _percentile(data, axis, percent, **kwargs):
# NB. scipy.stats.mstats.scoreatpercentile always works across just the
# first dimension of its input data, and returns a result that has one
# fewer dimension than the input.
# So shape=(3, 4, 5) -> shape(4, 5)
data = np.rollaxis(data, axis)
shape = data.shape[1:]
if shape:
data = data.reshape([data.shape[0], np.prod(shape)])
result = scipy.stats.mstats.scoreatpercentile(data, percent, **kwargs)
if not ma.isMaskedArray(data) and not ma.is_masked(result):
result = np.asarray(result)
if shape:
result = result.reshape(shape)
return result
def unwrap_masked(lon, centered=False, copy=True):
"""
Unwrap a sequence of longitudes or headings in degrees.
Parameters
----------
lon : array
Longtiudes or heading in degress. If masked output will also be
masked.
centered : bool, optional
Center the unwrapping as close to zero as possible.
copy : bool, optional.
True to return a copy, False will avoid a copy when possible.
Returns
-------
unwrap : array
Array of unwrapped longtitudes or headings, in degrees.
"""
masked_input = ma.isMaskedArray(lon)
if masked_input:
fill_value = lon.fill_value
# masked_invalid loses the original fill_value (ma bug, 2011/01/20)
lon = np.ma.masked_invalid(lon).astype(float)
if lon.ndim != 1:
raise ValueError("Only 1-D sequences are supported")
if lon.shape[0] < 2:
return lon
x = lon.compressed()
if len(x) < 2:
return lon
w = np.zeros(x.shape[0] - 1, int)
ld = np.diff(x)
np.putmask(w, ld > 180, -1)
np.putmask(w, ld < -180, 1)
x[1:] += (w.cumsum() * 360.0)
if centered:
x -= 360 * np.round(x.mean() / 360.0)
if lon.mask is ma.nomask:
lon[:] = x
else:
lon[~lon.mask] = x
if masked_input:
lon.fill_value = fill_value
return lon
else:
return lon.filled(np.nan)
def __init__(self, data, fill_value='none', realised_dtype=None):
"""
Create a data manager for the specified data.
Args:
* data:
The :class:`~numpy.ndarray` or :class:`~numpy.ma.core.MaskedArray`
real data, or :class:`~dask.array.core.Array` lazy data to be
managed.
Kwargs:
* fill_value:
The intended fill-value of :class:`~iris._data_manager.DataManager`
masked data. Note that, the fill-value is cast relative to the
dtype of the :class:`~iris._data_manager.DataManager`.
* realised_dtype:
The intended dtype of the specified lazy data, which must be
either integer or boolean. This is to handle the case of lazy
integer or boolean masked data.
"""
# Initialise the instance.
self._fill_value = None
self._lazy_array = None
self._real_array = None
self._realised_dtype = None
# Assign the data payload to be managed.
self.data = data
# Set the lazy data realised dtype, if appropriate.
self._realised_dtype_setter(realised_dtype)
default_fill_value = (isinstance(fill_value, six.string_types)
and fill_value == 'none')
# Set the fill-value, must be set after the realised dtype.
if ma.isMaskedArray(data) and default_fill_value:
self._propagate_masked_data_fill_value()
else:
self.fill_value = None if default_fill_value else fill_value
# Enforce the manager contract.
self._assert_axioms()
def make_bounds_discontiguous_at_point(cube, at_iy, at_ix):
"""
Meddle with the XY grid bounds of a cube to make the grid discontiguous.
Changes the points and bounds of a single gridcell, so that it becomes
discontinuous with the next gridcell to its right.
Also masks the cube data at the given point.
The cube must have bounded 2d 'x' and 'y' coordinates.
TODO: add a switch to make a discontiguity in the *y*-direction instead ?
"""
x_coord = cube.coord(axis='x')
y_coord = cube.coord(axis='y')
assert x_coord.shape == y_coord.shape
assert (coord.bounds.ndim == 3 and coord.shape[-1] == 4
for coord in (x_coord, y_coord))
# For both X and Y coord, move points + bounds to create a discontinuity.
def adjust_coord(coord):
pts, bds = coord.points, coord.bounds
# Fetch the 4 bounds (bottom-left, bottom-right, top-right, top-left)
bds_bl, bds_br, bds_tr, bds_tl = bds[at_iy, at_ix]
# Make a discontinuity "at" (iy, ix), by moving the right-hand edge of
# the cell to the midpoint of the existing left+right bounds.
new_bds_br = 0.5 * (bds_bl + bds_br)
new_bds_tr = 0.5 * (bds_tl + bds_tr)
bds_br, bds_tr = new_bds_br, new_bds_tr
bds[at_iy, at_ix] = [bds_bl, bds_br, bds_tr, bds_tl]
# Also reset the cell midpoint to the middle of the 4 new corners,
# in case having a midpoint outside the corners might cause a problem.
new_pt = 0.25 * sum([bds_bl, bds_br, bds_tr, bds_tl])
pts[at_iy, at_ix] = new_pt
# Write back the coord points+bounds (can only assign whole arrays).
coord.points, coord.bounds = pts, bds
adjust_coord(x_coord)
adjust_coord(y_coord)
# Also mask the relevant data point.
data = cube.data # N.B. fetch all the data.
if not ma.isMaskedArray(data):
# Promote to masked array, to avoid converting mask to NaN.
data = ma.masked_array(data)
data[at_iy, at_ix] = ma.masked
cube.data = data
def is_string_like(obj):
'Return True if *obj* looks like a string'
from numpy import ma
if isinstance(obj, six.string_types):
return True
# numpy strings are subclass of str, ma strings are not
if ma.isMaskedArray(obj):
if obj.ndim == 0 and obj.dtype.kind in 'SU':
return True
else:
return False
try:
obj + ''
except Exception:
return False
return True
def _propagate_masked_data_fill_value(self):
"""
Align the data manager fill-value with the real masked array
fill-value.
"""
data = self._real_array
if ma.isMaskedArray(data):
# Determine the default numpy fill-value.
np_fill_value = ma.masked_array(0, dtype=data.dtype).fill_value
if data.fill_value == np_fill_value:
# Never store the numpy default fill-value, rather
# represent this by clearing the data manager fill-value.
self.fill_value = None
else:
# Propagate the masked array fill-value to the data manager.
self.fill_value = data.fill_value
def as_data_frame(cube, copy=True):
"""
Convert a 2D cube to a Pandas DataFrame.
Args:
* cube - The cube to convert to a Pandas DataFrame.
Kwargs:
* copy - Whether to make a copy of the data.
Defaults to True. Must be True for masked data
and some data types (see notes below).
.. note::
This function will copy your data by default.
If you have a large array that cannot be copied,
make sure it is not masked and use copy=False.
.. note::
Pandas will sometimes make a copy of the array,
for example when creating from an int32 array.
Iris will detect this and raise an exception if copy=False.
"""
data = cube.data
if ma.isMaskedArray(data):
if not copy:
raise ValueError("Masked arrays must always be copied.")
data = data.astype('f').filled(np.nan)
elif copy:
data = data.copy()
index = columns = None
if cube.coords(dimensions=[0]):
index = _as_pandas_coord(cube.coord(dimensions=[0]))
if cube.coords(dimensions=[1]):
columns = _as_pandas_coord(cube.coord(dimensions=[1]))
data_frame = pandas.DataFrame(data, index, columns)
if not copy:
_assert_shared(data, data_frame)
return data_frame
def nhourly2day2monthly(a3in, nh, calc="mean",miss_in=None, miss_out=None):
# nh : n-hourly
# calc: "mean","sum"
# if miss_in !=None and miss_out==None, output will be filled with miss_in
#
dtype = a3in.dtype
per_day = int(24/nh)
nstep, ny, nx = a3in.shape
if nstep/per_day == 365:
year = 1999 # just for calc
elif nstep/per_day == 366:
year = 2000 # just for calc
if miss_in !=None:
a3in = ma.masked_equal(a3in, miss_in)
a3mon = np.empty([12,ny,nx], dtype=dtype)
for mon in range(1,12+1):
iday = 1
eday = calendar.monthrange(year,mon)[1]
idtime = datetime(year,mon,iday,0)
edtime = datetime(year,mon,eday,0)
idoy = ret_day_of_year(idtime)
edoy = ret_day_of_year(edtime)
ik = (idoy-1)
ek = (edoy-1+1)
a4tmp = a3in.reshape(-1,per_day,ny,nx)
if calc=="mean":
a2mon = a4tmp[ik:ek,:,:].mean(axis=1).mean(axis=0)
elif calc=="sum":
a2mon = a4tmp[ik:ek,:,:].mean(axis=1).sum(axis=0)
elif calc=="std":
a2mon = a4tmp[ik:ek,:,:].mean(axis=1).std(axis=0)
else:
print 'check calc', calc
if ma.isMaskedArray(a2mon):
if miss_out !=None:
a2mon = a2mon.filled(miss_out)
a3mon[mon-1,:,:] = a2mon
return a3mon
