def _mov_cov(x, y, span, ddof, dtype=None):
# helper function
denom = span - ddof
x = ma.asanyarray(x)
y = ma.asanyarray(y)
sum_prod = _mov_sum(x*y, span, dtype=dtype, type_num_double=True)
sum_x = _mov_sum(x, span, dtype=dtype, type_num_double=True)
sum_y = _mov_sum(y, span, dtype=dtype, type_num_double=True)
return sum_prod/denom - (sum_x * sum_y) / (span*denom)
def vec2comp(wdir, wspd, missing=MISSING):
'''
Convert direction and magnitude into U, V components
Parameters
----------
wdir : number, array_like
Angle in meteorological degrees
wspd : number, array_like
Magnitudes of wind vector (input units == output units)
missing : number (optional)
Optional missing parameter. If not given, assume default missing
value from sharppy.sharptab.constants.MISSING
Returns
-------
u : number, array_like (same as input)
U-component of the wind (units are the same as those of input speed)
v : number, array_like (same as input)
V-component of the wind (units are the same as those of input speed)
'''
if not QC(wdir) or not QC(wspd):
return ma.masked, ma.masked
wdir = ma.asanyarray(wdir).astype(np.float64)
wspd = ma.asanyarray(wspd).astype(np.float64)
wdir.set_fill_value(missing)
wspd.set_fill_value(missing)
assert wdir.shape == wspd.shape, 'wdir and wspd have different shapes'
if wdir.shape:
wdir[wdir == missing] = ma.masked
wspd[wspd == missing] = ma.masked
wdir[wspd.mask] = ma.masked
wspd[wdir.mask] = ma.masked
u, v = _vec2comp(wdir, wspd)
u[np.fabs(u) < TOL] = 0.
v[np.fabs(v) < TOL] = 0.
else:
if wdir == missing:
wdir = ma.masked
wspd = ma.masked
elif wspd == missing:
wdir = ma.masked
wspd = ma.masked
u, v = _vec2comp(wdir, wspd)
if ma.fabs(u) < TOL:
u = 0.
if ma.fabs(v) < TOL:
v = 0.
return u, v
def test_comp2vec_default_missing_val_array():
input_u = [MISSING, -10, 0, MISSING]
input_v= [MISSING, 0, 20, MISSING]
correct_wdir = [0, 90, 180, MISSING]
correct_wspd = [MISSING, 10, 20, 30]
correct_wdir = ma.asanyarray(correct_wdir).astype(np.float64)
correct_wspd = ma.asanyarray(correct_wspd).astype(np.float64)
correct_wdir[correct_wdir == MISSING] = ma.masked
correct_wspd[correct_wspd == MISSING] = ma.masked
correct_wdir[correct_wspd.mask] = ma.masked
correct_wspd[correct_wdir.mask] = ma.masked
correct_wdir.set_fill_value(MISSING)
correct_wspd.set_fill_value(MISSING)
returned_wdir, returned_wspd = utils.comp2vec(input_u, input_v)
npt.assert_almost_equal(returned_wdir, correct_wdir)
npt.assert_almost_equal(returned_wspd, correct_wspd)
def nan_mask(data: npt.ArrayLike) -> np.ndarray:
"""
Replaces any masked array values with NaNs.
As a consequence of filling the mask with NaNs, non-float arrays will be
cast to float.
Parameters
----------
data : ArrayLike
The masked array to be filled with NaNs.
Returns
-------
ndarray
The `data` with masked values replaced with NaNs.
Notes
-----
.. versionadded:: 0.1.0
"""
if np.ma.isMaskedArray(data):
if data.dtype.char not in np.typecodes["Float"]:
dmsg = (
f"converting from '{np.typename(data.dtype.char)}' "
f"to '{np.typename('f')}"
)
logger.debug(dmsg)
data = ma.asanyarray(data, dtype=float)
data = data.filled(np.nan)
return data
def compute_zscore(obs, random_stats):
"""
Parameters
----------
obs: numeral
original observation
random_stats : iterable
same observable in randomised versions
"""
random_stats = nma.asanyarray(random_stats)
nma.masked_invalid(random_stats, copy=False)
random_stats = numpy.ravel(random_stats)
if random_stats.size == 0:
return numpy.nan
mean = numpy.mean(random_stats.filled(0.0))
std = numpy.std(random_stats.filled(0.0))
nominator = obs- mean
if nominator == 0.0:
return nominator
if std == 0.0:
if nominator < 0.0:
return -numpy.inf
else:
return numpy.inf
else:
return (nominator / std)
def compute_zscore(obs, random_stats):
"""
Parameters
----------
obs: numeral
original observation
random_stats : iterable
same observable in randomised versions
"""
random_stats = nma.asanyarray(random_stats)
nma.masked_invalid(random_stats, copy=False)
random_stats = numpy.ravel(random_stats)
if random_stats.size == 0:
return numpy.nan
mean = numpy.mean(random_stats.filled(0.0))
std = numpy.std(random_stats.filled(0.0))
nominator = obs - mean
if nominator == 0.0:
return nominator
if std == 0.0:
if nominator < 0.0:
return -numpy.inf
else:
return numpy.inf
else:
return (nominator / std)
def _ttest_ind(Sample_mean_ArrayA,
Sample_var_ArrayA,
n_subjectsA,
Sample_mean_ArrayB,
Sample_var_ArrayB,
n_subjectsB,
equal_var=True,
sign=1):
if equal_var:
# force df to be an array for masked division not to throw a warning
df = ma.asanyarray(n_subjectsA + n_subjectsB - 2.0)
svar = ((n_subjectsA - 1) * Sample_var_ArrayA +
(n_subjectsB - 1) * Sample_var_ArrayB) / df
denom = ma.sqrt(
svar *
(1.0 / n_subjectsA + 1.0 / n_subjectsB)) # n-D computation here!
else:
vn1 = Sample_var_ArrayA / n_subjectsA
vn2 = Sample_var_ArrayB / n_subjectsB
df = (vn1 + vn2)**2 / (vn1**2 / (n_subjectsA - 1) + vn2**2 /
(n_subjectsB - 1))
# If df is undefined, variances are zero.
# It doesn't matter what df is as long as it is not NaN.
df = np.where(np.isnan(df), 1, df)
denom = ma.sqrt(vn1 + vn2)
with np.errstate(divide='ignore', invalid='ignore'):
ttest_ind = (Sample_mean_ArrayA - Sample_mean_ArrayB) * sign / denom
pvalues = special.betainc(0.5 * df, 0.5,
df / (df + ttest_ind * ttest_ind)).reshape(
ttest_ind.shape)
# ttest_ind, pvalues = ma.filled(ttest_ind), ma.filled(pvalues)
return ttest_ind, pvalues
def test_comp2vec_default_missing_val_array():
input_u = [MISSING, -10, 0, MISSING]
input_v = [MISSING, 0, 20, MISSING]
correct_wdir = [0, 90, 180, MISSING]
correct_wspd = [MISSING, 10, 20, 30]
correct_wdir = ma.asanyarray(correct_wdir).astype(np.float64)
correct_wspd = ma.asanyarray(correct_wspd).astype(np.float64)
correct_wdir[correct_wdir == MISSING] = ma.masked
correct_wspd[correct_wspd == MISSING] = ma.masked
correct_wdir[correct_wspd.mask] = ma.masked
correct_wspd[correct_wdir.mask] = ma.masked
correct_wdir.set_fill_value(MISSING)
correct_wspd.set_fill_value(MISSING)
returned_wdir, returned_wspd = utils.comp2vec(input_u, input_v)
npt.assert_almost_equal(returned_wdir, correct_wdir)
npt.assert_almost_equal(returned_wspd, correct_wspd)
def probability_bincount(obs, drop=True):
"""
Normalise the observed bincount from `obs` to sum up to unity.
Parameters
----------
obs: iterable
An iterable of integer numerals.
drop: bool (optional)
Determines whether or not bins with zero events are dropped from the
resulting list.
Returns
-------
A frequency distribution that is normalised to unity.
"""
obs = nma.asanyarray(obs, dtype="int32")
nma.masked_invalid(obs, copy=False)
obs = numpy.ravel(obs)
if obs.size == 0:
return list()
total = float(len(obs))
freq = numpy.bincount(obs[~obs.mask])
points = [(k, val / total) for (k, val) in enumerate(freq) if val > 0]
return points
def test_vec2comp_default_missing_val_array():
input_wdir = [0, 90, 180, MISSING]
input_wspd = [MISSING, 10, 20, 30]
correct_u = [MISSING, -10, 0, MISSING]
correct_v = [MISSING, 0, 20, MISSING]
correct_u = ma.asanyarray(correct_u).astype(np.float64)
correct_v = ma.asanyarray(correct_v).astype(np.float64)
correct_u[correct_u == MISSING] = ma.masked
correct_v[correct_v == MISSING] = ma.masked
correct_u[correct_v.mask] = ma.masked
correct_v[correct_u.mask] = ma.masked
correct_u.set_fill_value(MISSING)
correct_v.set_fill_value(MISSING)
returned_u, returned_v = utils.vec2comp(input_wdir, input_wspd)
npt.assert_almost_equal(returned_u, correct_u)
npt.assert_almost_equal(returned_v, correct_v)
def simple_3d_mask():
"""
Returns an abstract three dimensional cube that has data masked.
>>>print simple_3d_mask()
thingness / (1) (wibble: 2; latitude: 3; longitude: 4)
Dimension coordinates:
wibble x - -
latitude - x -
longitude - - x
>>> print simple_3d_mask().data
[[[-- -- -- --]
[-- -- -- --]
[-- 9 10 11]]
[[12 13 14 15]
[16 17 18 19]
[20 21 22 23]]]
"""
cube = simple_3d()
cube.data = ma.asanyarray(cube.data)
cube.data = ma.masked_less_equal(cube.data, 8.)
return cube
def test_vec2comp_default_missing_val_array():
input_wdir = [0, 90, 180, MISSING]
input_wspd = [MISSING, 10, 20, 30]
correct_u = [MISSING, -10, 0, MISSING]
correct_v= [MISSING, 0, 20, MISSING]
correct_u = ma.asanyarray(correct_u).astype(np.float64)
correct_v = ma.asanyarray(correct_v).astype(np.float64)
correct_u[correct_u == MISSING] = ma.masked
correct_v[correct_v == MISSING] = ma.masked
correct_u[correct_v.mask] = ma.masked
correct_v[correct_u.mask] = ma.masked
correct_u.set_fill_value(MISSING)
correct_v.set_fill_value(MISSING)
returned_u, returned_v = utils.vec2comp(input_wdir, input_wspd)
npt.assert_almost_equal(returned_u, correct_u)
npt.assert_almost_equal(returned_v, correct_v)
def simple_3d_mask():
"""
Returns an abstract three dimensional cube that has data masked.
>>> print(simple_3d_mask())
thingness / (1) (wibble: 2; latitude: 3; longitude: 4)
Dimension coordinates:
wibble x - -
latitude - x -
longitude - - x
>>> print(simple_3d_mask().data)
[[[-- -- -- --]
[-- -- -- --]
[-- 9 10 11]]
[[12 13 14 15]
[16 17 18 19]
[20 21 22 23]]]
"""
cube = simple_3d()
cube.data = ma.asanyarray(cube.data)
cube.data = ma.masked_less_equal(cube.data, 8.)
return cube
def probability_bincount(obs, drop=True):
"""
Normalise the observed bincount from `obs` to sum up to unity.
Parameters
----------
obs: iterable
An iterable of integer numerals.
drop: bool (optional)
Determines whether or not bins with zero events are dropped from the
resulting list.
Returns
-------
A frequency distribution that is normalised to unity.
"""
obs = nma.asanyarray(obs, dtype="int32")
nma.masked_invalid(obs, copy=False)
obs = numpy.ravel(obs)
if obs.size == 0:
return list()
total = float(len(obs))
freq = numpy.bincount(obs[~obs.mask])
points = [(k, val / total) for (k, val) in enumerate(freq) if val > 0]
return points
def crop(self, doy, depth, lat, lon, var):
""" Crop a subset of the dataset for each var
Given doy, depth, lat and lon, it returns the smallest subset
that still contains the requested coordinates inside it.
It handels special cases like a region around greenwich and
the international date line.
Accepts 0 to 360 and -180 to 180 longitude reference.
It extends time and longitude coordinates, so simplify the use
of series. For example, a ship track can be requested with
a longitude sequence like [352, 358, 364, 369, 380], and
the equivalent for day of year above 365.
"""
dims, idx = cropIndices(self.dims, lat, lon, depth)
dims['time'] = np.atleast_1d(doy)
idx['tn'] = np.arange(dims['time'].size)
# Temporary solution. Create an object for CARS dataset
xn = idx['xn']
yn = idx['yn']
zn = idx['zn']
tn = idx['tn']
subset = {}
for v in var:
if v == 'mn':
mn = []
for d in doy:
t = 2 * np.pi * d/366
# Naive solution
# FIXME: This is not an efficient solution.
value = self.ncs[0]['mean'][:, yn, xn]
value[:64] += self.ncs[0]['an_cos'][:, yn, xn] * np.cos(t) + \
self.ncs[0]['an_sin'][:, yn, xn] * np.sin(t)
value[:55] += self.ncs[0]['sa_cos'][:, yn, xn] * np.cos(2*t) + \
self.ncs[0]['sa_sin'][:, yn, xn] * np.sin(2*t)
mn.append(value[zn])
subset['mn'] = ma.asanyarray(mn)
else:
subset[v] = ma.asanyarray(
doy.size * [self[v][zn, yn, xn]])
return subset, dims
def test_comp2vec_user_missing_val_array():
missing = 50
input_u = [missing, -10, 0, missing]
input_v= [missing, 0, 20, missing]
correct_wdir = [0, 90, 180, missing]
correct_wspd = [missing, 10, 20, 30]
correct_wdir = ma.asanyarray(correct_wdir).astype(np.float64)
correct_wspd = ma.asanyarray(correct_wspd).astype(np.float64)
correct_wdir[correct_wdir == missing] = ma.masked
correct_wspd[correct_wspd == missing] = ma.masked
correct_wdir[correct_wspd.mask] = ma.masked
correct_wspd[correct_wdir.mask] = ma.masked
correct_wdir.set_fill_value(missing)
correct_wspd.set_fill_value(missing)
returned_wdir, returned_wspd = utils.comp2vec(input_u, input_v)
npt.assert_almost_equal(returned_wdir, correct_wdir)
npt.assert_almost_equal(returned_wspd, correct_wspd)
def test_vec2comp_user_missing_val_array():
missing = 50
input_wdir = [0, 90, 180, missing]
input_wspd = [missing, 10, 20, 30]
correct_u = [missing, -10, 0, missing]
correct_v= [missing, 0, 20, missing]
correct_u = ma.asanyarray(correct_u).astype(np.float64)
correct_v = ma.asanyarray(correct_v).astype(np.float64)
correct_u[correct_u == missing] = ma.masked
correct_v[correct_v == missing] = ma.masked
correct_u[correct_v.mask] = ma.masked
correct_v[correct_u.mask] = ma.masked
correct_u.set_fill_value(missing)
correct_v.set_fill_value(missing)
returned_u, returned_v = utils.vec2comp(input_wdir, input_wspd, missing)
npt.assert_almost_equal(returned_u, correct_u)
npt.assert_almost_equal(returned_v, correct_v)
def comp2vec(u, v, missing=MISSING):
'''
Convert U, V components into direction and magnitude
Parameters
----------
u : number, array_like
U-component of the wind
v : number, array_like
V-component of the wind
missing : number (optional)
Optional missing parameter. If not given, assume default missing
value from sharppy.sharptab.constants.MISSING
Returns
-------
wdir : number, array_like (same as input)
Angle in meteorological degrees
wspd : number, array_like (same as input)
Magnitudes of wind vector (input units == output units)
'''
if not QC(u) or not QC(v):
return ma.masked, ma.masked
u = ma.asanyarray(u).astype(np.float64)
v = ma.asanyarray(v).astype(np.float64)
u.set_fill_value(missing)
v.set_fill_value(missing)
wdir = np.degrees(np.arctan2(-u, -v))
if wdir.shape:
u[u == missing] = ma.masked
v[v == missing] = ma.masked
wdir[u.mask] = ma.masked
wdir[v.mask] = ma.masked
wdir[wdir < 0] += 360
wdir[np.fabs(wdir) < TOL] = 0.
else:
if u == missing or v == missing:
return ma.masked, ma.masked
if wdir < 0:
wdir += 360
if np.fabs(wdir) < TOL:
wdir = 0.
return wdir, mag(u, v)
def test_mag_array():
rt2 = np.sqrt(2)
input_u = [5, 10, 15]
input_v = [5, 10, 15]
correct_answer = [5*rt2, 10*rt2, 15*rt2]
correct_answer = ma.asanyarray(correct_answer)
returned_answer = utils.mag(input_u, input_v)
npt.assert_almost_equal(returned_answer, correct_answer)
def test_mag_array():
rt2 = np.sqrt(2)
input_u = [5, 10, 15]
input_v = [5, 10, 15]
correct_answer = [5 * rt2, 10 * rt2, 15 * rt2]
correct_answer = ma.asanyarray(correct_answer)
returned_answer = utils.mag(input_u, input_v)
npt.assert_almost_equal(returned_answer, correct_answer)
def test_comp2vec_user_missing_val_array():
missing = 50
input_u = [missing, -10, 0, missing]
input_v = [missing, 0, 20, missing]
correct_wdir = [0, 90, 180, missing]
correct_wspd = [missing, 10, 20, 30]
correct_wdir = ma.asanyarray(correct_wdir).astype(np.float64)
correct_wspd = ma.asanyarray(correct_wspd).astype(np.float64)
correct_wdir[correct_wdir == missing] = ma.masked
correct_wspd[correct_wspd == missing] = ma.masked
correct_wdir[correct_wspd.mask] = ma.masked
correct_wspd[correct_wdir.mask] = ma.masked
correct_wdir.set_fill_value(missing)
correct_wspd.set_fill_value(missing)
returned_wdir, returned_wspd = utils.comp2vec(input_u, input_v)
npt.assert_almost_equal(returned_wdir, correct_wdir)
npt.assert_almost_equal(returned_wspd, correct_wspd)
def test_vec2comp_user_missing_val_array():
missing = 50
input_wdir = [0, 90, 180, missing]
input_wspd = [missing, 10, 20, 30]
correct_u = [missing, -10, 0, missing]
correct_v = [missing, 0, 20, missing]
correct_u = ma.asanyarray(correct_u).astype(np.float64)
correct_v = ma.asanyarray(correct_v).astype(np.float64)
correct_u[correct_u == missing] = ma.masked
correct_v[correct_v == missing] = ma.masked
correct_u[correct_v.mask] = ma.masked
correct_v[correct_u.mask] = ma.masked
correct_u.set_fill_value(missing)
correct_v.set_fill_value(missing)
returned_u, returned_v = utils.vec2comp(input_wdir, input_wspd, missing)
npt.assert_almost_equal(returned_u, correct_u)
npt.assert_almost_equal(returned_v, correct_v)
def comp2vec(u, v, missing=MISSING):
'''
Convert U, V components into direction and magnitude
Parameters
----------
u : number, array_like
U-component of the wind
v : number, array_like
V-component of the wind
missing : number (optional)
Optional missing parameter. If not given, assume default missing
value from sharppy.sharptab.constants.MISSING
Returns
-------
wdir : number, array_like (same as input)
Angle in meteorological degrees
wspd : number, array_like (same as input)
Magnitudes of wind vector (input units == output units)
'''
if not QC(u) or not QC(v):
return ma.masked, ma.masked
u = ma.asanyarray(u).astype(np.float64)
v = ma.asanyarray(v).astype(np.float64)
u.set_fill_value(missing)
v.set_fill_value(missing)
wdir = np.degrees(np.arctan2(-u, -v))
if wdir.shape:
u[u == missing] = ma.masked
v[v == missing] = ma.masked
wdir[u.mask] = ma.masked
wdir[v.mask] = ma.masked
wdir[wdir < 0] += 360
wdir[np.fabs(wdir) < TOL] = 0.
else:
if u == missing or v == missing:
return ma.masked, ma.masked
if wdir < 0:
wdir += 360
if np.fabs(wdir) < TOL:
wdir = 0.
return wdir, mag(u, v)
def test_mag_default_missing_array():
rt2 = np.sqrt(2)
input_u = [MISSING, 10, 20, 30, 40]
input_v = [0, 10, 20, 30, MISSING]
correct_answer = [MISSING, 10*rt2, 20*rt2, 30*rt2, MISSING]
correct_answer = ma.asanyarray(correct_answer).astype(np.float64)
correct_answer[correct_answer == MISSING] = ma.masked
returned_answer = utils.mag(input_u, input_v)
npt.assert_almost_equal(returned_answer, correct_answer)
def _chk_asarray(a, axis):
# Always returns a masked array, raveled for axis=None
a = ma.asanyarray(a)
if axis is None:
a = ma.ravel(a)
outaxis = 0
else:
outaxis = axis
return a, outaxis
def ttest_ind(a, b, axis=0, equal_var=True):
"""
Calculates the T-test for the means of two independent samples of scores.
Parameters
----------
a, b : array_like
The arrays must have the same shape, except in the dimension
corresponding to `axis` (the first, by default).
axis : int or None, optional
Axis along which to compute test. If None, compute over the whole
arrays, `a`, and `b`.
equal_var : bool, optional
If True, perform a standard independent 2 sample test that assumes equal
population variances.
If False, perform Welch's t-test, which does not assume equal population
variance.
.. versionadded:: 0.17.0
Returns
-------
statistic : float or array
The calculated t-statistic.
pvalue : float or array
The two-tailed p-value.
Notes
-----
For more details on `ttest_ind`, see `stats.ttest_ind`.
"""
a, b, axis = _chk2_asarray(a, b, axis)
if a.size == 0 or b.size == 0:
return 'One of the vector is empty'
(mean1, mean2) = (a.mean(axis), b.mean(axis))
(var1, var2) = (a.var(axis=axis, ddof=1), b.var(axis=axis, ddof=1))
(n1, n2) = (a.count(axis), b.count(axis))
if equal_var:
# force df to be an array for masked division not to throw a warning
df = ma.asanyarray(n1 + n2 - 2.0)
svar = ((n1 - 1) * var1 + (n2 - 1) * var2) / df
denom = ma.sqrt(svar * (1.0 / n1 + 1.0 / n2)) # n-D computation here!
else:
vn1 = var1 / n1
vn2 = var2 / n2
with np.errstate(divide='ignore', invalid='ignore'):
df = (vn1 + vn2)**2 / (vn1**2 / (n1 - 1) + vn2**2 / (n2 - 1))
# If df is undefined, variances are zero.
# It doesn't matter what df is as long as it is not NaN.
df = np.where(np.isnan(df), 1, df)
denom = ma.sqrt(vn1 + vn2)
with np.errstate(divide='ignore', invalid='ignore'):
t = (mean1 - mean2) / denom
probs = special.betainc(0.5 * df, 0.5, df / (df + t * t)).reshape(t.shape)
return Ttest_ind(t, probs.squeeze())
def test_mag_default_missing_array():
rt2 = np.sqrt(2)
input_u = [MISSING, 10, 20, 30, 40]
input_v = [0, 10, 20, 30, MISSING]
correct_answer = [MISSING, 10 * rt2, 20 * rt2, 30 * rt2, MISSING]
correct_answer = ma.asanyarray(correct_answer).astype(np.float64)
correct_answer[correct_answer == MISSING] = ma.masked
returned_answer = utils.mag(input_u, input_v)
npt.assert_almost_equal(returned_answer, correct_answer)
def adaptive_distribution(obs,
binwidth,
limits=None,
factor=2.0,
k_max=21,
right_most=3):
"""
An adaptive binning technique that overlays multiple histograms with
increasing bin widths.
References
----------
[1] Liebovitch, L. S., A. T. Todorov, M. Zochowski, D. Scheurle, L. Colgin,
M. A. Wood, K. A. Ellenbogen, J. M. Herre, and R. C. Bernstein. 1999.
``Nonlinear Properties of Cardiac Rhythm Abnormalities.''
Physical Review E 59 (3): 3312–3319.
"""
obs = nma.asanyarray(obs)
nma.masked_invalid(obs, copy=False)
obs = numpy.ravel(obs)
if obs.size == 0:
return list()
if limits is None:
mn = obs.min()
mx = obs.max()
else:
(mn, mx) = limits
offset = binwidth * 0.5
if limits is None:
limits = (mn - offset, mx + offset)
num_bins = numpy.floor((mx - mn) / binwidth)
freq = numpy.histogram(obs[~obs.mask], num_bins, range=limits)[0]
total = float(freq.sum())
print(0)
print(binwidth)
print(freq[0:right_most + 1])
results = list()
iteration = 1
while len(freq) > right_most and not (freq[1:right_most] == 0).any():
for (k, count) in enumerate(freq[1:]):
if count == 0 or k == k_max:
break
results.append(
((k + 0.5) * binwidth + limits[0], count / (binwidth * total)))
binwidth *= factor
iteration += 1
# new loop
num_bins = numpy.floor((mx - mn) / binwidth)
freq = numpy.histogram(obs, num_bins, range=limits)[0]
total = float(freq.sum())
print(iteration)
print(binwidth)
print(freq[0:right_most + 1])
return sorted(results)
def _jprimes(x, i, x_bounds=None):
"""
Helper function to return the j' indices for the master curve fit
This function is a helper function for :py:func:`quality`. It is not
supposed to be called directly.
Parameters
----------
x : mapping to ndarrays
The x values.
i : int
The row index (finite size index)
x_bounds : 2-tuple, optional
bounds on x values
Returns
-------
ret : mapping to ndarrays
Has the same keys and shape as `x`.
Its element ``ret[i'][j]`` is the j' such that :math:`x_{i'j'} \leq
x_{ij} < x_{i'(j'+1)}`.
If no such j' exists, the element is np.nan.
Convert the element to int to use as an index.
"""
j_primes = -np.ones_like(x)
try:
x_masked = ma.masked_outside(x, x_bounds[0], x_bounds[1])
except (TypeError, IndexError):
x_masked = ma.asanyarray(x)
k, n = x.shape
# indices of lower and upper bounds
edges = ma.notmasked_edges(x_masked, axis=1)
x_lower = np.zeros(k, dtype=int)
x_upper = np.zeros(k, dtype=int)
x_lower[edges[0][0]] = edges[0][-1]
x_upper[edges[-1][0]] = edges[-1][-1]
for i_prime in range(k):
if i_prime == i:
j_primes[i_prime][:] = np.nan
continue
jprimes = np.searchsorted(x[i_prime], x[i],
side='right').astype(float) - 1
jprimes[np.logical_or(jprimes < x_lower[i_prime],
jprimes >= x_upper[i_prime])] = np.nan
j_primes[i_prime][:] = jprimes
return j_primes
def test_mag_user_missing_array():
missing = 50
rt2 = np.sqrt(2)
input_u = [missing, 10, 20, 30, 40]
input_v = [0, 10, 20, 30, missing]
correct_answer = [missing, 10*rt2, 20*rt2, 30*rt2, missing]
correct_answer = ma.asanyarray(correct_answer).astype(np.float64)
correct_answer[correct_answer == missing] = ma.masked
returned_answer = utils.mag(input_u, input_v, missing)
npt.assert_almost_equal(returned_answer, correct_answer)
def test_mag_user_missing_array():
missing = 50
rt2 = np.sqrt(2)
input_u = [missing, 10, 20, 30, 40]
input_v = [0, 10, 20, 30, missing]
correct_answer = [missing, 10 * rt2, 20 * rt2, 30 * rt2, missing]
correct_answer = ma.asanyarray(correct_answer).astype(np.float64)
correct_answer[correct_answer == missing] = ma.masked
returned_answer = utils.mag(input_u, input_v, missing)
npt.assert_almost_equal(returned_answer, correct_answer)
def adaptive_distribution(obs, binwidth, limits=None, factor=2.0, k_max=21, right_most=3):
"""
An adaptive binning technique that overlays multiple histograms with
increasing bin widths.
References
----------
[1] Liebovitch, L. S., A. T. Todorov, M. Zochowski, D. Scheurle, L. Colgin,
M. A. Wood, K. A. Ellenbogen, J. M. Herre, and R. C. Bernstein. 1999.
``Nonlinear Properties of Cardiac Rhythm Abnormalities.''
Physical Review E 59 (3): 3312–3319.
"""
obs = nma.asanyarray(obs)
nma.masked_invalid(obs, copy=False)
obs = numpy.ravel(obs)
if obs.size == 0:
return list()
if limits is None:
mn = obs.min()
mx = obs.max()
else:
(mn, mx) = limits
offset = binwidth * 0.5
if limits is None:
limits = (mn - offset, mx + offset)
num_bins = numpy.floor((mx - mn) / binwidth)
freq = numpy.histogram(obs[~obs.mask], num_bins, range=limits)[0]
total = float(freq.sum())
print 0
print binwidth
print freq[0:right_most + 1]
results = list()
iteration = 1
while len(freq) > right_most and not (freq[1:right_most] == 0).any():
for (k, count) in enumerate(freq[1:]):
if count == 0 or k == k_max:
break
results.append(((k + 0.5) * binwidth + limits[0], count / (binwidth * total)))
binwidth *= factor
iteration += 1
# new loop
num_bins = numpy.floor((mx - mn) / binwidth)
freq = numpy.histogram(obs, num_bins, range=limits)[0]
total = float(freq.sum())
print iteration
print binwidth
print freq[0:right_most + 1]
return sorted(results)
def __init__(self, parentNode, name, description=None,
title="", filters=None,
expectedrows=EXPECTED_ROWS_TABLE,
chunkshape=None, byteorder=None, _log=True):
new = description is None
if not new:
maskedarray = ma.asanyarray(description)
description = tabulate(maskedarray)
Table.__init__(self, parentNode, name,
description=description, title=title,
filters=filters,
expectedrows=expectedrows,
chunkshape=chunkshape, byteorder=byteorder,
_log=_log)
if not new:
self.attrs.special_attrs = self._update_special_attrs(maskedarray)
return
def __getitem__(self, item):
""" t, z, y, x
"""
tn, zn, yn, xn = item
#if type(zn) is not slice:
# zn = slice(zn, zn+1)
#zn_an = slice(zn.start, min(64, zn.stop), zn.step)
#zn_sa = slice(zn.start, min(55, zn.stop), zn.step)
output = []
d = 2 * np.pi * (np.arange(1, 367)[tn])/366
for t in np.atleast_1d(d):
tmp = self.nc['mean'][:, yn, xn]
tmp[:64] += self.nc['an_cos'][:, yn, xn] * np.cos(t) + \
self.nc['an_sin'][:, yn, xn] * np.sin(t)
tmp[:55] += self.nc['sa_cos'][:, yn, xn] * np.cos(2*t) + \
self.nc['sa_sin'][:, yn, xn] * np.sin(2*t)
output.append(tmp[zn])
return ma.asanyarray(output)
def ttest_1samp(population_mean, a, axis=0):
"""
Calculates the T-test for the mean of ONE group of scores.
Parameters
----------
a : array_like
sample observation
population_mean : float or array_like
expected value in null hypothesis, if array_like than it must have the
same shape as `a` excluding the axis dimension
axis : int or None, optional
Axis along which to compute test. If None, compute over the whole
array `a`.
Returns
-------
statistic : float or array
t-statistic
pvalue : float or array
two-tailed p-value
Notes
-----
For more details on `ttest_1samp`, see `stats.ttest_1samp`.
"""
a, axis = _chk_asarray(a, axis)
if a.size == 0:
return 'Empty Array'
sample_mean = a.mean(axis=axis)
sample_var = a.var(axis=axis, ddof=1)
sample_count = a.count(axis=axis)
# force df to be an array for masked division not to throw a warning
df = ma.asanyarray(sample_count - 1.0)
svar = ((sample_count - 1.0) * sample_var) / df
with np.errstate(divide='ignore', invalid='ignore'):
t = (sample_mean - population_mean) / ma.sqrt(svar / sample_count)
prob = special.betainc(0.5 * df, 0.5, df / (df + t * t))
return Ttest_1sampResult(t, prob)
def crop(self, doy, depth, lat, lon, var):
""" Crop a subset of the dataset for each var
Given doy, depth, lat and lon, it returns the smallest subset
that still contains the requested coordinates inside it.
It handels special cases like a region around greenwich and
the international date line.
Accepts 0 to 360 and -180 to 180 longitude reference.
It extends time and longitude coordinates, so simplify the use
of series. For example, a ship track can be requested with
a longitude sequence like [352, 358, 364, 369, 380], and
the equivalent for day of year above 365.
"""
dims, idx = cropIndices(self.dims, lat, lon, depth, doy)
subset = {}
for v in var:
subset[v] = ma.asanyarray([
self.ncs[tnn][v][0, idx['zn'], idx['yn'], idx['xn']] \
for tnn in idx['tn']])
return subset, dims
def crop(self, doy, depth, lat, lon, var):
""" Crop a subset of the dataset for each var
Given doy, depth, lat and lon, it returns the smallest subset
that still contains the requested coordinates inside it.
It handels special cases like a region around greenwich and
the international date line.
Accepts 0 to 360 and -180 to 180 longitude reference.
It extends time and longitude coordinates, so simplify the use
of series. For example, a ship track can be requested with
a longitude sequence like [352, 358, 364, 369, 380], and
the equivalent for day of year above 365.
"""
dims, idx = cropIndices(self.dims, lat, lon, depth, doy)
subset = {}
for v in var:
subset[v] = ma.asanyarray([
self.ncs[tnn][v][0, idx['zn'], idx['yn'], idx['xn']] \
for tnn in idx['tn']])
return subset, dims
def frequency_distribution(obs, num_bins=30, binwidth=None, limits=None,
weights=None, drop=True):
"""
Takes a number of events and counts the frequency of these events falling into
either a fixed number of equi-distant bins over the given range or bins of
fixed width over the range.
Parameters
----------
obs: iterable
An aggregation of numerical observations.
num_bins: int (optional)
The number of equi-distant bins used over the numeric range.
binwidth: float (optional)
Desired width of the bins. This overrides `num_bins`.
limits: tuple (lower, upper) (optional)
The lower and upper values for the range of the distribution.
If no value is given, a range slightly larger then the range of the
values in `obs` is used. Specifically ``(a.min() - s, a.max() + s)``,
where ``s = (1/2)(a.max() - a.min()) / num_bins``.
weights: iterable (optional)
Weights for the data points in `obs`. The default is `None` weighting
all points equal.
drop: bool (optional)
Determines whether or not bins with zero events are dropped from the
resulting list.
Returns
-------
A list of pairs, the first entry is the centre of the bin borders at which
the frequency of an event is measured, and the second entry is the frequency
itself.
"""
obs = nma.asanyarray(obs)
nma.masked_invalid(obs, copy=False)
obs = numpy.ravel(obs)
if obs.size == 0:
return list()
if limits is None:
mn = obs.min()
mx = obs.max()
else:
(mn, mx) = limits
if binwidth:
binwidth = float(binwidth)
num_bins = numpy.floor((mx - mn) / binwidth)
else:
num_bins = int(num_bins)
binwidth = (mx - mn) / num_bins
offset = binwidth * 0.5
if limits is None:
limits = (mn - offset, mx + offset)
# extend the numeric range slightly beyond the data range
(freq, bins) = numpy.histogram(obs[~obs.mask], bins=num_bins, range=limits,
weights=weights)
if drop:
points = [(bins[k] + offset, freq[k]) for k in range(freq.size)\
if freq[k] > 0]
else:
points = [(bins[k] + offset, freq[k]) for k in range(freq.size)]
return points
def simple_3d_mask():
cube = simple_3d()
cube.data = ma.asanyarray(cube.data)
cube.data = ma.masked_less_equal(cube.data, 8.0)
return cube
def rotate_winds(u_cube, v_cube, target_cs):
"""
Transform wind vectors to a different coordinate system.
The input cubes contain U and V components parallel to the local X and Y
directions of the input grid at each point.
The output cubes contain the same winds, at the same locations, but
relative to the grid directions of a different coordinate system.
Thus in vector terms, the magnitudes will always be the same, but the
angles can be different.
The outputs retain the original horizontal dimension coordinates, but
also have two 2-dimensional auxiliary coordinates containing the X and
Y locations in the target coordinate system.
Args:
* u_cube
An instance of :class:`iris.cube.Cube` that contains the x-component
of the vector.
* v_cube
An instance of :class:`iris.cube.Cube` that contains the y-component
of the vector.
* target_cs
An instance of :class:`iris.coord_systems.CoordSystem` that specifies
the new grid directions.
Returns:
A (u', v') tuple of :class:`iris.cube.Cube` instances that are the u
and v components in the requested target coordinate system.
The units are the same as the inputs.
.. note::
The U and V values relate to distance, with units such as 'm s-1'.
These are not the same as coordinate vectors, which transform in a
different manner.
.. note::
The names of the output cubes are those of the inputs, prefixed with
'transformed\_' (e.g. 'transformed_x_wind').
.. warning::
Conversion between rotated-pole and non-rotated systems can be
expressed analytically. However, this function always uses a numerical
approach. In locations where this numerical approach does not preserve
magnitude to an accuracy of 0.1%, the corresponding elements of the
returned cubes will be masked.
"""
# Check u_cube and v_cube have the same shape. We iterate through
# the u and v cube slices which relies on the shapes matching.
if u_cube.shape != v_cube.shape:
msg = 'Expected u and v cubes to have the same shape. ' \
'u cube has shape {}, v cube has shape {}.'
raise ValueError(msg.format(u_cube.shape, v_cube.shape))
# Check the u_cube and v_cube have the same x and y coords.
msg = 'Coordinates differ between u and v cubes. Coordinate {!r} from ' \
'u cube does not equal coordinate {!r} from v cube.'
if u_cube.coord(axis='x') != v_cube.coord(axis='x'):
raise ValueError(msg.format(u_cube.coord(axis='x').name(),
v_cube.coord(axis='x').name()))
if u_cube.coord(axis='y') != v_cube.coord(axis='y'):
raise ValueError(msg.format(u_cube.coord(axis='y').name(),
v_cube.coord(axis='y').name()))
# Check x and y coords have the same coordinate system.
x_coord = u_cube.coord(axis='x')
y_coord = u_cube.coord(axis='y')
if x_coord.coord_system != y_coord.coord_system:
msg = "Coordinate systems of x and y coordinates differ. " \
"Coordinate {!r} has a coord system of {!r}, but coordinate " \
"{!r} has a coord system of {!r}."
raise ValueError(msg.format(x_coord.name(), x_coord.coord_system,
y_coord.name(), y_coord.coord_system))
# Convert from iris coord systems to cartopy CRSs to access
# transform functionality. Use projection as cartopy
# transform_vectors relies on x_limits and y_limits.
if x_coord.coord_system is not None:
src_crs = x_coord.coord_system.as_cartopy_projection()
else:
# Default to Geodetic (but actually use PlateCarree as a
# projection is needed).
src_crs = ccrs.PlateCarree()
target_crs = target_cs.as_cartopy_projection()
# Check the number of dimensions of the x and y coords is the same.
# Subsequent logic assumes either both 1d or both 2d.
x = x_coord.points
y = y_coord.points
if x.ndim != y.ndim or x.ndim > 2 or y.ndim > 2:
msg = 'x and y coordinates must have the same number of dimensions ' \
'and be either 1D or 2D. The number of dimensions are {} and ' \
'{}, respectively.'.format(x.ndim, y.ndim)
raise ValueError(msg)
# Check the dimension mappings match between u_cube and v_cube.
if u_cube.coord_dims(x_coord) != v_cube.coord_dims(x_coord):
raise ValueError('Dimension mapping of x coordinate differs '
'between u and v cubes.')
if u_cube.coord_dims(y_coord) != v_cube.coord_dims(y_coord):
raise ValueError('Dimension mapping of y coordinate differs '
'between u and v cubes.')
x_dims = u_cube.coord_dims(x_coord)
y_dims = u_cube.coord_dims(y_coord)
# Convert points to 2D, if not already, and determine dims.
if x.ndim == y.ndim == 1:
x, y = _meshgrid(x, y)
dims = (y_dims[0], x_dims[0])
else:
dims = x_dims
# Transpose x, y 2d arrays to match the order in cube's data
# array so that x, y and the sliced data all line up.
if dims[0] > dims[1]:
x = x.transpose()
y = y.transpose()
# Create resulting cubes.
ut_cube = u_cube.copy()
vt_cube = v_cube.copy()
ut_cube.rename('transformed_{}'.format(u_cube.name()))
vt_cube.rename('transformed_{}'.format(v_cube.name()))
# Get distance scalings for source crs.
ds_dx1, ds_dy1 = _crs_distance_differentials(src_crs, x, y)
# Get distance scalings for target crs.
x2, y2 = _transform_xy(src_crs, x, y, target_crs)
ds_dx2, ds_dy2 = _crs_distance_differentials(target_crs, x2, y2)
ds = DistanceDifferential(ds_dx1, ds_dy1, ds_dx2, ds_dy2)
# Calculate coordinate partial differentials from source crs to target crs.
dx2_dx1, dy2_dx1, dx2_dy1, dy2_dy1 = _inter_crs_differentials(src_crs,
x, y,
target_crs)
dx2 = PartialDifferential(dx2_dx1, dx2_dy1)
dy2 = PartialDifferential(dy2_dx1, dy2_dy1)
# Calculate mask based on preservation of magnitude.
mask = _transform_distance_vectors_tolerance_mask(src_crs, x, y,
target_crs,
ds, dx2, dy2)
apply_mask = mask.any()
if apply_mask:
# Make masked arrays to accept masking.
ut_cube.data = ma.asanyarray(ut_cube.data)
vt_cube.data = ma.asanyarray(vt_cube.data)
# Project vectors with u, v components one horiz slice at a time and
# insert into the resulting cubes.
shape = list(u_cube.shape)
for dim in dims:
shape[dim] = 1
ndindex = np.ndindex(*shape)
for index in ndindex:
index = list(index)
for dim in dims:
index[dim] = slice(None, None)
index = tuple(index)
u = u_cube.data[index]
v = v_cube.data[index]
ut, vt = _transform_distance_vectors(u, v, ds, dx2, dy2)
if apply_mask:
ut = ma.asanyarray(ut)
ut[mask] = ma.masked
vt = ma.asanyarray(vt)
vt[mask] = ma.masked
ut_cube.data[index] = ut
vt_cube.data[index] = vt
# Calculate new coords of locations in target coordinate system.
xyz_tran = target_crs.transform_points(src_crs, x, y)
xt = xyz_tran[..., 0].reshape(x.shape)
yt = xyz_tran[..., 1].reshape(y.shape)
# Transpose xt, yt 2d arrays to match the dim order
# of the original x an y arrays - i.e. undo the earlier
# transpose (if applied).
if dims[0] > dims[1]:
xt = xt.transpose()
yt = yt.transpose()
xt_coord = iris.coords.AuxCoord(xt,
standard_name='projection_x_coordinate',
coord_system=target_cs)
yt_coord = iris.coords.AuxCoord(yt,
standard_name='projection_y_coordinate',
coord_system=target_cs)
# Set units based on coord_system.
if isinstance(target_cs, (iris.coord_systems.GeogCS,
iris.coord_systems.RotatedGeogCS)):
xt_coord.units = yt_coord.units = 'degrees'
else:
xt_coord.units = yt_coord.units = 'm'
ut_cube.add_aux_coord(xt_coord, dims)
ut_cube.add_aux_coord(yt_coord, dims)
vt_cube.add_aux_coord(xt_coord.copy(), dims)
vt_cube.add_aux_coord(yt_coord.copy(), dims)
return ut_cube, vt_cube
def rotate_winds(u_cube, v_cube, target_cs):
r"""
Transform wind vectors to a different coordinate system.
The input cubes contain U and V components parallel to the local X and Y
directions of the input grid at each point.
The output cubes contain the same winds, at the same locations, but
relative to the grid directions of a different coordinate system.
Thus in vector terms, the magnitudes will always be the same, but the
angles can be different.
The outputs retain the original horizontal dimension coordinates, but
also have two 2-dimensional auxiliary coordinates containing the X and
Y locations in the target coordinate system.
Args:
* u_cube
An instance of :class:`iris.cube.Cube` that contains the x-component
of the vector.
* v_cube
An instance of :class:`iris.cube.Cube` that contains the y-component
of the vector.
* target_cs
An instance of :class:`iris.coord_systems.CoordSystem` that specifies
the new grid directions.
Returns:
A (u', v') tuple of :class:`iris.cube.Cube` instances that are the u
and v components in the requested target coordinate system.
The units are the same as the inputs.
.. note::
The U and V values relate to distance, with units such as 'm s-1'.
These are not the same as coordinate vectors, which transform in a
different manner.
.. note::
The names of the output cubes are those of the inputs, prefixed with
'transformed\_' (e.g. 'transformed_x_wind').
.. warning::
Conversion between rotated-pole and non-rotated systems can be
expressed analytically. However, this function always uses a numerical
approach. In locations where this numerical approach does not preserve
magnitude to an accuracy of 0.1%, the corresponding elements of the
returned cubes will be masked.
"""
# Check u_cube and v_cube have the same shape. We iterate through
# the u and v cube slices which relies on the shapes matching.
if u_cube.shape != v_cube.shape:
msg = ("Expected u and v cubes to have the same shape. "
"u cube has shape {}, v cube has shape {}.")
raise ValueError(msg.format(u_cube.shape, v_cube.shape))
# Check the u_cube and v_cube have the same x and y coords.
msg = ("Coordinates differ between u and v cubes. Coordinate {!r} from "
"u cube does not equal coordinate {!r} from v cube.")
if u_cube.coord(axis="x") != v_cube.coord(axis="x"):
raise ValueError(
msg.format(
u_cube.coord(axis="x").name(),
v_cube.coord(axis="x").name()))
if u_cube.coord(axis="y") != v_cube.coord(axis="y"):
raise ValueError(
msg.format(
u_cube.coord(axis="y").name(),
v_cube.coord(axis="y").name()))
# Check x and y coords have the same coordinate system.
x_coord = u_cube.coord(axis="x")
y_coord = u_cube.coord(axis="y")
if x_coord.coord_system != y_coord.coord_system:
msg = ("Coordinate systems of x and y coordinates differ. "
"Coordinate {!r} has a coord system of {!r}, but coordinate "
"{!r} has a coord system of {!r}.")
raise ValueError(
msg.format(
x_coord.name(),
x_coord.coord_system,
y_coord.name(),
y_coord.coord_system,
))
# Convert from iris coord systems to cartopy CRSs to access
# transform functionality. Use projection as cartopy
# transform_vectors relies on x_limits and y_limits.
if x_coord.coord_system is not None:
src_crs = x_coord.coord_system.as_cartopy_projection()
else:
# Default to Geodetic (but actually use PlateCarree as a
# projection is needed).
src_crs = ccrs.PlateCarree()
target_crs = target_cs.as_cartopy_projection()
# Check the number of dimensions of the x and y coords is the same.
# Subsequent logic assumes either both 1d or both 2d.
x = x_coord.points
y = y_coord.points
if x.ndim != y.ndim or x.ndim > 2 or y.ndim > 2:
msg = ("x and y coordinates must have the same number of dimensions "
"and be either 1D or 2D. The number of dimensions are {} and "
"{}, respectively.".format(x.ndim, y.ndim))
raise ValueError(msg)
# Check the dimension mappings match between u_cube and v_cube.
if u_cube.coord_dims(x_coord) != v_cube.coord_dims(x_coord):
raise ValueError("Dimension mapping of x coordinate differs "
"between u and v cubes.")
if u_cube.coord_dims(y_coord) != v_cube.coord_dims(y_coord):
raise ValueError("Dimension mapping of y coordinate differs "
"between u and v cubes.")
x_dims = u_cube.coord_dims(x_coord)
y_dims = u_cube.coord_dims(y_coord)
# Convert points to 2D, if not already, and determine dims.
if x.ndim == y.ndim == 1:
x, y = _meshgrid(x, y)
dims = (y_dims[0], x_dims[0])
else:
dims = x_dims
# Transpose x, y 2d arrays to match the order in cube's data
# array so that x, y and the sliced data all line up.
if dims[0] > dims[1]:
x = x.transpose()
y = y.transpose()
# Create resulting cubes.
ut_cube = u_cube.copy()
vt_cube = v_cube.copy()
ut_cube.rename("transformed_{}".format(u_cube.name()))
vt_cube.rename("transformed_{}".format(v_cube.name()))
# Get distance scalings for source crs.
ds_dx1, ds_dy1 = _crs_distance_differentials(src_crs, x, y)
# Get distance scalings for target crs.
x2, y2 = _transform_xy(src_crs, x, y, target_crs)
ds_dx2, ds_dy2 = _crs_distance_differentials(target_crs, x2, y2)
ds = DistanceDifferential(ds_dx1, ds_dy1, ds_dx2, ds_dy2)
# Calculate coordinate partial differentials from source crs to target crs.
dx2_dx1, dy2_dx1, dx2_dy1, dy2_dy1 = _inter_crs_differentials(
src_crs, x, y, target_crs)
dx2 = PartialDifferential(dx2_dx1, dx2_dy1)
dy2 = PartialDifferential(dy2_dx1, dy2_dy1)
# Calculate mask based on preservation of magnitude.
mask = _transform_distance_vectors_tolerance_mask(src_crs, x, y,
target_crs, ds, dx2, dy2)
apply_mask = mask.any()
if apply_mask:
# Make masked arrays to accept masking.
ut_cube.data = ma.asanyarray(ut_cube.data)
vt_cube.data = ma.asanyarray(vt_cube.data)
# Project vectors with u, v components one horiz slice at a time and
# insert into the resulting cubes.
shape = list(u_cube.shape)
for dim in dims:
shape[dim] = 1
ndindex = np.ndindex(*shape)
for index in ndindex:
index = list(index)
for dim in dims:
index[dim] = slice(None, None)
index = tuple(index)
u = u_cube.data[index]
v = v_cube.data[index]
ut, vt = _transform_distance_vectors(u, v, ds, dx2, dy2)
if apply_mask:
ut = ma.asanyarray(ut)
ut[mask] = ma.masked
vt = ma.asanyarray(vt)
vt[mask] = ma.masked
ut_cube.data[index] = ut
vt_cube.data[index] = vt
# Calculate new coords of locations in target coordinate system.
xyz_tran = target_crs.transform_points(src_crs, x, y)
xt = xyz_tran[..., 0].reshape(x.shape)
yt = xyz_tran[..., 1].reshape(y.shape)
# Transpose xt, yt 2d arrays to match the dim order
# of the original x an y arrays - i.e. undo the earlier
# transpose (if applied).
if dims[0] > dims[1]:
xt = xt.transpose()
yt = yt.transpose()
xt_coord = iris.coords.AuxCoord(xt,
standard_name="projection_x_coordinate",
coord_system=target_cs)
yt_coord = iris.coords.AuxCoord(yt,
standard_name="projection_y_coordinate",
coord_system=target_cs)
# Set units based on coord_system.
if isinstance(
target_cs,
(iris.coord_systems.GeogCS, iris.coord_systems.RotatedGeogCS),
):
xt_coord.units = yt_coord.units = "degrees"
else:
xt_coord.units = yt_coord.units = "m"
ut_cube.add_aux_coord(xt_coord, dims)
ut_cube.add_aux_coord(yt_coord, dims)
vt_cube.add_aux_coord(xt_coord.copy(), dims)
vt_cube.add_aux_coord(yt_coord.copy(), dims)
return ut_cube, vt_cube
def __init__(self, **kwargs):
## set the missing variable
self.missing = kwargs.get('missing', MISSING)
self.profile = kwargs.get('profile')
self.latitude = kwargs.get('latitude', ma.masked)
self.strictQC = kwargs.get('strictQC', False)
## get the data and turn them into arrays
self.pres = ma.asanyarray(kwargs.get('pres'), dtype=float)
self.hght = ma.asanyarray(kwargs.get('hght'), dtype=float)
self.tmpc = ma.asanyarray(kwargs.get('tmpc'), dtype=float)
self.dwpc = ma.asanyarray(kwargs.get('dwpc'), dtype=float)
assert self.pres.ndim == 1 and self.hght.ndim == 1 and self.tmpc.ndim == 1 and self.dwpc.ndim == 1,\
"The dimensions of the pres, hght, tmpc, and dwpc arrays passed to the Profile object constructor are not all one dimensional."
assert len(self.pres) > 1 and len(self.hght) > 1 and len(self.tmpc) > 1 and len(self.dwpc) > 1,\
"The length of the pres, hght, tmpc, and dwpc arrays passed to Profile object constructor must all have a length greater than 1."
assert len(self.pres) == len(self.hght) == len(self.tmpc) == len(self.dwpc),\
"The pres, hght, tmpc, or dwpc arrays passed to the Profile object constructor must all have the same length."
if np.ma.max(self.pres) <= 100:
warnings.warn("The pressure values passed to the profile object are below 100 mb. This may cause some the SHARPpy routines not to behave as expected.")
if 'wdir' in kwargs and 'wspd' in kwargs:
self.wdir = ma.asanyarray(kwargs.get('wdir'), dtype=float)
self.wspd = ma.asanyarray(kwargs.get('wspd'), dtype=float)
assert len(self.wdir) == len(self.wspd) == len(self.pres), "The wdir and wspd arrays passed to the Profile constructor must have the same length as the pres array."
assert self.wdir.ndim == 1 and self.wspd.ndim == 1, "The wdir and wspd arrays passed to the Profile constructor are not one dimensional."
#self.u, self.v = utils.vec2comp(self.wdir, self.wspd)
self.u = None
self.v = None
## did the user provide the wind in u,v form?
elif 'u' in kwargs and 'v' in kwargs:
self.u = ma.asanyarray(kwargs.get('u'), dtype=float)
self.v = ma.asanyarray(kwargs.get('v'), dtype=float)
assert len(self.u) == len(self.v) == len(self.pres), "The u and v arrays passed to the Profile constructor must have the same length as the pres array."
assert self.u.ndim == 1 and self.v.ndim == 1, "The wdir and wspd arrays passed to the Profile constructor are not one dimensional."
#self.wdir, self.wspd = utils.comp2vec(self.u, self.v)
self.wdir = None
self.wspd = None
else:
warnings.warn("No wind data (wdir/wspd or u/v) passed to the Profile object constructor. This may cause some of the SHARPpy routines to not behave as expected.")
## check if any standard deviation data was supplied
if 'tmp_stdev' in kwargs:
self.dew_stdev = ma.asanyarray(kwargs.get('dew_stdev'), dtype=float)
self.tmp_stdev = ma.asanyarray(kwargs.get('tmp_stdev'), dtype=float)
else:
self.dew_stdev = None
self.tmp_stdev = None
if kwargs.get('omeg', None) is not None:
## get the omega data and turn into arrays
self.omeg = ma.asanyarray(kwargs.get('omeg'))
assert len(self.omeg) == len(self.pres), "Length of omeg array passed to constructor is not the same length as the pres array."
assert self.omeg.ndim == 1, "omeg array is not one dimensional."
assert len(self.omeg) > 1, "omeg array length must have a length greater than 1."
else:
self.omeg = None
## optional keyword argument for location
self.location = kwargs.get('location', None)
self.date = kwargs.get('date', None)
if self.strictQC is True:
self.checkDataIntegrity()
def subset(self, doy, depth, lat, lon, var):
""" Subset the necessary data to interpolate in the right position
Special cases that should be handled here:
0 to 360 versus -180 to 180
position near grenwich, or international date line
"""
dims = {}
zn = slice(
np.nonzero(self.dims['depth'] <= depth.min())[0].max(),
np.nonzero(
self.dims['depth'] >=
min(self.dims['depth'].max(), depth.max())
)[0].min() + 1
)
# If a higher degree interpolation system uses more than one data
# point in the edge, I should extend this selection one point on
# each side, without go beyond 0
# if zn.start < 0:
# zn = slice(0, zn.stop, zn.step)
dims['depth'] = np.atleast_1d(self.dims['depth'][zn])
yn = slice(
np.nonzero(self.dims['lat'] <= lat.min())[0].max(),
np.nonzero(self.dims['lat'] >= lat.max())[0].min() + 1)
dims['lat'] = np.atleast_1d(self.dims['lat'][yn])
lon_ext = np.array(
(self.dims['lon'] - 360).tolist() +
self.dims['lon'].tolist() +
(self.dims['lon']+360).tolist())
xn_ext = np.array(3 * list(range(self.dims['lon'].shape[0])))
xn_start = np.nonzero(lon_ext <= lon.min())[0].max()
xn_end = np.nonzero(lon_ext >= lon.max())[0].min()
xn = xn_ext[xn_start:xn_end+1]
dims['lon'] = np.atleast_1d(lon_ext[xn_start:xn_end+1])
#if self.dims['time'].shape == (1,):
# tn = 0
# dims['time'] = self.dims['time']
#else:
# time_ext = np.array(
# [self.dims['time'][-1] - 365.25] +
# self.dims['time'].tolist() +
# [self.dims['time'][0] + 365.25])
# tn_ext = list(range(self.dims['time'].size))
# tn_ext = [tn_ext[-1]] + tn_ext + [tn_ext[0]]
# tn_start = np.nonzero(time_ext <= doy.min())[0].max()
# tn_end = np.nonzero(time_ext >= doy.max())[0].min()
# tn = tn_ext[tn_start:tn_end+1]
# dims['time'] = np.atleast_1d(time_ext[tn_start:tn_end+1])
dims['time'] = np.atleast_1d(doy)
# messy way to accept t_mn or mn
varin = []
#for v in var:
# if v in self.KEYS:
# varin.append(v)
# elif self.KEYS[0][:2] + v in self.KEYS:
# varin.append(self.KEYS[0][:2] + v)
import re
#for v in var:
# if re.search('(?:[t,s]_)?mn', v):
# varin.append('mn')
# elif re.search('(?:[t,s]_)?sd', v):
# varin.append('sd')
# elif re.search('(?:[t,s]_)?dd', v):
# varin.append('sd')
subset = {}
#for v, vin in zip(var, varin):
# subset[v] = ma.asanyarray(
# [self.ncs[tnn][vin][0, zn, yn, xn] for tnn in tn])
for v in var:
if v == 'mn':
mn = []
for d in doy:
t = 2 * np.pi * d/366
# Naive solution
# FIXME: This is not an efficient solution.
value = self.ncs[0]['mean'][:, yn, xn]
value[:64] += self.ncs[0]['an_cos'][:, yn, xn] * np.cos(t) + \
self.ncs[0]['an_sin'][:, yn, xn] * np.sin(t)
value[:55] += self.ncs[0]['sa_cos'][:, yn, xn] * np.cos(2*t) + \
self.ncs[0]['sa_sin'][:, yn, xn] * np.sin(2*t)
mn.append(value[zn])
subset['mn'] = ma.asanyarray(mn)
else:
subset[v] = ma.asanyarray(
doy.size * [self[v][zn, yn, xn]])
return subset, dims
''' Create the Sounding (Profile) Object '''
def __init__(self, **kwargs):
## set the missing variable
self.missing = kwargs.get('missing', MISSING)
self.profile = kwargs.get('profile')
## get the data and turn them into arrays
self.pres = ma.asanyarray(kwargs.get('pres'), dtype=float)
self.hght = ma.asanyarray(kwargs.get('hght'), dtype=float)
self.tmpc = ma.asanyarray(kwargs.get('tmpc'), dtype=float)
self.dwpc = ma.asanyarray(kwargs.get('dwpc'), dtype=float)
if 'wdir' in kwargs:
self.wdir = ma.asanyarray(kwargs.get('wdir'), dtype=float)
self.wspd = ma.asanyarray(kwargs.get('wspd'), dtype=float)
self.u = None
self.v = None
## did the user provide the wind in u,v form?
elif 'u' in kwargs:
self.u = ma.asanyarray(kwargs.get('u'), dtype=float)
self.v = ma.asanyarray(kwargs.get('v'), dtype=float)
self.wdir = None
self.wspd = None
## check if any standard deviation data was supplied
if 'tmp_stdev' in kwargs:
self.dew_stdev = ma.asanyarray(kwargs.get('dew_stdev'), dtype=float)
self.tmp_stdev = ma.asanyarray(kwargs.get('tmp_stdev'), dtype=float)
else:
self.dew_stdev = None
self.tmp_stdev = None
if kwargs.get('omeg', None) is not None:
## get the omega data and turn into arrays
self.omeg = ma.asanyarray(kwargs.get('omeg'))
else:
self.omeg = None
## optional keyword argument for location
self.location = kwargs.get('location', None)
def __init__(self, **kwargs):
'''
Create the sounding data object
Parameters
----------
Mandatory Keywords
pres : array_like
The pressure values (Hectopaschals)
hght : array_like
The corresponding height values (Meters)
tmpc : array_like
The corresponding temperature values (Celsius)
dwpc : array_like
The corresponding dewpoint temperature values (Celsius)
Optional Keyword Pairs (must use one or the other)
wdir : array_like
The direction from which the wind is blowing in
meteorological degrees
wspd : array_like
The speed of the wind
OR
u : array_like
The U-component of the direction from which the wind
is blowing
v : array_like
The V-component of the direction from which the wind
is blowing.
Optional Keywords
missing : number (default: sharppy.sharptab.constants.MISSING)
The value of the missing flag
Returns
-------
A profile object
'''
self.missing = kwargs.get('missing', MISSING)
self.masked = ma.masked
self.pres = ma.asanyarray(kwargs.get('pres'))
self.hght = ma.asanyarray(kwargs.get('hght'))
self.tmpc = ma.asanyarray(kwargs.get('tmpc'))
self.dwpc = ma.asanyarray(kwargs.get('dwpc'))
self.pres[self.pres == self.missing] = ma.masked
self.hght[self.hght == self.missing] = ma.masked
self.tmpc[self.tmpc == self.missing] = ma.masked
self.dwpc[self.dwpc == self.missing] = ma.masked
self.logp = np.log10(self.pres.copy())
if 'wdir' in kwargs:
self.wdir = ma.asanyarray(kwargs.get('wdir'))
self.wspd = ma.asanyarray(kwargs.get('wspd'))
self.wdir[self.wdir == self.missing] = ma.masked
self.wspd[self.wspd == self.missing] = ma.masked
self.wdir[self.wspd.mask] = ma.masked
self.wspd[self.wdir.mask] = ma.masked
self.u, self.v = utils.vec2comp(self.wdir, self.wspd)
elif 'u' in kwargs:
self.u = ma.asanyarray(kwargs.get('u'))
self.v = ma.asanyarray(kwargs.get('v'))
self.u[self.u == self.missing] = ma.masked
self.v[self.v == self.missing] = ma.masked
self.u[self.v.mask] = ma.masked
self.v[self.u.mask] = ma.masked
self.wdir, self.wspd = utils.comp2vec(self.u, self.v)
self.pres.set_fill_value(self.missing)
self.hght.set_fill_value(self.missing)
self.tmpc.set_fill_value(self.missing)
self.dwpc.set_fill_value(self.missing)
self.wdir.set_fill_value(self.missing)
self.wspd.set_fill_value(self.missing)
self.u.set_fill_value(self.missing)
self.v.set_fill_value(self.missing)
self.sfc = self.get_sfc()
def subset(self, doy, depth, lat, lon, var):
""" Subset the necessary data to interpolate in the right position
Special cases that should be handled here:
0 to 360 versus -180 to 180
position near grenwich, or international date line
"""
dims = {}
zn = slice(
np.nonzero(self.dims['depth'] <= depth.min())[0].max(),
np.nonzero(
self.dims['depth'] >= \
min(self.dims['depth'].max(), depth.max())
)[0].min() + 1
)
# If a higher degree interpolation system uses more than one data
# point in the edge, I should extend this selection one point on
# each side, without go beyond 0
#if zn.start < 0:
# zn = slice(0, zn.stop, zn.step)
dims['depth'] = np.atleast_1d(self.dims['depth'][zn])
yn = slice(
np.nonzero(self.dims['lat'] <= lat.min())[0].max(),
np.nonzero(self.dims['lat'] >= lat.max())[0].min() + 1)
dims['lat'] = np.atleast_1d(self.dims['lat'][yn])
lon_ext = np.array(
(self.dims['lon'] - 360).tolist() + \
self.dims['lon'].tolist() + \
(self.dims['lon']+360).tolist())
xn_ext = np.array(3 * list(range(self.dims['lon'].shape[0])))
xn_start = np.nonzero(lon_ext <= lon.min())[0].max()
xn_end = np.nonzero(lon_ext >= lon.max())[0].min()
xn = xn_ext[xn_start:xn_end+1]
dims['lon'] = np.atleast_1d(lon_ext[xn_start:xn_end+1])
if self.dims['time'].shape == (1,):
tn = 0
dims['time'] = self.dims['time']
else:
time_ext = np.array(
[self.dims['time'][-1] - 365.25] + \
self.dims['time'].tolist() + \
[self.dims['time'][0] + 365.25])
tn_ext = list(range(self.dims['time'].size))
tn_ext = [tn_ext[-1]] + tn_ext + [tn_ext[0]]
tn_start = np.nonzero(time_ext <= doy.min())[0].max()
tn_end = np.nonzero(time_ext >= doy.max())[0].min()
tn = tn_ext[tn_start:tn_end+1]
dims['time'] = np.atleast_1d(time_ext[tn_start:tn_end+1])
# messy way to accept t_mn or mn
varin = []
for v in var:
if v in self.KEYS:
varin.append(v)
elif self.KEYS[0][:2] + v in self.KEYS:
varin.append(self.KEYS[0][:2] + v)
subset = {}
for v, vin in zip(var, varin):
subset[v] = ma.asanyarray(
[self.ncs[tnn][vin][0, zn, yn, xn] for tnn in tn])
return subset, dims
def frequency_distribution(obs,
num_bins=30,
binwidth=None,
limits=None,
weights=None,
drop=True):
"""
Takes a number of events and counts the frequency of these events falling into
either a fixed number of equi-distant bins over the given range or bins of
fixed width over the range.
Parameters
----------
obs: iterable
An aggregation of numerical observations.
num_bins: int (optional)
The number of equi-distant bins used over the numeric range.
binwidth: float (optional)
Desired width of the bins. This overrides `num_bins`.
limits: tuple (lower, upper) (optional)
The lower and upper values for the range of the distribution.
If no value is given, a range slightly larger then the range of the
values in `obs` is used. Specifically ``(a.min() - s, a.max() + s)``,
where ``s = (1/2)(a.max() - a.min()) / num_bins``.
weights: iterable (optional)
Weights for the data points in `obs`. The default is `None` weighting
all points equal.
drop: bool (optional)
Determines whether or not bins with zero events are dropped from the
resulting list.
Returns
-------
A list of pairs, the first entry is the centre of the bin borders at which
the frequency of an event is measured, and the second entry is the frequency
itself.
"""
obs = nma.asanyarray(obs)
nma.masked_invalid(obs, copy=False)
obs = numpy.ravel(obs)
if obs.size == 0:
return list()
if limits is None:
mn = obs.min()
mx = obs.max()
else:
(mn, mx) = limits
if binwidth:
binwidth = float(binwidth)
num_bins = numpy.floor((mx - mn) / binwidth)
else:
num_bins = int(num_bins)
binwidth = (mx - mn) / num_bins
offset = binwidth * 0.5
if limits is None:
limits = (mn - offset, mx + offset)
# extend the numeric range slightly beyond the data range
(freq, bins) = numpy.histogram(obs[~obs.mask],
bins=num_bins,
range=limits,
weights=weights)
if drop:
points = [(bins[k] + offset, freq[k]) for k in range(freq.size)\
if freq[k] > 0]
else:
points = [(bins[k] + offset, freq[k]) for k in range(freq.size)]
return points
def subset(self, doy, depth, lat, lon, var):
""" Subset the necessary data to interpolate in the right position
Special cases that should be handled here:
0 to 360 versus -180 to 180
position near grenwich, or international date line
"""
dims = {}
zn = slice(
np.nonzero(self.dims['depth'] <= depth.min())[0].max(),
np.nonzero(self.dims['depth'] >= min(self.dims['depth'].max(),
depth.max()))[0].min() + 1)
# If a higher degree interpolation system uses more than one data
# point in the edge, I should extend this selection one point on
# each side, without go beyond 0
# if zn.start < 0:
# zn = slice(0, zn.stop, zn.step)
dims['depth'] = np.atleast_1d(self.dims['depth'][zn])
yn = slice(
np.nonzero(self.dims['lat'] <= lat.min())[0].max(),
np.nonzero(self.dims['lat'] >= lat.max())[0].min() + 1)
dims['lat'] = np.atleast_1d(self.dims['lat'][yn])
lon_ext = np.array((self.dims['lon'] - 360).tolist() +
self.dims['lon'].tolist() +
(self.dims['lon'] + 360).tolist())
xn_ext = np.array(3 * list(range(self.dims['lon'].shape[0])))
xn_start = np.nonzero(lon_ext <= lon.min())[0].max()
xn_end = np.nonzero(lon_ext >= lon.max())[0].min()
xn = xn_ext[xn_start:xn_end + 1]
dims['lon'] = np.atleast_1d(lon_ext[xn_start:xn_end + 1])
if self.dims['time'].shape == (1, ):
tn = 0
dims['time'] = self.dims['time']
else:
time_ext = np.array([self.dims['time'][-1] - 365.25] +
self.dims['time'].tolist() +
[self.dims['time'][0] + 365.25])
tn_ext = list(range(self.dims['time'].size))
tn_ext = [tn_ext[-1]] + tn_ext + [tn_ext[0]]
tn_start = np.nonzero(time_ext <= doy.min())[0].max()
tn_end = np.nonzero(time_ext >= doy.max())[0].min()
tn = tn_ext[tn_start:tn_end + 1]
dims['time'] = np.atleast_1d(time_ext[tn_start:tn_end + 1])
# messy way to accept t_mn or mn
varin = []
for v in var:
if v in self.KEYS:
varin.append(v)
elif self.KEYS[0][:2] + v in self.KEYS:
varin.append(self.KEYS[0][:2] + v)
subset = {}
for v, vin in zip(var, varin):
subset[v] = ma.asanyarray(
[self.ncs[tnn][vin][0, zn, yn, xn] for tnn in tn])
return subset, dims
def __init__(self, **kwargs):
## set the missing variable
self.missing = kwargs.get('missing', MISSING)
self.profile = kwargs.get('profile')
self.latitude = kwargs.get('latitude', ma.masked)
## get the data and turn them into arrays
self.pres = ma.asanyarray(kwargs.get('pres'), dtype=float)
self.hght = ma.asanyarray(kwargs.get('hght'), dtype=float)
self.tmpc = ma.asanyarray(kwargs.get('tmpc'), dtype=float)
self.dwpc = ma.asanyarray(kwargs.get('dwpc'), dtype=float)
if 'wdir' in kwargs:
self.wdir = ma.asanyarray(kwargs.get('wdir'), dtype=float)
self.wspd = ma.asanyarray(kwargs.get('wspd'), dtype=float)
self.u = None
self.v = None
## did the user provide the wind in u,v form?
elif 'u' in kwargs:
self.u = ma.asanyarray(kwargs.get('u'), dtype=float)
self.v = ma.asanyarray(kwargs.get('v'), dtype=float)
self.wdir = None
self.wspd = None
## check if any standard deviation data was supplied
if 'tmp_stdev' in kwargs:
self.dew_stdev = ma.asanyarray(kwargs.get('dew_stdev'),
dtype=float)
self.tmp_stdev = ma.asanyarray(kwargs.get('tmp_stdev'),
dtype=float)
else:
self.dew_stdev = None
self.tmp_stdev = None
if kwargs.get('omeg', None) is not None:
## get the omega data and turn into arrays
self.omeg = ma.asanyarray(kwargs.get('omeg'))
else:
self.omeg = None
## optional keyword argument for location
self.location = kwargs.get('location', None)
self.date = kwargs.get('date', None)
def _jprimes(x, i, x_bounds=None):
"""
Helper function to return the j' indices for the master curve fit
This function is a helper function for :py:func:`quality`. It is not
supposed to be called directly.
Parameters
----------
x : mapping to ndarrays
The x values.
i : int
The row index (finite size index)
x_bounds : 2-tuple, optional
bounds on x values
Returns
-------
ret : mapping to ndarrays
Has the same keys and shape as `x`.
Its element ``ret[i'][j]`` is the j' such that :math:`x_{i'j'} \leq
x_{ij} < x_{i'(j'+1)}`.
If no such j' exists, the element is np.nan.
Convert the element to int to use as an index.
"""
j_primes = - np.ones_like(x)
try:
x_masked = ma.masked_outside(x, x_bounds[0], x_bounds[1])
except (TypeError, IndexError):
x_masked = ma.asanyarray(x)
k, n = x.shape
# indices of lower and upper bounds
edges = ma.notmasked_edges(x_masked, axis=1)
x_lower = np.zeros(k, dtype=int)
x_upper = np.zeros(k, dtype=int)
x_lower[edges[0][0]] = edges[0][-1]
x_upper[edges[-1][0]] = edges[-1][-1]
for i_prime in range(k):
if i_prime == i:
j_primes[i_prime][:] = np.nan
continue
jprimes = np.searchsorted(
x[i_prime], x[i], side='right'
).astype(float) - 1
jprimes[
np.logical_or(
jprimes < x_lower[i_prime],
jprimes >= x_upper[i_prime]
)
] = np.nan
j_primes[i_prime][:] = jprimes
return j_primes
