def _add_iris_coord(cube, name, points, dim, calendar=None):
"""
Add a Coord to a Cube from a Pandas index or columns array.
If no calendar is specified for a time series, Gregorian is assumed.
"""
units = Unit("unknown")
if calendar is None:
calendar = cf_units.CALENDAR_GREGORIAN
# Convert pandas datetime objects to python datetime obejcts.
if isinstance(points, DatetimeIndex):
points = np.array([i.to_datetime() for i in points])
# Convert datetime objects to Iris' current datetime representation.
if points.dtype == object:
dt_types = (datetime.datetime, netcdftime.datetime)
if all([isinstance(i, dt_types) for i in points]):
units = Unit("hours since epoch", calendar=calendar)
points = units.date2num(points)
points = np.array(points)
if (np.issubdtype(points.dtype, np.number) and
iris.util.monotonic(points, strict=True)):
coord = DimCoord(points, units=units)
coord.rename(name)
cube.add_dim_coord(coord, dim)
else:
coord = AuxCoord(points, units=units)
coord.rename(name)
cube.add_aux_coord(coord, dim)
def units_convertible(units1, units2, reftimeistime=True):
"""Return True if a Unit representing the string units1 can be converted
to a Unit representing the string units2, else False."""
try:
u1 = Unit(units1)
u2 = Unit(units2)
except ValueError:
return False
return u1.is_convertible(units2)
def _convert_coord_unit_to_datetime(coord, dt):
"""Converts a datetime to be in the unit of a specified Coord.
"""
if isinstance(coord, iris.coords.Coord):
# The unit class is then cf_units.Unit.
iris_unit = coord.units
else:
iris_unit = Unit(coord.units)
return iris_unit.num2date(dt)
def unit_converter(data, inunit, outunit):
'''
Unit converter. Takes an (numpy) array, valid udunits inunits and outunits
as strings, and returns the array in outunits.
Parameters
----------
data : array_like
inunit : string
unit to convert from, must be UDUNITS-compatible string
outunit : string
unit to conver to, must be UDUNITS-compatible string
Returns
-------
out : array_like
Example
-------
>>> import numpy as np
>>> c = Converter("kg","Gt")
>>> out = c(np.array([1,2])*1e12)
>>> out = array([ 1., 2.])
'''
inunit = str(inunit)
outunit = str(outunit)
if isinstance(data, np.ma.MaskedArray):
mask = data.mask
else:
mask = None
data = np.array(data)
if not (inunit == outunit):
try:
try:
from cf_units import Unit
in_unit = Unit(inunit)
out_unit = Unit(outunit)
outdata = in_unit.convert(data, out_unit)
except:
from udunits2 import Converter, System, Unit
sys = System()
c = Converter((Unit(sys, inunit), Unit(sys, outunit)))
outdata = c(data)
except:
print(
"Neither cf_units or udunits2 module found, you're on your own.")
c = 1. / 1e3
outdata = c * data
else:
outdata = data
if mask is not None:
return np.ma.array(outdata, mask=mask)
else:
return outdata
def test_360_day_calendar(self):
n = 360
calendar = '360_day'
time_unit = Unit('days since 1970-01-01 00:00', calendar=calendar)
time_coord = AuxCoord(np.arange(n), 'time', units=time_unit)
expected_ydata = np.array([CalendarDateTime(time_unit.num2date(point),
calendar)
for point in time_coord.points])
line1, = iplt.plot(time_coord)
result_ydata = line1.get_ydata()
self.assertArrayEqual(expected_ydata, result_ydata)
def test_gregorian_calendar_conversion_setup(self):
# Reproduces a situation where a unit's gregorian calendar would not
# match (using the `is` operator) to the literal string 'gregorian',
# causing an `is not` test to return a false negative.
cal_str = cf_units.CALENDAR_GREGORIAN
calendar = self.MyStr(cal_str)
self.assertIsNot(calendar, cal_str)
u1 = Unit('hours since 1970-01-01 00:00:00', calendar=calendar)
u2 = Unit('hours since 1969-11-30 00:00:00', calendar=calendar)
u1point = np.array([8.], dtype=np.float32)
expected = np.array([776.], dtype=np.float32)
result = u1.convert(u1point, u2)
return expected, result
def test_360_day_calendar(self):
n = 360
calendar = '360_day'
time_unit = Unit('days since 1970-01-01 00:00', calendar=calendar)
time_coord = AuxCoord(np.arange(n), 'time', units=time_unit)
times = [time_unit.num2date(point) for point in time_coord.points]
times = [netcdftime.datetime(atime.year, atime.month, atime.day,
atime.hour, atime.minute, atime.second)
for atime in times]
expected_ydata = np.array([CalendarDateTime(time, calendar)
for time in times])
line1, = iplt.plot(time_coord)
result_ydata = line1.get_ydata()
self.assertArrayEqual(expected_ydata, result_ydata)
def __init__(self):
self.u = Unit('days since 1600-01-01 00:00:00', calendar=CALENDAR_STANDARD)
self.points = np.arange(1, 5, 1)
self.coord = iris.coords.DimCoord(self.points, units=self.u)
self.start = datetime.datetime(2000, 1, 1)
self.end = datetime.datetime(2003, 4, 24)
self.start = self.u.date2num(self.start)
self.end = self.u.date2num(self.end)
def setUp(self):
self.section = {'year': 2007,
'month': 1,
'day': 15,
'hour': 0,
'minute': 3,
'second': 0}
self.unit = Unit('hours since epoch', calendar=CALENDAR_GREGORIAN)
dt = datetime(self.section['year'], self.section['month'],
self.section['day'], self.section['hour'],
self.section['minute'], self.section['second'])
self.point = self.unit.date2num(dt)
def setUp(self):
self.section = {"year": 2007, "month": 1, "day": 15, "hour": 0, "minute": 3, "second": 0}
self.unit = Unit("hours since epoch", calendar=CALENDAR_GREGORIAN)
dt = datetime(
self.section["year"],
self.section["month"],
self.section["day"],
self.section["hour"],
self.section["minute"],
self.section["second"],
)
self.point = self.unit.date2num(dt)
class Test(tests.IrisTest):
def setUp(self):
self.section = {'year': 2007,
'month': 1,
'day': 15,
'hour': 0,
'minute': 3,
'second': 0}
self.unit = Unit('hours since epoch', calendar=CALENDAR_GREGORIAN)
dt = datetime(self.section['year'], self.section['month'],
self.section['day'], self.section['hour'],
self.section['minute'], self.section['second'])
self.point = self.unit.date2num(dt)
def _check(self, section, standard_name=None):
expected = DimCoord(self.point, standard_name=standard_name,
units=self.unit)
# The call being tested.
coord = reference_time_coord(section)
self.assertEqual(coord, expected)
def test_start_of_forecast_0(self):
section = deepcopy(self.section)
section['significanceOfReferenceTime'] = 0
self._check(section, 'forecast_reference_time')
def test_start_of_forecast_1(self):
section = deepcopy(self.section)
section['significanceOfReferenceTime'] = 1
self._check(section, 'forecast_reference_time')
def test_observation_time(self):
section = deepcopy(self.section)
section['significanceOfReferenceTime'] = 3
self._check(section, 'time')
def test_unknown_significance(self):
section = deepcopy(self.section)
section['significanceOfReferenceTime'] = 5
emsg = 'unsupported significance'
with self.assertRaisesRegexp(TranslationError, emsg):
self._check(section)
class Test(tests.IrisTest):
def setUp(self):
self.section = {"year": 2007, "month": 1, "day": 15, "hour": 0, "minute": 3, "second": 0}
self.unit = Unit("hours since epoch", calendar=CALENDAR_GREGORIAN)
dt = datetime(
self.section["year"],
self.section["month"],
self.section["day"],
self.section["hour"],
self.section["minute"],
self.section["second"],
)
self.point = self.unit.date2num(dt)
def _check(self, section, standard_name=None):
expected = DimCoord(self.point, standard_name=standard_name, units=self.unit)
# The call being tested.
coord = reference_time_coord(section)
self.assertEqual(coord, expected)
def test_start_of_forecast(self):
section = deepcopy(self.section)
section["significanceOfReferenceTime"] = 1
self._check(section, "forecast_reference_time")
def test_observation_time(self):
section = deepcopy(self.section)
section["significanceOfReferenceTime"] = 3
self._check(section, "time")
def test_unknown_significance(self):
section = deepcopy(self.section)
section["significanceOfReferenceTime"] = 0
emsg = "unsupported significance"
with self.assertRaisesRegexp(TranslationError, emsg):
self._check(section)
def units(self, u):
"""Units of the data"""
self._optinfo['units'] = u if isinstance(u, Unit) else Unit(u)
def process(self, input_cube):
"""Convert each point to a truth value based on provided threshold
values. The truth value may or may not be fuzzy depending upon if
fuzzy_bounds are supplied. If the plugin has a "threshold_units"
member, this is used to convert both thresholds and fuzzy bounds into
the units of the input cube.
Args:
input_cube (iris.cube.Cube):
Cube to threshold. The code is dimension-agnostic.
Returns:
cube (iris.cube.Cube):
Cube after a threshold has been applied. The data within this
cube will contain values between 0 and 1 to indicate whether
a given threshold has been exceeded or not.
The cube meta-data will contain:
* input_cube name prepended with `probability_of_`
* threshold dimension coordinate with same units as input_cube
* threshold attribute (above or below threshold)
* cube units set to (1).
Raises:
ValueError: if a np.nan value is detected within the input cube.
"""
# Record input cube data type to ensure consistent output, though
# integer data must become float to enable fuzzy thresholding.
input_cube_dtype = input_cube.dtype
if input_cube.dtype.kind == 'i':
input_cube_dtype = np.float32
thresholded_cubes = iris.cube.CubeList()
if np.isnan(input_cube.data).any():
raise ValueError("Error: NaN detected in input cube data")
# if necessary, convert thresholds and fuzzy bounds into cube units
if self.threshold_units is not None:
self.thresholds = [
self.threshold_units.convert(threshold, input_cube.units)
for threshold in self.thresholds
]
self.fuzzy_bounds = [
tuple([
self.threshold_units.convert(threshold, input_cube.units)
for threshold in bounds
]) for bounds in self.fuzzy_bounds
]
# apply fuzzy thresholding
for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds):
cube = input_cube.copy()
# if upper and lower bounds are equal, set a deterministic 0/1
# probability based on exceedance of the threshold
if bounds[0] == bounds[1]:
truth_value = cube.data > threshold
# otherwise, scale exceedance probabilities linearly between 0/1
# at the min/max fuzzy bounds and 0.5 at the threshold value
else:
truth_value = np.where(
cube.data < threshold,
rescale(cube.data,
data_range=(bounds[0], threshold),
scale_range=(0., 0.5),
clip=True),
rescale(cube.data,
data_range=(threshold, bounds[1]),
scale_range=(0.5, 1.),
clip=True),
)
truth_value = truth_value.astype(input_cube_dtype)
# if requirement is for probabilities below threshold (rather than
# above), invert the exceedance probability
if self.below_thresh_ok:
truth_value = 1. - truth_value
cube.data = truth_value
coord = iris.coords.DimCoord(threshold,
long_name="threshold",
units=cube.units)
cube.add_aux_coord(coord)
cube = iris.util.new_axis(cube, 'threshold')
thresholded_cubes.append(cube)
cube, = thresholded_cubes.concatenate()
# TODO: Correct when formal cf-standards exists
# Force the metadata to temporary conventions
if self.below_thresh_ok:
cube.attributes.update({'relative_to_threshold': 'below'})
else:
cube.attributes.update({'relative_to_threshold': 'above'})
cube.rename("probability_of_{}".format(cube.name()))
cube.units = Unit(1)
cube = enforce_coordinate_ordering(cube,
["realization", "percentile_over"])
return cube
def __init__(self,
thresholds,
fuzzy_factor=None,
fuzzy_bounds=None,
threshold_units=None,
below_thresh_ok=False):
"""
Set up for processing an in-or-out of threshold field, including the
generation of fuzzy_bounds which are required to threshold an input
cube (through self.process(cube)). If fuzzy_factor is not None, fuzzy
bounds are calculated using the threshold value in the units in which
it is provided.
The usage of fuzzy_factor is exemplified as follows:
For a 6 mm/hr threshold with a 0.75 fuzzy factor, a range of 25%
around this threshold (between (6*0.75=) 4.5 and (6*(2-0.75)=) 7.5)
would be generated. The probabilities of exceeding values within this
range are scaled linearly, so that 4.5 mm/hr yields a thresholded value
of 0 and 7.5 mm/hr yields a thresholded value of 1. Therefore, in this
case, the thresholded exceedance probabilities between 4.5 mm/hr and
7.5 mm/hr would follow the pattern:
::
Data value | Probability
------------|-------------
4.5 | 0
5.0 | 0.167
5.5 | 0.333
6.0 | 0.5
6.5 | 0.667
7.0 | 0.833
7.5 | 1.0
Args:
thresholds (list of floats or float):
The threshold points for 'significant' datapoints.
Keyword Args:
fuzzy_factor (float):
Specifies lower bound for fuzzy membership value when
multiplied by each threshold. Upper bound is equivalent linear
distance above threshold. If None, no fuzzy_factor is applied.
fuzzy_bounds (list of tuples):
Lower and upper bounds for fuzziness.
List should be of same length as thresholds.
Each entry in list should be a tuple of two floats
representing the lower and upper bounds respectively.
If None, no fuzzy_bounds are applied.
threshold_units (string):
Units of the threshold values. If not provided the units are
assumed to be the same as those of the input cube.
below_thresh_ok (boolean):
True to count points as significant if *below* the threshold,
False to count points as significant if *above* the threshold.
Raises:
ValueError: If a threshold of 0.0 is requested when using a fuzzy
factor.
ValueError: If the fuzzy_factor is not greater than 0 and less
than 1.
ValueError: If both fuzzy_factor and fuzzy_bounds are set
as this is ambiguous.
"""
# ensure threshold is a list, even if only a single value is provided
self.thresholds = thresholds
if np.isscalar(thresholds):
self.thresholds = [thresholds]
# if necessary, set threshold units
if threshold_units is None:
self.threshold_units = None
else:
self.threshold_units = Unit(threshold_units)
# initialise threshold coordinate name as None
self.threshold_coord_name = None
# read fuzzy factor or set (default) to 1 (no smoothing)
fuzzy_factor_loc = 1.
if fuzzy_factor is not None:
if fuzzy_bounds is not None:
raise ValueError(
"Invalid combination of keywords. Cannot specify "
"fuzzy_factor and fuzzy_bounds together")
if not 0 < fuzzy_factor < 1:
raise ValueError(
"Invalid fuzzy_factor: must be >0 and <1: {}".format(
fuzzy_factor))
if 0 in self.thresholds:
raise ValueError(
"Invalid threshold with fuzzy factor: cannot use a "
"multiplicative fuzzy factor with threshold == 0")
fuzzy_factor_loc = fuzzy_factor
# Set fuzzy-bounds. If neither fuzzy_factor nor fuzzy_bounds is set,
# both lower_thr and upper_thr default to the threshold value. A test
# of this equality is used later to determine whether to process with
# a sharp threshold or fuzzy bounds.
if fuzzy_bounds is None:
self.fuzzy_bounds = []
for thr in self.thresholds:
lower_thr = thr * fuzzy_factor_loc
upper_thr = thr * (2. - fuzzy_factor_loc)
if thr < 0:
lower_thr, upper_thr = upper_thr, lower_thr
self.fuzzy_bounds.append((lower_thr, upper_thr))
else:
self.fuzzy_bounds = fuzzy_bounds
# ensure fuzzy_bounds is a list of tuples
if isinstance(fuzzy_bounds, tuple):
self.fuzzy_bounds = [fuzzy_bounds]
# check that thresholds and fuzzy_bounds are self-consistent
for thr, bounds in zip(self.thresholds, self.fuzzy_bounds):
assert len(bounds) == 2, ("Invalid bounds for one threshold: {}. "
"Expected 2 floats.".format(bounds))
bounds_msg = ("Threshold must be within bounds: "
"!( {} <= {} <= {} )".format(bounds[0], thr,
bounds[1]))
assert bounds[0] <= thr, bounds_msg
assert bounds[1] >= thr, bounds_msg
self.below_thresh_ok = below_thresh_ok
def test_fix_data(self):
cube = self.fix.fix_data(self.cube)
self.assertEqual(cube.data[0], 12.0 / 44.0)
self.assertEqual(cube.units, Unit('J'))
def units_temporal(units):
u = Unit(units)
return u.is_time_reference()
def add_coordinate(
incube: Cube,
coord_points: List,
coord_name: str,
coord_units: Optional[str] = None,
dtype: Type = np.float32,
order: Optional[List[int]] = None,
is_datetime: bool = False,
attributes: Optional[Dict[str, Any]] = None,
) -> Cube:
"""
Function to duplicate a sample cube with an additional coordinate to create
a cubelist. The cubelist is merged to create a single cube, which can be
reordered to place the new coordinate in the required position.
Args:
incube:
Cube to be duplicated.
coord_points:
Values for the coordinate.
coord_name:
Long name of the coordinate to be added.
coord_units:
Coordinate unit required.
dtype:
Datatype for coordinate points.
order:
Optional list of integers to reorder the dimensions on the new
merged cube. For example, if the new coordinate is required to
be in position 1 on a 4D cube, use order=[1, 0, 2, 3] to swap the
new coordinate position with that of the original leading
coordinate.
is_datetime:
If "true", the leading coordinate points have been given as a
list of datetime objects and need converting. In this case the
"coord_units" argument is overridden and the time points provided
in seconds. The "dtype" argument is overridden and set to int64.
attributes:
Optional coordinate attributes.
Returns:
Cube containing an additional dimension coordinate.
"""
# if the coordinate already exists as a scalar coordinate, remove it
cube = incube.copy()
try:
cube.remove_coord(coord_name)
except CoordinateNotFoundError:
pass
# if new coordinate points are provided as datetimes, convert to seconds
if is_datetime:
coord_units = TIME_COORDS["time"].units
dtype = TIME_COORDS["time"].dtype
new_coord_points = [_create_time_point(val) for val in coord_points]
coord_points = new_coord_points
cubes = iris.cube.CubeList([])
for val in coord_points:
temp_cube = cube.copy()
temp_cube.add_aux_coord(
DimCoord(
np.array([val], dtype=dtype),
long_name=coord_name,
units=coord_units,
attributes=attributes,
))
# recalculate forecast period if time or frt have been updated
if (coord_name in ["time", "forecast_reference_time"]
and coord_units is not None
and Unit(coord_units).is_time_reference()):
forecast_period = forecast_period_coord(
temp_cube, force_lead_time_calculation=True)
try:
temp_cube.replace_coord(forecast_period)
except CoordinateNotFoundError:
temp_cube.add_aux_coord(forecast_period)
cubes.append(temp_cube)
new_cube = cubes.merge_cube()
if order is not None:
new_cube.transpose(order)
return new_cube
def set_up_probability_cube(
data: ndarray,
thresholds: Union[List[float], ndarray],
variable_name: str = "air_temperature",
threshold_units: str = "K",
spp__relative_to_threshold: str = "greater_than",
**kwargs: Any,
) -> Cube:
"""
Set up a cube containing probabilities at thresholds with:
- x/y spatial dimensions (equal area or lat / lon)
- leading "threshold" dimension
- "time", "forecast_reference_time" and "forecast_period" scalar coords
- option to specify additional scalar coordinates
- "spp__relative_to_threshold" attribute (default "greater_than")
- default or configurable attributes
- configurable cube data, name conforms to
"probability_of_X_above(or below)_threshold" convention
Args:
data:
3D (threshold-y-x ordered) array of data to put into the cube
thresholds:
List of int / float threshold values whose length must match the
first dimension on the input data cube
variable_name:
Name of the underlying variable to which the probability field
applies, eg "air_temperature". NOT name of probability field.
threshold_units:
Units of the underlying variable / threshold.
spp__relative_to_threshold:
Value of the attribute "spp__relative_to_threshold" which is
required for IMPROVER probability cubes.
**kwargs:
Additional keyword arguments passed to 'set_up_variable_cube' function
Returns:
Cube containing probabilities at thresholds
"""
# create a "relative to threshold" attribute
coord_attributes = {
"spp__relative_to_threshold": spp__relative_to_threshold
}
if spp__relative_to_threshold in (
"above",
"greater_than",
"greater_than_or_equal_to",
):
name = "probability_of_{}_above_threshold".format(variable_name)
elif spp__relative_to_threshold in ("below", "less_than",
"less_than_or_equal_to"):
name = "probability_of_{}_below_threshold".format(variable_name)
else:
msg = ("The spp__relative_to_threshold attribute MUST be set for "
"IMPROVER probability cubes")
raise ValueError(msg)
cube = set_up_variable_cube(
data,
name=name,
units="1",
realizations=thresholds,
**kwargs,
)
threshold_name = variable_name.replace("_in_vicinity", "")
cube.coord("realization").rename(threshold_name)
cube.coord(threshold_name).var_name = "threshold"
cube.coord(threshold_name).attributes.update(coord_attributes)
cube.coord(threshold_name).units = Unit(threshold_units)
if len(thresholds) == 1:
cube = next(cube.slices_over(threshold_name))
return cube
def main(argv=None):
"""Load in arguments and ensure they are set correctly.
Then run Triangular weighted blending across the given coordinate."""
parser = ArgParser(
description='Use the TriangularWeightedBlendAcrossAdjacentPoints to '
'blend across a particular coordinate. It does not '
'collapse the coordinate, but instead blends across '
'adjacent points and puts the blended values back in the '
'original coordinate, with adjusted bounds.')
parser.add_argument('coordinate', type=str,
metavar='COORDINATE_TO_BLEND_OVER',
help='The coordinate over which the blending '
'will be applied.')
parser.add_argument('central_point', metavar='CENTRAL_POINT', type=float,
help='Central point at which the output from the '
'triangular weighted blending will be '
'calculated. This should be in the units of the '
'units argument that is passed in. '
'This value should be a point on the '
'coordinate for blending over.')
parser.add_argument('--units', metavar='UNIT_STRING', required=True,
help='Units of the the central_point and width.')
parser.add_argument('--calendar', metavar='CALENDAR',
default='gregorian',
help='Calendar for parameter_unit if required. '
'Default=gregorian')
parser.add_argument('--width', metavar='TRIANGLE_WIDTH', type=float,
required=True,
help='Width of the triangular weighting function used '
'in the blending, in the units of the '
'units argument passed in.')
parser.add_argument('--blend_time_using_forecast_period',
default=False, action='store_true', help='Flag that '
'we are blending over time but using the forecast '
'period coordinate as a proxy. Note this should only '
'be used when time and forecast_period share a '
'dimension: ie when all files provided are from the '
'same forecast cycle.')
parser.add_argument('input_filepaths', metavar='INPUT_FILES', nargs="+",
help='Paths to input NetCDF files including and '
'surrounding the central_point.')
parser.add_argument('output_filepath', metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
args = parser.parse_args(args=argv)
# TriangularWeightedBlendAcrossAdjacentPoints can't currently handle
# blending over times where iris reads the coordinate points as datetime
# objects. Fail here to avoid unhelpful errors downstream.
if "time" in args.coordinate:
msg = ("Cannot blend over {} coordinate (points encoded as datetime "
"objects)".format(args.coordinate))
raise ValueError(msg)
# This is left as a placeholder for when we have this capability
if args.coordinate == 'time':
units = Unit(args.units, args.calendar)
else:
units = args.units
cubelist = load_cubelist(args.input_filepaths)
if (args.blend_time_using_forecast_period and
args.coordinate == 'forecast_period'):
cube = MergeCubes().process(cubelist, check_time_bounds_ranges=True)
elif args.blend_time_using_forecast_period:
msg = ('"--blend_time_using_forecast_period" can only be used with '
'"forecast_period" coordinate')
raise ValueError(msg)
else:
cube = MergeCubes().process(cubelist)
BlendingPlugin = TriangularWeightedBlendAcrossAdjacentPoints(
args.coordinate, args.central_point, units, args.width)
result = BlendingPlugin.process(cube)
save_netcdf(result, args.output_filepath)
def test_eta_incompatible_units(self):
self.eta.units = Unit("km")
with self.assertRaises(ValueError):
OceanSigmaFactory(**self.kwargs)
def test_eta_incompatible_units(self):
new_eta = mock.Mock(units=Unit("Pa"), nbounds=0)
with self.assertRaises(ValueError):
self.factory.update(self.eta, new_eta)
def process(cubelist,
coordinate,
central_point,
units,
width,
calendar='gregorian',
blend_time_using_forecast_period=False):
"""Runs weighted blending across adjacent points.
Uses the TriangularWeightedBlendAcrossAdjacentPoints to blend across
a particular coordinate. It does not collapse the coordinate, but
instead blends across adjacent points and puts the blended values back
in the original coordinate, with adjusted bounds.
Args:
cubelist (iris.cube.CubeList):
CubeList including and surrounding the central point.
coordinate (str):
The coordinate over which the blending will be applied.
central_point (float):
Central point at which the output from the triangular weighted
blending will be calculated. This should be in the units of the
units argument that is passed in. This value should be a point
on the coordinate for blending over.
units (str):
Units of the central_point and width
width (float):
Width of the triangular weighting function used in the blending,
in the units of the units argument.
calendar (str)
Calendar for parameter_unit if required.
Default is 'gregorian'.
blend_time_using_forecast_period (bool):
If True, we are blending over time but using the forecast
period coordinate as a proxy. Note, this should only be used when
time and forecast_period share a dimension: i.e when all cubes
provided are from the same forecast cycle.
Default is False.
Returns:
iris.cube.Cube:
A processed Cube
Raises:
ValueError:
If coordinate has "time" in it.
ValueError:
If blend_time_forecast_period is not used with forecast_period
coordinate.
"""
# TriangularWeightedBlendAcrossAdjacentPoints can't currently handle
# blending over times where iris reads the coordinate points as datetime
# objects. Fail here to avoid unhelpful errors downstream.
if "time" in coordinate:
msg = ("Cannot blend over {} coordinate (points encoded as datetime "
"objects)".format(coordinate))
raise ValueError(msg)
# This is left as a placeholder for when we have this capability
if coordinate == 'time':
units = Unit(units, calendar)
if blend_time_using_forecast_period and coordinate == 'forecast_period':
cube = MergeCubes().process(cubelist, check_time_bounds_ranges=True)
elif blend_time_using_forecast_period:
msg = ('"--blend_time_using_forecast_period" can only be used with '
'"forecast_period" coordinate')
raise ValueError(msg)
else:
cube = MergeCubes().process(cubelist)
blending_plugin = TriangularWeightedBlendAcrossAdjacentPoints(
coordinate, central_point, units, width)
result = blending_plugin.process(cube)
return result
def units_temporal(units):
try:
u = Unit(units)
except ValueError:
return False
return u.is_time_reference()
def __init__(self,
thresholds,
fuzzy_factor=None,
fuzzy_bounds=None,
threshold_units=None,
comparison_operator='>'):
"""
Set up for processing an in-or-out of threshold field, including the
generation of fuzzy_bounds which are required to threshold an input
cube (through self.process(cube)). If fuzzy_factor is not None, fuzzy
bounds are calculated using the threshold value in the units in which
it is provided.
The usage of fuzzy_factor is exemplified as follows:
For a 6 mm/hr threshold with a 0.75 fuzzy factor, a range of 25%
around this threshold (between (6*0.75=) 4.5 and (6*(2-0.75)=) 7.5)
would be generated. The probabilities of exceeding values within this
range are scaled linearly, so that 4.5 mm/hr yields a thresholded value
of 0 and 7.5 mm/hr yields a thresholded value of 1. Therefore, in this
case, the thresholded exceedance probabilities between 4.5 mm/hr and
7.5 mm/hr would follow the pattern:
::
Data value | Probability
------------|-------------
4.5 | 0
5.0 | 0.167
5.5 | 0.333
6.0 | 0.5
6.5 | 0.667
7.0 | 0.833
7.5 | 1.0
Args:
thresholds (list of float or float):
The threshold points for 'significant' datapoints.
fuzzy_factor (float):
Specifies lower bound for fuzzy membership value when
multiplied by each threshold. Upper bound is equivalent linear
distance above threshold. If None, no fuzzy_factor is applied.
fuzzy_bounds (list of tuple):
Lower and upper bounds for fuzziness.
List should be of same length as thresholds.
Each entry in list should be a tuple of two floats
representing the lower and upper bounds respectively.
If None, no fuzzy_bounds are applied.
threshold_units (str):
Units of the threshold values. If not provided the units are
assumed to be the same as those of the input cube.
comparison_operator (str):
Indicates the comparison_operator to use with the threshold.
e.g. 'ge' or '>=' to evaluate data >= threshold or '<' to
evaluate data < threshold. When using fuzzy thresholds, there
is no difference between < and <= or > and >=.
Valid choices: > >= < <= gt ge lt le.
Raises:
ValueError: If a threshold of 0.0 is requested when using a fuzzy
factor.
ValueError: If the fuzzy_factor is not greater than 0 and less
than 1.
ValueError: If both fuzzy_factor and fuzzy_bounds are set
as this is ambiguous.
"""
# ensure threshold is a list, even if only a single value is provided
self.thresholds = thresholds
if np.isscalar(thresholds):
self.thresholds = [thresholds]
# if necessary, set threshold units
if threshold_units is None:
self.threshold_units = None
else:
self.threshold_units = Unit(threshold_units)
# initialise threshold coordinate name as None
self.threshold_coord_name = None
# read fuzzy factor or set (default) to 1 (no smoothing)
fuzzy_factor_loc = 1.
if fuzzy_factor is not None:
if fuzzy_bounds is not None:
raise ValueError(
"Invalid combination of keywords. Cannot specify "
"fuzzy_factor and fuzzy_bounds together")
if not 0 < fuzzy_factor < 1:
raise ValueError(
"Invalid fuzzy_factor: must be >0 and <1: {}".format(
fuzzy_factor))
if 0 in self.thresholds:
raise ValueError(
"Invalid threshold with fuzzy factor: cannot use a "
"multiplicative fuzzy factor with threshold == 0")
fuzzy_factor_loc = fuzzy_factor
# Set fuzzy-bounds. If neither fuzzy_factor nor fuzzy_bounds is set,
# both lower_thr and upper_thr default to the threshold value. A test
# of this equality is used later to determine whether to process with
# a sharp threshold or fuzzy bounds.
if fuzzy_bounds is None:
self.fuzzy_bounds = []
for thr in self.thresholds:
lower_thr = thr * fuzzy_factor_loc
upper_thr = thr * (2. - fuzzy_factor_loc)
if thr < 0:
lower_thr, upper_thr = upper_thr, lower_thr
self.fuzzy_bounds.append((lower_thr, upper_thr))
else:
self.fuzzy_bounds = fuzzy_bounds
# ensure fuzzy_bounds is a list of tuples
if isinstance(fuzzy_bounds, tuple):
self.fuzzy_bounds = [fuzzy_bounds]
# check that thresholds and fuzzy_bounds are self-consistent
for thr, bounds in zip(self.thresholds, self.fuzzy_bounds):
if len(bounds) != 2:
raise ValueError("Invalid bounds for one threshold: {}."
" Expected 2 floats.".format(bounds))
if bounds[0] > thr or bounds[1] < thr:
bounds_msg = ("Threshold must be within bounds: "
"!( {} <= {} <= {} )".format(
bounds[0], thr, bounds[1]))
raise ValueError(bounds_msg)
# Dict of known logical comparisons. Each key contains a dict of
# {'function': The operator function for this comparison_operator,
# 'spp_string': Comparison_Operator string for use in CF-convention
# meta-data}
self.comparison_operator_dict = {}
self.comparison_operator_dict.update(
dict.fromkeys(['ge', 'GE', '>='], {
'function': operator.ge,
'spp_string': 'above'
}))
self.comparison_operator_dict.update(
dict.fromkeys(['gt', 'GT', '>'], {
'function': operator.gt,
'spp_string': 'above'
}))
self.comparison_operator_dict.update(
dict.fromkeys(['le', 'LE', '<='], {
'function': operator.le,
'spp_string': 'below'
}))
self.comparison_operator_dict.update(
dict.fromkeys(['lt', 'LT', '<'], {
'function': operator.lt,
'spp_string': 'below'
}))
self.comparison_operator_string = comparison_operator
self._decode_comparison_operator_string()
def check_ancils(
a_over_s_cube: Cube,
sigma_cube: Cube,
z0_cube: Optional[Cube],
pp_oro_cube: Cube,
model_oro_cube: Cube,
) -> ndarray:
"""Check ancils grid and units.
Check if ancil cubes are on the same grid and if they have the
expected units. The testing for "same grid" might be replaced
if there is a general utils function made for it or so.
Args:
a_over_s_cube:
holding the silhouette roughness field
sigma_cube:
holding the standard deviation of height in a grid cell
z0_cube:
holding the vegetative roughness field
pp_oro_cube:
holding the post processing grid orography
model_oro_cube:
holding the model orography on post processing grid
Returns:
Containing bools describing whether or not the tests passed
"""
ancil_list = [a_over_s_cube, sigma_cube, pp_oro_cube, model_oro_cube]
unwanted_coord_list = [
"time",
"height",
"model_level_number",
"forecast_time",
"forecast_reference_time",
"forecast_period",
]
for field, exp_unit in zip(
ancil_list,
[1, Unit("m"), Unit("m"), Unit("m")]):
for unwanted_coord in unwanted_coord_list:
try:
field.remove_coord(unwanted_coord)
except CoordinateNotFoundError:
pass
if field.units != exp_unit:
msg = ("{} ancil field has unexpected unit:"
" {} (expected) vs. {} (actual)")
raise ValueError(
msg.format(field.name(), exp_unit, field.units))
if z0_cube is not None:
ancil_list.append(z0_cube)
for unwanted_coord in unwanted_coord_list:
try:
z0_cube.remove_coord(unwanted_coord)
except CoordinateNotFoundError:
pass
if z0_cube.units != Unit("m"):
msg = "z0 ancil has unexpected unit: should be {} " "is {}"
raise ValueError(msg.format(Unit("m"), z0_cube.units))
permutated_ancil_list = list(itertools.permutations(ancil_list, 2))
oklist = []
for entry in permutated_ancil_list:
x_axis_flag = entry[0].coord(axis="y") == entry[1].coord(axis="y")
y_axis_flag = entry[0].coord(axis="x") == entry[1].coord(axis="x")
oklist.append(x_axis_flag & y_axis_flag)
# HybridHeightToPhenomOnPressure._cube_compatibility_check(entr[0],
# entr[1])
return np.array(oklist).all() # replace by a return value of True
def NAME_to_cube(filenames, callback):
"""
Returns a generator of cubes given a list of filenames and a callback.
"""
for filename in filenames:
header, column_headings, data_arrays = load_NAME_III(filename)
for i, data_array in enumerate(data_arrays):
# turn the dictionary of column headers with a list of header
# information for each field into a dictionary of headers for just
# this field. Ignore the first 4 columns of grid position (data was
# located with the data array).
field_headings = dict((k, v[i + 4])
for k, v in column_headings.items())
# make an cube
cube = iris.cube.Cube(data_array)
# define the name and unit
name = ('%s %s' % (field_headings['species'],
field_headings['quantity']))
name = name.upper().replace(' ', '_')
cube.rename(name)
# Some units are badly encoded in the file, fix this by putting a
# space in between. (if gs is not found, then the string will be
# returned unchanged)
cube.units = field_headings['unit'].replace('gs', 'g s')
# define and add the singular coordinates of the field (flight
# level, time etc.)
cube.add_aux_coord(icoords.AuxCoord(field_headings['z_level'],
long_name='flight_level',
units='1'))
# define the time unit and use it to serialise the datetime for the
# time coordinate
time_unit = Unit('hours since epoch', calendar=CALENDAR_GREGORIAN)
time_coord = icoords.AuxCoord(
time_unit.date2num(field_headings['time']),
standard_name='time',
units=time_unit)
cube.add_aux_coord(time_coord)
# build a coordinate system which can be referenced by latitude and
# longitude coordinates
lat_lon_coord_system = icoord_systems.GeogCS(6371229)
# build regular latitude and longitude coordinates which have
# bounds
start = header['X grid origin'] + header['X grid resolution']
step = header['X grid resolution']
count = header['X grid size']
pts = start + np.arange(count, dtype=np.float32) * step
lon_coord = icoords.DimCoord(pts, standard_name='longitude',
units='degrees',
coord_system=lat_lon_coord_system)
lon_coord.guess_bounds()
start = header['Y grid origin'] + header['Y grid resolution']
step = header['Y grid resolution']
count = header['Y grid size']
pts = start + np.arange(count, dtype=np.float32) * step
lat_coord = icoords.DimCoord(pts, standard_name='latitude',
units='degrees',
coord_system=lat_lon_coord_system)
lat_coord.guess_bounds()
# add the latitude and longitude coordinates to the cube, with
# mappings to data dimensions
cube.add_dim_coord(lat_coord, 0)
cube.add_dim_coord(lon_coord, 1)
# implement standard iris callback capability. Although callbacks
# are not used in this example, the standard mechanism for a custom
# loader to implement a callback is shown:
cube = iris.io.run_callback(callback, cube,
[header, field_headings, data_array],
filename)
# yield the cube created (the loop will continue when the next()
# element is requested)
yield cube
def setUp(self):
self.sigma = mock.Mock(units=Unit('1'), nbounds=0)
self.eta = mock.Mock(units=Unit('m'), nbounds=0)
self.depth = mock.Mock(units=Unit('m'), nbounds=0)
self.kwargs = dict(sigma=self.sigma, eta=self.eta, depth=self.depth)
self.factory = OceanSigmaFactory(**self.kwargs)
class BasicThreshold(object):
"""Apply a threshold truth criterion to a cube.
Calculate the threshold truth values based on a linear membership function
around the threshold values provided. A cube will be returned with a new
threshold dimension coordinate.
Can operate on multiple time sequences within a cube.
"""
def __init__(self,
thresholds,
fuzzy_factor=None,
fuzzy_bounds=None,
threshold_units=None,
below_thresh_ok=False):
"""
Set up for processing an in-or-out of threshold field, including the
generation of fuzzy_bounds which are required to threshold an input
cube (through self.process(cube)). If fuzzy_factor is not None, fuzzy
bounds are calculated using the threshold value in the units in which
it is provided.
The usage of fuzzy_factor is exemplified as follows:
For a 6 mm/hr threshold with a 0.75 fuzzy factor, a range of 25%
around this threshold (between (6*0.75=) 4.5 and (6*(2-0.75)=) 7.5)
would be generated. The probabilities of exceeding values within this
range are scaled linearly, so that 4.5 mm/hr yields a thresholded value
of 0 and 7.5 mm/hr yields a thresholded value of 1. Therefore, in this
case, the thresholded exceedance probabilities between 4.5 mm/hr and
7.5 mm/hr would follow the pattern:
::
Data value | Probability
------------|-------------
4.5 | 0
5.0 | 0.167
5.5 | 0.333
6.0 | 0.5
6.5 | 0.667
7.0 | 0.833
7.5 | 1.0
Args:
thresholds (list of floats or float):
The threshold points for 'significant' datapoints.
Keyword Args:
fuzzy_factor (float):
Specifies lower bound for fuzzy membership value when
multiplied by each threshold. Upper bound is equivalent linear
distance above threshold. If None, no fuzzy_factor is applied.
fuzzy_bounds (list of tuples):
Lower and upper bounds for fuzziness.
List should be of same length as thresholds.
Each entry in list should be a tuple of two floats
representing the lower and upper bounds respectively.
If None, no fuzzy_bounds are applied.
threshold_units (string):
Units of the threshold values. If not provided the units are
assumed to be the same as those of the input cube.
below_thresh_ok (boolean):
True to count points as significant if *below* the threshold,
False to count points as significant if *above* the threshold.
Raises:
ValueError: If a threshold of 0.0 is requested when using a fuzzy
factor.
ValueError: If the fuzzy_factor is not greater than 0 and less
than 1.
ValueError: If both fuzzy_factor and fuzzy_bounds are set
as this is ambiguous.
"""
# ensure threshold is a list, even if only a single value is provided
self.thresholds = thresholds
if np.isscalar(thresholds):
self.thresholds = [thresholds]
# if necessary, set threshold units
if threshold_units is None:
self.threshold_units = None
else:
self.threshold_units = Unit(threshold_units)
# initialise threshold coordinate name as None
self.threshold_coord_name = None
# read fuzzy factor or set (default) to 1 (no smoothing)
fuzzy_factor_loc = 1.
if fuzzy_factor is not None:
if fuzzy_bounds is not None:
raise ValueError(
"Invalid combination of keywords. Cannot specify "
"fuzzy_factor and fuzzy_bounds together")
if not 0 < fuzzy_factor < 1:
raise ValueError(
"Invalid fuzzy_factor: must be >0 and <1: {}".format(
fuzzy_factor))
if 0 in self.thresholds:
raise ValueError(
"Invalid threshold with fuzzy factor: cannot use a "
"multiplicative fuzzy factor with threshold == 0")
fuzzy_factor_loc = fuzzy_factor
# Set fuzzy-bounds. If neither fuzzy_factor nor fuzzy_bounds is set,
# both lower_thr and upper_thr default to the threshold value. A test
# of this equality is used later to determine whether to process with
# a sharp threshold or fuzzy bounds.
if fuzzy_bounds is None:
self.fuzzy_bounds = []
for thr in self.thresholds:
lower_thr = thr * fuzzy_factor_loc
upper_thr = thr * (2. - fuzzy_factor_loc)
if thr < 0:
lower_thr, upper_thr = upper_thr, lower_thr
self.fuzzy_bounds.append((lower_thr, upper_thr))
else:
self.fuzzy_bounds = fuzzy_bounds
# ensure fuzzy_bounds is a list of tuples
if isinstance(fuzzy_bounds, tuple):
self.fuzzy_bounds = [fuzzy_bounds]
# check that thresholds and fuzzy_bounds are self-consistent
for thr, bounds in zip(self.thresholds, self.fuzzy_bounds):
assert len(bounds) == 2, ("Invalid bounds for one threshold: {}. "
"Expected 2 floats.".format(bounds))
bounds_msg = ("Threshold must be within bounds: "
"!( {} <= {} <= {} )".format(bounds[0], thr,
bounds[1]))
assert bounds[0] <= thr, bounds_msg
assert bounds[1] >= thr, bounds_msg
self.below_thresh_ok = below_thresh_ok
def __repr__(self):
"""Represent the configured plugin instance as a string."""
return ('<BasicThreshold: thresholds {}, ' + 'fuzzy_bounds {}, ' +
'below_thresh_ok: {}>').format(self.thresholds,
self.fuzzy_bounds,
self.below_thresh_ok)
def _add_threshold_coord(self, cube, threshold):
"""
Add a scalar threshold-type coordinate to a cube containing
thresholded data and promote the new coordinate to be the
leading dimension of the cube.
Args:
cube (iris.cube.Cube):
Cube containing thresholded data (1s and 0s)
threshold (np.float32):
Value at which the data has been thresholded
Returns:
iris.cube.Cube:
With new "threshold" axis
"""
try:
coord = iris.coords.DimCoord(
np.array([threshold], dtype=np.float32),
standard_name=self.threshold_coord_name,
var_name="threshold",
units=cube.units)
except ValueError as cause:
if 'is not a valid standard_name' in str(cause):
coord = iris.coords.DimCoord(
np.array([threshold], dtype=np.float32),
long_name=self.threshold_coord_name,
var_name="threshold",
units=cube.units)
else:
raise ValueError(cause)
# Use an spp__relative_to_threshold attribute, as an extension to the
# CF-conventions.
if self.below_thresh_ok:
coord.attributes.update({'spp__relative_to_threshold': 'below'})
else:
coord.attributes.update({'spp__relative_to_threshold': 'above'})
cube.add_aux_coord(coord)
return iris.util.new_axis(cube, coord)
def process(self, input_cube):
"""Convert each point to a truth value based on provided threshold
values. The truth value may or may not be fuzzy depending upon if
fuzzy_bounds are supplied. If the plugin has a "threshold_units"
member, this is used to convert both thresholds and fuzzy bounds into
the units of the input cube.
Args:
input_cube (iris.cube.Cube):
Cube to threshold. The code is dimension-agnostic.
Returns:
cube (iris.cube.Cube):
Cube after a threshold has been applied. The data within this
cube will contain values between 0 and 1 to indicate whether
a given threshold has been exceeded or not.
The cube meta-data will contain:
* Input_cube name prepended with
probability_of_X_above(or below)_threshold (where X is
the diagnostic under consideration)
* Threshold dimension coordinate with same units as input_cube
* Threshold attribute (above or below threshold)
* Cube units set to (1).
Raises:
ValueError: if a np.nan value is detected within the input cube.
"""
# Record input cube data type to ensure consistent output, though
# integer data must become float to enable fuzzy thresholding.
input_cube_dtype = input_cube.dtype
if input_cube.dtype.kind == 'i':
input_cube_dtype = np.float32
thresholded_cubes = iris.cube.CubeList()
if np.isnan(input_cube.data).any():
raise ValueError("Error: NaN detected in input cube data")
# if necessary, convert thresholds and fuzzy bounds into cube units
if self.threshold_units is not None:
self.thresholds = [
self.threshold_units.convert(threshold, input_cube.units)
for threshold in self.thresholds
]
self.fuzzy_bounds = [
tuple([
self.threshold_units.convert(threshold, input_cube.units)
for threshold in bounds
]) for bounds in self.fuzzy_bounds
]
# set name of threshold coordinate to match input diagnostic
self.threshold_coord_name = input_cube.name()
# apply fuzzy thresholding
for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds):
cube = input_cube.copy()
# if upper and lower bounds are equal, set a deterministic 0/1
# probability based on exceedance of the threshold
if bounds[0] == bounds[1]:
truth_value = cube.data > threshold
# otherwise, scale exceedance probabilities linearly between 0/1
# at the min/max fuzzy bounds and 0.5 at the threshold value
else:
truth_value = np.where(
cube.data < threshold,
rescale(cube.data,
data_range=(bounds[0], threshold),
scale_range=(0., 0.5),
clip=True),
rescale(cube.data,
data_range=(threshold, bounds[1]),
scale_range=(0.5, 1.),
clip=True),
)
truth_value = truth_value.astype(input_cube_dtype)
# if requirement is for probabilities below threshold (rather than
# above), invert the exceedance probability
if self.below_thresh_ok:
truth_value = 1. - truth_value
cube.data = truth_value
# Overwrite masked values that have been thresholded
# with the un-thresholded values from the input cube.
if np.ma.is_masked(cube.data):
cube.data[input_cube.data.mask] = (
input_cube.data[input_cube.data.mask])
cube = self._add_threshold_coord(cube, threshold)
thresholded_cubes.append(cube)
cube, = thresholded_cubes.concatenate()
if self.below_thresh_ok:
cube.rename("probability_of_{}_below_threshold".format(
cube.name()))
else:
cube.rename("probability_of_{}_above_threshold".format(
cube.name()))
cube.units = Unit(1)
cube = enforce_coordinate_ordering(cube, ["realization", "percentile"])
return cube
def test_sigma(self):
new_sigma = mock.Mock(units=Unit('1'), nbounds=0)
self.factory.update(self.sigma, new_sigma)
self.assertIs(self.factory.sigma, new_sigma)
def process(
self,
orography: Cube,
thresholds_dict: Dict[str, List[float]],
landmask: Optional[Cube] = None,
) -> Cube:
"""Calculate the weights depending upon where the orography point is
within the topographic zones.
Args:
orography:
Orography on standard grid.
thresholds_dict:
Definition of orography bands required.
The expected format of the dictionary is e.g.
`{'bounds': [[0, 50], [50, 200]], 'units': 'm'}`
landmask:
Land mask on standard grid, with land points set to one and
sea points set to zero. If provided sea points are masked
out in the output array.
Returns:
Cube containing the weights depending upon where the orography
point is within the topographic zones.
"""
# Check that orography is a 2d cube.
if len(orography.shape) != 2:
msg = ("The input orography cube should be two-dimensional."
"The input orography cube has {} dimensions".format(
len(orography.shape)))
raise InvalidCubeError(msg)
# Find bands and midpoints from bounds.
bands = np.array(thresholds_dict["bounds"], dtype=np.float32)
threshold_units = thresholds_dict["units"]
# Create topographic_zone_cube first, so that a cube is created for
# each band. This will allow the data for neighbouring bands to be
# put into the cube.
mask_data = np.zeros(orography.shape, dtype=np.float32)
topographic_zone_cubes = iris.cube.CubeList([])
for band in bands:
sea_points_included = not landmask
topographic_zone_cube = _make_mask_cube(
mask_data,
orography.coords(),
band,
threshold_units,
sea_points_included=sea_points_included,
)
topographic_zone_cubes.append(topographic_zone_cube)
topographic_zone_weights = topographic_zone_cubes.concatenate_cube()
topographic_zone_weights.data = topographic_zone_weights.data.astype(
np.float32)
# Ensure topographic_zone coordinate units is equal to orography units.
topographic_zone_weights.coord("topographic_zone").convert_units(
orography.units)
# Read bands from cube, now that they can be guaranteed to be in the
# same units as the orography. The bands are converted to a list, so
# that they can be iterated through.
bands = list(topographic_zone_weights.coord("topographic_zone").bounds)
midpoints = topographic_zone_weights.coord("topographic_zone").points
# Raise a warning, if orography extremes are outside the extremes of
# the bands.
if np.max(orography.data) > np.max(bands):
msg = ("The maximum orography is greater than the uppermost band. "
"This will potentially cause the topographic zone weights "
"to not sum to 1 for a given grid point.")
warnings.warn(msg)
if np.min(orography.data) < np.min(bands):
msg = ("The minimum orography is lower than the lowest band. "
"This will potentially cause the topographic zone weights "
"to not sum to 1 for a given grid point.")
warnings.warn(msg)
# Insert the appropriate weights into the topographic zone cube. This
# includes the weights from the band that a point is in, as well as
# the contribution from an adjacent band.
for band_number, band in enumerate(bands):
# Determine the points that are within the specified band.
mask_y, mask_x = np.where((orography.data > band[0])
& (orography.data <= band[1]))
orography_band = np.full(orography.shape, np.nan, dtype=np.float32)
orography_band[mask_y, mask_x] = orography.data[mask_y, mask_x]
# Calculate the weights. This involves calculating the
# weights for all the orography but only inserting weights
# that are within the band into the topographic_zone_weights cube.
weights = self.calculate_weights(orography_band, band)
topographic_zone_weights.data[band_number, mask_y,
mask_x] = weights[mask_y, mask_x]
# Calculate the contribution to the weights from the adjacent
# lower band.
topographic_zone_weights.data = self.add_weight_to_lower_adjacent_band(
topographic_zone_weights.data,
orography_band,
midpoints[band_number],
band_number,
)
# Calculate the contribution to the weights from the adjacent
# upper band.
topographic_zone_weights.data = self.add_weight_to_upper_adjacent_band(
topographic_zone_weights.data,
orography_band,
midpoints[band_number],
band_number,
len(bands) - 1,
)
# Metadata updates
topographic_zone_weights.rename("topographic_zone_weights")
topographic_zone_weights.units = Unit("1")
# Mask output weights using a land-sea mask.
topographic_zone_masked_weights = iris.cube.CubeList([])
for topographic_zone_slice in topographic_zone_weights.slices_over(
"topographic_zone"):
if landmask:
topographic_zone_slice.data = GenerateOrographyBandAncils(
).sea_mask(landmask.data, topographic_zone_slice.data)
topographic_zone_masked_weights.append(topographic_zone_slice)
topographic_zone_weights = topographic_zone_masked_weights.merge_cube()
return topographic_zone_weights
def test_sigma_too_many_bounds(self):
new_sigma = mock.Mock(units=Unit('1'), nbounds=4)
with self.assertRaises(ValueError):
self.factory.update(self.sigma, new_sigma)
def test_sigma_incompatible_units(self):
new_sigma = mock.Mock(units=Unit('Pa'), nbounds=0)
with self.assertRaises(ValueError):
self.factory.update(self.sigma, new_sigma)
def test_eta(self):
new_eta = mock.Mock(units=Unit('m'), nbounds=0)
self.factory.update(self.eta, new_eta)
self.assertIs(self.factory.eta, new_eta)
def units(self, units):
new_units = units if isinstance(units, Unit) else Unit(units)
if not new_units.is_no_unit():
raise UnitsError('CharArrays cannot have units.')
self._optinfo['units'] = new_units
def test_depth(self):
new_depth = mock.Mock(units=Unit('m'), nbounds=0)
self.factory.update(self.depth, new_depth)
self.assertIs(self.factory.depth, new_depth)
def test_non_gregorian_calendar_conversion_dtype(self):
data = np.arange(4, dtype=np.float32)
u1 = Unit('hours since 2000-01-01 00:00:00', calendar='360_day')
u2 = Unit('hours since 2000-01-02 00:00:00', calendar='360_day')
result = u1.convert(data, u2)
self.assertEqual(result.dtype, np.float32)
q = iris.load('p_lvl_data.nc', 'q')[0][step] # Specific humidity
# --------------------------------------------------------------------
# --------------------- Define Other Variables -----------------------
pressure_levels = [300, 350, 400, 450, 500, 550, 600, 650, 700, 750,
800, 850, 900, 925, 950, 975, 1000]
g = 9.807 # Gravity
e0 = 6.11 # Saturation vapour pressure at T0 in hPa
eps = 0.622 # Ratio of Rd/Rv in kg/kg
# --------------------------------------------------------------------
# ------------------- Calculate Derived Variables --------------------
# Create cube with all entries zero
s_zero = (t2m.copy() * 0) # Same dimensions as single level data
p_zero = (t.copy() * 0) # Same dimensions as pressure level data
s_zero.units = Unit('Unknown') # Erase units
p_zero.units = Unit('Unknown') # Erase units
# Calculate variables
e_sfc = s_zero.copy()
q_sfc = s_zero.copy()
e_sfc.data = e0 * np.exp((17.27 * d2m.data)/(d2m.data + 237.3))
q_sfc.data = (eps * e_sfc.data)/(sp.data - ((1-eps) * e_sfc.data))
# --------------------------------------------------------------------
p = p_zero
for i in range(17):
p.data[i] = pressure_levels[i]
# ------------------------ Calculate Variable ------------------------
# Calculate integrand for max saturation on pressure levels
integrand_p = p_zero.copy()
def _check_variable_attrs(dataset, var_name, required_attributes=None):
'''
Convenience method to check a variable attributes based on the
expected_vars dict
'''
score = 0
out_of = 0
messages = []
if var_name not in dataset.variables:
# No need to check the attrs if the variable doesn't exist
return (score, out_of, messages)
var = dataset.variables[var_name]
# Get the expected attrs to check
check_attrs = required_attributes or required_var_attrs.get(var_name, {})
var_attrs = set(var.ncattrs())
for attr in check_attrs:
if attr == 'dtype':
# dtype check is special, see above
continue
out_of += 1
score += 1
# Check if the attribute is present
if attr not in var_attrs:
messages.append('Variable {} must contain attribute: {}'
''.format(var_name, attr))
score -= 1
continue
# Attribute exists, let's check if there was a value we need to compare against
if check_attrs[attr] is not None:
if getattr(var, attr) != check_attrs[attr]:
# No match, this may be an error, but first an exception for units
if attr == 'units':
msg = ('Variable {} units attribute must be '
'convertible to {}'.format(var_name, check_attrs[attr]))
try:
cur_unit = Unit(var.units)
comp_unit = Unit(check_attrs[attr])
if not cur_unit.is_convertible(comp_unit):
messages.append(msg)
score -= 1
except ValueError:
messages.append(msg)
score -= 1
else:
messages.append('Variable {} attribute {} must be {}'.format(var_name, attr, check_attrs[attr]))
score -= 1
else:
# Final check to make sure the attribute isn't an empty string
try:
# try stripping whitespace, and return an error if empty
att_strip = getattr(var, attr).strip()
if not att_strip:
messages.append('Variable {} attribute {} is empty'
''.format(var_name, attr))
score -= 1
except AttributeError:
pass
return (score, out_of, messages)
def test_depth_incompatible_units(self):
self.depth.units = Unit('km')
with self.assertRaises(ValueError):
OceanSigmaFactory(**self.kwargs)
class TestCategoriseCoordFunctionForTime:
def __init__(self):
self.u = Unit('days since 1600-01-01 00:00:00', calendar=CALENDAR_STANDARD)
self.points = np.arange(1, 5, 1)
self.coord = iris.coords.DimCoord(self.points, units=self.u)
self.start = datetime.datetime(2000, 1, 1)
self.end = datetime.datetime(2003, 4, 24)
self.start = self.u.date2num(self.start)
self.end = self.u.date2num(self.end)
def setup_func(self):
self.__init__()
@with_setup(setup_func)
def test_categorise_coord_function_time_year_only(self):
delta = date_delta_creator(1)
result_function = categorise_coord_function(self.start, self.end, delta, True)
expected = np.array([self.u.date2num(datetime.datetime(2000, 7, 1, 0, 0, 0)),
self.u.date2num(datetime.datetime(2001, 7, 1, 0, 0, 0)),
self.u.date2num(datetime.datetime(2002, 7, 1, 0, 0, 0))])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2000, 1, 1, 0, 0, 0))), expected[0])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2001, 3, 3, 0, 0, 0))), expected[1])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2002, 5, 8, 0, 0, 0))), expected[2])
@with_setup(setup_func)
def test_categorise_coord_function_time_year_month(self):
delta = date_delta_creator(1, 1)
result_function = categorise_coord_function(self.start, self.end, delta, True)
expected = np.array([self.u.date2num(datetime.datetime(2000, 7, 15, 0, 0, 0)),
self.u.date2num(datetime.datetime(2001, 8, 15, 0, 0, 0)),
self.u.date2num(datetime.datetime(2002, 9, 15, 0, 0, 0))])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2000, 1, 1, 0, 0, 0))), expected[0])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2001, 3, 3, 0, 0, 0))), expected[1])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2002, 5, 8, 0, 0, 0))), expected[2])
@with_setup(setup_func)
def test_categorise_coord_function_with_month_going_past_december(self):
start = datetime.datetime(2000, 11, 3)
start = self.u.date2num(start)
end = datetime.datetime(2004, 11, 3)
end = self.u.date2num(end)
delta = date_delta_creator(1, 2)
result_function = categorise_coord_function(start, end, delta, True)
expected = np.array([self.u.date2num(datetime.datetime(2001, 6, 3, 0, 0, 0)),
self.u.date2num(datetime.datetime(2002, 8, 3, 0, 0, 0)),
self.u.date2num(datetime.datetime(2003, 10, 3, 0, 0, 0))])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(1999, 1, 1, 0, 0, 0))), expected[0])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2001, 3, 3, 0, 0, 0))), expected[0])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2002, 3, 3, 0, 0, 0))), expected[1])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2005, 5, 8, 0, 0, 0))), expected[2])
@with_setup(setup_func)
def test_categorise_coord_function_time_year_month_day_hour_minute_second(self):
delta = date_delta_creator(1, 3, 2, 4, 5, 6)
result_function = categorise_coord_function(self.start, self.end, delta, True)
expected = np.array([self.u.date2num(datetime.datetime(2000, 8, 16, 2, 2, 33)),
self.u.date2num(datetime.datetime(2001, 11, 18, 6, 7, 39)),
self.u.date2num(datetime.datetime(2003, 2, 20, 10, 12, 45))])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2000, 1, 1, 0, 0, 0))), expected[0])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2001, 7, 3, 0, 0, 0))), expected[1])
assert_equal(result_function(self.coord, self.u.date2num(datetime.datetime(2002, 9, 8, 0, 0, 0))), expected[2])
def setUp(self):
self.sigma = mock.Mock(units=Unit('1'), nbounds=0)
self.eta = mock.Mock(units=Unit('m'), nbounds=0)
self.depth = mock.Mock(units=Unit('m'), nbounds=0)
self.kwargs = dict(sigma=self.sigma, eta=self.eta, depth=self.depth)
def test_fail(self):
units = Unit('days since 1970-01-02 00:00:00', calendar='gregorian')
cube2 = Cube([1, 2, 3], 'air_temperature', units='K')
cube2.add_dim_coord(DimCoord([0, 1, 2], 'time', units=units), 0)
with self.assertRaises(iris.exceptions.ConcatenateError):
CubeList([self.cube1, cube2]).concatenate_cube()
def set_up_cube(num_time_points=1,
num_grid_points=1,
num_height_levels=7,
data=None,
name=None,
unit=None,
height=None):
"""Set up a UK model standard grid (UKVX) cube."""
cubel = iris.cube.CubeList()
tunit = Unit("hours since 1970-01-01 00:00:00", "gregorian")
t_0 = 402192.5
if isinstance(num_grid_points, int):
num_grid_points_x = num_grid_points_y = num_grid_points
else:
num_grid_points_x = num_grid_points[0]
num_grid_points_y = num_grid_points[1]
for i_idx in range(num_height_levels):
cubel1 = iris.cube.CubeList()
for j_idx in range(num_time_points):
cube = _make_ukvx_grid()
cube = cube[:num_grid_points_x, :num_grid_points_y]
cube.add_aux_coord(AuxCoord(t_0 + j_idx, "time", units=tunit))
if height is None:
cube.add_aux_coord(AuxCoord(i_idx, "model_level_number"))
elif isinstance(height, float) or isinstance(height, int):
cube.add_aux_coord(
AuxCoord(height, "height", units=Unit("meter")))
else:
cube.add_aux_coord(
AuxCoord(height[i_idx], "height", units=Unit("meter")))
cubel1.append(cube)
cubel.append(cubel1.merge_cube())
cubel = cubel.merge(0)
cube = cubel[0]
if data is not None:
try:
data = np.array(data, dtype=np.float32)
cube.data = data.reshape(cube.data.shape)
except ValueError as err:
if err == "total size of new array must be unchanged":
msg = ("supplied data does not fit the cube."
"cube dimensions: {} vs. supplied data {}")
raise ValueError(msg.format(cube.shape, data.shape))
raise ValueError(err)
if name is not None:
try:
cube.standard_name = name
except ValueError as err:
msg = ("error trying to set the supplied name as cube data name: "
"{}".format(err))
raise ValueError(msg)
except TypeError as err:
msg = ("error trying to set the supplied name as cube data name: "
"the name should be string and have a valid variable name "
"{}".format(err))
raise ValueError(msg)
if unit is not None:
try:
cube.units = Unit(unit)
except ValueError as err:
msg = "error trying to set Units to cube. supplied unit: {}"
raise ValueError(msg.format(unit))
return cube
def setUp(self):
self.units = Unit('days since 1970-01-01 00:00:00',
calendar='gregorian')
self.cube1 = Cube([1, 2, 3], 'air_temperature', units='K')
self.cube1.add_dim_coord(DimCoord([0, 1, 2], 'time', units=self.units),
0)
def process(self, input_cube):
"""Convert each point to a truth value based on provided threshold
values. The truth value may or may not be fuzzy depending upon if
fuzzy_bounds are supplied. If the plugin has a "threshold_units"
member, this is used to convert both thresholds and fuzzy bounds into
the units of the input cube.
Args:
input_cube (iris.cube.Cube):
Cube to threshold. The code is dimension-agnostic.
Returns:
cube (iris.cube.Cube):
Cube after a threshold has been applied. The data within this
cube will contain values between 0 and 1 to indicate whether
a given threshold has been exceeded or not.
The cube meta-data will contain:
* Input_cube name prepended with
probability_of_X_above(or below)_threshold (where X is
the diagnostic under consideration)
* Threshold dimension coordinate with same units as input_cube
* Threshold attribute (above or below threshold)
* Cube units set to (1).
Raises:
ValueError: if a np.nan value is detected within the input cube.
"""
# Record input cube data type to ensure consistent output, though
# integer data must become float to enable fuzzy thresholding.
input_cube_dtype = input_cube.dtype
if input_cube.dtype.kind == 'i':
input_cube_dtype = np.float32
thresholded_cubes = iris.cube.CubeList()
if np.isnan(input_cube.data).any():
raise ValueError("Error: NaN detected in input cube data")
# if necessary, convert thresholds and fuzzy bounds into cube units
if self.threshold_units is not None:
self.thresholds = [
self.threshold_units.convert(threshold, input_cube.units)
for threshold in self.thresholds
]
self.fuzzy_bounds = [
tuple([
self.threshold_units.convert(threshold, input_cube.units)
for threshold in bounds
]) for bounds in self.fuzzy_bounds
]
# set name of threshold coordinate to match input diagnostic
self.threshold_coord_name = input_cube.name()
# apply fuzzy thresholding
for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds):
cube = input_cube.copy()
# if upper and lower bounds are equal, set a deterministic 0/1
# probability based on exceedance of the threshold
if bounds[0] == bounds[1]:
truth_value = cube.data > threshold
# otherwise, scale exceedance probabilities linearly between 0/1
# at the min/max fuzzy bounds and 0.5 at the threshold value
else:
truth_value = np.where(
cube.data < threshold,
rescale(cube.data,
data_range=(bounds[0], threshold),
scale_range=(0., 0.5),
clip=True),
rescale(cube.data,
data_range=(threshold, bounds[1]),
scale_range=(0.5, 1.),
clip=True),
)
truth_value = truth_value.astype(input_cube_dtype)
# if requirement is for probabilities below threshold (rather than
# above), invert the exceedance probability
if self.below_thresh_ok:
truth_value = 1. - truth_value
cube.data = truth_value
# Overwrite masked values that have been thresholded
# with the un-thresholded values from the input cube.
if np.ma.is_masked(cube.data):
cube.data[input_cube.data.mask] = (
input_cube.data[input_cube.data.mask])
cube = self._add_threshold_coord(cube, threshold)
thresholded_cubes.append(cube)
cube, = thresholded_cubes.concatenate()
if self.below_thresh_ok:
cube.rename("probability_of_{}_below_threshold".format(
cube.name()))
else:
cube.rename("probability_of_{}_above_threshold".format(
cube.name()))
cube.units = Unit(1)
cube = enforce_coordinate_ordering(cube, ["realization", "percentile"])
return cube
def test_depth_incompatible_units(self):
new_depth = mock.Mock(units=Unit('Pa'), nbounds=0)
with self.assertRaises(ValueError):
self.factory.update(self.depth, new_depth)
def test_basic(self):
"""Test that the result is a numpy array."""
cube_name = "air_temperature"
cube_units = Unit("degreesC")
result = get_bounds_of_distribution(cube_name, cube_units)
self.assertIsInstance(result, np.ndarray)
