def test_less_than(self):
""" Tests the case where spp__relative_threshold is less_than"""
cube = set_up_probability_cube(
self.data, self.threshold_points, spp__relative_to_threshold="less_than"
)
result = probability_is_above_or_below(cube)
self.assertEqual(result, "below")
def _get_multiplier(self) -> float:
"""
Check whether the cube contains "above" or "below" threshold
probabilities. For "above", the probability of occurrence between
thresholds is the difference between probabilities at the lower
and higher thresholds: P(lower) - P(higher). For "below" it is the
inverse of this: P(higher) - P(lower), which is implemented by
multiplying the difference by -1.
Returns:
1. or -1.
Raises:
ValueError: If the spp__relative_to_threshold attribute is
not recognised
"""
relative_to_threshold = probability_is_above_or_below(self.cube)
if relative_to_threshold == "above":
multiplier = 1.0
elif relative_to_threshold == "below":
multiplier = -1.0
else:
raise ValueError("Input cube must contain probabilities of "
"occurrence above or below threshold")
return multiplier
def test_incorrect_attribute(self):
"""Tests it returns None if the spp__relative_to_threshold
attribute has an invalid value."""
cube = set_up_probability_cube(self.data, self.threshold_points,)
cube.coord("air_temperature").attributes = {
"spp__relative_to_threshold": "higher"
}
result = probability_is_above_or_below(cube)
self.assertEqual(result, None)
def test_no_spp__relative_to_threshold(self):
"""Tests it returns None if there is no spp__relative_to_threshold
attribute."""
cube = set_up_probability_cube(self.data, self.threshold_points,)
cube.coord("air_temperature").attributes = {
"relative_to_threshold": "greater_than"
}
result = probability_is_above_or_below(cube)
self.assertEqual(result, None)
def test_greater_than_or_equal_to(self):
""" Tests the case where spp__relative_threshold is
greater_than_or_equal_to"""
cube = set_up_probability_cube(
self.data,
self.threshold_points,
spp__relative_to_threshold="greater_than_or_equal_to",
)
result = probability_is_above_or_below(cube)
self.assertEqual(result, "above")
def _update_metadata(self, cube: Cube) -> None:
"""Rename the cube and add attributes to the threshold coordinate
after merging
"""
threshold_coord = cube.coord(self.threshold_coord_name)
threshold_coord.attributes.update(
{"spp__relative_to_threshold": self.comparison_operator.spp_string}
)
if cube.cell_methods:
format_cell_methods_for_probability(cube, self.threshold_coord_name)
cube.rename(
"probability_of_{parameter}_{relative_to}_threshold".format(
parameter=self.threshold_coord_name,
relative_to=probability_is_above_or_below(cube),
)
)
cube.units = Unit(1)
def _update_metadata(self, output_cube, original_units):
"""
Update output cube name and threshold coordinate
Args:
output_cube (iris.cube.Cube):
Cube containing new "between_thresholds" probabilities
original_units (str):
Required threshold-type coordinate units
"""
new_name = self.cube.name().replace(
"{}_threshold".format(probability_is_above_or_below(self.cube)),
"between_thresholds",
)
output_cube.rename(new_name)
new_thresh_coord = output_cube.coord(self.thresh_coord.name())
new_thresh_coord.convert_units(original_units)
new_thresh_coord.attributes[
"spp__relative_to_threshold"] = "between_thresholds"
def check_input_cubes(self, cubes: CubeList) -> Optional[Dict[str, Any]]:
"""
Check that the input cubes contain all the diagnostics and thresholds
required by the decision tree. Sets self.coord_named_threshold to
"True" if threshold-type coordinates have the name "threshold" (as
opposed to the standard name of the diagnostic), for backward
compatibility.
Args:
cubes:
A CubeList containing the input diagnostic cubes.
Returns:
A dictionary of (keyword) nodes names where the diagnostic
data is missing and (values) node associated with
diagnostic_missing_action.
Raises:
IOError:
Raises an IOError if any of the required input data is missing.
The error includes details of which fields are missing.
"""
optional_node_data_missing = {}
missing_data = []
for key, query in self.queries.items():
diagnostics = get_parameter_names(
expand_nested_lists(query, "diagnostic_fields")
)
thresholds = expand_nested_lists(query, "diagnostic_thresholds")
conditions = expand_nested_lists(query, "diagnostic_conditions")
for diagnostic, threshold, condition in zip(
diagnostics, thresholds, conditions
):
# First we check the diagnostic name and units, performing
# a conversion is required and possible.
test_condition = iris.Constraint(name=diagnostic)
matched_cube = cubes.extract(test_condition)
if not matched_cube:
if "diagnostic_missing_action" in query:
optional_node_data_missing.update(
{key: query[query["diagnostic_missing_action"]]}
)
else:
missing_data.append([diagnostic, threshold, condition])
continue
cube_threshold_units = find_threshold_coordinate(matched_cube[0]).units
threshold.convert_units(cube_threshold_units)
# Then we check if the required threshold is present in the
# cube, and that the thresholding is relative to it correctly.
threshold = threshold.points.item()
threshold_name = find_threshold_coordinate(matched_cube[0]).name()
# Set flag to check for old threshold coordinate names
if threshold_name == "threshold" and not self.coord_named_threshold:
self.coord_named_threshold = True
# Check threshold == 0.0
if abs(threshold) < self.float_abs_tolerance:
coord_constraint = {
threshold_name: lambda cell: np.isclose(
cell.point, 0, rtol=0, atol=self.float_abs_tolerance
)
}
else:
coord_constraint = {
threshold_name: lambda cell: np.isclose(
cell.point, threshold, rtol=self.float_tolerance, atol=0
)
}
# Checks whether the spp__relative_to_threshold attribute is above
# or below a threshold and and compares to the diagnostic_condition.
test_condition = iris.Constraint(
coord_values=coord_constraint,
cube_func=lambda cube: (
probability_is_above_or_below(cube) == condition
),
)
matched_threshold = matched_cube.extract(test_condition)
if not matched_threshold:
missing_data.append([diagnostic, threshold, condition])
if missing_data:
msg = (
"Weather Symbols input cubes are missing"
" the following required"
" input fields:\n"
)
dyn_msg = "name: {}, threshold: {}, " "spp__relative_to_threshold: {}\n"
for item in missing_data:
msg = msg + dyn_msg.format(*item)
raise IOError(msg)
if not optional_node_data_missing:
optional_node_data_missing = None
return optional_node_data_missing
def test_leading_dimension(cube_type, spp__relative_to_threshold):
""" Tests cube generated with leading dimension specified using percentile and
probability flags, and different values for spp__relative_to_threshold """
if cube_type == "other":
# Tests that error is raised when cube type isn't supported
msg = ("Cube type {} not supported. "
'Specify one of "variable", "percentile" or "probability".'
).format(cube_type)
with pytest.raises(ValueError, match=msg):
generate_metadata(
MANDATORY_ATTRIBUTE_DEFAULTS,
cube_type=cube_type,
spp__relative_to_threshold=spp__relative_to_threshold,
)
else:
leading_dimension = [10, 20, 30, 40, 50, 60, 70, 80]
if spp__relative_to_threshold is not None:
cube = generate_metadata(
MANDATORY_ATTRIBUTE_DEFAULTS,
leading_dimension=leading_dimension,
cube_type=cube_type,
spp__relative_to_threshold=spp__relative_to_threshold,
)
else:
cube = generate_metadata(
MANDATORY_ATTRIBUTE_DEFAULTS,
leading_dimension=leading_dimension,
cube_type=cube_type,
)
spp__relative_to_threshold = RELATIVE_TO_THRESHOLD_DEFAULT
if cube_type == "percentile":
cube_name = NAME_DEFAULT
coord_name = "percentile"
elif cube_type == "probability":
cube_name = "probability_of_{}_{}_threshold".format(
NAME_DEFAULT, probability_is_above_or_below(cube))
coord_name = NAME_DEFAULT
assert cube.coord(coord_name).attributes == {
"spp__relative_to_threshold": spp__relative_to_threshold
}
cube.coord(coord_name).attributes = {}
else:
cube_name = NAME_DEFAULT
coord_name = "realization"
assert cube.name() == cube_name
assert cube.ndim == 3
assert cube.shape == (len(leading_dimension), NPOINTS_DEFAULT,
NPOINTS_DEFAULT)
assert cube.coords()[0].name() == coord_name
np.testing.assert_array_equal(
cube.coord(coord_name).points, leading_dimension)
# Assert that no other values have unexpectedly changed by returning changed
# values to defaults and comparing against default cube
default_cube = generate_metadata(MANDATORY_ATTRIBUTE_DEFAULTS)
cube.coord(coord_name).points = default_cube.coord(
"realization").points
cube.coord(coord_name).rename("realization")
cube.coord("realization").units = "1"
cube.rename(default_cube.standard_name)
cube.units = default_cube.units
assert cube == default_cube
def process(self, input_cube):
"""Convert each point to a truth value based on provided threshold
values. The truth value may or may not be fuzzy depending upon if
fuzzy_bounds are supplied. If the plugin has a "threshold_units"
member, this is used to convert both thresholds and fuzzy bounds into
the units of the input cube.
Args:
input_cube (iris.cube.Cube):
Cube to threshold. The code is dimension-agnostic.
Returns:
iris.cube.Cube:
Cube after a threshold has been applied. The data within this
cube will contain values between 0 and 1 to indicate whether
a given threshold has been exceeded or not.
The cube meta-data will contain:
* Input_cube name prepended with
probability_of_X_above(or below)_threshold (where X is
the diagnostic under consideration)
* Threshold dimension coordinate with same units as input_cube
* Threshold attribute ("greater_than",
"greater_than_or_equal_to", "less_than", or
less_than_or_equal_to" depending on the operator)
* Cube units set to (1).
Raises:
ValueError: if a np.nan value is detected within the input cube.
"""
# Record input cube data type to ensure consistent output, though
# integer data must become float to enable fuzzy thresholding.
input_cube_dtype = input_cube.dtype
if input_cube.dtype.kind == "i":
input_cube_dtype = FLOAT_DTYPE
thresholded_cubes = iris.cube.CubeList()
if np.isnan(input_cube.data).any():
raise ValueError("Error: NaN detected in input cube data")
# if necessary, convert thresholds and fuzzy bounds into cube units
if self.threshold_units is not None:
self.thresholds = [
self.threshold_units.convert(threshold, input_cube.units)
for threshold in self.thresholds
]
self.fuzzy_bounds = [
tuple([
self.threshold_units.convert(threshold, input_cube.units)
for threshold in bounds
]) for bounds in self.fuzzy_bounds
]
# set name of threshold coordinate to match input diagnostic
self.threshold_coord_name = input_cube.name()
# apply fuzzy thresholding
for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds):
cube = input_cube.copy()
# if upper and lower bounds are equal, set a deterministic 0/1
# probability based on exceedance of the threshold
if bounds[0] == bounds[1]:
truth_value = self.comparison_operator["function"](cube.data,
threshold)
# otherwise, scale exceedance probabilities linearly between 0/1
# at the min/max fuzzy bounds and 0.5 at the threshold value
else:
truth_value = np.where(
cube.data < threshold,
rescale(
cube.data,
data_range=(bounds[0], threshold),
scale_range=(0.0, 0.5),
clip=True,
),
rescale(
cube.data,
data_range=(threshold, bounds[1]),
scale_range=(0.5, 1.0),
clip=True,
),
)
# if requirement is for probabilities less_than or
# less_than_or_equal_to the threshold (rather than
# greater_than or greater_than_or_equal_to), invert
# the exceedance probability
if "less_than" in self.comparison_operator["spp_string"]:
truth_value = 1.0 - truth_value
truth_value = np.ma.masked_where(np.ma.getmask(cube.data),
truth_value)
truth_value = truth_value.astype(input_cube_dtype)
cube.data = truth_value
# Overwrite masked values that have been thresholded
# with the un-thresholded values from the input cube.
if np.ma.is_masked(cube.data):
cube.data[input_cube.data.mask] = input_cube.data[
input_cube.data.mask]
cube = self._add_threshold_coord(cube, threshold)
for func in self.each_threshold_func:
cube = func(cube)
thresholded_cubes.append(cube)
(cube, ) = thresholded_cubes.concatenate()
if len(self.thresholds) == 1:
# if only one threshold has been provided, this should be scalar
cube = next(cube.slices_over(cube.coord(var_name="threshold")))
cube.rename("probability_of_{}_{}_threshold".format(
cube.name(), probability_is_above_or_below(cube)))
cube.units = Unit(1)
enforce_coordinate_ordering(cube, ["realization", "percentile"])
return cube
