def test_process_return_type(forecast_grid, truth_grid):
"""Test the process method returns a reliability table cube."""
result = Plugin().process(forecast_grid, truth_grid)
assert isinstance(result, iris.cube.Cube)
assert result.name() == "reliability_calibration_table"
assert result.coord("air_temperature") == forecast_grid.coord(var_name="threshold")
assert result.coord_dims("air_temperature")[0] == 0
def setUp(self):
"""Create reliability calibration tables for testing."""
super().setUp()
reliability_cube_format = CalPlugin()._create_reliability_table_cube(
self.forecasts, self.expected_threshold_coord)
self.reliability_cube = reliability_cube_format.copy(
data=self.expected_table)
self.different_frt = self.reliability_cube.copy()
new_frt = self.different_frt.coord("forecast_reference_time")
new_frt.points = new_frt.points + 48 * 3600
new_frt.bounds = new_frt.bounds + 48 * 3600
self.masked_reliability_cube = self.reliability_cube.copy()
masked_array = np.zeros(self.reliability_cube.shape, dtype=bool)
masked_array[:, :, 0, :2] = True
self.masked_reliability_cube.data = np.ma.array(
self.reliability_cube.data, mask=masked_array)
self.masked_different_frt = self.different_frt.copy()
masked_array = np.zeros(self.different_frt.shape, dtype=bool)
masked_array[:, :, :2, 0] = True
self.masked_different_frt.data = np.ma.array(self.different_frt.data,
mask=masked_array)
self.overlapping_frt = self.reliability_cube.copy()
new_frt = self.overlapping_frt.coord("forecast_reference_time")
new_frt.points = new_frt.points + 6 * 3600
new_frt.bounds = new_frt.bounds + 6 * 3600
self.lat_lon_collapse = np.array([
[0.0, 0.0, 1.0, 2.0, 1.0],
[0.0, 0.375, 1.5, 1.625, 1.0],
[1.0, 2.0, 3.0, 2.0, 1.0],
])
def test_return_type(self):
"""Test the process method returns a reliability table cube."""
result = Plugin().process(self.forecasts, self.truths)
self.assertIsInstance(result, iris.cube.Cube)
self.assertEqual(result.name(), "reliability_calibration_table")
self.assertEqual(result.coord("air_temperature"), self.expected_threshold_coord)
self.assertEqual(result.coord_dims("air_temperature")[0], 0)
def reliability_cube(forecast_grid, expected_table):
from improver.calibration.reliability_calibration import (
ConstructReliabilityCalibrationTables as CalPlugin,
)
rel_cube_format = CalPlugin()._create_reliability_table_cube(
forecast_grid, forecast_grid.coord(var_name="threshold")
)
rel_cube = rel_cube_format.copy(data=expected_table)
return rel_cube
def test_valid_inputs(self):
"""Test the cube returned has the structure expected."""
forecast_slice = next(self.forecast_1.slices_over("air_temperature"))
result = Plugin()._create_reliability_table_cube(
forecast_slice, forecast_slice.coord(var_name="threshold"))
self.assertIsInstance(result, iris.cube.Cube)
self.assertSequenceEqual(result.shape, self.expected_table_shape)
self.assertEqual(result.name(), "reliability_calibration_table")
self.assertEqual(result.attributes, self.expected_attributes)
def setUp(self):
"""Create reliability calibration table and forecast cubes for
testing."""
forecast_probabilities_0 = np.linspace(0.5, 1, 9, dtype=np.float32)
forecast_probabilities_1 = np.linspace(0, 0.4, 9, dtype=np.float32)
thresholds = [275.0, 280.0]
forecast_probabilities = np.stack(
[forecast_probabilities_0, forecast_probabilities_1]).reshape(
(2, 3, 3))
self.forecast = set_up_probability_cube(forecast_probabilities,
thresholds)
self.forecast_thresholds = iris.cube.CubeList()
for forecast_slice in self.forecast.slices_over("air_temperature"):
self.forecast_thresholds.append(forecast_slice)
reliability_cube_format = CalPlugin()._create_reliability_table_cube(
self.forecast[0], self.forecast.coord(var_name="threshold"))
reliability_cube_format = reliability_cube_format.collapsed(
[
reliability_cube_format.coord(axis="x"),
reliability_cube_format.coord(axis="y"),
],
iris.analysis.SUM,
)
# Over forecasting exceeding 275K.
reliability_data_0 = np.array(
[
[0, 0, 250, 500, 750], # Observation count
[0, 250, 500, 750, 1000], # Sum of forecast probability
[1000, 1000, 1000, 1000, 1000], # Forecast count
],
dtype=np.float32,
)
# Under forecasting exceeding 280K.
reliability_data_1 = np.array(
[
[250, 500, 750, 1000, 1000], # Observation count
[0, 250, 500, 750, 1000], # Sum of forecast probability
[1000, 1000, 1000, 1000, 1000], # Forecast count
],
dtype=np.float32,
)
r0 = reliability_cube_format.copy(data=reliability_data_0)
r1 = reliability_cube_format.copy(data=reliability_data_1)
r1.replace_coord(self.forecast[1].coord(var_name="threshold"))
self.reliability_cubelist = iris.cube.CubeList([r0, r1])
self.reliability_cube = self.reliability_cubelist.merge_cube()
self.threshold = self.forecast.coord(var_name="threshold")
self.plugin = Plugin()
self.plugin.threshold_coord = self.threshold
def test_coordinate_no_single_value_bins(self):
"""Test the probability_bins coordinate has the expected values and
type with no single value lower and upper bins."""
expected_bounds = np.array([[0, 0.5], [0.5, 1]])
expected_points = np.mean(expected_bounds, axis=1)
plugin = Plugin(n_probability_bins=2, )
result = plugin._create_probability_bins_coord()
self.assertIsInstance(result, iris.coords.DimCoord)
assert_allclose(result.points, expected_points)
assert_allclose(result.bounds, expected_bounds)
def test_valid_inputs(create_rel_table_inputs, expected_attributes):
"""Tests correct reliability cube generated. Parameterized using
`create_rel_table_inputs` fixture."""
forecast_1 = create_rel_table_inputs.forecast[0]
forecast_slice = next(forecast_1.slices_over("air_temperature"))
result = Plugin()._create_reliability_table_cube(
forecast_slice, forecast_slice.coord(var_name="threshold"))
assert isinstance(result, Cube)
assert result.shape == create_rel_table_inputs.expected_shape
assert result.name() == "reliability_calibration_table"
assert result.attributes == expected_attributes
def reliability_table_agg(forecast_grid, truth_grid, reliability_data):
reliability_cube_format = CalPlugin().process(forecast_grid, truth_grid)
reliability_cube_format = reliability_cube_format.collapsed(
[
reliability_cube_format.coord(axis="x"),
reliability_cube_format.coord(axis="y"),
],
iris.analysis.SUM,
)
reliability_table_agg = reliability_cube_format.copy(data=reliability_data)
return reliability_table_agg
def setUp(self):
"""Set up forecast count"""
# Set up a reliabilty table with a single threshold
thresholds = [275.0]
self.forecast = set_up_probability_cube(
np.ones((1, 3, 3), dtype=np.float32), thresholds
)
reliability_cube_format = CalPlugin()._create_reliability_table_cube(
self.forecast, self.forecast.coord(var_name="threshold")
)
reliability_cube_format = reliability_cube_format.collapsed(
[
reliability_cube_format.coord(axis="x"),
reliability_cube_format.coord(axis="y"),
],
iris.analysis.SUM,
)
self.obs_count = np.array([0, 0, 250, 500, 750], dtype=np.float32)
self.forecast_probability_sum = np.array(
[0, 250, 500, 750, 1000], dtype=np.float32
)
self.forecast_count = np.array([1000, 1000, 1000, 1000, 1000], dtype=np.float32)
reliability_data_0 = np.stack(
[self.obs_count, self.forecast_probability_sum, self.forecast_count]
)
self.reliability_table = reliability_cube_format.copy(data=reliability_data_0)
self.probability_bin_coord = self.reliability_table.coord("probability_bin")
# Set up a reliablity table cube with two thresholds
reliability_data_1 = np.array(
[
[250, 500, 750, 1000, 1000], # Observation count
[0, 250, 500, 750, 1000], # Sum of forecast probability
[1000, 1000, 1000, 1000, 1000], # Forecast count
],
dtype=np.float32,
)
reliability_table_1 = self.reliability_table.copy(data=reliability_data_1)
reliability_table_1.coord("air_temperature").points = np.array(
278.0, dtype=np.float32
)
self.multi_threshold_rt = iris.cube.CubeList(
[self.reliability_table, reliability_table_1]
).merge_cube()
# Set up expected resulting reliablity table data for Test__combine_bin_pair
self.expected_enforced_monotonic = np.array(
[
[0, 250, 500, 1750], # Observation count
[0, 250, 500, 1750], # Sum of forecast probability
[1000, 1000, 1000, 2000], # Forecast count
]
)
def test_metadata_with_incomplete_inputs(forecast_grid, expected_attributes):
"""Test the metadata returned is complete and as expected when the
forecast cube does not contain all the required metadata to copy."""
forecast_1 = forecast_grid[0]
result = Plugin._define_metadata(forecast_1)
assert isinstance(result, dict)
assert result == expected_attributes
def test_metadata_with_incomplete_inputs(self):
"""Test the metadata returned is complete and as expected when the
forecast cube does not contain all the required metadata to copy."""
result = Plugin._define_metadata(self.forecast_1)
self.assertIsInstance(result, dict)
self.assertEqual(result, self.expected_attributes)
def process(
*cubes: cli.inputcube,
truth_attribute,
n_probability_bins: int = 5,
single_value_lower_limit: bool = False,
single_value_upper_limit: bool = False,
aggregate_coordinates: cli.comma_separated_list = None,
):
"""Populate reliability tables for use in reliability calibration.
Loads historical forecasts and gridded truths that are compared to build
reliability tables. Reliability tables are returned as a cube with a
leading threshold dimension that matches that of the forecast probability
cubes and the thresholded truth.
Args:
cubes (list of iris.cube.Cube):
A list of cubes containing the historical probability forecasts and
corresponding truths used for calibration. These cubes must include
the same diagnostic name in their names, and must both have
equivalent threshold coordinates. The cubes will be distinguished
using the user provided truth attribute.
truth_attribute (str):
An attribute and its value in the format of "attribute=value",
which must be present on truth cubes.
n_probability_bins (int):
The total number of probability bins required in the reliability
tables. If single value limits are turned on, these are included in
this total. If using single_value_limits this value must be at
least 3.
single_value_lower_limit (bool):
Mandates that the lowest bin should be single valued, with a small
precision tolerance, defined as 1.0E-6. The bin is thus 0 to 1.0E-6.
single_value_upper_limit (bool):
Mandates that the highest bin should be single valued, with a small
precision tolerance, defined as 1.0E-6. The bin is thus (1 - 1.0E-6) to 1.
aggregate_coordinates (List[str]):
An optional list of coordinates over which to aggregate the reliability
calibration table using summation. This is equivalent to constructing
then using aggregate-reliability-tables but with reduced memory
usage due to avoiding large intermediate data.
Returns:
iris.cube.Cube:
Reliability tables for the forecast diagnostic with a leading
threshold coordinate.
"""
from improver.calibration import split_forecasts_and_truth
from improver.calibration.reliability_calibration import (
ConstructReliabilityCalibrationTables, )
forecast, truth, _ = split_forecasts_and_truth(cubes, truth_attribute)
return ConstructReliabilityCalibrationTables(
n_probability_bins=n_probability_bins,
single_value_lower_limit=single_value_lower_limit,
single_value_upper_limit=single_value_upper_limit,
)(forecast, truth, aggregate_coordinates)
def test_metadata_with_complete_inputs(forecast_grid, expected_attributes):
"""Test the metadata returned is complete and as expected when the
forecast cube contains the required metadata to copy."""
forecast_1 = forecast_grid[0]
forecast_1.attributes["institution"] = "Kitten Inc"
expected_attributes["institution"] = "Kitten Inc"
result = Plugin._define_metadata(forecast_1)
assert isinstance(result, dict)
assert result == expected_attributes
def test_metadata_with_complete_inputs(self):
"""Test the metadata returned is complete and as expected when the
forecast cube contains the required metadata to copy."""
self.forecast_1.attributes["institution"] = "Kitten Inc"
self.expected_attributes["institution"] = "Kitten Inc"
result = Plugin._define_metadata(self.forecast_1)
self.assertIsInstance(result, dict)
self.assertEqual(result, self.expected_attributes)
def test_coordinate_single_value_bins(self):
"""Test the probability_bins coordinate has the expected values and
type when using the single value lower and upper bins."""
expected_bounds = np.array([
[0.0000000e00, 1.0000000e-06],
[1.0000001e-06, 4.9999997e-01],
[5.0000000e-01, 9.9999893e-01],
[9.9999899e-01, 1.0000000e00],
])
expected_points = np.mean(expected_bounds, axis=1)
plugin = Plugin(
n_probability_bins=4,
single_value_lower_limit=True,
single_value_upper_limit=True,
)
result = plugin._create_probability_bins_coord()
self.assertIsInstance(result, iris.coords.DimCoord)
assert_allclose(result.points, expected_points)
assert_allclose(result.bounds, expected_bounds)
def setUp(self):
"""Create reliability calibration tables for testing."""
super().setUp()
reliability_cube_format = CalPlugin()._create_reliability_table_cube(
self.forecasts, self.expected_threshold_coord)
self.reliability_cube = reliability_cube_format.copy(
data=self.expected_table)
self.different_frt = self.reliability_cube.copy()
new_frt = self.different_frt.coord('forecast_reference_time')
new_frt.points = new_frt.points + 48 * 3600
new_frt.bounds = new_frt.bounds + 48 * 3600
self.overlapping_frt = self.reliability_cube.copy()
new_frt = self.overlapping_frt.coord('forecast_reference_time')
new_frt.points = new_frt.points + 6 * 3600
new_frt.bounds = new_frt.bounds + 6 * 3600
self.lat_lon_collapse = np.array([[0., 0., 1., 2., 1.],
[0., 0.375, 1.5, 1.625, 1.],
[1., 2., 3., 2., 1.]])
def process(
*cubes: cli.inputcube,
truth_attribute,
n_probability_bins: int = 5,
single_value_limits: bool = True,
):
"""Populate reliability tables for use in reliability calibration.
Loads historical forecasts and gridded truths that are compared to build
reliability tables. Reliability tables are returned as a cube with a
leading threshold dimension that matches that of the forecast probability
cubes and the thresholded truth.
Args:
cubes (list of iris.cube.Cube):
A list of cubes containing the historical probability forecasts and
corresponding truths used for calibration. These cubes must include
the same diagnostic name in their names, and must both have
equivalent threshold coordinates. The cubes will be distinguished
using the user provided truth attribute.
truth_attribute (str):
An attribute and its value in the format of "attribute=value",
which must be present on truth cubes.
n_probability_bins (int):
The total number of probability bins required in the reliability
tables. If single value limits are turned on, these are included in
this total. If using single_value_limits this value must be at
least 3.
single_value_limits (bool):
Mandates that the extrema bins (0 and 1) should be single valued,
with a small precision tolerance of 1.0E-6, e.g. 0 to 1.0E-6 for
the lowest bin, and 1 - 1.0E-6 to 1 for the highest bin.
Returns:
iris.cube.Cube:
Reliability tables for the forecast diagnostic with a leading
threshold coordinate.
"""
from improver.calibration import split_forecasts_and_truth
from improver.calibration.reliability_calibration import (
ConstructReliabilityCalibrationTables, )
forecast, truth, _ = split_forecasts_and_truth(cubes, truth_attribute)
return ConstructReliabilityCalibrationTables(
n_probability_bins=n_probability_bins,
single_value_limits=single_value_limits)(forecast, truth)
def reliability_table_point_spot(forecast_spot, truth_spot, reliability_data):
reliability_cube_format = CalPlugin().process(forecast_spot, truth_spot)
data = np.stack([reliability_data] * 9, axis=-1)
reliability_table = reliability_cube_format.copy(data=data)
return reliability_table
def reliability_table_point_grid(forecast_grid, truth_grid, reliability_data):
reliability_cube_format = CalPlugin().process(forecast_grid, truth_grid)
data = np.stack([np.stack([reliability_data] * 3, axis=-1)] * 3, axis=-1)
reliability_table = reliability_cube_format.copy(data=data)
return reliability_table