def test_aggregating_over_single_cube_coordinates(self):
"""Test of aggregating over coordinates of a single cube. In this
instance the latitude and longitude coordinates are collapsed."""
frt = "forecast_reference_time"
expected_points = self.reliability_cube.coord(frt).points
expected_bounds = self.reliability_cube.coord(frt).bounds
plugin = Plugin()
result = plugin.process(
[self.reliability_cube], coordinates=["latitude", "longitude"]
)
assert_array_equal(result.data, self.lat_lon_collapse)
self.assertEqual(result.coord(frt).points, expected_points)
assert_array_equal(result.coord(frt).bounds, expected_bounds)
def test_process_table_values(create_rel_table_inputs, expected_table):
"""Test that cube values are as expected, when process has
sliced the inputs up for processing and then summed the contributions
from the two dates. Note that the values tested here are for only one
of the two processed thresholds (283K). The results contain
contributions from two forecast/truth pairs.
Parameterized using `create_rel_table_inputs` fixture."""
expected = np.sum([expected_table, expected_table], axis=0)
result = Plugin(
single_value_lower_limit=True, single_value_upper_limit=True
).process(create_rel_table_inputs.forecast, create_rel_table_inputs.truth)
assert_array_equal(
result[0].data, expected.reshape(create_rel_table_inputs.expected_shape)
)
def test_dpb_with_both_single_value_limits_too_few_bins():
"""In this test both lower and uppper single_value_limits are requested
whilst also trying to use 2 bins. This would leave no bins to cover the
range 0 to 1, so an error is raised."""
msg = (
"Cannot use both single_value_lower_limit and "
"single_value_upper_limit with 2 or fewer probability bins."
)
with pytest.raises(ValueError, match=msg):
Plugin()._define_probability_bins(
n_probability_bins=2,
single_value_lower_limit=True,
single_value_upper_limit=True,
)
def test_table_values_masked_truth(self):
"""Test, similar to test_table_values, using masked arrays. The
mask is different for different timesteps, reflecting the potential
for masked areas in e.g. a radar truth to differ between timesteps.
At timestep 1, two grid points are masked. At timestep 2, two
grid points are also masked with one masked grid point in common
between timesteps. As a result, only one grid point is masked (
within the upper left corner) within the resulting reliability table."""
expected_table_for_second_mask = np.array(
[
[
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]],
],
[
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]],
[[0.0, 0.125, 0.25], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 0.5, 0.625], [0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.75, 0.875, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]],
],
[
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]],
[[0.0, 1.0, 1.0], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 1.0, 1.0], [0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 1.0, 0.0]],
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]],
],
],
dtype=np.float32,
)
expected = np.sum(
[self.expected_table_for_mask, expected_table_for_second_mask],
axis=0)
expected_mask = np.zeros(expected.shape, dtype=bool)
expected_mask[:, :, 0, 0] = True
result = Plugin(single_value_lower_limit=True,
single_value_upper_limit=True).process(
self.forecasts, self.masked_truths)
self.assertIsInstance(result.data, np.ma.MaskedArray)
assert_array_equal(result[0].data.data, expected)
assert_array_equal(result[0].data.mask, expected_mask)
# Different thresholds must have the same mask.
assert_array_equal(result[0].data.mask, result[1].data.mask)
def test_with_both_single_value_limits():
"""Test the generation of probability bins with both upper and lower
single value end bins. The range 0 to 1 will be divided into 2 equally
sized bins, with 2 end bins holding values approximately equal to 0 and 1."""
expected = np.array([
[0.0000000e00, 1.0000000e-06],
[1.0000001e-06, 4.9999997e-01],
[5.0000000e-01, 9.9999893e-01],
[9.9999899e-01, 1.0000000e00],
])
result = Plugin()._define_probability_bins(
n_probability_bins=4,
single_value_lower_limit=True,
single_value_upper_limit=True,
)
assert_allclose(result, expected)
def test_process_aggregating_multiple_cubes(reliability_cube, different_frt,
expected_table):
"""Test of aggregating two cubes without any additional coordinate
collapsing."""
frt = "forecast_reference_time"
expected_points = different_frt.coord(frt).points
expected_bounds = [[
reliability_cube.coord(frt).bounds[0][0],
different_frt.coord(frt).bounds[-1][1],
]]
plugin = Plugin()
result = plugin.process([reliability_cube, different_frt])
assert_array_equal(result.data, expected_table * 2)
assert_array_equal(result.shape, (3, 5, 3, 3))
assert_array_equal(result.coord(frt).points, expected_points)
assert_array_equal(result.coord(frt).bounds, expected_bounds)
def test_one_non_monotonic_bin_pair(self):
"""Test one bin pair is combined, if one bin pair is non-monotonic."""
obs_count = np.array([0, 250, 500, 1000, 750], dtype=np.float32)
result = Plugin()._combine_bin_pair(
obs_count,
self.forecast_probability_sum,
self.forecast_count,
self.probability_bin_coord,
)
assert_array_equal(result[:3], self.expected_enforced_monotonic)
expected_bin_coord_points = np.array([0.1, 0.3, 0.5, 0.8], dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.2], [0.2, 0.4], [0.4, 0.6], [0.6, 1.0]], dtype=np.float32,
)
assert_allclose(expected_bin_coord_points, result[3].points)
assert_allclose(expected_bin_coord_bounds, result[3].bounds)
def test_table_values_masked_truth(self):
"""Test the reliability table returned has the expected values when a
masked truth is input."""
forecast_slice = next(self.forecast_1.slices_over("air_temperature"))
truth_slice = next(self.masked_truth_1.slices_over("air_temperature"))
result = Plugin(
single_value_lower_limit=True, single_value_upper_limit=True
)._populate_masked_reliability_bins(forecast_slice.data, truth_slice.data)
self.assertSequenceEqual(result.shape, self.expected_table_shape)
self.assertTrue(np.ma.is_masked(result))
assert_array_equal(result.data, self.expected_table_for_mask)
expected_mask = np.zeros(self.expected_table_for_mask.shape, dtype=bool)
expected_mask[:, :, 0, :2] = True
assert_array_equal(result.mask, expected_mask)
def test_process_no_change_point(create_rel_tables_point):
"""Test with no changes required to preserve monotonicity. Parameterized
using `create_rel_tables` fixture."""
rel_table = create_rel_tables_point.table
result = Plugin(point_by_point=True).process(rel_table.copy())
assert all(len(result_list) == 9 for result_list in result)
expected = rel_table.data[create_rel_tables_point.indices0]
assert all([np.array_equal(cube.data, expected) for cube in result[0]])
coords_exclude = ["latitude", "longitude", "spot_index", "wmo_id"]
coords_table = [
c for c in rel_table[0].coords() if c.name() not in coords_exclude
]
# Ensure coords are in the same order
coords_result = [result[0][0].coords(c.name())[0] for c in coords_table]
assert coords_table == coords_result
def test_cub_monotonic_no_undersampled_bins(default_obs_counts,
default_fcst_counts,
probability_bin_coord):
"""Test no bins are combined when no bins are under-sampled and all bin
pairs are monotonic."""
obs_count = forecast_probability_sum = default_obs_counts
result = Plugin()._combine_undersampled_bins(
obs_count,
forecast_probability_sum,
default_fcst_counts,
probability_bin_coord,
)
assert_array_equal(
result[:3], [obs_count, forecast_probability_sum, default_fcst_counts])
assert result[3] == probability_bin_coord
def setUp(self):
"""Create reliability calibration table and forecast cubes for
testing."""
forecast_probabilities_0 = np.linspace(0.5, 1, 9, dtype=np.float32)
forecast_probabilities_1 = np.linspace(0, 0.4, 9, dtype=np.float32)
thresholds = [275.0, 280.0]
forecast_probabilities = np.stack(
[forecast_probabilities_0, forecast_probabilities_1]).reshape(
(2, 3, 3))
self.forecast = set_up_probability_cube(forecast_probabilities,
thresholds)
reliability_cube_format = CalPlugin()._create_reliability_table_cube(
self.forecast[0], self.forecast.coord(var_name="threshold"))
reliability_cube_format = reliability_cube_format.collapsed(
[
reliability_cube_format.coord(axis="x"),
reliability_cube_format.coord(axis="y"),
],
iris.analysis.SUM,
)
# Over forecasting exceeding 275K.
relability_data_0 = np.array([
[0, 0, 250, 500, 750], # Observation count
[0, 250, 500, 750, 1000], # Sum of forecast probability
[1000, 1000, 1000, 1000, 1000],
]) # Forecast count
# Under forecasting exceeding 280K.
relability_data_1 = np.array([
[250, 500, 750, 1000, 1000], # Observation count
[0, 250, 500, 750, 1000], # Sum of forecast probability
[1000, 1000, 1000, 1000, 1000],
]) # Forecast count
r0 = reliability_cube_format.copy(data=relability_data_0)
r1 = reliability_cube_format.copy(data=relability_data_1)
r1.replace_coord(self.forecast[1].coord(var_name="threshold"))
reliability_cube = iris.cube.CubeList([r0, r1])
self.reliability_cube = reliability_cube.merge_cube()
self.threshold = self.forecast.coord(var_name="threshold")
self.plugin = Plugin()
self.plugin.threshold_coord = self.threshold
def test_aggregating_over_cubes_and_coordinates(self):
"""Test of aggregating over coordinates and cubes in a single call. In
this instance the latitude and longitude coordinates are collapsed and
the values from two input cube combined."""
frt = 'forecast_reference_time'
expected_points = self.different_frt.coord(frt).points
expected_bounds = [[
self.reliability_cube.coord(frt).bounds[0][0],
self.different_frt.coord(frt).bounds[-1][1]
]]
plugin = Plugin()
result = plugin.process([self.reliability_cube, self.different_frt],
coordinates=['latitude', 'longitude'])
assert_array_equal(result.data, self.lat_lon_collapse * 2)
self.assertEqual(result.coord(frt).points, expected_points)
assert_array_equal(result.coord(frt).bounds, expected_bounds)
def test_two_non_monotonic_bin_pairs(self):
"""Test one bin pair is combined, if two bin pairs are non-monotonic.
As only a single bin pair is combined, the resulting observation
count will still yield a non-monotonic observation frequency."""
obs_count = np.array([0, 750, 500, 1000, 750], dtype=np.float32)
self.expected_enforced_monotonic[0][1] = 750 # Amend observation count
result = Plugin()._combine_bin_pair(
obs_count,
self.forecast_probability_sum,
self.forecast_count,
self.probability_bin_coord,
)
assert_array_equal(result[:3], self.expected_enforced_monotonic)
expected_bin_coord_points = np.array([0.1, 0.3, 0.5, 0.8], dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.2], [0.2, 0.4], [0.4, 0.6], [0.6, 1.0]], dtype=np.float32,
)
assert_allclose(expected_bin_coord_points, result[3].points)
assert_allclose(expected_bin_coord_bounds, result[3].bounds)
def test_pmrb_table_values_masked_truth(
forecast_grid, masked_truths, expected_table_for_mask, expected_table_shape_grid
):
"""Test the reliability table returned has the expected values when a
masked truth is input."""
forecast_1 = forecast_grid[0]
masked_truth_1 = masked_truths[0]
forecast_slice = next(forecast_1.slices_over("air_temperature"))
truth_slice = next(masked_truth_1.slices_over("air_temperature"))
result = Plugin(
single_value_lower_limit=True, single_value_upper_limit=True
)._populate_masked_reliability_bins(forecast_slice.data, truth_slice.data)
assert result.shape == expected_table_shape_grid
assert np.ma.is_masked(result)
assert_array_equal(result.data, expected_table_for_mask)
expected_mask = np.zeros(expected_table_for_mask.shape, dtype=bool)
expected_mask[:, :, 0, :2] = True
assert_array_equal(result.mask, expected_mask)
def test_unmatching_forecast_periods(self):
"""Test case in which forecasts do not share consistent forecast
periods."""
forecasts = iris.cube.CubeList()
forecast = self.forecasts[0].copy()
forecast.coord("forecast_period").points = [3600]
forecasts.append(forecast)
forecasts.append(self.forecasts[1])
forecasts = MergeCubes()(forecasts)
msg = ("Forecasts have been provided from differing cycle hours "
"or forecast periods, or without these coordinates. These "
"coordinates should be present and consistent between "
"forecasts. Number of cycle hours found: 1, number of "
"forecast periods found: 2.")
with self.assertRaisesRegex(ValueError, msg):
Plugin()._check_forecast_consistency(forecasts)
def test_coordinate_single_value_bins(self):
"""Test the probability_bins coordinate has the expected values and
type when using the single value lower and upper bins."""
expected_bounds = np.array([
[0.0000000e00, 1.0000000e-06],
[1.0000001e-06, 4.9999997e-01],
[5.0000000e-01, 9.9999893e-01],
[9.9999899e-01, 1.0000000e00],
])
expected_points = np.mean(expected_bounds, axis=1)
plugin = Plugin(
n_probability_bins=4,
single_value_lower_limit=True,
single_value_upper_limit=True,
)
result = plugin._create_probability_bins_coord()
self.assertIsInstance(result, iris.coords.DimCoord)
assert_allclose(result.points, expected_points)
assert_allclose(result.bounds, expected_bounds)
def test_with_upper_single_value_limit():
"""Test the generation of probability bins with only the upper single value
limit bin. The range 0 to 1 will be divided into 4 equally sized bins,
with 1 upper bin holding values approximately equal to 1."""
expected = np.array(
[
[0.0, 0.3333333],
[0.33333334, 0.6666666],
[0.6666667, 0.9999989],
[0.999999, 1.0],
],
dtype=np.float32,
)
result = Plugin()._define_probability_bins(
n_probability_bins=4,
single_value_lower_limit=False,
single_value_upper_limit=True,
)
assert_allclose(result, expected)
def test_with_lower_single_value_limit():
"""Test the generation of probability bins with only the lower single value
limit bin. The range 0 to 1 will be divided into 4 equally sized bins,
with 1 lower bin holding values approximately equal to 0."""
expected = np.array(
[
[0.0000000e00, 1.0000000e-06],
[1.0000001e-06, 3.3333331e-01],
[3.3333334e-01, 6.6666663e-01],
[6.6666669e-01, 1.0000000e00],
],
dtype=np.float32,
)
result = Plugin()._define_probability_bins(
n_probability_bins=4,
single_value_lower_limit=True,
single_value_upper_limit=False,
)
assert_allclose(result, expected)
def test_values(self):
"""Test expected values are returned when two or more bins are
available for interpolation."""
expected_0 = (
np.array([0.0, 0.25, 0.5, 0.75, 1.0]),
np.array([0.0, 0.0, 0.25, 0.5, 0.75]),
)
expected_1 = (
np.array([0.0, 0.25, 0.5, 0.75, 1.0]),
np.array([0.25, 0.5, 0.75, 1.0, 1.0]),
)
plugin = Plugin()
threshold_0 = plugin._calculate_reliability_probabilities(
self.reliability_cube[0])
threshold_1 = plugin._calculate_reliability_probabilities(
self.reliability_cube[1])
assert_array_equal(threshold_0, expected_0)
assert_array_equal(threshold_1, expected_1)
def test_process_aggregating_over_cubes_and_coordinates(
reliability_cube, different_frt, lat_lon_collapse):
"""Test of aggregating over coordinates and cubes in a single call. In
this instance the latitude and longitude coordinates are collapsed and
the values from two input cube combined."""
frt = "forecast_reference_time"
expected_points = different_frt.coord(frt).points
expected_bounds = [[
reliability_cube.coord(frt).bounds[0][0],
different_frt.coord(frt).bounds[-1][1],
]]
plugin = Plugin()
result = plugin.process(
[reliability_cube, different_frt],
coordinates=["latitude", "longitude"],
)
assert_array_equal(result.data, lat_lon_collapse * 2)
assert_array_equal(result.coord(frt).points, expected_points)
assert_array_equal(result.coord(frt).bounds, expected_bounds)
def test_emcam_upper_bins_non_monotonic(reliability_table_slice):
"""Test expected values are returned where the upper observation
count bins are non-monotonic."""
expected_data = np.array([[0, 375, 375, 750], [0, 250, 500, 1750],
[1000, 1000, 1000, 2000]])
expected_bin_coord_points = np.array([0.1, 0.3, 0.5, 0.8],
dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.2], [0.2, 0.4], [0.4, 0.6], [0.6, 1.0]],
dtype=np.float32,
)
reliability_table_slice.data = np.array([
[0, 1000, 750, 500, 250], # Observation count
[0, 250, 500, 750, 1000], # Sum of forecast probability
[1000, 1000, 1000, 1000, 1000], # Forecast count
])
result = Plugin()._enforce_min_count_and_montonicity(
reliability_table_slice.copy())
assert_array_equal(result.data, expected_data)
assert_allclose(
result.coord("probability_bin").points, expected_bin_coord_points)
assert_allclose(
result.coord("probability_bin").bounds, expected_bin_coord_bounds)
def setUp(self):
"""Set up data for testing the interpolate method."""
self.reliability_probabilities = np.array([0.0, 0.4, 0.8])
self.observation_frequencies = np.array([0.2, 0.6, 1.0])
self.plugin = Plugin()
def test_init_with_invalid_minimum_forecast_count():
"""Test an exception is raised if the minimum_forecast_count value is
less than 1."""
msg = "The minimum_forecast_count must be at least 1"
with pytest.raises(ValueError, match=msg):
Plugin(minimum_forecast_count=0)
def test_init_with_arguments():
"""Test init with specified arguments."""
plugin = Plugin(minimum_forecast_count=100)
assert plugin.minimum_forecast_count == 100
def test_init_using_defaults():
"""Test init without providing any arguments."""
plugin = Plugin()
assert plugin.minimum_forecast_count == 200
def test_valid_bounds(self):
"""Test that no exception is raised if the input cubes have forecast
reference time bounds that do not overlap."""
plugin = Plugin()
plugin._check_frt_coord([self.reliability_cube, self.different_frt])
def test_basic(self):
"""Test repr is as expected."""
self.assertEqual(str(Plugin()),
"<AggregateReliabilityCalibrationTables>")
def test_monotonic(self):
"""Test no change to observation frequency, if already monotonic."""
result = Plugin()._assume_constant_observation_frequency(
self.obs_count, self.forecast_count)
assert_array_equal(result.data, self.obs_count)
def test_using_defaults(self):
"""Test without providing any arguments."""
plugin = Plugin()
self.assertIsNone(plugin.threshold_coord, None)
def test_using_defaults(self):
"""Test without providing any arguments."""
plugin = Plugin()
self.assertEqual(len(plugin.probability_bins), 5)
self.assertEqual(plugin.expected_table_shape, (3, 5))
