def test_process_and_aggregate(create_rel_table_inputs):
"""Test that aggregation during construction produces the same result as
applying the two plugins sequentially."""
# use the spatial coordinates for aggregation - input is a parameterised fixture
if create_rel_table_inputs.forecast.coords("spot_index"):
agg_coords = ["spot_index"]
else:
agg_coords = ["longitude", "latitude"]
# construct and aggregate as two separate plugins
constructed = Plugin(
single_value_lower_limit=True, single_value_upper_limit=True
).process(create_rel_table_inputs.forecast, create_rel_table_inputs.truth)
aggregated = AggregateReliabilityCalibrationTables().process(
[constructed], agg_coords
)
# construct plugin with aggregate_coords option
constructed_with_agg = Plugin(
single_value_lower_limit=True, single_value_upper_limit=True
).process(
create_rel_table_inputs.forecast, create_rel_table_inputs.truth, agg_coords
)
# check that the two cubes are identical
assert constructed_with_agg == aggregated
def test_combine_undersampled_bins_non_monotonic(self):
"""Test expected values are returned when a bin is below the minimum
forecast count when the observed frequency is non-monotonic."""
expected_data = np.array([[1000, 425, 1000], [1000, 425, 1000],
[2000, 600, 1000]])
expected_bin_coord_points = np.array([0.2, 0.6, 0.9], dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.4], [0.4, 0.8], [0.8, 1.0]],
dtype=np.float32,
)
self.multi_threshold_rt.data[1] = np.array([
[750, 250, 50, 375, 1000], # Observation count
[750, 250, 50, 375, 1000], # Sum of forecast probability
[1000, 1000, 100, 500, 1000], # Forecast count
])
result = Plugin().process(self.multi_threshold_rt.copy())
assert_array_equal(result[0].data, self.multi_threshold_rt[0].data)
self.assertEqual(result[0].coords(),
self.multi_threshold_rt[0].coords())
assert_array_equal(result[1].data, expected_data)
assert_allclose(result[1].coord("probability_bin").points,
expected_bin_coord_points)
assert_allclose(result[1].coord("probability_bin").bounds,
expected_bin_coord_bounds)
def test_cub_poorly_sampled_bins(probability_bin_coord):
"""Test when all bins are poorly sampled and the minimum forecast count
cannot be reached."""
obs_count = forecast_probability_sum = np.array([0, 2, 5, 8, 10],
dtype=np.float32)
forecast_count = np.array([10, 10, 10, 10, 10], dtype=np.float32)
expected = np.array([
[25], # Observation count
[25], # Sum of forecast probability
[50], # Forecast count
])
result = Plugin()._combine_undersampled_bins(
obs_count,
forecast_probability_sum,
forecast_count,
probability_bin_coord,
)
assert_array_equal(result[:3], expected)
expected_bin_coord_points = np.array([0.5], dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 1.0]],
dtype=np.float32,
)
assert_allclose(expected_bin_coord_points, result[3].points)
assert_allclose(expected_bin_coord_bounds, result[3].bounds)
def test_with_arguments(self):
"""Test with specified arguments."""
plugin = Plugin(minimum_forecast_count=100)
self.assertEqual(plugin.minimum_forecast_count, 100)
self.assertIsNone(plugin.threshold_coord, None)
def test_using_defaults(self):
"""Test without providing any arguments."""
plugin = Plugin()
self.assertEqual(plugin.minimum_forecast_count, 200)
self.assertIsNone(plugin.threshold_coord, None)
def test_process_mismatching_threshold_coordinates(truth_grid, forecast_grid):
"""Test that an exception is raised if the forecast and truth cubes
have differing threshold coordinates."""
truths_grid = truth_grid[:, 0, ...]
msg = "Threshold coordinates differ between forecasts and truths."
with pytest.raises(ValueError, match=msg):
Plugin().process(forecast_grid, truths_grid)
def test_lowest_bin_non_monotonic(self):
"""Test expected values are returned where the lowest observation
count bin is non-monotonic."""
expected_data = np.array([[1000, 500, 500, 750], [250, 500, 750, 1000],
[2000, 1000, 1000, 1000]])
expected_bin_coord_points = np.array([0.2, 0.5, 0.7, 0.9],
dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.4], [0.4, 0.6], [0.6, 0.8], [0.8, 1.0]],
dtype=np.float32,
)
self.multi_threshold_rt.data[1] = np.array([
[1000, 0, 250, 500, 750], # Observation count
[0, 250, 500, 750, 1000], # Sum of forecast probability
[1000, 1000, 1000, 1000, 1000], # Forecast count
])
result = Plugin().process(self.multi_threshold_rt.copy())
assert_array_equal(result[0].data, self.multi_threshold_rt[0].data)
self.assertEqual(result[0].coords(),
self.multi_threshold_rt[0].coords())
assert_array_equal(result[1].data, expected_data)
assert_allclose(result[1].coord("probability_bin").points,
expected_bin_coord_points)
assert_allclose(result[1].coord("probability_bin").bounds,
expected_bin_coord_bounds)
def test_basic(self):
"""Test repr is as expected."""
plugin = Plugin(n_probability_bins=2, single_value_limits=False)
self.assertEqual(
str(plugin),
'<ConstructReliabilityCalibrationTables: probability_bins: '
'[0.00 --> 0.50], [0.50 --> 1.00]>')
def test_aggregating_over_masked_cubes_and_coordinates(self):
"""Test of aggregating over coordinates and cubes in a single call
using a masked reliability table. In this instance the latitude and
longitude coordinates are collapsed and the values from two input cube
combined."""
frt = "forecast_reference_time"
expected_points = self.masked_different_frt.coord(frt).points
expected_bounds = [[
self.masked_reliability_cube.coord(frt).bounds[0][0],
self.masked_different_frt.coord(frt).bounds[-1][1],
]]
expected_result = np.array([
[0.0, 0.0, 2.0, 4.0, 2.0],
[0.0, 0.625, 2.625, 3.25, 2.0],
[0.0, 3.0, 5.0, 4.0, 2.0],
])
plugin = Plugin()
result = plugin.process(
[self.masked_reliability_cube, self.masked_different_frt],
coordinates=["latitude", "longitude"],
)
self.assertIsInstance(result.data, np.ma.MaskedArray)
assert_array_equal(result.data, expected_result)
self.assertEqual(result.coord(frt).points, expected_points)
assert_array_equal(result.coord(frt).bounds, expected_bounds)
def test_process_no_change_agg(reliability_table_agg):
"""Test with no changes required to preserve monotonicity."""
result = Plugin().process(reliability_table_agg.copy())
assert_array_equal(result[0].data, reliability_table_agg[0].data)
assert result[0].coords() == reliability_table_agg[0].coords()
assert_array_equal(result[1].data, reliability_table_agg[1].data)
assert result[1].coords() == reliability_table_agg[1].coords()
def test_emcam_combine_undersampled_bins_non_monotonic(
reliability_table_slice):
"""Test expected values are returned when a bin is below the minimum
forecast count when the observed frequency is non-monotonic."""
expected_data = np.array([[1000, 425, 1000], [1000, 425, 1000],
[2000, 600, 1000]])
expected_bin_coord_points = np.array([0.2, 0.6, 0.9], dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.4], [0.4, 0.8], [0.8, 1.0]],
dtype=np.float32,
)
reliability_table_slice.data = np.array(
[
[750, 250, 50, 375, 1000], # Observation count
[750, 250, 50, 375, 1000], # Sum of forecast probability
[1000, 1000, 100, 500, 1000], # Forecast count
],
dtype=np.float32,
)
result = Plugin()._enforce_min_count_and_montonicity(
reliability_table_slice.copy())
assert_array_equal(result.data, expected_data)
assert_allclose(
result.coord("probability_bin").points, expected_bin_coord_points)
assert_allclose(
result.coord("probability_bin").bounds, expected_bin_coord_bounds)
def test_cbp_two_non_monotonic_bin_pairs(
default_obs_counts,
default_fcst_counts,
probability_bin_coord,
expected_enforced_monotonic,
):
"""Test one bin pair is combined, if two bin pairs are non-monotonic.
As only a single bin pair is combined, the resulting observation
count will still yield a non-monotonic observation frequency."""
obs_count = np.array([0, 750, 500, 1000, 750], dtype=np.float32)
forecast_probability_sum = default_obs_counts
expected_enforced_monotonic[0][1] = 750 # Amend observation count
result = Plugin()._combine_bin_pair(
obs_count,
forecast_probability_sum,
default_fcst_counts,
probability_bin_coord,
)
assert_array_equal(result[:3], expected_enforced_monotonic)
expected_bin_coord_points = np.array([0.1, 0.3, 0.5, 0.8],
dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.2], [0.2, 0.4], [0.4, 0.6], [0.6, 1.0]],
dtype=np.float32,
)
assert_allclose(expected_bin_coord_points, result[3].points)
assert_allclose(expected_bin_coord_bounds, result[3].bounds)
def test_cub_two_equal_undersampled_bins(probability_bin_coord):
"""Test when two bins are under-sampled and the under-sampled bins have
an equal forecast count."""
obs_count = np.array([0, 25, 250, 75, 250], dtype=np.float32)
forecast_probability_sum = np.array([0, 25, 250, 75, 250],
dtype=np.float32)
forecast_count = np.array([1000, 100, 500, 100, 250], dtype=np.float32)
expected = np.array([
[0, 275, 325], # Observation count
[0, 275, 325], # Sum of forecast probability
[1000, 600, 350], # Forecast count
])
result = Plugin()._combine_undersampled_bins(
obs_count,
forecast_probability_sum,
forecast_count,
probability_bin_coord,
)
assert_array_equal(result[:3], expected)
expected_bin_coord_points = np.array([0.1, 0.4, 0.8], dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.2], [0.2, 0.6], [0.6, 1.0]],
dtype=np.float32,
)
assert_allclose(expected_bin_coord_points, result[3].points)
assert_allclose(expected_bin_coord_bounds, result[3].bounds)
def test_cub_three_equal_undersampled_bin_neighbours(probability_bin_coord):
"""Test when three neighbouring bins are under-sampled."""
obs_count = np.array([0, 25, 50, 75, 250], dtype=np.float32)
forecast_probability_sum = np.array([0, 25, 50, 75, 250], dtype=np.float32)
forecast_count = np.array([1000, 100, 100, 100, 250], dtype=np.float32)
expected = np.array([
[0, 150, 250], # Observation count
[0, 150, 250], # Sum of forecast probability
[1000, 300, 250], # Forecast count
])
result = Plugin()._combine_undersampled_bins(
obs_count,
forecast_probability_sum,
forecast_count,
probability_bin_coord,
)
assert_array_equal(result[:3], expected)
expected_bin_coord_points = np.array([0.1, 0.5, 0.9], dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.2], [0.2, 0.8], [0.8, 1.0]],
dtype=np.float32,
)
assert_allclose(expected_bin_coord_points, result[3].points)
assert_allclose(expected_bin_coord_bounds, result[3].bounds)
def setUp(self):
"""Set up data for testing the interpolate method."""
self.reliability_probabilities = np.array([0.0, 0.4, 0.8])
self.observation_frequencies = np.array([0.2, 0.6, 1.0])
self.plugin = Plugin()
def test_cub_one_undersampled_bin_upper_neighbour(probability_bin_coord):
"""Test for one under-sampled bin that is combined with its upper
neighbour."""
obs_count = np.array([0, 500, 50, 750, 1000], dtype=np.float32)
forecast_probability_sum = np.array([0, 500, 50, 750, 1000],
dtype=np.float32)
forecast_count = np.array([1000, 2000, 100, 1000, 1000], dtype=np.float32)
expected = np.array([
[0, 500, 800, 1000], # Observation count
[0, 500, 800, 1000], # Sum of forecast probability
[1000, 2000, 1100, 1000], # Forecast count
])
result = Plugin()._combine_undersampled_bins(
obs_count,
forecast_probability_sum,
forecast_count,
probability_bin_coord,
)
assert_array_equal(result[:3], expected)
expected_bin_coord_points = np.array([0.1, 0.3, 0.6, 0.9],
dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.2], [0.2, 0.4], [0.4, 0.8], [0.8, 1.0]],
dtype=np.float32,
)
assert_allclose(expected_bin_coord_points, result[3].points)
assert_allclose(expected_bin_coord_bounds, result[3].bounds)
def test_process_return_type(forecast_grid, truth_grid):
"""Test the process method returns a reliability table cube."""
result = Plugin().process(forecast_grid, truth_grid)
assert isinstance(result, iris.cube.Cube)
assert result.name() == "reliability_calibration_table"
assert result.coord("air_temperature") == forecast_grid.coord(var_name="threshold")
assert result.coord_dims("air_temperature")[0] == 0
def test_process_undersampled_non_monotonic_point(create_rel_tables_point):
"""Test expected values are returned when one slice contains a bin that is
below the minimum forecast count, whilst the observed frequency is
non-monotonic. Test that remaining data, which requires no change, is not
changed. Parameterized using `create_rel_tables` fixture."""
expected_data = np.array([[1000, 425, 1000], [1000, 425, 1000],
[2000, 600, 1000]])
expected_bin_coord_points = np.array([0.2, 0.6, 0.9], dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.4], [0.4, 0.8], [0.8, 1.0]],
dtype=np.float32,
)
rel_table = create_rel_tables_point.table
rel_table.data[create_rel_tables_point.indices0] = np.array([
[750, 250, 50, 375, 1000], # Observation count
[750, 250, 50, 375, 1000], # Sum of forecast probability
[1000, 1000, 100, 500, 1000], # Forecast count
])
result = Plugin(point_by_point=True).process(rel_table.copy())
assert_array_equal(result[0][0].data, expected_data)
assert_allclose(result[0][0].coord("probability_bin").points,
expected_bin_coord_points)
assert_allclose(result[0][0].coord("probability_bin").bounds,
expected_bin_coord_bounds)
# Check the unchanged data remains unchanged
expected = rel_table.data[create_rel_tables_point.indices1]
assert all([np.array_equal(cube.data, expected) for cube in result[0][1:]])
def test_no_change(self):
"""Test with no changes required to preserve monotonicity"""
result = Plugin().process(self.multi_threshold_rt.copy())
assert_array_equal(result[0].data, self.multi_threshold_rt[0].data)
self.assertEqual(result[0].coords(), self.multi_threshold_rt[0].coords())
assert_array_equal(result[1].data, self.multi_threshold_rt[1].data)
self.assertEqual(result[1].coords(), self.multi_threshold_rt[1].coords())
def test_non_monotonic_equal_forecast_count(self):
"""Test enforcement of monotonicity for observation frequency."""
obs_count = np.array([0, 750, 500, 1000, 750], dtype=np.float32)
expected_result = np.array([0, 750, 750, 1000, 1000], dtype=np.float32)
result = Plugin()._assume_constant_observation_frequency(
obs_count, self.forecast_count)
assert_array_equal(result.data, expected_result)
def setUp(self):
"""Set up monotonic bins as default and plugin for testing."""
super().setUp()
self.obs_count = np.array([0, 250, 500, 750, 1000], dtype=np.float32)
self.forecast_probability_sum = np.array([0, 250, 500, 750, 1000],
dtype=np.float32)
self.plugin = Plugin()
def test_with_invalid_minimum_forecast_count(self):
"""Test an exception is raised if the minimum_forecast_count value is
less than 1."""
msg = "The minimum_forecast_count must be at least 1"
with self.assertRaisesRegex(ValueError, msg):
Plugin(minimum_forecast_count=0)
def test_emcam_lowest_bin_non_monotonic(reliability_table_slice):
"""Test expected values are returned where the lowest observation
count bin is non-monotonic."""
expected_data = np.array([[1000, 500, 500, 750], [250, 500, 750, 1000],
[2000, 1000, 1000, 1000]])
expected_bin_coord_points = np.array([0.2, 0.5, 0.7, 0.9],
dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.4], [0.4, 0.6], [0.6, 0.8], [0.8, 1.0]],
dtype=np.float32,
)
reliability_table_slice.data = np.array([
[1000, 0, 250, 500, 750], # Observation count
[0, 250, 500, 750, 1000], # Sum of forecast probability
[1000, 1000, 1000, 1000, 1000], # Forecast count
])
result = Plugin()._enforce_min_count_and_montonicity(
reliability_table_slice.copy())
assert_array_equal(result.data, expected_data)
assert_allclose(
result.coord("probability_bin").points, expected_bin_coord_points)
assert_allclose(
result.coord("probability_bin").bounds, expected_bin_coord_bounds)
def test_cub_one_undersampled_bin_at_bottom(default_obs_counts,
probability_bin_coord):
"""Test when the lowest probability bin is under-sampled."""
obs_count = forecast_probability_sum = default_obs_counts
forecast_count = np.array([100, 1000, 1000, 1000, 1000], dtype=np.float32)
expected = np.array([
[250, 500, 750, 1000], # Observation count
[250, 500, 750, 1000], # Sum of forecast probability
[1100, 1000, 1000, 1000], # Forecast count
])
result = Plugin()._combine_undersampled_bins(
obs_count,
forecast_probability_sum,
forecast_count,
probability_bin_coord,
)
assert_array_equal(result[:3], expected)
expected_bin_coord_points = np.array([0.2, 0.5, 0.7, 0.9],
dtype=np.float32)
expected_bin_coord_bounds = np.array(
[[0.0, 0.4], [0.4, 0.6], [0.6, 0.8], [0.8, 1.0]],
dtype=np.float32,
)
assert_allclose(expected_bin_coord_points, result[3].points)
assert_allclose(expected_bin_coord_bounds, result[3].bounds)
def test_acof_monotonic(default_fcst_counts):
"""Test no change to observation frequency, if already monotonic."""
obs_count = np.array([0, 0, 250, 500, 750], dtype=np.float32)
result = Plugin()._assume_constant_observation_frequency(
obs_count,
default_fcst_counts,
)
assert_array_equal(result.data, obs_count)
def test_mismatching_threshold_coordinates(self):
"""Test that an exception is raised if the forecast and truth cubes
have differing threshold coordinates."""
self.truths = self.truths[:, 0, ...]
msg = "Threshold coordinates differ between forecasts and truths."
with self.assertRaisesRegex(ValueError, msg):
Plugin().process(self.forecasts, self.truths)
def test_single_cube(reliability_cube):
"""Test the plugin returns an unaltered cube if only one is passed in
and no coordinates are given."""
plugin = Plugin()
expected = reliability_cube.copy()
result = plugin.process([reliability_cube])
assert result == expected
def test_frt_coord_invalid_bounds(reliability_cube, overlapping_frt):
"""Test that an exception is raised if the input cubes have forecast
reference time bounds that overlap."""
plugin = Plugin()
msg = "Reliability calibration tables have overlapping"
with pytest.raises(ValueError, match=msg):
plugin._check_frt_coord([reliability_cube, overlapping_frt])
def test_invalid_bounds(self):
"""Test that an exception is raised if the input cubes have forecast
reference time bounds that overlap."""
plugin = Plugin()
msg = "Reliability calibration tables have overlapping"
with self.assertRaisesRegex(ValueError, msg):
plugin._check_frt_coord([self.reliability_cube, self.overlapping_frt])
def test_without_single_value_limits():
"""Test the generation of probability bins without single value end
bins. The range 0 to 1 will be divided into 4 equally sized bins."""
expected = np.array([[0.0, 0.24999999], [0.25, 0.49999997],
[0.5, 0.74999994], [0.75, 1.0]])
result = Plugin()._define_probability_bins(n_probability_bins=4,
single_value_limits=False)
assert_allclose(result, expected)
