def test_point_by_point_with_nans(self):
"""
Test that the expected coefficients are generated when the ensemble
mean is the predictor for a normal distribution and coefficients are
calculated independently at each grid point with one grid point
having NaN values for the truth.
"""
predictor = "mean"
distribution = "norm"
self.truth.data[:, 0, 0] = np.nan
plugin = Plugin(predictor,
tolerance=self.tolerance,
point_by_point=True)
result = plugin.process(
self.initial_guess_spot_mean,
self.forecast_predictor_mean,
self.truth,
self.forecast_variance,
distribution,
)
self.expected_mean_coefficients_point_by_point[:, 0,
0] = self.initial_guess_for_mean
self.assertEMOSCoefficientsAlmostEqual(
result, self.expected_mean_coefficients_point_by_point)
def test_realizations_predictor_max_iterations(self):
"""
Test that the plugin returns a list of coefficients
equal to specific values, when the ensemble realizations are the
predictor assuming a truncated normal distribution and the value
specified for the max_iterations is overridden. The coefficients are
calculated by minimising the CRPS and using a set default value for
the initial guess.
"""
predictor = "realizations"
max_iterations = 1000
distribution = "truncnorm"
plugin = Plugin(tolerance=self.tolerance,
max_iterations=max_iterations)
result = plugin.process(
self.initial_guess_for_realization,
self.forecast_predictor_realizations,
self.truth,
self.forecast_variance,
predictor,
distribution,
)
self.assertEMOSCoefficientsAlmostEqual(
result, self.expected_realizations_coefficients)
def test_coefficient_values_for_truncnorm_distribution(self):
"""Ensure that the values for the optimised_coefficients match the
expected values, and the coefficient names also match
expected values for a truncated normal distribution. In this case,
a linear least-squares regression is used to construct the initial
guess."""
distribution = "truncnorm"
plugin = Plugin(distribution)
result = plugin.process(
self.historic_wind_speed_forecast_cube, self.wind_speed_truth_cube
)
self.assertEMOSCoefficientsAlmostEqual(
np.array([cube.data for cube in result]),
self.expected_mean_predictor_truncnorm,
)
self.assertArrayEqual(
[cube.name() for cube in result], self.expected_coeff_names
)
for cube in result:
self.assertArrayEqual(
cube.attributes["shape_parameters"],
np.array([0, np.inf], dtype=np.float32),
)
def test_realizations_predictor_estimate_coefficients_masked_halo(self):
"""
Test that the plugin returns the expected values for the initial guess
for the calibration coefficients, when the ensemble mean is used
as the predictor. The coefficients are estimated using a linear model.
In this case, the result of the linear regression is for an intercept
of 0.333333 with different weights for the realizations because
some of the realizations are closer to the truth, in this instance. In
this case the original data has been surrounded by a halo of masked
nans, which gives the same coefficients as the original data.
"""
predictor = "realizations"
estimate_coefficients_from_linear_model_flag = True
plugin = Plugin(self.distribution, self.desired_units)
result = plugin.compute_initial_guess(
self.truth_masked_halo,
self.current_forecast_predictor_realizations_masked_halo,
predictor,
estimate_coefficients_from_linear_model_flag,
no_of_realizations=self.no_of_realizations,
)
self.assertArrayAlmostEqual(
self.expected_realizations_predictor_with_linear_model, result
)
def test_mean_predictor_point_by_point(self):
"""
Test that the expected coefficients are generated when the ensemble
mean is the predictor for a normal distribution and coefficients are
calculated independently at each grid point. The coefficients are
calculated by minimising the CRPS.
"""
predictor = "mean"
distribution = "norm"
initial_guess = np.broadcast_to(
self.initial_guess_for_mean,
(
len(self.truth.coord(axis="y").points) *
len(self.truth.coord(axis="x").points),
len(self.initial_guess_for_mean),
),
)
plugin = Plugin(tolerance=self.tolerance, point_by_point=True)
result = plugin.process(
initial_guess,
self.forecast_predictor_mean,
self.truth,
self.forecast_variance,
predictor,
distribution,
)
self.assertEMOSCoefficientsAlmostEqual(
result, self.expected_mean_coefficients_point_by_point)
def setUp(self):
"""Set-up coefficients and plugin for testing."""
super().setUp()
self.plugin = Plugin()
self.plugin.current_forecast = self.current_temperature_forecast_cube
self.plugin.coefficients_cubelist = self.coeffs_from_mean
def test_mean_predictor_point_by_point_sites(self):
"""
Test that the expected coefficients are generated when the ensemble
mean is the predictor for a normal distribution and coefficients are
calculated independently at each site location. The coefficients are
calculated by minimising the CRPS.
"""
forecast_spot_cube = self.historic_forecast_spot_cube.collapsed(
"realization", iris.analysis.MEAN)
forecast_var_spot_cube = forecast_spot_cube.copy()
forecast_var_spot_cube.data = forecast_var_spot_cube.data / 10.0
predictor = "mean"
distribution = "norm"
initial_guess = np.broadcast_to(
self.initial_guess_for_mean,
(
len(self.truth.coord(axis="y").points) *
len(self.truth.coord(axis="x").points),
len(self.initial_guess_for_mean),
),
)
plugin = Plugin(tolerance=self.tolerance, point_by_point=True)
result = plugin.process(
initial_guess,
forecast_spot_cube,
self.truth_spot_cube,
forecast_var_spot_cube,
predictor,
distribution,
)
self.assertEMOSCoefficientsAlmostEqual(
result, self.expected_mean_coefficients_point_by_point_sites)
def test_realizations_predictor_point_by_point(self):
"""
Test that the expected coefficients are generated when the ensemble
realizations are the predictor for a normal distribution and
coefficients are calculated independently at each grid point. The
coefficients are calculated by minimising the CRPS.
"""
predictor = "realizations"
distribution = "norm"
initial_guess = np.broadcast_to(
self.initial_guess_for_realization,
(
len(self.truth.coord(axis="y").points) *
len(self.truth.coord(axis="x").points),
len(self.initial_guess_for_realization),
),
)
# Use a larger value for the tolerance to terminate sooner to avoid
# minimising in computational noise.
plugin = Plugin(tolerance=0.01, point_by_point=True)
result = plugin.process(
initial_guess,
self.forecast_predictor_realizations,
self.truth,
self.forecast_variance,
predictor,
distribution,
)
self.assertArrayAlmostEqual(
result,
self.expected_realizations_coefficients_point_by_point,
decimal=2)
def test_catch_warnings_percentage_change(self, warning_list=None):
"""
Test that two warnings are generated if the minimisation
does not result in a convergence. The first warning reports a that
the minimisation did not result in convergence, whilst the second
warning reports that the percentage change in the final iteration was
greater than the tolerated value.
The ensemble mean is the predictor.
"""
initial_guess = np.array([0, 1, 5000, 1], dtype=np.float64)
predictor = "mean"
distribution = "truncnorm"
plugin = Plugin(tolerance=self.tolerance, max_iterations=5)
plugin.process(
initial_guess,
self.forecast_predictor_mean,
self.truth,
self.forecast_variance,
predictor,
distribution,
)
warning_msg_min = "Minimisation did not result in convergence after"
warning_msg_iter = "The final iteration resulted in a percentage "
self.assertTrue(
any(item.category == UserWarning for item in warning_list))
self.assertTrue(
any(warning_msg_min in str(item) for item in warning_list))
self.assertTrue(
any(warning_msg_iter in str(item) for item in warning_list))
def test_basic(self):
"""A simple tests for the __repr__ method."""
result = str(Plugin())
msg = ("<ContinuousRankedProbabilityScoreMinimisers: "
"minimisation_dict: {'norm': 'calculate_normal_crps', "
"'truncnorm': 'calculate_truncated_normal_crps'}; "
"tolerance: 0.02; max_iterations: 1000>")
self.assertEqual(result, msg)
def test_update_kwargs(self):
"""A test to update the available keyword argument."""
result = str(Plugin(tolerance=10, max_iterations=10))
msg = ("<ContinuousRankedProbabilityScoreMinimisers: "
"minimisation_dict: {'norm': 'calculate_normal_crps', "
"'truncnorm': 'calculate_truncated_normal_crps'}; "
"tolerance: 10; max_iterations: 10>")
self.assertEqual(result, msg)
def setUp(self):
"""Set up inputs for testing."""
super().setUp()
self.tolerance = 1e-4
self.mean_plugin = Plugin("mean", tolerance=self.tolerance)
self.realizations_plugin = Plugin("realizations", tolerance=self.tolerance)
self.sqrt_pi = np.sqrt(np.pi).astype(np.float64)
self.initial_guess_for_mean = np.array([0, 1, 0, 1], dtype=np.float64)
self.initial_guess_for_realization = np.array(
[0, np.sqrt(1 / 3.0), np.sqrt(1 / 3.0), np.sqrt(1 / 3.0), 0, 1],
dtype=np.float64,
)
self.initial_guess_mean_additional_predictor = np.array(
[0, 0.5, 0.5, 0, 1], dtype=np.float64
)
def setUp(self):
"""Set up expected output."""
super().setUp()
self.tolerance = 1e-4
self.plugin = Plugin(tolerance=self.tolerance)
self.expected_mean_coefficients = ([0.0459, 0.6047, 0.3965, 0.958])
self.expected_realizations_coefficients = ([
0.0265, 0.2175, 0.2692, 0.0126, 0.5965, 0.7952
])
def test_basic(self):
"""Ensure that the optimised_coefficients are returned as a cube,
with the expected number of coefficients."""
plugin = Plugin(self.distribution)
result = plugin.process(
self.historic_temperature_forecast_cube, self.temperature_truth_cube
)
self.assertIsInstance(result, iris.cube.CubeList)
self.assertEqual(len(result), len(self.coeff_names))
def test_missing_cube(self):
"""Test that an exception is raised if either of the historic
forecasts or truth were missing."""
self.historic_temperature_forecast_cube.convert_units("Fahrenheit")
plugin = Plugin(self.distribution)
msg = ".*cubes must be provided"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(self.historic_temperature_forecast_cube, None)
def test_statsmodels_mean(self, warning_list=None):
"""
Test that the plugin raises no warnings if the statsmodels module
is not found for when the predictor is the ensemble mean.
"""
predictor = "mean"
statsmodels_warning = "The statsmodels module cannot be imported"
Plugin(self.distribution, self.desired_units, predictor=predictor)
self.assertNotIn(statsmodels_warning, warning_list)
def setUp(self):
"""Set-up coefficients and plugin for testing."""
super().setUp()
self.optimised_coeffs = (dict(
zip(
self.coeffs_from_mean.coord("coefficient_name").points,
self.coeffs_from_mean.data)))
self.plugin = Plugin()
self.plugin.current_forecast = self.current_temperature_forecast_cube
def setUp(self):
"""Set up additional cube for land-sea mask."""
super().setUp()
mask_data = np.array([[0, 1, 0], [0, 1, 1], [1, 1, 0]], dtype=np.int32)
self.mask_cube = set_up_variable_cube(mask_data,
name="land_binary_mask",
units="1")
self.plugin = Plugin("norm", "20171110T0000Z")
# Copy a few slices of the temperature truth cube to test on.
self.cube3D = self.temperature_truth_cube[0:2, ...].copy()
def setUp(self):
"""Set up the plugin and cubes for testing."""
super().setUp()
frt_dt = datetime.datetime(2017, 11, 10, 0, 0)
time_dt = datetime.datetime(2017, 11, 10, 4, 0)
data = np.ones((3, 3), dtype=np.float32)
self.historic_forecast = _create_historic_forecasts(
data, time_dt, frt_dt,
).merge_cube()
data_with_realizations = np.ones((3, 3, 3), dtype=np.float32)
self.historic_forecast_with_realizations = _create_historic_forecasts(
data_with_realizations, time_dt, frt_dt, realizations=[0, 1, 2],
).merge_cube()
self.optimised_coeffs = np.array([0, 1, 2, 3], np.int32)
self.distribution = "norm"
self.desired_units = "degreesC"
self.predictor = "mean"
self.plugin = Plugin(
distribution=self.distribution,
desired_units=self.desired_units,
predictor=self.predictor,
)
self.expected_frt = (
self.historic_forecast.coord("forecast_reference_time").cell(-1).point
)
self.expected_x_coord_points = np.median(
self.historic_forecast.coord(axis="x").points
)
self.historic_forecast.coord(axis="x").guess_bounds()
self.expected_x_coord_bounds = np.array(
[
[
np.min(self.historic_forecast.coord(axis="x").bounds),
np.max(self.historic_forecast.coord(axis="x").bounds),
]
]
)
self.expected_y_coord_points = np.median(
self.historic_forecast.coord(axis="y").points
)
self.historic_forecast.coord(axis="y").guess_bounds()
self.expected_y_coord_bounds = np.array(
[
[
np.min(self.historic_forecast.coord(axis="y").bounds),
np.max(self.historic_forecast.coord(axis="y").bounds),
]
]
)
self.attributes = generate_mandatory_attributes([self.historic_forecast])
self.attributes["diagnostic_standard_name"] = self.historic_forecast.name()
self.attributes["distribution"] = self.distribution
self.attributes["title"] = "Ensemble Model Output Statistics coefficients"
def setUp(self):
"""Set up expected output.
The coefficients are in the order [gamma, delta, alpha, beta].
"""
super().setUp()
self.tolerance = 1e-4
self.plugin = Plugin(tolerance=self.tolerance)
self.expected_mean_coefficients = ([0.0023, 0.8070, -0.0008, 1.0009])
self.expected_realizations_coefficients = ([
-0.1373, 0.1141, 0.0409, 0.414, 0.2056, 0.8871
])
def test_non_matching_units(self):
"""Test that an exception is raised if the historic forecasts and truth
have non matching units."""
self.historic_temperature_forecast_cube.convert_units("Fahrenheit")
plugin = Plugin(self.distribution)
msg = "The historic forecast units"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(
self.historic_temperature_forecast_cube, self.temperature_truth_cube
)
def test_statsmodels_realizations(self, warning_list=None):
"""
Test that the plugin raises the desired warning if the statsmodels
module is not found for when the predictor is the ensemble
realizations.
"""
predictor = "realizations"
Plugin(self.distribution, self.desired_units, predictor=predictor)
warning_msg = "The statsmodels module cannot be imported"
self.assertTrue(any(item.category == ImportWarning for item in warning_list))
self.assertTrue(any(warning_msg in str(item) for item in warning_list))
def test_basic(self):
"""Test without specifying keyword arguments"""
result = str(Plugin(self.distribution))
msg = ("<EstimateCoefficientsForEnsembleCalibration: "
"distribution: norm; "
"desired_units: None; "
"predictor: mean; "
"minimiser: <class 'improver.calibration.ensemble_calibration."
"ContinuousRankedProbabilityScoreMinimisers'>; "
"coeff_names: ['alpha', 'beta', 'gamma', 'delta']; "
"tolerance: 0.02; "
"max_iterations: 1000>")
self.assertEqual(result, msg)
def setUp(self):
"""Set up expected output."""
super().setUp()
self.tolerance = 1e-4
self.plugin = Plugin(tolerance=self.tolerance)
self.expected_mean_coefficients = [0.3958, 0.9854, -0.0, 0.621]
self.expected_realizations_coefficients = [
0.1898,
-0.1558,
0.4452,
0.8877,
-0.1331,
-0.0002,
]
def test_historic_forecast_unit_conversion(self):
"""Ensure the expected optimised coefficients are generated,
even if the input historic forecast cube has different units."""
self.historic_temperature_forecast_cube.convert_units("Fahrenheit")
desired_units = "Kelvin"
plugin = Plugin(self.distribution, desired_units=desired_units)
result = plugin.process(
self.historic_temperature_forecast_cube, self.temperature_truth_cube
)
self.assertEMOSCoefficientsAlmostEqual(
np.array([cube.data for cube in result]), self.expected_mean_predictor_norm,
)
def test_coeff_names(self):
"""Test that the plugin instance defines the expected
coefficient names."""
expected = ["alpha", "beta", "gamma", "delta"]
predictor = "mean"
tolerance = 10
max_iterations = 10
plugin = Plugin(
self.distribution,
self.desired_units,
predictor=predictor,
tolerance=tolerance,
max_iterations=max_iterations,
)
self.assertEqual(plugin.coeff_names, expected)
def test_too_few_coefficients(self):
"""Test that an exception is raised if the number of coefficients
provided for creating the coefficients cube is not equal to the
number of coefficient names."""
distribution = "truncnorm"
desired_units = "Fahrenheit"
predictor = "mean"
optimised_coeffs = [1, 2, 3]
plugin = Plugin(
distribution=distribution, desired_units=desired_units, predictor=predictor,
)
msg = "The number of coefficients in"
with self.assertRaisesRegex(ValueError, msg):
plugin.create_coefficients_cubelist(
optimised_coeffs, self.historic_forecast
)
def test_basic_realizations_predictor(self):
"""
Test that the plugin returns a numpy float value. The ensemble
realizations are the predictor. The result indicates the minimum value
for the CRPS that was achieved by the minimisation.
"""
predictor = "realizations"
plugin = Plugin()
result = plugin.calculate_truncated_normal_crps(
self.initial_guess_for_realization,
self.forecast_predictor_data_realizations, self.truth_data,
self.forecast_variance_data, self.sqrt_pi, predictor)
self.assertIsInstance(result, np.float64)
self.assertAlmostEqual(result, 0.1670167)
def test_catch_warnings(self, warning_list=None):
"""
Test that a warning is generated if the minimisation
does not result in a convergence. The ensemble mean is the predictor.
"""
predictor = "mean"
distribution = "truncated_gaussian"
plugin = Plugin(tolerance=self.tolerance, max_iterations=10)
plugin.process(self.initial_guess_for_mean,
self.forecast_predictor_mean, self.truth,
self.forecast_variance, predictor, distribution)
warning_msg = "Minimisation did not result in convergence after"
self.assertTrue(
any(item.category == UserWarning for item in warning_list))
self.assertTrue(any(warning_msg in str(item) for item in warning_list))
def setUp(self):
"""Set up expected output.
The coefficients are in the order [alpha, beta, gamma, delta].
"""
super().setUp()
self.tolerance = 1e-4
self.plugin = Plugin(tolerance=self.tolerance)
self.expected_mean_coefficients = [-0.0008, 1.0009, 0.0023, 0.8070]
self.expected_realizations_coefficients = [
0.0427,
0.4117,
0.1946,
0.8907,
-0.1435,
0.037,
]
