def test_point_by_point_with_nans(self): """ Test that the expected coefficients are generated when the ensemble mean is the predictor for a normal distribution and coefficients are calculated independently at each grid point with one grid point having NaN values for the truth. """ predictor = "mean" distribution = "norm" self.truth.data[:, 0, 0] = np.nan plugin = Plugin(predictor, tolerance=self.tolerance, point_by_point=True) result = plugin.process( self.initial_guess_spot_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, distribution, ) self.expected_mean_coefficients_point_by_point[:, 0, 0] = self.initial_guess_for_mean self.assertEMOSCoefficientsAlmostEqual( result, self.expected_mean_coefficients_point_by_point)
def test_realizations_predictor_max_iterations(self): """ Test that the plugin returns a list of coefficients equal to specific values, when the ensemble realizations are the predictor assuming a truncated normal distribution and the value specified for the max_iterations is overridden. The coefficients are calculated by minimising the CRPS and using a set default value for the initial guess. """ predictor = "realizations" max_iterations = 1000 distribution = "truncnorm" plugin = Plugin(tolerance=self.tolerance, max_iterations=max_iterations) result = plugin.process( self.initial_guess_for_realization, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_realizations_coefficients)
def test_coefficient_values_for_truncnorm_distribution(self): """Ensure that the values for the optimised_coefficients match the expected values, and the coefficient names also match expected values for a truncated normal distribution. In this case, a linear least-squares regression is used to construct the initial guess.""" distribution = "truncnorm" plugin = Plugin(distribution) result = plugin.process( self.historic_wind_speed_forecast_cube, self.wind_speed_truth_cube ) self.assertEMOSCoefficientsAlmostEqual( np.array([cube.data for cube in result]), self.expected_mean_predictor_truncnorm, ) self.assertArrayEqual( [cube.name() for cube in result], self.expected_coeff_names ) for cube in result: self.assertArrayEqual( cube.attributes["shape_parameters"], np.array([0, np.inf], dtype=np.float32), )
def test_realizations_predictor_estimate_coefficients_masked_halo(self): """ Test that the plugin returns the expected values for the initial guess for the calibration coefficients, when the ensemble mean is used as the predictor. The coefficients are estimated using a linear model. In this case, the result of the linear regression is for an intercept of 0.333333 with different weights for the realizations because some of the realizations are closer to the truth, in this instance. In this case the original data has been surrounded by a halo of masked nans, which gives the same coefficients as the original data. """ predictor = "realizations" estimate_coefficients_from_linear_model_flag = True plugin = Plugin(self.distribution, self.desired_units) result = plugin.compute_initial_guess( self.truth_masked_halo, self.current_forecast_predictor_realizations_masked_halo, predictor, estimate_coefficients_from_linear_model_flag, no_of_realizations=self.no_of_realizations, ) self.assertArrayAlmostEqual( self.expected_realizations_predictor_with_linear_model, result )
def test_mean_predictor_point_by_point(self): """ Test that the expected coefficients are generated when the ensemble mean is the predictor for a normal distribution and coefficients are calculated independently at each grid point. The coefficients are calculated by minimising the CRPS. """ predictor = "mean" distribution = "norm" initial_guess = np.broadcast_to( self.initial_guess_for_mean, ( len(self.truth.coord(axis="y").points) * len(self.truth.coord(axis="x").points), len(self.initial_guess_for_mean), ), ) plugin = Plugin(tolerance=self.tolerance, point_by_point=True) result = plugin.process( initial_guess, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_mean_coefficients_point_by_point)
def setUp(self): """Set-up coefficients and plugin for testing.""" super().setUp() self.plugin = Plugin() self.plugin.current_forecast = self.current_temperature_forecast_cube self.plugin.coefficients_cubelist = self.coeffs_from_mean
def test_mean_predictor_point_by_point_sites(self): """ Test that the expected coefficients are generated when the ensemble mean is the predictor for a normal distribution and coefficients are calculated independently at each site location. The coefficients are calculated by minimising the CRPS. """ forecast_spot_cube = self.historic_forecast_spot_cube.collapsed( "realization", iris.analysis.MEAN) forecast_var_spot_cube = forecast_spot_cube.copy() forecast_var_spot_cube.data = forecast_var_spot_cube.data / 10.0 predictor = "mean" distribution = "norm" initial_guess = np.broadcast_to( self.initial_guess_for_mean, ( len(self.truth.coord(axis="y").points) * len(self.truth.coord(axis="x").points), len(self.initial_guess_for_mean), ), ) plugin = Plugin(tolerance=self.tolerance, point_by_point=True) result = plugin.process( initial_guess, forecast_spot_cube, self.truth_spot_cube, forecast_var_spot_cube, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_mean_coefficients_point_by_point_sites)
def test_realizations_predictor_point_by_point(self): """ Test that the expected coefficients are generated when the ensemble realizations are the predictor for a normal distribution and coefficients are calculated independently at each grid point. The coefficients are calculated by minimising the CRPS. """ predictor = "realizations" distribution = "norm" initial_guess = np.broadcast_to( self.initial_guess_for_realization, ( len(self.truth.coord(axis="y").points) * len(self.truth.coord(axis="x").points), len(self.initial_guess_for_realization), ), ) # Use a larger value for the tolerance to terminate sooner to avoid # minimising in computational noise. plugin = Plugin(tolerance=0.01, point_by_point=True) result = plugin.process( initial_guess, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertArrayAlmostEqual( result, self.expected_realizations_coefficients_point_by_point, decimal=2)
def test_catch_warnings_percentage_change(self, warning_list=None): """ Test that two warnings are generated if the minimisation does not result in a convergence. The first warning reports a that the minimisation did not result in convergence, whilst the second warning reports that the percentage change in the final iteration was greater than the tolerated value. The ensemble mean is the predictor. """ initial_guess = np.array([0, 1, 5000, 1], dtype=np.float64) predictor = "mean" distribution = "truncnorm" plugin = Plugin(tolerance=self.tolerance, max_iterations=5) plugin.process( initial_guess, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) warning_msg_min = "Minimisation did not result in convergence after" warning_msg_iter = "The final iteration resulted in a percentage " self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue( any(warning_msg_min in str(item) for item in warning_list)) self.assertTrue( any(warning_msg_iter in str(item) for item in warning_list))
def test_basic(self): """A simple tests for the __repr__ method.""" result = str(Plugin()) msg = ("<ContinuousRankedProbabilityScoreMinimisers: " "minimisation_dict: {'norm': 'calculate_normal_crps', " "'truncnorm': 'calculate_truncated_normal_crps'}; " "tolerance: 0.02; max_iterations: 1000>") self.assertEqual(result, msg)
def test_update_kwargs(self): """A test to update the available keyword argument.""" result = str(Plugin(tolerance=10, max_iterations=10)) msg = ("<ContinuousRankedProbabilityScoreMinimisers: " "minimisation_dict: {'norm': 'calculate_normal_crps', " "'truncnorm': 'calculate_truncated_normal_crps'}; " "tolerance: 10; max_iterations: 10>") self.assertEqual(result, msg)
def setUp(self): """Set up inputs for testing.""" super().setUp() self.tolerance = 1e-4 self.mean_plugin = Plugin("mean", tolerance=self.tolerance) self.realizations_plugin = Plugin("realizations", tolerance=self.tolerance) self.sqrt_pi = np.sqrt(np.pi).astype(np.float64) self.initial_guess_for_mean = np.array([0, 1, 0, 1], dtype=np.float64) self.initial_guess_for_realization = np.array( [0, np.sqrt(1 / 3.0), np.sqrt(1 / 3.0), np.sqrt(1 / 3.0), 0, 1], dtype=np.float64, ) self.initial_guess_mean_additional_predictor = np.array( [0, 0.5, 0.5, 0, 1], dtype=np.float64 )
def setUp(self): """Set up expected output.""" super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = ([0.0459, 0.6047, 0.3965, 0.958]) self.expected_realizations_coefficients = ([ 0.0265, 0.2175, 0.2692, 0.0126, 0.5965, 0.7952 ])
def test_basic(self): """Ensure that the optimised_coefficients are returned as a cube, with the expected number of coefficients.""" plugin = Plugin(self.distribution) result = plugin.process( self.historic_temperature_forecast_cube, self.temperature_truth_cube ) self.assertIsInstance(result, iris.cube.CubeList) self.assertEqual(len(result), len(self.coeff_names))
def test_missing_cube(self): """Test that an exception is raised if either of the historic forecasts or truth were missing.""" self.historic_temperature_forecast_cube.convert_units("Fahrenheit") plugin = Plugin(self.distribution) msg = ".*cubes must be provided" with self.assertRaisesRegex(ValueError, msg): plugin.process(self.historic_temperature_forecast_cube, None)
def test_statsmodels_mean(self, warning_list=None): """ Test that the plugin raises no warnings if the statsmodels module is not found for when the predictor is the ensemble mean. """ predictor = "mean" statsmodels_warning = "The statsmodels module cannot be imported" Plugin(self.distribution, self.desired_units, predictor=predictor) self.assertNotIn(statsmodels_warning, warning_list)
def setUp(self): """Set-up coefficients and plugin for testing.""" super().setUp() self.optimised_coeffs = (dict( zip( self.coeffs_from_mean.coord("coefficient_name").points, self.coeffs_from_mean.data))) self.plugin = Plugin() self.plugin.current_forecast = self.current_temperature_forecast_cube
def setUp(self): """Set up additional cube for land-sea mask.""" super().setUp() mask_data = np.array([[0, 1, 0], [0, 1, 1], [1, 1, 0]], dtype=np.int32) self.mask_cube = set_up_variable_cube(mask_data, name="land_binary_mask", units="1") self.plugin = Plugin("norm", "20171110T0000Z") # Copy a few slices of the temperature truth cube to test on. self.cube3D = self.temperature_truth_cube[0:2, ...].copy()
def setUp(self): """Set up the plugin and cubes for testing.""" super().setUp() frt_dt = datetime.datetime(2017, 11, 10, 0, 0) time_dt = datetime.datetime(2017, 11, 10, 4, 0) data = np.ones((3, 3), dtype=np.float32) self.historic_forecast = _create_historic_forecasts( data, time_dt, frt_dt, ).merge_cube() data_with_realizations = np.ones((3, 3, 3), dtype=np.float32) self.historic_forecast_with_realizations = _create_historic_forecasts( data_with_realizations, time_dt, frt_dt, realizations=[0, 1, 2], ).merge_cube() self.optimised_coeffs = np.array([0, 1, 2, 3], np.int32) self.distribution = "norm" self.desired_units = "degreesC" self.predictor = "mean" self.plugin = Plugin( distribution=self.distribution, desired_units=self.desired_units, predictor=self.predictor, ) self.expected_frt = ( self.historic_forecast.coord("forecast_reference_time").cell(-1).point ) self.expected_x_coord_points = np.median( self.historic_forecast.coord(axis="x").points ) self.historic_forecast.coord(axis="x").guess_bounds() self.expected_x_coord_bounds = np.array( [ [ np.min(self.historic_forecast.coord(axis="x").bounds), np.max(self.historic_forecast.coord(axis="x").bounds), ] ] ) self.expected_y_coord_points = np.median( self.historic_forecast.coord(axis="y").points ) self.historic_forecast.coord(axis="y").guess_bounds() self.expected_y_coord_bounds = np.array( [ [ np.min(self.historic_forecast.coord(axis="y").bounds), np.max(self.historic_forecast.coord(axis="y").bounds), ] ] ) self.attributes = generate_mandatory_attributes([self.historic_forecast]) self.attributes["diagnostic_standard_name"] = self.historic_forecast.name() self.attributes["distribution"] = self.distribution self.attributes["title"] = "Ensemble Model Output Statistics coefficients"
def setUp(self): """Set up expected output. The coefficients are in the order [gamma, delta, alpha, beta]. """ super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = ([0.0023, 0.8070, -0.0008, 1.0009]) self.expected_realizations_coefficients = ([ -0.1373, 0.1141, 0.0409, 0.414, 0.2056, 0.8871 ])
def test_non_matching_units(self): """Test that an exception is raised if the historic forecasts and truth have non matching units.""" self.historic_temperature_forecast_cube.convert_units("Fahrenheit") plugin = Plugin(self.distribution) msg = "The historic forecast units" with self.assertRaisesRegex(ValueError, msg): plugin.process( self.historic_temperature_forecast_cube, self.temperature_truth_cube )
def test_statsmodels_realizations(self, warning_list=None): """ Test that the plugin raises the desired warning if the statsmodels module is not found for when the predictor is the ensemble realizations. """ predictor = "realizations" Plugin(self.distribution, self.desired_units, predictor=predictor) warning_msg = "The statsmodels module cannot be imported" self.assertTrue(any(item.category == ImportWarning for item in warning_list)) self.assertTrue(any(warning_msg in str(item) for item in warning_list))
def test_basic(self): """Test without specifying keyword arguments""" result = str(Plugin(self.distribution)) msg = ("<EstimateCoefficientsForEnsembleCalibration: " "distribution: norm; " "desired_units: None; " "predictor: mean; " "minimiser: <class 'improver.calibration.ensemble_calibration." "ContinuousRankedProbabilityScoreMinimisers'>; " "coeff_names: ['alpha', 'beta', 'gamma', 'delta']; " "tolerance: 0.02; " "max_iterations: 1000>") self.assertEqual(result, msg)
def setUp(self): """Set up expected output.""" super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = [0.3958, 0.9854, -0.0, 0.621] self.expected_realizations_coefficients = [ 0.1898, -0.1558, 0.4452, 0.8877, -0.1331, -0.0002, ]
def test_historic_forecast_unit_conversion(self): """Ensure the expected optimised coefficients are generated, even if the input historic forecast cube has different units.""" self.historic_temperature_forecast_cube.convert_units("Fahrenheit") desired_units = "Kelvin" plugin = Plugin(self.distribution, desired_units=desired_units) result = plugin.process( self.historic_temperature_forecast_cube, self.temperature_truth_cube ) self.assertEMOSCoefficientsAlmostEqual( np.array([cube.data for cube in result]), self.expected_mean_predictor_norm, )
def test_coeff_names(self): """Test that the plugin instance defines the expected coefficient names.""" expected = ["alpha", "beta", "gamma", "delta"] predictor = "mean" tolerance = 10 max_iterations = 10 plugin = Plugin( self.distribution, self.desired_units, predictor=predictor, tolerance=tolerance, max_iterations=max_iterations, ) self.assertEqual(plugin.coeff_names, expected)
def test_too_few_coefficients(self): """Test that an exception is raised if the number of coefficients provided for creating the coefficients cube is not equal to the number of coefficient names.""" distribution = "truncnorm" desired_units = "Fahrenheit" predictor = "mean" optimised_coeffs = [1, 2, 3] plugin = Plugin( distribution=distribution, desired_units=desired_units, predictor=predictor, ) msg = "The number of coefficients in" with self.assertRaisesRegex(ValueError, msg): plugin.create_coefficients_cubelist( optimised_coeffs, self.historic_forecast )
def test_basic_realizations_predictor(self): """ Test that the plugin returns a numpy float value. The ensemble realizations are the predictor. The result indicates the minimum value for the CRPS that was achieved by the minimisation. """ predictor = "realizations" plugin = Plugin() result = plugin.calculate_truncated_normal_crps( self.initial_guess_for_realization, self.forecast_predictor_data_realizations, self.truth_data, self.forecast_variance_data, self.sqrt_pi, predictor) self.assertIsInstance(result, np.float64) self.assertAlmostEqual(result, 0.1670167)
def test_catch_warnings(self, warning_list=None): """ Test that a warning is generated if the minimisation does not result in a convergence. The ensemble mean is the predictor. """ predictor = "mean" distribution = "truncated_gaussian" plugin = Plugin(tolerance=self.tolerance, max_iterations=10) plugin.process(self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution) warning_msg = "Minimisation did not result in convergence after" self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue(any(warning_msg in str(item) for item in warning_list))
def setUp(self): """Set up expected output. The coefficients are in the order [alpha, beta, gamma, delta]. """ super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = [-0.0008, 1.0009, 0.0023, 0.8070] self.expected_realizations_coefficients = [ 0.0427, 0.4117, 0.1946, 0.8907, -0.1435, 0.037, ]