def test_realizations_predictor_max_iterations(self): """ Test that the plugin returns a list of coefficients equal to specific values, when the ensemble realizations are the predictor assuming a truncated normal distribution and the value specified for the max_iterations is overridden. The coefficients are calculated by minimising the CRPS and using a set default value for the initial guess. """ predictor = "realizations" max_iterations = 1000 distribution = "truncnorm" plugin = Plugin(tolerance=self.tolerance, max_iterations=max_iterations) result = plugin.process( self.initial_guess_for_realization, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_realizations_coefficients)
def test_realizations_predictor_point_by_point(self): """ Test that the expected coefficients are generated when the ensemble realizations are the predictor for a normal distribution and coefficients are calculated independently at each grid point. The coefficients are calculated by minimising the CRPS. """ predictor = "realizations" distribution = "norm" initial_guess = np.broadcast_to( self.initial_guess_for_realization, ( len(self.truth.coord(axis="y").points) * len(self.truth.coord(axis="x").points), len(self.initial_guess_for_realization), ), ) # Use a larger value for the tolerance to terminate sooner to avoid # minimising in computational noise. plugin = Plugin(tolerance=0.01, point_by_point=True) result = plugin.process( initial_guess, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertArrayAlmostEqual( result, self.expected_realizations_coefficients_point_by_point, decimal=2)
def test_mean_predictor_point_by_point_sites(self): """ Test that the expected coefficients are generated when the ensemble mean is the predictor for a normal distribution and coefficients are calculated independently at each site location. The coefficients are calculated by minimising the CRPS. """ forecast_spot_cube = self.historic_forecast_spot_cube.collapsed( "realization", iris.analysis.MEAN) forecast_var_spot_cube = forecast_spot_cube.copy() forecast_var_spot_cube.data = forecast_var_spot_cube.data / 10.0 predictor = "mean" distribution = "norm" initial_guess = np.broadcast_to( self.initial_guess_for_mean, ( len(self.truth.coord(axis="y").points) * len(self.truth.coord(axis="x").points), len(self.initial_guess_for_mean), ), ) plugin = Plugin(tolerance=self.tolerance, point_by_point=True) result = plugin.process( initial_guess, forecast_spot_cube, self.truth_spot_cube, forecast_var_spot_cube, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_mean_coefficients_point_by_point_sites)
def test_mean_predictor_point_by_point(self): """ Test that the expected coefficients are generated when the ensemble mean is the predictor for a normal distribution and coefficients are calculated independently at each grid point. The coefficients are calculated by minimising the CRPS. """ predictor = "mean" distribution = "norm" initial_guess = np.broadcast_to( self.initial_guess_for_mean, ( len(self.truth.coord(axis="y").points) * len(self.truth.coord(axis="x").points), len(self.initial_guess_for_mean), ), ) plugin = Plugin(tolerance=self.tolerance, point_by_point=True) result = plugin.process( initial_guess, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_mean_coefficients_point_by_point)
def test_catch_warnings_percentage_change(self, warning_list=None): """ Test that two warnings are generated if the minimisation does not result in a convergence. The first warning reports a that the minimisation did not result in convergence, whilst the second warning reports that the percentage change in the final iteration was greater than the tolerated value. The ensemble mean is the predictor. """ initial_guess = np.array([0, 1, 5000, 1], dtype=np.float64) predictor = "mean" distribution = "truncnorm" plugin = Plugin(tolerance=self.tolerance, max_iterations=5) plugin.process( initial_guess, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) warning_msg_min = "Minimisation did not result in convergence after" warning_msg_iter = "The final iteration resulted in a percentage " self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue( any(warning_msg_min in str(item) for item in warning_list)) self.assertTrue( any(warning_msg_iter in str(item) for item in warning_list))
def test_point_by_point_with_nans(self): """ Test that the expected coefficients are generated when the ensemble mean is the predictor for a normal distribution and coefficients are calculated independently at each grid point with one grid point having NaN values for the truth. """ predictor = "mean" distribution = "norm" self.truth.data[:, 0, 0] = np.nan plugin = Plugin(predictor, tolerance=self.tolerance, point_by_point=True) result = plugin.process( self.initial_guess_spot_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, distribution, ) self.expected_mean_coefficients_point_by_point[:, 0, 0] = self.initial_guess_for_mean self.assertEMOSCoefficientsAlmostEqual( result, self.expected_mean_coefficients_point_by_point)
def test_catch_warnings(self, warning_list=None): """ Test that a warning is generated if the minimisation does not result in a convergence. The ensemble mean is the predictor. """ predictor = "mean" distribution = "truncated_gaussian" plugin = Plugin(tolerance=self.tolerance, max_iterations=10) plugin.process(self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution) warning_msg = "Minimisation did not result in convergence after" self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue(any(warning_msg in str(item) for item in warning_list))
def test_basic_realizations_predictor(self): """ Test that the plugin returns a numpy float value. The ensemble realizations are the predictor. The result indicates the minimum value for the CRPS that was achieved by the minimisation. """ predictor = "realizations" plugin = Plugin() result = plugin.calculate_truncated_normal_crps( self.initial_guess_for_realization, self.forecast_predictor_data_realizations, self.truth_data, self.forecast_variance_data, self.sqrt_pi, predictor) self.assertIsInstance(result, np.float64) self.assertAlmostEqual(result, 0.1670167)
def test_basic_mean_predictor(self): """ Test that the plugin returns a numpy float value with the mean as the predictor. The result indicates the minimum value for the CRPS that was achieved by the minimisation. """ predictor = "mean" plugin = Plugin() result = plugin.calculate_normal_crps(self.initial_guess_for_mean, self.forecast_predictor_data, self.truth_data, self.forecast_variance_data, self.sqrt_pi, predictor) self.assertIsInstance(result, np.float64) self.assertAlmostEqual(result, 0.2609063)
def setUp(self): """Set up expected output.""" super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = ([0.0459, 0.6047, 0.3965, 0.958]) self.expected_realizations_coefficients = ([ 0.0265, 0.2175, 0.2692, 0.0126, 0.5965, 0.7952 ])
def test_basic_mean_predictor_bad_value(self): """ Test that the plugin returns a numpy float64 value and that the value matches the BAD_VALUE, when the appropriate condition is found. The ensemble mean is the predictor. The initial guess is specifically set to float32 precision for the purpose for generating the BAD_VALUE for the unit test. """ initial_guess = np.array([1e65, 1e65, 1e65, 1e65], dtype=np.float32) predictor = "mean" plugin = Plugin() result = plugin.calculate_truncated_normal_crps( initial_guess, self.forecast_predictor_data, self.truth_data, self.forecast_variance_data, self.sqrt_pi, predictor) self.assertIsInstance(result, np.float64) self.assertAlmostEqual(result, plugin.BAD_VALUE)
def test_mean_predictor_point_by_point_sites_additional_predictor(self): """ Test that the plugin returns a numpy array with the expected coefficients. Coefficients are calculated independently at each site. The ensemble mean and altitude are the predictors. """ predictor = "mean" distribution = "norm" plugin = Plugin(predictor, tolerance=self.tolerance, point_by_point=True) result = plugin.process( self.ig_spot_mean_additional_predictor, self.fp_additional_predictor_spot, self.truth_spot_cube, self.forecast_variance_spot, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_point_by_point_sites_additional_predictor )
def setUp(self): """Set up expected output. The coefficients are in the order [gamma, delta, alpha, beta]. """ super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = ([0.0023, 0.8070, -0.0008, 1.0009]) self.expected_realizations_coefficients = ([ -0.1373, 0.1141, 0.0409, 0.414, 0.2056, 0.8871 ])
def test_mean_predictor_point_by_point_sites(self): """ Test that the expected coefficients are generated when the ensemble mean is the predictor for a normal distribution and coefficients are calculated independently at each site location. The coefficients are calculated by minimising the CRPS. """ predictor = "mean" distribution = "norm" plugin = Plugin(predictor, tolerance=self.tolerance, point_by_point=True) result = plugin.process( self.initial_guess_spot_mean, self.forecast_predictor_spot, self.truth_spot_cube, self.forecast_variance_spot, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_mean_coefficients_point_by_point_sites )
def setUp(self): """Set up expected output.""" super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = [0.3958, 0.9854, -0.0, 0.621] self.expected_realizations_coefficients = [ 0.1898, -0.1558, 0.4452, 0.8877, -0.1331, -0.0002, ]
def test_mean_predictor_max_iterations(self): """ Test that the plugin returns a list of coefficients equal to specific values, when the ensemble mean is the predictor assuming a normal distribution and the value specified for the max_iterations is overridden. The coefficients are calculated by minimising the CRPS. """ predictor = "mean" max_iterations = 400 distribution = "norm" plugin = Plugin(tolerance=self.tolerance, max_iterations=max_iterations) result = plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual(result, self.expected_mean_coefficients)
def setUp(self): """Set up expected output. The coefficients are in the order [alpha, beta, gamma, delta]. """ super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = [-0.0008, 1.0009, 0.0023, 0.8070] self.expected_realizations_coefficients = [ 0.0427, 0.4117, 0.1946, 0.8907, -0.1435, 0.037, ]
def setUp(self): """Set up plugin.""" super().setUp() self.precision = 4 self.plugin = Plugin(tolerance=1e-4)
def setUp(self): """Set up values for tests.""" self.distribution = "norm" self.minimiser = repr(ContinuousRankedProbabilityScoreMinimisers()) self.coeff_names = ["alpha", "beta", "gamma", "delta"]
class Test_calculate_normal_crps(SetupNormalInputs): """ Test minimising the CRPS for a normal distribution. Either the ensemble mean or the individual ensemble realizations are used as the predictors. """ def setUp(self): """Set up plugin.""" super().setUp() self.precision = 4 self.plugin = Plugin(tolerance=1e-4) @ManageWarnings( ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_basic_mean_predictor(self): """ Test that the plugin returns a numpy float value with the mean as the predictor. The result indicates the minimum value for the CRPS that was achieved by the minimisation. """ predictor = "mean" result = self.plugin.calculate_normal_crps( self.initial_guess_for_mean, self.forecast_predictor_data, self.truth_data, self.forecast_variance_data, self.sqrt_pi, predictor, ) self.assertIsInstance(result, np.float64) self.assertAlmostEqual(result, 0.3006, places=self.precision) @ManageWarnings( ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_basic_realizations_predictor(self): """ Test that the plugin returns a numpy float value with the ensemble realizations as the predictor. The result indicates the minimum value for the CRPS that was achieved by the minimisation. """ predictor = "realizations" result = self.plugin.calculate_normal_crps( self.initial_guess_for_realization, self.forecast_predictor_data_realizations, self.truth_data, self.forecast_variance_data, self.sqrt_pi, predictor, ) self.assertIsInstance(result, np.float64) self.assertAlmostEqual(result, 0.3006, places=self.precision) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "invalid value encountered in", ], warning_types=[UserWarning, RuntimeWarning], ) def test_basic_mean_predictor_bad_value(self): """ Test that the plugin returns a numpy float64 value and that the value matches the BAD_VALUE, when the appropriate condition is found. The ensemble mean is the predictor. The initial guess is specifically set to float32 precision for the purpose for generating the BAD_VALUE for the unit test. """ initial_guess = np.array([1e65, 1e65, 1e65, 1e65], dtype=np.float32) predictor = "mean" result = self.plugin.calculate_normal_crps( initial_guess, self.forecast_predictor_data, self.truth_data, self.forecast_variance_data, self.sqrt_pi, predictor, ) self.assertIsInstance(result, np.float64) self.assertAlmostEqual(result, self.plugin.BAD_VALUE, self.precision)
class Test_process_truncated_normal_distribution(SetupTruncatedNormalInputs, EnsembleCalibrationAssertions ): """ Test minimising the CRPS for a truncated normal distribution. Either the ensemble mean or the individual ensemble realizations are used as the predictors. """ def setUp(self): """Set up expected output.""" super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = [0.3958, 0.9854, -0.0, 0.621] self.expected_realizations_coefficients = [ 0.1898, -0.1558, 0.4452, 0.8877, -0.1331, -0.0002, ] @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "The final iteration resulted in", "invalid value encountered in", "divide by zero encountered in", ], warning_types=[ UserWarning, UserWarning, RuntimeWarning, RuntimeWarning ], ) def test_basic_mean_predictor(self): """ Test that the plugin returns a numpy array. The ensemble mean is the predictor. """ predictor = "mean" distribution = "truncnorm" result = self.plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertIsInstance(result, np.ndarray) self.assertEMOSCoefficientsAlmostEqual(result, self.expected_mean_coefficients) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "invalid value encountered in", "divide by zero encountered in", ], warning_types=[ UserWarning, UserWarning, RuntimeWarning, RuntimeWarning ], ) def test_basic_realizations_predictor(self): """ Test that the plugin returns a numpy array with the expected coefficients. The ensemble realizations are the predictor. """ predictor = "realizations" distribution = "truncnorm" result = self.plugin.process( self.initial_guess_for_realization, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertIsInstance(result, np.ndarray) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_realizations_coefficients) @ManageWarnings( ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_mean_predictor_keyerror(self): """ Test that an exception is raised when the distribution requested is not an available option when the predictor is the ensemble mean. """ predictor = "mean" distribution = "foo" msg = "Distribution requested" with self.assertRaisesRegex(KeyError, msg): self.plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "The final iteration resulted in", "invalid value encountered in", "divide by zero encountered in", ], warning_types=[ UserWarning, UserWarning, UserWarning, RuntimeWarning, RuntimeWarning, ], ) def test_mean_predictor_max_iterations(self): """ Test that the plugin returns a list of coefficients equal to specific values, when the ensemble mean is the predictor assuming a truncated normal distribution and the value specified for the max_iterations is overridden. The coefficients are calculated by minimising the CRPS. """ predictor = "mean" max_iterations = 400 distribution = "truncnorm" plugin = Plugin(tolerance=self.tolerance, max_iterations=max_iterations) result = plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual(result, self.expected_mean_coefficients) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "invalid value encountered in", "divide by zero encountered in", ], warning_types=[ UserWarning, UserWarning, RuntimeWarning, RuntimeWarning ], ) def test_realizations_predictor_max_iterations(self): """ Test that the plugin returns a list of coefficients equal to specific values, when the ensemble realizations are the predictor assuming a truncated normal distribution and the value specified for the max_iterations is overridden. The coefficients are calculated by minimising the CRPS. """ predictor = "realizations" max_iterations = 1000 distribution = "truncnorm" plugin = Plugin(tolerance=self.tolerance, max_iterations=max_iterations) result = plugin.process( self.initial_guess_for_realization, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_realizations_coefficients) @ManageWarnings( record=True, ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_catch_warnings(self, warning_list=None): """ Test that a warning is generated if the minimisation does not result in a convergence. The ensemble mean is the predictor. """ predictor = "mean" distribution = "truncnorm" plugin = Plugin(tolerance=self.tolerance, max_iterations=10) plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) warning_msg = "Minimisation did not result in convergence after" self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue(any(warning_msg in str(item) for item in warning_list)) @ManageWarnings( record=True, ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_catch_warnings_percentage_change(self, warning_list=None): """ Test that two warnings are generated if the minimisation does not result in a convergence. The first warning reports a that the minimisation did not result in convergence, whilst the second warning reports that the percentage change in the final iteration was greater than the tolerated value. The ensemble mean is the predictor. """ initial_guess = np.array([0, 1, 5000, 1], dtype=np.float64) predictor = "mean" distribution = "truncnorm" plugin = Plugin(tolerance=self.tolerance, max_iterations=5) plugin.process( initial_guess, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) warning_msg_min = "Minimisation did not result in convergence after" warning_msg_iter = "The final iteration resulted in a percentage " self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue( any(warning_msg_min in str(item) for item in warning_list)) self.assertTrue( any(warning_msg_iter in str(item) for item in warning_list))
class Test_process_normal_distribution(SetupNormalInputs, EnsembleCalibrationAssertions): """ Test minimising the CRPS for a normal distribution. Either the ensemble mean or the individual ensemble realizations are used as the predictors. """ def setUp(self): """Set up expected output. The coefficients are in the order [alpha, beta, gamma, delta]. """ super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = [-0.0008, 1.0009, 0.0023, 0.8070] self.expected_realizations_coefficients = [ 0.0427, 0.4117, 0.1946, 0.8907, -0.1435, 0.037, ] @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", ], warning_types=[UserWarning, UserWarning, RuntimeWarning], ) def test_basic_mean_predictor(self): """ Test that the plugin returns a numpy array with the expected coefficients. The ensemble mean is the predictor. """ predictor = "mean" distribution = "norm" result = self.plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertIsInstance(result, np.ndarray) self.assertEqual(result.dtype, np.float32) self.assertEMOSCoefficientsAlmostEqual(result, self.expected_mean_coefficients) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", "invalid value encountered in", ], warning_types=[ UserWarning, UserWarning, RuntimeWarning, RuntimeWarning ], ) def test_basic_realizations_predictor(self): """ Test that the plugin returns a numpy array with the expected coefficients. The ensemble realizations are the predictor. """ predictor = "realizations" distribution = "norm" result = self.plugin.process( self.initial_guess_for_realization, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertIsInstance(result, np.ndarray) self.assertEqual(result.dtype, np.float32) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_realizations_coefficients) @ManageWarnings( ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_mean_predictor_keyerror(self): """ Test that the minimisation has resulted in a KeyError, if the distribution that has been requested was not within the dictionary containing the minimisation functions. """ predictor = "mean" distribution = "foo" msg = "Distribution requested" with self.assertRaisesRegex(KeyError, msg): self.plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", ], warning_types=[UserWarning, UserWarning, RuntimeWarning], ) def test_mean_predictor_max_iterations(self): """ Test that the plugin returns a list of coefficients equal to specific values, when the ensemble mean is the predictor assuming a normal distribution and the value specified for the max_iterations is overridden. The coefficients are calculated by minimising the CRPS and using a set default value for the initial guess. """ predictor = "mean" max_iterations = 400 distribution = "norm" plugin = Plugin(tolerance=self.tolerance, max_iterations=max_iterations) result = plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual(result, self.expected_mean_coefficients) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", "invalid value encountered in", ], warning_types=[ UserWarning, UserWarning, RuntimeWarning, RuntimeWarning ], ) def test_realizations_predictor_max_iterations(self): """ Test that the plugin returns a list of coefficients equal to specific values, when the ensemble realizations are the predictor assuming a truncated normal distribution and the value specified for the MAX_ITERATIONS is overridden. The coefficients are calculated by minimising the CRPS and using a set default value for the initial guess. """ predictor = "realizations" max_iterations = 1000 distribution = "norm" plugin = Plugin(tolerance=self.tolerance, max_iterations=max_iterations) result = plugin.process( self.initial_guess_for_realization, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_realizations_coefficients) @ManageWarnings( record=True, ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_catch_warnings(self, warning_list=None): """ Test that a warning is generated if the minimisation does not result in a convergence. The ensemble mean is the predictor. """ predictor = "mean" distribution = "norm" plugin = Plugin(tolerance=self.tolerance, max_iterations=10) plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) warning_msg = "Minimisation did not result in convergence after" self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue(any(warning_msg in str(item) for item in warning_list)) @ManageWarnings( record=True, ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_catch_warnings_percentage_change(self, warning_list=None): """ Test that two warnings are generated if the minimisation does not result in a convergence. The first warning reports a that the minimisation did not result in convergence, whilst the second warning reports that the percentage change in the final iteration was greater than the tolerated value. The ensemble mean is the predictor. """ initial_guess = np.array([5000, 1, 0, 1], dtype=np.float64) predictor = "mean" distribution = "norm" plugin = Plugin(tolerance=self.tolerance, max_iterations=5) plugin.process( initial_guess, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) warning_msg_min = "Minimisation did not result in convergence after" warning_msg_iter = "The final iteration resulted in a percentage " self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue( any(warning_msg_min in str(item) for item in warning_list)) self.assertTrue( any(warning_msg_iter in str(item) for item in warning_list))
class Test_process_normal_distribution(SetupNormalInputs, EnsembleCalibrationAssertions): """ Test minimising the CRPS for a normal distribution. Either the ensemble mean or the individual ensemble realizations are used as the predictors. """ def setUp(self): """Set up expected output. The coefficients are in the order [alpha, beta, gamma, delta]. """ super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = [-0.0003, 1.0013, 0.0012, 0.5945] self.expected_realizations_coefficients = [ 0.0254, 0.4349, 0.39, 0.8122, -0.0016, 0.2724, ] self.expected_mean_coefficients_point_by_point = np.array( [ [ [0.0015, 0.0037, -0.002], [-0.0009, 0.0008, 0.0015], [-0.0046, 0.0053, -0.0038], ], [ [1.0039, 1.0035, 1.0009], [1.0013, 1.0011, 1.001], [1.002, 1.0015, 1.0008], ], [ [0.0017, -0.0009, -0.0002], [0.0054, 0.0003, -0.0002], [-0.0001, -0.0018, 0.0002], ], [ [-0.0, 0.0007, -0.0009], [0.0003, -0.0001, -0.001], [-0.0013, 0.0, 0.0006], ], ], dtype=np.float32, ) self.expected_mean_coefficients_point_by_point_sites = np.array( [ [0.0017, 0.0017, 0.0017, 0.0017], [1.0036, 1.0036, 1.0036, 1.0036], [0.0017, 0.0017, 0.0017, 0.0017], [-0.0, -0.0, -0.0, 0.0], ], dtype=np.float32, ) self.expected_realizations_coefficients_point_by_point = np.array( [ [ [0.0001, 0.0001, 0.0001], [0.0001, 0.0001, 0.0], [0.0001, 0.0001, 0.0001], ], [ [0.579, 0.5793, 0.5782], [0.5782, 0.5778, 0.5781], [0.5786, 0.5782, 0.5783], ], [ [0.5795, 0.5786, 0.5782], [0.5783, 0.578, 0.5767], [0.5791, 0.578, 0.5763], ], [ [0.5773, 0.5769, 0.5763], [0.5769, 0.5771, 0.5782], [0.5764, 0.5773, 0.5783], ], [ [0.0001, 0.0001, 0.0001], [0.0001, 0.0001, 0.0001], [0.0001, 0.0001, 0.0], ], [ [1.0194, 1.0143, 1.0199], [1.0199, 1.02, 1.013], [1.0144, 0.9885, 1.0246], ], ], dtype=np.float32, ) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", ], warning_types=[UserWarning, UserWarning, RuntimeWarning], ) def test_basic_mean_predictor(self): """ Test that the plugin returns a numpy array with the expected coefficients. The ensemble mean is the predictor. """ predictor = "mean" distribution = "norm" result = self.plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertIsInstance(result, np.ndarray) self.assertEqual(result.dtype, np.float32) self.assertEMOSCoefficientsAlmostEqual(result, self.expected_mean_coefficients) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", "invalid value encountered in", ], warning_types=[ UserWarning, UserWarning, RuntimeWarning, RuntimeWarning ], ) def test_basic_realizations_predictor(self): """ Test that the plugin returns a numpy array with the expected coefficients. The ensemble realizations are the predictor. """ predictor = "realizations" distribution = "norm" result = self.plugin.process( self.initial_guess_for_realization, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertIsInstance(result, np.ndarray) self.assertEqual(result.dtype, np.float32) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_realizations_coefficients) @ManageWarnings( ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_mean_predictor_keyerror(self): """ Test that the minimisation has resulted in a KeyError, if the distribution that has been requested was not within the dictionary containing the minimisation functions. """ predictor = "mean" distribution = "foo" msg = "Distribution requested" with self.assertRaisesRegex(KeyError, msg): self.plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", ], warning_types=[UserWarning, UserWarning, RuntimeWarning], ) def test_mean_predictor_max_iterations(self): """ Test that the plugin returns a list of coefficients equal to specific values, when the ensemble mean is the predictor assuming a normal distribution and the value specified for the max_iterations is overridden. The coefficients are calculated by minimising the CRPS. """ predictor = "mean" max_iterations = 400 distribution = "norm" plugin = Plugin(tolerance=self.tolerance, max_iterations=max_iterations) result = plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual(result, self.expected_mean_coefficients) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", "invalid value encountered in", ], warning_types=[ UserWarning, UserWarning, RuntimeWarning, RuntimeWarning ], ) def test_realizations_predictor_max_iterations(self): """ Test that the plugin returns a list of coefficients equal to specific values, when the ensemble realizations are the predictor assuming a truncated normal distribution and the value specified for the MAX_ITERATIONS is overridden. The coefficients are calculated by minimising the CRPS. """ predictor = "realizations" max_iterations = 1000 distribution = "norm" plugin = Plugin(tolerance=self.tolerance, max_iterations=max_iterations) result = plugin.process( self.initial_guess_for_realization, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_realizations_coefficients) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", ], warning_types=[UserWarning, UserWarning, RuntimeWarning], ) def test_mean_predictor_point_by_point(self): """ Test that the expected coefficients are generated when the ensemble mean is the predictor for a normal distribution and coefficients are calculated independently at each grid point. The coefficients are calculated by minimising the CRPS. """ predictor = "mean" distribution = "norm" initial_guess = np.broadcast_to( self.initial_guess_for_mean, ( len(self.truth.coord(axis="y").points) * len(self.truth.coord(axis="x").points), len(self.initial_guess_for_mean), ), ) plugin = Plugin(tolerance=self.tolerance, point_by_point=True) result = plugin.process( initial_guess, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_mean_coefficients_point_by_point) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", ], warning_types=[UserWarning, UserWarning, RuntimeWarning], ) def test_mean_predictor_point_by_point_sites(self): """ Test that the expected coefficients are generated when the ensemble mean is the predictor for a normal distribution and coefficients are calculated independently at each site location. The coefficients are calculated by minimising the CRPS. """ forecast_spot_cube = self.historic_forecast_spot_cube.collapsed( "realization", iris.analysis.MEAN) forecast_var_spot_cube = forecast_spot_cube.copy() forecast_var_spot_cube.data = forecast_var_spot_cube.data / 10.0 predictor = "mean" distribution = "norm" initial_guess = np.broadcast_to( self.initial_guess_for_mean, ( len(self.truth.coord(axis="y").points) * len(self.truth.coord(axis="x").points), len(self.initial_guess_for_mean), ), ) plugin = Plugin(tolerance=self.tolerance, point_by_point=True) result = plugin.process( initial_guess, forecast_spot_cube, self.truth_spot_cube, forecast_var_spot_cube, predictor, distribution, ) self.assertEMOSCoefficientsAlmostEqual( result, self.expected_mean_coefficients_point_by_point_sites) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", "invalid value encountered in", ], warning_types=[ UserWarning, UserWarning, RuntimeWarning, RuntimeWarning ], ) def test_realizations_predictor_point_by_point(self): """ Test that the expected coefficients are generated when the ensemble realizations are the predictor for a normal distribution and coefficients are calculated independently at each grid point. The coefficients are calculated by minimising the CRPS. """ predictor = "realizations" distribution = "norm" initial_guess = np.broadcast_to( self.initial_guess_for_realization, ( len(self.truth.coord(axis="y").points) * len(self.truth.coord(axis="x").points), len(self.initial_guess_for_realization), ), ) # Use a larger value for the tolerance to terminate sooner to avoid # minimising in computational noise. plugin = Plugin(tolerance=0.01, point_by_point=True) result = plugin.process( initial_guess, self.forecast_predictor_realizations, self.truth, self.forecast_variance, predictor, distribution, ) self.assertArrayAlmostEqual( result, self.expected_realizations_coefficients_point_by_point, decimal=2) @ManageWarnings( ignored_messages=[ "Collapsing a non-contiguous coordinate.", "Minimisation did not result in convergence", "divide by zero encountered in", ], warning_types=[UserWarning, UserWarning, RuntimeWarning], ) @ManageWarnings( record=True, ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_catch_warnings(self, warning_list=None): """ Test that a warning is generated if the minimisation does not result in a convergence. The ensemble mean is the predictor. """ predictor = "mean" distribution = "norm" plugin = Plugin(tolerance=self.tolerance, max_iterations=10) plugin.process( self.initial_guess_for_mean, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) warning_msg = "Minimisation did not result in convergence after" self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue(any(warning_msg in str(item) for item in warning_list)) @ManageWarnings( record=True, ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_catch_warnings_percentage_change(self, warning_list=None): """ Test that two warnings are generated if the minimisation does not result in a convergence. The first warning reports a that the minimisation did not result in convergence, whilst the second warning reports that the percentage change in the final iteration was greater than the tolerated value. The ensemble mean is the predictor. """ initial_guess = np.array([5000, 1, 0, 1], dtype=np.float64) predictor = "mean" distribution = "norm" plugin = Plugin(tolerance=self.tolerance, max_iterations=5) plugin.process( initial_guess, self.forecast_predictor_mean, self.truth, self.forecast_variance, predictor, distribution, ) warning_msg_min = "Minimisation did not result in convergence after" warning_msg_iter = "The final iteration resulted in a percentage " self.assertTrue( any(item.category == UserWarning for item in warning_list)) self.assertTrue( any(warning_msg_min in str(item) for item in warning_list)) self.assertTrue( any(warning_msg_iter in str(item) for item in warning_list))
def setUp(self): """Set up expected output. The coefficients are in the order [alpha, beta, gamma, delta]. """ super().setUp() self.tolerance = 1e-4 self.plugin = Plugin(tolerance=self.tolerance) self.expected_mean_coefficients = [-0.0003, 1.0013, 0.0012, 0.5945] self.expected_realizations_coefficients = [ 0.0254, 0.4349, 0.39, 0.8122, -0.0016, 0.2724, ] self.expected_mean_coefficients_point_by_point = np.array( [ [ [0.0015, 0.0037, -0.002], [-0.0009, 0.0008, 0.0015], [-0.0046, 0.0053, -0.0038], ], [ [1.0039, 1.0035, 1.0009], [1.0013, 1.0011, 1.001], [1.002, 1.0015, 1.0008], ], [ [0.0017, -0.0009, -0.0002], [0.0054, 0.0003, -0.0002], [-0.0001, -0.0018, 0.0002], ], [ [-0.0, 0.0007, -0.0009], [0.0003, -0.0001, -0.001], [-0.0013, 0.0, 0.0006], ], ], dtype=np.float32, ) self.expected_mean_coefficients_point_by_point_sites = np.array( [ [0.0017, 0.0017, 0.0017, 0.0017], [1.0036, 1.0036, 1.0036, 1.0036], [0.0017, 0.0017, 0.0017, 0.0017], [-0.0, -0.0, -0.0, 0.0], ], dtype=np.float32, ) self.expected_realizations_coefficients_point_by_point = np.array( [ [ [0.0001, 0.0001, 0.0001], [0.0001, 0.0001, 0.0], [0.0001, 0.0001, 0.0001], ], [ [0.579, 0.5793, 0.5782], [0.5782, 0.5778, 0.5781], [0.5786, 0.5782, 0.5783], ], [ [0.5795, 0.5786, 0.5782], [0.5783, 0.578, 0.5767], [0.5791, 0.578, 0.5763], ], [ [0.5773, 0.5769, 0.5763], [0.5769, 0.5771, 0.5782], [0.5764, 0.5773, 0.5783], ], [ [0.0001, 0.0001, 0.0001], [0.0001, 0.0001, 0.0001], [0.0001, 0.0001, 0.0], ], [ [1.0194, 1.0143, 1.0199], [1.0199, 1.02, 1.013], [1.0144, 0.9885, 1.0246], ], ], dtype=np.float32, )