def test_lots_of_probability_thresholds(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values for the percentiles, if there are lots of thresholds.
"""
input_probs_1d = np.linspace(1, 0, 30)
input_probs = np.tile(input_probs_1d, (3, 3, 1, 1)).T
data = np.array([[[[2.9, 2.9, 2.9],
[2.9, 2.9, 2.9],
[2.9, 2.9, 2.9]]],
[[[14.5, 14.5, 14.5],
[14.5, 14.5, 14.5],
[14.5, 14.5, 14.5]]],
[[[26.099998, 26.099998, 26.099998],
[26.099998, 26.099998, 26.099998],
[26.099998, 26.099998, 26.099998]]]],
dtype=np.float32)
temperature_values = np.arange(0, 30)
cube = (
add_forecast_reference_time_and_forecast_period(
set_up_probability_threshold_cube(
input_probs, "air_temperature", "degreesC",
forecast_thresholds=temperature_values,
spp__relative_to_threshold='above')))
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
self.assertArrayAlmostEqual(result.data, data)
def test_check_single_threshold(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values for the percentiles, if a single threshold is used for
constructing the percentiles.
"""
data = np.array([[[[12.2, 8., 12.2],
[-16., 8., -30.4],
[-30.4, -34., -35.2]]],
[[[29., 26.66666667, 29.],
[23.75, 26.66666667, 8.],
[8., -10., -16.]]],
[[[45.8, 45.33333333, 45.8],
[44.75, 45.33333333, 41.6],
[41.6, 29., 3.2]]]], dtype=np.float32)
threshold_coord = find_threshold_coordinate(
self.current_temperature_forecast_cube)
for acube in self.current_temperature_forecast_cube.slices_over(
threshold_coord):
cube = acube
break
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
self.assertArrayAlmostEqual(result.data, data, decimal=5)
def test_probabilities_not_monotonically_increasing(self,
warning_list=None):
"""
Test that the plugin raises a Warning when the probabilities
of the Cumulative Distribution Function are not monotonically
increasing.
"""
data = np.array([0.05, 0.7, 0.95])
data = data[:, np.newaxis, np.newaxis, np.newaxis]
self.current_temperature_forecast_cube = (
add_forecast_reference_time_and_forecast_period(
set_up_probability_threshold_cube(
data, "air_temperature", "degreesC",
forecast_thresholds=[8, 10, 12], y_dimension_length=1,
x_dimension_length=1, spp__relative_to_threshold='above')))
cube = self.current_temperature_forecast_cube
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
warning_msg = "The probability values used to construct the"
plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
self.assertTrue(any(warning_msg in str(item)
for item in warning_list))
def test_lots_of_percentiles(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values for the percentiles, if lots of percentile values are
requested.
"""
data = np.array([[[[13.9, -16., 10.2], [-28., -16., -35.2],
[-35.2, -37., -37.6]]],
[[[17.7, 8.25, 10.6], [-4., 8.25, -25.6],
[-25.6, -31., -32.8]]],
[[[21.5, 8.75, 11.], [8.33333333, 8.75, -16.],
[-16., -25., -28.]]],
[[[25.3, 9.25, 11.4], [9., 9.25, -6.4],
[-6.4, -19., -23.2]]],
[[[29.1, 9.75, 11.8], [9.66666667, 9.75, 3.2],
[3.2, -13., -18.4]]],
[[[32.9, 10.33333333, 15.8], [10.33333333, 10.2, 8.5],
[8.33333333, -7., -13.6]]],
[[[36.7, 11., 23.4], [11., 10.6, 9.5],
[9., -1., -8.8]]],
[[[40.5, 11.66666667, 31.], [11.66666667, 11., 10.5],
[9.66666667, 5., -4.]]],
[[[44.3, 21.5, 38.6], [21.5, 11.4, 11.5],
[10.5, 8.5, 0.8]]],
[[[48.1, 40.5, 46.2], [40.5, 11.8, 31.],
[11.5, 9.5, 5.6]]]])
cube = self.current_temperature_forecast_cube
percentiles = np.arange(5, 100, 10)
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(cube, percentiles,
bounds_pairing)
self.assertArrayAlmostEqual(result.data, data)
def test_simple_check_data_below(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values for the percentiles when input probabilities are given
for being below a threshold.
The input cube contains probabilities that values are below a given
threshold.
"""
expected = np.array([8.4, 10.61538462, 11.84615385])
expected = expected[:, np.newaxis, np.newaxis, np.newaxis]
data = np.array([0.95, 0.3, 0.05])[::-1]
data = data[:, np.newaxis, np.newaxis, np.newaxis]
self.current_temperature_forecast_cube = (
add_forecast_reference_time_and_forecast_period(
set_up_probability_threshold_cube(
data, "air_temperature", "degreesC",
forecast_thresholds=[8, 10, 12], y_dimension_length=1,
x_dimension_length=1, spp__relative_to_threshold='above')))
cube = self.current_temperature_forecast_cube
cube.coord(var_name="threshold"
).attributes["spp__relative_to_threshold"] = "below"
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
self.assertArrayAlmostEqual(result.data, expected)
def test_check_data_multiple_timesteps(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values for the percentiles.
"""
expected = np.array([[[[8., 8.], [-8., 8.66666667]],
[[8., -16.], [8., -16.]]],
[[[12., 12.], [12., 12.]],
[[10.5, 10.], [10.5, 10.]]],
[[[31., 31.], [31., 31.]],
[[11.5, 11.33333333], [11.5, 12.]]]])
data = np.array([[[[0.8, 0.8], [0.7, 0.9]], [[0.8, 0.6], [0.8, 0.6]]],
[[[0.6, 0.6], [0.6, 0.6]], [[0.5, 0.4], [0.5, 0.4]]],
[[[0.4, 0.4], [0.4, 0.4]], [[0.1, 0.1], [0.1, 0.2]]]])
cube = set_up_probability_above_threshold_cube(data,
"air_temperature",
"degreesC",
timesteps=2,
x_dimension_length=2,
y_dimension_length=2)
self.probability_cube = (
add_forecast_reference_time_and_forecast_period(
cube,
time_point=np.array([402295.0, 402296.0]),
fp_point=[2.0, 3.0]))
cube = self.probability_cube
percentiles = [20, 60, 80]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(cube, percentiles,
bounds_pairing)
self.assertArrayAlmostEqual(result.data, expected)
def test_probabilities_not_monotonically_increasing(self):
"""
Test that the plugin raises a ValueError when the probabilities
of the Cumulative Distribution Function are not monotonically
increasing.
"""
data = np.array([0.05, 0.7, 0.95])
data = data[:, np.newaxis, np.newaxis, np.newaxis]
self.current_temperature_forecast_cube = (
add_forecast_reference_time_and_forecast_period(
set_up_probability_above_threshold_cube(
data,
"air_temperature",
"1",
forecast_thresholds=[8, 10, 12],
y_dimension_length=1,
x_dimension_length=1)))
cube = self.current_temperature_forecast_cube
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
msg = "The probability values used to construct the"
with self.assertRaisesRegexp(ValueError, msg):
plugin._probabilities_to_percentiles(cube, percentiles,
bounds_pairing)
def test_simple_check_data_above(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values for the percentiles when input probabilities are given
for being above a threshold.
The input cube contains probabilities that values are above a given
threshold.
"""
expected = np.array([8.15384615, 9.38461538, 11.6])
expected = expected[:, np.newaxis, np.newaxis, np.newaxis]
data = np.array([0.95, 0.3, 0.05])
data = data[:, np.newaxis, np.newaxis, np.newaxis]
self.current_temperature_forecast_cube = (
add_forecast_reference_time_and_forecast_period(
set_up_probability_above_threshold_cube(
data,
"air_temperature",
"1",
forecast_thresholds=[8, 10, 12],
y_dimension_length=1,
x_dimension_length=1)))
cube = self.current_temperature_forecast_cube
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(cube, percentiles,
bounds_pairing)
self.assertArrayAlmostEqual(result.data, expected)
def test_transpose_cube_dimensions(self):
"""
Test that the plugin returns an the expected data, when comparing
input cubes which have dimensions in a different order.
"""
# Calculate result for nontransposed cube.
cube = self.current_temperature_forecast_cube
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
nontransposed_result = plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
# Calculate result for transposed cube.
# Original cube dimensions are [P, T, Y, X].
# Transposed cube dimensions are [X, Y, T, P].
cube.transpose([3, 2, 1, 0])
transposed_result = plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
# Result cube will be [P, X, Y, T]
# Transpose cube to be [P, T, Y, X]
transposed_result.transpose([0, 3, 2, 1])
self.assertArrayAlmostEqual(
nontransposed_result.data, transposed_result.data)
def test_basic(self):
"""Test that the plugin returns an Iris.cube.Cube."""
cube = self.current_temperature_forecast_cube
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
self.assertIsInstance(result, Cube)
def test_check_data_specifying_single_percentile_not_as_list(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values for a specific percentile passed in as a value.
"""
expected_data = np.array([self.percentile_25])
cube = self.current_temperature_forecast_cube
plugin = Plugin()
result = plugin.process(cube, percentiles=25)
self.assertArrayAlmostEqual(result.data, expected_data, decimal=5)
def test_basic(self):
"""Test that the plugin returns two numpy arrays."""
cube = self.current_temperature_forecast_cube
probabilities_for_cdf = cube.data.reshape(3, 9)
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._add_bounds_to_thresholds_and_probabilities(
self.threshold_points, probabilities_for_cdf, bounds_pairing)
self.assertIsInstance(result[0], np.ndarray)
self.assertIsInstance(result[1], np.ndarray)
def test_unknown_thresholding(self):
"""Test that the plugin returns an Iris.cube.Cube."""
cube = self.current_temperature_forecast_cube
cube.attributes["relative_to_threshold"] = "between"
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
msg = "Probabilities to percentiles only implemented for"
with self.assertRaisesRegexp(NotImplementedError, msg):
plugin._probabilities_to_percentiles(cube, percentiles,
bounds_pairing)
def test_check_data_not_specifying_percentiles(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values without specifying the number of percentiles.
"""
expected_data = np.array(
[self.percentile_25, self.percentile_50, self.percentile_75])
cube = self.current_temperature_forecast_cube
plugin = Plugin()
result = plugin.process(cube)
self.assertArrayAlmostEqual(result.data, expected_data, decimal=5)
def test_return_name(self):
"""Test that the plugin returns an Iris.cube.Cube with an appropriate
name.
"""
cube = self.current_temperature_forecast_cube
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
self.assertEqual(result.name(), 'air_temperature')
def test_new_endpoints_generation(self):
"""Test that the plugin re-applies the threshold bounds using the
maximum and minimum threshold points values when the original bounds
have been exceeded and ecc_bounds_warning has been set."""
probabilities_for_cdf = np.array([[0.05, 0.7, 0.95]])
threshold_points = np.array([-50, 10, 60])
bounds_pairing = (-40, 50)
plugin = Plugin(ecc_bounds_warning=True)
result = plugin._add_bounds_to_thresholds_and_probabilities(
threshold_points, probabilities_for_cdf, bounds_pairing)
self.assertEqual(max(result[0]), max(threshold_points))
self.assertEqual(min(result[0]), min(threshold_points))
def test_check_data_specifying_single_percentile_not_as_list(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values for a specific percentile passed in as a value.
"""
data = np.array([[[[21.5, 8.75, 11.], [8.33333333, 8.75, -16.],
[-16., -25., -28.]]]])
cube = self.current_temperature_forecast_cube
percentiles = 25
plugin = Plugin()
result = plugin.process(cube, percentiles=percentiles)
self.assertArrayAlmostEqual(result.data, data)
def test_return_coord_units(self):
"""Test that the plugin returns an Iris.cube.Cube with an appropriate
percentile coordinate with suitable units.
"""
cube = self.current_temperature_forecast_cube
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
self.assertIsInstance(result.coord('percentile'), DimCoord)
self.assertArrayEqual(result.coord('percentile').points, percentiles)
self.assertEqual(result.coord('percentile').units, unit.Unit("%"))
def test_result_cube_has_no_air_temperature_threshold_coordinate(self):
"""
Test that the plugin returns a cube with coordinates that
do not include the air_temperature_threshold coordinate.
"""
cube = self.current_temperature_forecast_cube
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(
cube, percentiles, bounds_pairing)
for coord in result.coords():
self.assertNotEqual(coord.name(), "threshold")
def test_check_data_over_specifying_percentiles(self):
"""
Test that the plugin raises a suitable error when both a number and set
or percentiles are specified.
"""
no_of_percentiles = 3
percentiles = [25, 50, 75]
cube = self.current_temperature_forecast_cube
plugin = Plugin()
msg = "Cannot specify both no_of_percentiles and percentiles"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(cube, no_of_percentiles=no_of_percentiles,
percentiles=percentiles)
def test_bounds_of_threshold_points(self):
"""
Test that the plugin returns the expected results for the
threshold_points, where they've been padded with the values from
the bounds_pairing.
"""
cube = self.current_temperature_forecast_cube
probabilities_for_cdf = cube.data.reshape(3, 9)
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._add_bounds_to_thresholds_and_probabilities(
self.threshold_points, probabilities_for_cdf, bounds_pairing)
self.assertArrayAlmostEqual(result[0][0], bounds_pairing[0])
self.assertArrayAlmostEqual(result[0][-1], bounds_pairing[1])
def test_endpoints_of_distribution_exceeded(self):
"""
Test that the plugin raises a ValueError when the constant
end points of the distribution are exceeded by a threshold value
used in the forecast.
"""
probabilities_for_cdf = np.array([[0.05, 0.7, 0.95]])
threshold_points = np.array([8, 10, 60])
bounds_pairing = (-40, 50)
plugin = Plugin()
msg = "The calculated threshold values"
with self.assertRaisesRegex(ValueError, msg):
plugin._add_bounds_to_thresholds_and_probabilities(
threshold_points, probabilities_for_cdf, bounds_pairing)
def test_check_data_not_specifying_percentiles(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values without specifying the number of percentiles.
"""
data = np.array([[[[21.5, 8.75, 11.], [8.33333333, 8.75, -16.],
[-16., -25., -28.]]],
[[[31., 10., 12.], [10., 10., 8.], [8., -10., -16.]]],
[[[40.5, 11.66666667, 31.], [11.66666667, 11., 10.5],
[9.66666667, 5., -4.]]]])
cube = self.current_temperature_forecast_cube
plugin = Plugin()
result = plugin.process(cube)
self.assertArrayAlmostEqual(result.data, data)
def test_probability_data(self):
"""
Test that the plugin returns the expected results for the
probabilities, where they've been padded with zeros and ones to
represent the extreme ends of the Cumulative Distribution Function.
"""
cube = self.current_temperature_forecast_cube
probabilities_for_cdf = cube.data.reshape(3, 9)
zero_array = np.zeros(probabilities_for_cdf[:, 0].shape)
one_array = np.ones(probabilities_for_cdf[:, 0].shape)
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._add_bounds_to_thresholds_and_probabilities(
self.threshold_points, probabilities_for_cdf, bounds_pairing)
self.assertArrayAlmostEqual(result[1][:, 0], zero_array)
self.assertArrayAlmostEqual(result[1][:, -1], one_array)
def test_endpoints_of_distribution_exceeded_warning(
self, warning_list=None):
"""
Test that the plugin raises a warning message when the constant
end points of the distribution are exceeded by a threshold value
used in the forecast and the ecc_bounds_warning keyword argument
has been specified.
"""
probabilities_for_cdf = np.array([[0.05, 0.7, 0.95]])
threshold_points = np.array([8, 10, 60])
bounds_pairing = (-40, 50)
plugin = Plugin(ecc_bounds_warning=True)
warning_msg = "The calculated threshold values"
plugin._add_bounds_to_thresholds_and_probabilities(
threshold_points, probabilities_for_cdf, bounds_pairing)
self.assertTrue(any(warning_msg in str(item) for item in warning_list))
def test_check_data_spot_forecasts(self):
"""
Test that the plugin returns an Iris.cube.Cube with the expected
data values for the percentiles for spot forecasts.
"""
data = np.array(
[[[15.8, 8., 10.4, -16., 8., -30.4, -30.4, -34., -35.2]],
[[31., 10., 12., 10., 10., 8., 8., -10., -16.]],
[[46.2, 31., 42.4, 31., 11.6, 12., 11., 9., 3.2]]])
cube = self.current_temperature_spot_forecast_cube
percentiles = [10, 50, 90]
bounds_pairing = (-40, 50)
plugin = Plugin()
result = plugin._probabilities_to_percentiles(cube, percentiles,
bounds_pairing)
self.assertArrayAlmostEqual(result.data, data)
def main(argv=None):
"""Load in arguments for applying coefficients for Ensemble Model Output
Statistics (EMOS), otherwise known as Non-homogeneous Gaussian
Regression (NGR). The coefficients are applied to the forecast
that is supplied, so as to calibrate the forecast. The calibrated
forecast is written to a netCDF file.
"""
parser = ArgParser(
description='Apply coefficients for Ensemble Model Output '
'Statistics (EMOS), otherwise known as Non-homogeneous '
'Gaussian Regression (NGR). The supported input formats '
'are realizations, probabilities and percentiles. '
'The forecast will be converted to realizations before '
'applying the coefficients and then converted back to '
'match the input format.')
# Filepaths for the forecast, EMOS coefficients and the output.
parser.add_argument(
'forecast_filepath',
metavar='FORECAST_FILEPATH',
help='A path to an input NetCDF file containing the forecast to be '
'calibrated. The input format could be either realizations, '
'probabilities or percentiles.')
parser.add_argument('coefficients_filepath',
metavar='COEFFICIENTS_FILEPATH',
help='A path to an input NetCDF file containing the '
'coefficients used for calibration.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILEPATH',
help='The output path for the processed NetCDF')
# Optional arguments.
parser.add_argument(
'--num_realizations',
metavar='NUMBER_OF_REALIZATIONS',
default=None,
type=np.int32,
help='Optional argument to specify the number of '
'ensemble realizations to produce. '
'If the current forecast is input as probabilities or '
'percentiles then this argument is used to create the requested '
'number of realizations. In addition, this argument is used to '
'construct the requested number of realizations from the mean '
'and variance output after applying the EMOS coefficients.'
'Default will be the number of realizations in the raw input '
'file, if realizations are provided as input, otherwise if the '
'input format is probabilities or percentiles, then an error '
'will be raised if no value is provided.')
parser.add_argument(
'--random_ordering',
default=False,
action='store_true',
help='Option to reorder the post-processed forecasts randomly. If not '
'set, the ordering of the raw ensemble is used. This option is '
'only valid when the input format is realizations.')
parser.add_argument(
'--random_seed',
metavar='RANDOM_SEED',
default=None,
help='Option to specify a value for the random seed for testing '
'purposes, otherwise, the default random seed behaviour is '
'utilised. The random seed is used in the generation of the '
'random numbers used for either the random_ordering option to '
'order the input percentiles randomly, rather than use the '
'ordering from the raw ensemble, or for splitting tied values '
'within the raw ensemble, so that the values from the input '
'percentiles can be ordered to match the raw ensemble.')
parser.add_argument(
'--ecc_bounds_warning',
default=False,
action='store_true',
help='If True, where the percentiles exceed the ECC bounds range, '
'raise a warning rather than an exception. This occurs when the '
'current forecast is in the form of probabilities and is '
'converted to percentiles, as part of converting the input '
'probabilities into realizations.')
parser.add_argument(
'--predictor_of_mean',
metavar='PREDICTOR_OF_MEAN',
choices=['mean', 'realizations'],
default='mean',
help='String to specify the predictor used to calibrate the forecast '
'mean. Currently the ensemble mean ("mean") and the ensemble '
'realizations ("realizations") are supported as options. '
'Default: "mean".')
args = parser.parse_args(args=argv)
current_forecast = load_cube(args.forecast_filepath)
coeffs = load_cube(args.coefficients_filepath)
original_current_forecast = current_forecast.copy()
msg = ("The current forecast has been provided as {0}. "
"These {0} need to be converted to realizations "
"for ensemble calibration. The args.num_realizations "
"argument is used to define the number of realizations "
"to construct from the input {0}, so if the "
"current forecast is provided as {0} then "
"args.num_realizations must be defined.")
try:
find_percentile_coordinate(current_forecast)
input_forecast_type = "percentiles"
except CoordinateNotFoundError:
input_forecast_type = "realizations"
if current_forecast.name().startswith("probability_of"):
input_forecast_type = "probabilities"
# If probabilities, convert to percentiles.
conversion_plugin = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=args.ecc_bounds_warning)
elif input_forecast_type == "percentiles":
# If percentiles, resample percentiles so that the percentiles are
# evenly spaced.
conversion_plugin = ResamplePercentiles(
ecc_bounds_warning=args.ecc_bounds_warning)
# If percentiles, resample percentiles and then rebadge.
# If probabilities, generate percentiles and then rebadge.
if input_forecast_type in ["percentiles", "probabilities"]:
if not args.num_realizations:
raise ValueError(msg.format(input_forecast_type))
current_forecast = conversion_plugin.process(
current_forecast, no_of_percentiles=args.num_realizations)
current_forecast = (
RebadgePercentilesAsRealizations().process(current_forecast))
# Default number of ensemble realizations is the number in
# the raw forecast.
if not args.num_realizations:
args.num_realizations = len(
current_forecast.coord('realization').points)
# Apply coefficients as part of Ensemble Model Output Statistics (EMOS).
ac = ApplyCoefficientsFromEnsembleCalibration(
current_forecast,
coeffs,
predictor_of_mean_flag=args.predictor_of_mean)
calibrated_predictor, calibrated_variance = ac.process()
# If input forecast is probabilities, convert output into probabilities.
# If input forecast is percentiles, convert output into percentiles.
# If input forecast is realizations, convert output into realizations.
if input_forecast_type == "probabilities":
result = GenerateProbabilitiesFromMeanAndVariance().process(
calibrated_predictor, calibrated_variance,
original_current_forecast)
elif input_forecast_type == "percentiles":
perc_coord = find_percentile_coordinate(original_current_forecast)
result = GeneratePercentilesFromMeanAndVariance().process(
calibrated_predictor,
calibrated_variance,
percentiles=perc_coord.points)
elif input_forecast_type == "realizations":
# Ensemble Copula Coupling to generate realizations
# from mean and variance.
percentiles = GeneratePercentilesFromMeanAndVariance().process(
calibrated_predictor,
calibrated_variance,
no_of_percentiles=args.num_realizations)
result = EnsembleReordering().process(
percentiles,
current_forecast,
random_ordering=args.random_ordering,
random_seed=args.random_seed)
save_netcdf(result, args.output_filepath)
def main(argv=None):
"""Load in arguments and get going."""
parser = ArgParser(
description="Calculate percentiled data over a given coordinate by "
"collapsing that coordinate. Typically used to convert realization "
"data into percentiled data, but may calculate over any "
"dimension coordinate. Alternatively, calling this CLI with a dataset"
" containing probabilities will convert those to percentiles using "
"the ensemble copula coupling plugin. If no particular percentiles "
"are given at which to calculate values and no 'number of percentiles'"
" to calculate are specified, the following defaults will be used: "
"[0, 5, 10, 20, 25, 30, 40, 50, 60, 70, 75, 80, 90, 95, 100]")
parser.add_argument("input_filepath",
metavar="INPUT_FILE",
help="A path to an input NetCDF file to be processed")
parser.add_argument("output_filepath",
metavar="OUTPUT_FILE",
help="The output path for the processed NetCDF")
parser.add_argument("--coordinates",
metavar="COORDINATES_TO_COLLAPSE",
nargs="+",
help="Coordinate or coordinates over which to collapse"
" data and calculate percentiles; e.g. "
"'realization' or 'latitude longitude'. This argument "
"must be provided when collapsing a coordinate or "
"coordinates to create percentiles, but is redundant "
"when converting probabilities to percentiles and may "
"be omitted. This coordinate(s) will be removed "
"and replaced by a percentile coordinate.")
parser.add_argument('--ecc_bounds_warning',
default=False,
action='store_true',
help='If True, where calculated percentiles are '
'outside the ECC bounds range, raise a warning '
'rather than an exception.')
group = parser.add_mutually_exclusive_group(required=False)
group.add_argument("--percentiles",
metavar="PERCENTILES",
nargs="+",
default=None,
type=float,
help="Optional definition of percentiles at which to "
"calculate data, e.g. --percentiles 0 33.3 66.6 100")
group.add_argument('--no-of-percentiles',
default=None,
type=int,
metavar='NUMBER_OF_PERCENTILES',
help="Optional definition of the number of percentiles "
"to be generated, these distributed regularly with the "
"aim of dividing into blocks of equal probability.")
args = parser.parse_args(args=argv)
cube = load_cube(args.input_filepath)
percentiles = args.percentiles
if args.no_of_percentiles is not None:
percentiles = choose_set_of_percentiles(args.no_of_percentiles,
sampling="quantile")
# TODO: Correct when formal cf-standards exists
if 'probability_of_' in cube.name():
if args.coordinates:
warnings.warn("Converting probabilities to percentiles. The "
"provided COORDINATES_TO_COLLAPSE variable will "
"not be used.")
result = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=args.ecc_bounds_warning).process(
cube, percentiles=percentiles)
else:
if not args.coordinates:
raise ValueError("To collapse a coordinate to calculate "
"percentiles, a coordinate or list of "
"coordinates must be provided.")
# Switch back to use the slow scipy method if the cube contains masked
# data which the numpy method cannot handle.
fast_percentile_method = True
if np.ma.is_masked(cube.data):
# Check for masked points:
fast_percentile_method = False
elif np.ma.isMaskedArray(cube.data):
# Check if we have a masked array with an empty mask. If so,
# replace it with a non-masked array:
cube.data = cube.data.data
result = PercentileConverter(
args.coordinates,
percentiles=percentiles,
fast_percentile_method=fast_percentile_method).process(cube)
save_netcdf(result, args.output_filepath)
def process(cube,
coordinates=None,
ecc_bounds_warning=False,
percentiles=None,
no_of_percentiles=None):
r"""Collapses cube coordinates and calculate percentiled data.
Calculate percentiled data over a given coordinate by collapsing that
coordinate. Typically used to convert realization data into percentiled
data, but may calculate over any dimension coordinate. Alternatively
calling this with a dataset containing probabilities will convert those
to percentiles using the ensemble coupla coupling plugin. If no particular
percentiles are given at which to calculate values and no
'number of percentiles' to calculate are specified, the
following defaults will be used.
'[0, 5, 10, 20, 25, 30, 40, 50, 60, 70, 75, 80, 90, 95, 100]'
Args:
cube (iris.cube.Cube):
A Cube for processing.
coordinates (str or list):
Coordinate or coordinates over which to collapse data and
calculate percentiles; e.g. 'realization' or 'latitude longitude'.
This argument must be provided when collapsing a coordinate or
coordinates to create percentiles, but is redundant when
converting probabilities to percentiles and may be omitted. This
coordinate(s) will be removed and replaced by a percentile
coordinate.
Default is None.
ecc_bounds_warning (bool):
If True, where calculated percentiles are outside the ECC bounds
range, raises a warning rather than an exception.
Default is False.
percentiles (list or None):
Optional definition of percentiles at which to calculate data.
Default is None.
no_of_percentiles (int):
Optional definition of the number of percentiles to be generated,
these distributed regularly with the aim of dividing into blocks
of equal probability.
Default is None.
Returns:
result (iris.cube.Cube):
The processed Cube.
Raises:
ValueError:
If the cube name does not contain 'probability_of\_' and
coordinates isn't used.
Warns:
Warning:
If 'probability_of\_' is in the cube name and coordinates is used.
"""
if no_of_percentiles is not None:
percentiles = choose_set_of_percentiles(no_of_percentiles,
sampling="quantile")
# TODO: Correct when formal cf-standards exists
if 'probability_of_' in cube.name():
result = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=ecc_bounds_warning).process(
cube, percentiles=percentiles)
if coordinates:
warnings.warn("Converting probabilities to percentiles. The "
"provided COORDINATES_TO_COLLAPSE variable will "
"not be used.")
else:
if not coordinates:
raise ValueError("To collapse a coordinate to calculate "
"percentiles, a coordinate or list of "
"coordinates must be provided.")
# Switch back to use the slow scipy method if the cube contains masked
# data which the numpy method cannot handle.
fast_percentile_method = True
if np.ma.is_masked(cube.data):
# Check for masked points:
fast_percentile_method = False
elif np.ma.isMaskedArray(cube.data):
# Check if we have a masked array with an empty mask. If so,
# replace it with a non-masked array:
cube.data = cube.data.data
result = PercentileConverter(
coordinates,
percentiles=percentiles,
fast_percentile_method=fast_percentile_method).process(cube)
return result
def main(argv=None):
"""Load in arguments and start spotdata extraction process."""
parser = ArgParser(
description="Extract diagnostic data from gridded fields for spot data"
" sites. It is possible to apply a temperature lapse rate adjustment"
" to temperature data that helps to account for differences between"
" the spot sites real altitude and that of the grid point from which"
" the temperature data is extracted.")
# Input and output files required.
parser.add_argument("neighbour_filepath", metavar="NEIGHBOUR_FILEPATH",
help="Path to a NetCDF file of spot-data neighbours. "
"This file also contains the spot site information.")
parser.add_argument("diagnostic_filepath", metavar="DIAGNOSTIC_FILEPATH",
help="Path to a NetCDF file containing the diagnostic "
"data to be extracted.")
parser.add_argument("temperature_lapse_rate_filepath",
metavar="LAPSE_RATE_FILEPATH", nargs='?',
help="(Optional) Filepath to a NetCDF file containing"
" temperature lapse rates. If this cube is provided,"
" and a screen temperature cube is being processed,"
" the lapse rates will be used to adjust the"
" temperatures to better represent each spot's"
" site-altitude.")
parser.add_argument("output_filepath", metavar="OUTPUT_FILEPATH",
help="The output path for the resulting NetCDF")
parser.add_argument(
"--apply_lapse_rate_correction",
default=False, action="store_true",
help="If the option is set and a lapse rate cube has been "
"provided, extracted screen temperatures will be adjusted to "
"better match the altitude of the spot site for which they have "
"been extracted.")
method_group = parser.add_argument_group(
title="Neighbour finding method",
description="If none of these options are set, the nearest grid point "
"to a spot site will be used without any other constraints.")
method_group.add_argument(
"--land_constraint", default=False, action='store_true',
help="If set the neighbour cube will be interrogated for grid point"
" neighbours that were identified using a land constraint. This means"
" that the grid points should be land points except for sites where"
" none were found within the search radius when the neighbour cube was"
" created. May be used with minimum_dz.")
method_group.add_argument(
"--minimum_dz", default=False, action='store_true',
help="If set the neighbour cube will be interrogated for grid point"
" neighbours that were identified using a minimum height difference"
" constraint. These are grid points that were found to be the closest"
" in altitude to the spot site within the search radius defined when"
" the neighbour cube was created. May be used with land_constraint.")
percentile_group = parser.add_argument_group(
title="Extract percentiles",
description="Extract particular percentiles from probabilistic, "
"percentile, or realization inputs. If deterministic input is "
"provided a warning is raised and all leading dimensions are included "
"in the returned spot-data cube.")
percentile_group.add_argument(
"--extract_percentiles", default=None, nargs='+', type=int,
help="If set to a percentile value or a list of percentile values, "
"data corresponding to those percentiles will be returned. For "
"example setting '--extract_percentiles 25 50 75' will result in the "
"25th, 50th, and 75th percentiles being returned from a cube of "
"probabilities, percentiles, or realizations. Note that for "
"percentile inputs, the desired percentile(s) must exist in the input "
"cube.")
parser.add_argument(
"--ecc_bounds_warning", default=False, action="store_true",
help="If True, where calculated percentiles are outside the ECC "
"bounds range, raise a warning rather than an exception.")
meta_group = parser.add_argument_group("Metadata")
meta_group.add_argument(
"--metadata_json", metavar="METADATA_JSON", default=None,
help="If provided, this JSON file can be used to modify the metadata "
"of the returned netCDF file. Defaults to None.")
output_group = parser.add_argument_group("Suppress Verbose output")
# This CLI may be used to prepare data for verification without knowing the
# form of the input, be it deterministic, realizations or probabilistic.
# A warning is normally raised when attempting to extract a percentile from
# deterministic data as this is not possible; the spot-extraction of the
# entire cube is returned. When preparing data for verification we know
# that we will produce a large number of these warnings when passing in
# deterministic data. This option to suppress warnings is provided to
# reduce the amount of unneeded logging information that is written out.
output_group.add_argument(
"--suppress_warnings", default=False, action="store_true",
help="Suppress warning output. This option should only be used if "
"it is known that warnings will be generated but they are not "
"required.")
args = parser.parse_args(args=argv)
neighbour_cube = load_cube(args.neighbour_filepath)
diagnostic_cube = load_cube(args.diagnostic_filepath)
neighbour_selection_method = NeighbourSelection(
land_constraint=args.land_constraint,
minimum_dz=args.minimum_dz).neighbour_finding_method_name()
plugin = SpotExtraction(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(neighbour_cube, diagnostic_cube)
# If a probability or percentile diagnostic cube is provided, extract
# the given percentile if available. This is done after the spot-extraction
# to minimise processing time; usually there are far fewer spot sites than
# grid points.
if args.extract_percentiles:
try:
perc_coordinate = find_percentile_coordinate(result)
except CoordinateNotFoundError:
if 'probability_of_' in result.name():
result = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=args.ecc_bounds_warning).process(
result, percentiles=args.extract_percentiles)
result = iris.util.squeeze(result)
elif result.coords('realization', dim_coords=True):
fast_percentile_method = (
False if np.ma.isMaskedArray(result.data) else True)
result = PercentileConverter(
'realization', percentiles=args.extract_percentiles,
fast_percentile_method=fast_percentile_method).process(
result)
else:
msg = ('Diagnostic cube is not a known probabilistic type. '
'The {} percentile could not be extracted. Extracting '
'data from the cube including any leading '
'dimensions.'.format(
args.extract_percentiles))
if not args.suppress_warnings:
warnings.warn(msg)
else:
constraint = ['{}={}'.format(perc_coordinate.name(),
args.extract_percentiles)]
perc_result = extract_subcube(result, constraint)
if perc_result is not None:
result = perc_result
else:
msg = ('The percentile diagnostic cube does not contain the '
'requested percentile value. Requested {}, available '
'{}'.format(args.extract_percentiles,
perc_coordinate.points))
raise ValueError(msg)
# Check whether a lapse rate cube has been provided and we are dealing with
# temperature data and the lapse-rate option is enabled.
if (args.temperature_lapse_rate_filepath and
args.apply_lapse_rate_correction):
if not result.name() == "air_temperature":
msg = ("A lapse rate cube was provided, but the diagnostic being "
"processed is not air temperature and cannot be adjusted.")
raise ValueError(msg)
lapse_rate_cube = load_cube(args.temperature_lapse_rate_filepath)
if not lapse_rate_cube.name() == 'air_temperature_lapse_rate':
msg = ("A cube has been provided as a lapse rate cube but does "
"not have the expected name air_temperature_lapse_rate: "
"{}".format(lapse_rate_cube.name()))
raise ValueError(msg)
try:
lapse_rate_height_coord = lapse_rate_cube.coord("height")
except (ValueError, CoordinateNotFoundError):
msg = ("Lapse rate cube does not contain a single valued height "
"coordinate. This is required to ensure it is applied to "
"equivalent temperature data.")
raise ValueError(msg)
# Check the height of the temperature data matches that used to
# calculate the lapse rates. If so, adjust temperatures using the lapse
# rate values.
if diagnostic_cube.coord("height") == lapse_rate_height_coord:
plugin = SpotLapseRateAdjust(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(result, neighbour_cube, lapse_rate_cube)
else:
msg = ("A lapse rate cube was provided, but the height of "
"the temperature data does not match that of the data used "
"to calculate the lapse rates. As such the temperatures "
"were not adjusted with the lapse rates.")
if not args.suppress_warnings:
warnings.warn(msg)
elif (args.apply_lapse_rate_correction and
not args.temperature_lapse_rate_filepath):
msg = ("A lapse rate cube was not provided, but the option to "
"apply the lapse rate correction was enabled. No lapse rate "
"correction could be applied.")
if not args.suppress_warnings:
warnings.warn(msg)
# Modify final metadata as described by provided JSON file.
if args.metadata_json:
with open(args.metadata_json, 'r') as input_file:
metadata_dict = json.load(input_file)
result = amend_metadata(result, **metadata_dict)
# Remove the internal model_grid_hash attribute if present.
result.attributes.pop('model_grid_hash', None)
# Save the spot data cube.
save_netcdf(result, args.output_filepath)
