def process(self, calibrated_forecast_predictor_and_variance,
no_of_percentiles):
"""
Generate ensemble percentiles from the mean and variance.
Args:
calibrated_forecast_predictor_and_variance (Iris CubeList):
CubeList containing the calibrated forecast predictor and
calibrated forecast variance.
raw_forecast (Iris Cube or CubeList):
Cube or CubeList that is expected to be the raw
(uncalibrated) forecast.
Returns:
calibrated_forecast_percentiles (iris.cube.Cube):
Cube for calibrated percentiles.
The percentile coordinate is always the zeroth dimension.
"""
(calibrated_forecast_predictor, calibrated_forecast_variance) = (
calibrated_forecast_predictor_and_variance)
calibrated_forecast_predictor = concatenate_cubes(
calibrated_forecast_predictor)
calibrated_forecast_variance = concatenate_cubes(
calibrated_forecast_variance)
percentiles = choose_set_of_percentiles(no_of_percentiles)
calibrated_forecast_percentiles = (
self._mean_and_variance_to_percentiles(
calibrated_forecast_predictor,
calibrated_forecast_variance,
percentiles))
return calibrated_forecast_percentiles
def process(
self, post_processed_forecast, raw_forecast,
random_ordering=False, random_seed=None):
"""
Reorder post-processed forecast using the ordering of the
raw ensemble.
Parameters
----------
post_processed_forecast : Iris Cube or CubeList
The cube or cubelist containing the post-processed
forecast members.
raw_forecast : Iris Cube or CubeList
The cube or cubelist containing the raw (not post-processed)
forecast.
random_ordering : Logical
If random_ordering is True, the post-processed forecasts are
reordered randomly, rather than using the ordering of the
raw ensemble.
random_seed : Integer or None
If random_seed is an integer, the integer value is used for
the random seed.
If random_seed is None, no random seed is set, so the random
values generated are not reproducible.
Returns
-------
post-processed_forecast_members : cube
Cube containing the new ensemble members where all points within
the dataset have been reordered in comparison to the input
percentiles.
"""
post_processed_forecast_percentiles = concatenate_cubes(
post_processed_forecast,
coords_to_slice_over=["percentile_over_realization", "time"])
post_processed_forecast_percentiles = (
ensure_dimension_is_the_zeroth_dimension(
post_processed_forecast_percentiles,
"percentile_over_realization"))
raw_forecast_members = concatenate_cubes(raw_forecast)
raw_forecast_members = ensure_dimension_is_the_zeroth_dimension(
raw_forecast_members, "realization")
raw_forecast_members = (
self._recycle_raw_ensemble_members(
post_processed_forecast_percentiles, raw_forecast_members))
post_processed_forecast_members = self.rank_ecc(
post_processed_forecast_percentiles, raw_forecast_members,
random_ordering=random_ordering,
random_seed=random_seed)
post_processed_forecast_members = (
RebadgePercentilesAsMembers.process(
post_processed_forecast_members))
post_processed_forecast_members = (
ensure_dimension_is_the_zeroth_dimension(
post_processed_forecast_members,
"realization"))
return post_processed_forecast_members
def test_identical_cubes(self):
"""
Test that the utility returns the expected error message,
if an attempt is made to concatenate identical cubes.
"""
cubes = iris.cube.CubeList([self.cube, self.cube])
msg = "An unexpected problem prevented concatenation."
with self.assertRaisesRegex(ConcatenateError, msg):
concatenate_cubes(cubes)
def _recycle_raw_ensemble_members(
post_processed_forecast_percentiles, raw_forecast_members):
"""
Function to determine whether there is a mismatch between the number
of percentiles and the number of raw forecast members. If more
percentiles are requested than ensemble members, then the ensemble
members are recycled. This assumes that the identity of the ensemble
members within the raw ensemble forecast is random, such that the
raw ensemble members are exchangeable. If fewer percentiles are
requested than ensemble members, then only the first n ensemble
members are used.
Parameters
----------
post_processed_forecast_percentiles : cube
Cube for post-processed percentiles. The percentiles are assumed
to be in ascending order.
raw_forecast_members : cube
Cube containing the raw (not post-processed) forecasts.
Returns
-------
Iris cube
Cube for the raw ensemble forecast, where the raw ensemble members
have either been recycled or constrained, depending upon the
number of percentiles present in the post-processed forecast cube.
"""
plen = len(
post_processed_forecast_percentiles.coord(
"percentile_over_realization").points)
mlen = len(raw_forecast_members.coord("realization").points)
if plen == mlen:
pass
else:
raw_forecast_members_extended = iris.cube.CubeList()
realization_list = []
mpoints = raw_forecast_members.coord("realization").points
# Loop over the number of percentiles and finding the
# corresponding ensemble member number. The ensemble member
# numbers are recycled e.g. 1, 2, 3, 1, 2, 3, etc.
for index in range(plen):
realization_list.append(mpoints[index % len(mpoints)])
# Assume that the ensemble members are ascending linearly.
new_member_numbers = realization_list[0] + range(plen)
# Extract the members required in the realization_list from
# the raw_forecast_members. Edit the member number as appropriate
# and append to a cubelist containing rebadged raw ensemble
# members.
for realization, index in zip(
realization_list, new_member_numbers):
constr = iris.Constraint(realization=realization)
raw_forecast_member = raw_forecast_members.extract(constr)
raw_forecast_member.coord("realization").points = index
raw_forecast_members_extended.append(raw_forecast_member)
raw_forecast_members = (
concatenate_cubes(raw_forecast_members_extended))
return raw_forecast_members
def test_works_two_thresh(self):
"""Test that the plugin works with a cube that contains multiple
thresholds."""
width = 2.0
thresh_cube = self.cube.copy()
thresh_cube.remove_coord("forecast_reference_time")
changes = {'points': [0.25], 'units': '1', 'var_name': 'threshold'}
cube_with_thresh1 = add_coord(thresh_cube.copy(),
'precipitation_amount', changes)
changes = {'points': [0.5], 'units': '1', 'var_name': 'threshold'}
cube_with_thresh2 = add_coord(thresh_cube.copy(),
'precipitation_amount', changes)
changes = {'points': [0.75], 'units': '1', 'var_name': 'threshold'}
cube_with_thresh3 = add_coord(thresh_cube.copy(),
'precipitation_amount', changes)
cubelist = iris.cube.CubeList(
[cube_with_thresh1, cube_with_thresh2, cube_with_thresh3])
thresh_cubes = concatenate_cubes(
cubelist, coords_to_slice_over='precipitation_amount')
plugin = TriangularWeightedBlendAcrossAdjacentPoints(
'forecast_period', self.forecast_period, 'hours', width)
result = plugin(thresh_cubes)
# Test that the result cube retains threshold co-ordinates
# from original cube.
self.assertEqual(thresh_cubes.coord('precipitation_amount'),
result.coord('precipitation_amount'))
def test_too_many_coefficients(self):
"""
Test that the plugin returns values for the coefficients,
which match the expected values.
"""
cube = self.current_temperature_forecast_cube
cube1 = cube.copy()
cube2 = cube.copy()
cube2.coord("time").points = cube2.coord("time").points + 3
cube2.data += 3
cube = concatenate_cubes(CubeList([cube1, cube2]))
optimised_coeffs = {}
for time_slice in cube.slices_over("time"):
the_date = datetime_from_timestamp(time_slice.coord("time").points)
optimised_coeffs[the_date] = self.default_optimised_coeffs
predictor_cube = cube.collapsed("realization", iris.analysis.MEAN)
variance_cube = cube.collapsed("realization", iris.analysis.VARIANCE)
coeff_names = ["cat", "dog", "elephant", "frog", "giraffe"]
predictor_of_mean_flag = "mean"
plugin = Plugin(self.cube, optimised_coeffs, coeff_names)
msg = "Number of coefficient names"
with self.assertRaisesRegex(ValueError, msg):
dummy_result = plugin._apply_params(predictor_cube, variance_cube,
optimised_coeffs, coeff_names,
predictor_of_mean_flag)
def rank_ecc(post_processed_forecast_percentiles,
raw_forecast_members,
random_ordering=False,
random_seed=None):
"""
Function to apply Ensemble Copula Coupling. This ranks the
post-processed forecast members based on a ranking determined from
the raw forecast members.
Args:
post_processed_forecast_percentiles (cube):
Cube for post-processed percentiles. The percentiles are
assumed to be in ascending order.
raw_forecast_members (cube):
Cube containing the raw (not post-processed) forecasts.
The probabilistic dimension is assumed to be the zeroth
dimension.
random_ordering (Logical):
If random_ordering is True, the post-processed forecasts are
reordered randomly, rather than using the ordering of the
raw ensemble.
random_seed (Integer or None):
If random_seed is an integer, the integer value is used for
the random seed.
If random_seed is None, no random seed is set, so the random
values generated are not reproducible.
Returns:
iris.cube.Cube:
Cube for post-processed members where at a particular grid
point, the ranking of the values within the ensemble matches
the ranking from the raw ensemble.
"""
results = iris.cube.CubeList([])
for rawfc, calfc in zip(
raw_forecast_members.slices_over("time"),
post_processed_forecast_percentiles.slices_over("time")):
if random_seed is not None:
random_seed = int(random_seed)
random_seed = np.random.RandomState(random_seed)
random_data = random_seed.rand(*rawfc.data.shape)
if random_ordering:
# Returns the indices that would sort the array.
# As these indices are from a random dataset, only an argsort
# is used.
ranking = np.argsort(random_data, axis=0)
else:
# Lexsort returns the indices sorted firstly by the
# primary key, the raw forecast data (unless random_ordering
# is enabled), and secondly by the secondary key, an array of
# random data, in order to split tied values randomly.
sorting_index = np.lexsort((random_data, rawfc.data), axis=0)
# Returns the indices that would sort the array.
ranking = np.argsort(sorting_index, axis=0)
# Index the post-processed forecast data using the ranking array.
# np.choose allows indexing of a 3d array using a 3d array,
calfc.data = np.choose(ranking, calfc.data)
results.append(calfc)
return concatenate_cubes(results)
def test_two_dates(self):
"""
Test that the plugin returns a tuple when two dates are present
within the input cube.
"""
cube = self.current_temperature_forecast_cube
cube1 = cube.copy()
cube2 = cube.copy()
cube2.coord("time").points = cube2.coord("time").points + 3
cube2.data += 3
cube = concatenate_cubes(CubeList([cube1, cube2]))
optimised_coeffs = {}
for time_slice in cube.slices_over("time"):
the_date = datetime_from_timestamp(time_slice.coord("time").points)
optimised_coeffs[the_date] = self.default_optimised_coeffs
predictor_cube = cube.collapsed("realization", iris.analysis.MEAN)
variance_cube = cube.collapsed("realization", iris.analysis.VARIANCE)
predictor_of_mean_flag = "mean"
plugin = Plugin(cube, optimised_coeffs, self.coeff_names)
forecast_predictor, forecast_variance, coefficients = (
plugin._apply_params(predictor_cube, variance_cube,
optimised_coeffs, self.coeff_names,
predictor_of_mean_flag))
for result in [forecast_predictor, forecast_variance]:
self.assertEqual(len(result), 2)
self.assertEqual(len(coefficients), 8)
def test_calibrated_predictor(self):
"""
Test that the plugin returns values for the calibrated predictor (the
calibrated mean), which match the expected values.
"""
data = np.array([[231.15002913, 242.40003036, 253.6500316],
[264.90003284, 276.15003408, 287.40003531],
[298.65003655, 309.90003779, 321.15003903]])
cube = self.current_temperature_forecast_cube
cube1 = cube.copy()
cube2 = cube.copy()
cube2.coord("time").points = cube2.coord("time").points + 3
cube2.data += 3
cube = concatenate_cubes(CubeList([cube1, cube2]))
optimised_coeffs = {}
for time_slice in cube.slices_over("time"):
the_date = datetime_from_timestamp(time_slice.coord("time").points)
optimised_coeffs[the_date] = self.default_optimised_coeffs
predictor_cube = cube.collapsed("realization", iris.analysis.MEAN)
variance_cube = cube.collapsed("realization", iris.analysis.VARIANCE)
predictor_of_mean_flag = "mean"
plugin = Plugin(self.cube, optimised_coeffs, self.coeff_names)
forecast_predictor, _, _ = plugin._apply_params(
predictor_cube, variance_cube, optimised_coeffs, self.coeff_names,
predictor_of_mean_flag)
self.assertArrayAlmostEqual(forecast_predictor[0].data, data)
def test_cubelist_with_forecast_reference_time_only(self):
"""
Test that the utility returns an iris.cube.Cube with the expected
resulting data, if a CubeList containing cubes with different
forecast_reference_time coordinates is passed in as the input.
This makes sure that the forecast_reference_time from the input cubes
is maintained within the output cube, after concatenation.
"""
cube1 = self.cube.copy()
cube2 = self.cube.copy()
cube2.coord("time").points = np.array([412230.0], dtype=np.float64)
time_origin = "hours since 1970-01-01 00:00:00"
calendar = "gregorian"
tunit = Unit(time_origin, calendar)
cube1.add_aux_coord(
DimCoord([412227.0], "forecast_reference_time", units=tunit))
cube2.add_aux_coord(
DimCoord([412230.0], "forecast_reference_time", units=tunit))
cubelist = iris.cube.CubeList([cube1, cube2])
result = concatenate_cubes(cubelist)
self.assertArrayAlmostEqual(
result.coord("forecast_reference_time").points,
[412227.0, 412230.0])
def test_works_two_thresh(self):
"""Test that the plugin works with a cube that contains multiple
thresholds."""
width = 2.0
changes = {'points': [0.25], 'units': '1'}
cube_with_thresh1 = add_coord(self.cube.copy(), 'threshold', changes)
changes = {'points': [0.5], 'units': '1'}
cube_with_thresh2 = add_coord(self.cube.copy(), 'threshold', changes)
changes = {'points': [0.75], 'units': '1'}
cube_with_thresh3 = add_coord(self.cube.copy(), 'threshold', changes)
cubelist = iris.cube.CubeList([cube_with_thresh1, cube_with_thresh2,
cube_with_thresh3])
thresh_cubes = concatenate_cubes(cubelist,
coords_to_slice_over='threshold')
plugin = TriangularWeightedBlendAcrossAdjacentPoints(
'forecast_period', self.forecast_period, 'hours', width,
'weighted_mean')
result = plugin.process(thresh_cubes)
# Test that the result cube retains threshold co-ordinates
# from origonal cube.
self.assertEqual(thresh_cubes.coord('threshold'),
result.coord('threshold'))
def test_coefficients(self):
"""
Test that the plugin returns values for the coefficients,
which match the expected values.
"""
data = np.array([4.55819380e-06])
cube = self.current_temperature_forecast_cube
cube1 = cube.copy()
cube2 = cube.copy()
cube2.coord("time").points = cube2.coord("time").points + 3
cube2.data += 3
cube = concatenate_cubes(CubeList([cube1, cube2]))
optimised_coeffs = {}
for time_slice in cube.slices_over("time"):
the_date = datetime_from_timestamp(time_slice.coord("time").points)
optimised_coeffs[the_date] = self.default_optimised_coeffs
predictor_cube = cube.collapsed("realization", iris.analysis.MEAN)
variance_cube = cube.collapsed("realization", iris.analysis.VARIANCE)
predictor_of_mean_flag = "mean"
plugin = Plugin(self.cube, optimised_coeffs, self.coeff_names)
_, _, coefficients = plugin._apply_params(predictor_cube,
variance_cube,
optimised_coeffs,
self.coeff_names,
predictor_of_mean_flag)
self.assertArrayAlmostEqual(coefficients[0].data, data)
def test_coefficients_realizations(self):
"""
Test that the plugin returns values for the calibrated forecasts,
which match the expected values when the individual ensemble
realizations are used as the predictor.
"""
data = np.array([5.0])
cube = self.current_temperature_forecast_cube
cube1 = cube.copy()
cube2 = cube.copy()
cube2.coord("time").points = cube2.coord("time").points + 3
cube2.data += 3
cube = concatenate_cubes(CubeList([cube1, cube2]))
optimised_coeffs = {}
for time_slice in cube.slices_over("time"):
the_date = datetime_from_timestamp(time_slice.coord("time").points)
optimised_coeffs[the_date] = np.array([5, 1, 0, 0.57, 0.6, 0.6])
self.coeff_names = ["gamma", "delta", "a", "beta"]
predictor_cube = cube.copy()
variance_cube = cube.collapsed("realization", iris.analysis.VARIANCE)
predictor_of_mean_flag = "realizations"
plugin = Plugin(self.cube, optimised_coeffs, self.coeff_names)
_, _, coefficients = plugin._apply_params(predictor_cube,
variance_cube,
optimised_coeffs,
self.coeff_names,
predictor_of_mean_flag)
self.assertArrayAlmostEqual(coefficients[0].data, data)
def _create_historic_forecasts(cube, number_of_days=5):
"""
Function to create a set of pseudo historic forecast cubes, based on the
input cube, and assuming that there will be one forecast per day at the
same hour of the day.
"""
historic_forecasts = CubeList([])
no_of_hours_in_day = 24
time_range = np.linspace(no_of_hours_in_day,
no_of_hours_in_day * number_of_days,
num=number_of_days,
endpoint=True)
for index in time_range:
temp_cube = cube.copy()
# TODO: Remove conversion to hours, once all ensemble calibration
# unit tests have been upgraded to use set_up_variable_cube.
for coord_name in ["forecast_reference_time", "time"]:
orig_units = temp_cube.coord(coord_name).units
temp_cube.coord(coord_name).convert_units(
"hours since 1970-01-01 00:00:00")
temp_cube.coord(coord_name).points = (
temp_cube.coord(coord_name).points - index)
temp_cube.coord(coord_name).convert_units(orig_units)
temp_cube.data -= 2
historic_forecasts.append(temp_cube)
historic_forecast = concatenate_cubes(
historic_forecasts,
coords_to_slice_over=["time"],
coordinates_for_association=["forecast_reference_time"])
return historic_forecast
def process(self, forecast_at_percentiles, no_of_percentiles=None,
sampling="quantile"):
"""
1. Concatenates cubes with a percentile coordinate.
2. Creates a list of percentiles.
3. Accesses the lower and upper bound pair of the forecast values,
in order to specify lower and upper bounds for the percentiles.
4. Interpolate the percentile coordinate into an alternative
set of percentiles using linear interpolation.
Args:
forecast_at_percentiles (Iris CubeList or Iris Cube):
Cube or CubeList expected to contain a percentile coordinate.
no_of_percentiles (Integer or None):
Number of percentiles
If None, the number of percentiles within the input
forecast_at_percentiles cube is used as the
number of percentiles.
sampling (String):
Type of sampling of the distribution to produce a set of
percentiles e.g. quantile or random.
Accepted options for sampling are:
* Quantile: A regular set of equally-spaced percentiles aimed
at dividing a Cumulative Distribution Function into
blocks of equal probability.
* Random: A random set of ordered percentiles.
Returns:
forecast_at_percentiles (iris.cube.Cube):
Cube with forecast values at the desired set of percentiles.
The percentile coordinate is always the zeroth dimension.
"""
forecast_at_percentiles = concatenate_cubes(forecast_at_percentiles)
percentile_coord = (
find_percentile_coordinate(forecast_at_percentiles).name())
if no_of_percentiles is None:
no_of_percentiles = (
len(forecast_at_percentiles.coord(
percentile_coord).points))
percentiles = choose_set_of_percentiles(
no_of_percentiles, sampling=sampling)
cube_units = forecast_at_percentiles.units
bounds_pairing = (
get_bounds_of_distribution(
forecast_at_percentiles.name(), cube_units))
forecast_at_percentiles = self._interpolate_percentiles(
forecast_at_percentiles, percentiles, bounds_pairing,
percentile_coord)
return forecast_at_percentiles
def process(self,
forecast_probabilities,
no_of_percentiles=None,
sampling="quantile"):
"""
1. Concatenates cubes with a threshold coordinate.
2. Creates a list of percentiles.
3. Accesses the lower and upper bound pair to find the ends of the
cumulative distribution function.
4. Convert the threshold coordinate into
values at a set of percentiles using linear interpolation,
see Figure 1 from Flowerdew, 2014.
Parameters
----------
forecast_probabilities : Iris CubeList or Iris Cube
Cube or CubeList expected to contain a threshold coordinate.
no_of_percentiles : Integer or None
Number of percentiles
If None, the number of thresholds within the input
forecast_probabilities cube is used as the number of percentiles.
sampling : String
Type of sampling of the distribution to produce a set of
percentiles e.g. quantile or random.
Accepted options for sampling are:
Quantile: A regular set of equally-spaced percentiles aimed
at dividing a Cumulative Distribution Function into
blocks of equal probability.
Random: A random set of ordered percentiles.
Returns
-------
forecast_at_percentiles : Iris cube
Cube with forecast values at the desired set of percentiles.
The threshold coordinate is always the zeroth dimension.
"""
forecast_probabilities = concatenate_cubes(forecast_probabilities)
threshold_coord = forecast_probabilities.coord("threshold")
phenom_name = (forecast_probabilities.name().replace(
"probability_of_", ""))
if no_of_percentiles is None:
no_of_percentiles = (len(
forecast_probabilities.coord(threshold_coord.name()).points))
percentiles = choose_set_of_percentiles(no_of_percentiles,
sampling=sampling)
cube_units = (forecast_probabilities.coord(
threshold_coord.name()).units)
bounds_pairing = (get_bounds_of_distribution(phenom_name, cube_units))
forecast_at_percentiles = self._probabilities_to_percentiles(
forecast_probabilities, percentiles, bounds_pairing)
return forecast_at_percentiles
def test_cubelist_different_var_names(self):
"""
Test that the utility returns an iris.cube.Cube, if a CubeList
containing non-identical cubes is passed in as the input.
"""
self.cube.coord("time").var_name = "time_0"
self.later_cube.coord("time").var_name = "time_1"
cubelist = iris.cube.CubeList([self.cube, self.later_cube])
result = concatenate_cubes(cubelist)
self.assertIsInstance(result, Cube)
def test_cubelist_different_number_of_realizations_time(self):
"""
Test that the utility returns the expected error message, if a
CubeList containing cubes with different numbers of realizations are
passed in as the input, and the slicing done, in order to help the
concatenation is only done over time.
"""
cube1 = self.cube.copy()
cube3 = iris.cube.CubeList([])
for cube in cube1.slices_over("realization"):
if cube.coord("realization").points == 0:
cube2 = cube
elif cube.coord("realization").points in [1, 2]:
cube3.append(cube)
cube3 = cube3.merge_cube()
cubelist = iris.cube.CubeList([cube2, cube3])
msg = "failed to concatenate into a single cube"
with self.assertRaisesRegex(ConcatenateError, msg):
concatenate_cubes(cubelist, coords_to_slice_over=["time"])
def test_cubelist_slice_over_realization_only(self):
"""
Test that the utility returns an iris.cube.Cube with the expected
realization coordinate, if a CubeList containing cubes with different
realizations is passed in as the input.
"""
cubelist = iris.cube.CubeList([self.cube, self.later_cube])
result = concatenate_cubes(
cubelist, coords_to_slice_over=["realization"])
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(
result.coord("realization").points, [0, 1, 2])
def process(self, cubelist):
"""
Take an input cubelist containing forecasts from different cycles and
merges them into a single cube.
The steps taken are:
1. Update forecast reference time and period to match the latest
contributing cycle.
2. Check for duplicate realization numbers. If a duplicate is
found, renumber all of the realizations uniquely.
3. Concatenate into one cube along the realization axis.
Args:
cubelist (iris.cube.CubeList or list of iris.cube.Cube):
List of input forecasts
Returns:
iris.cube.Cube:
Concatenated forecasts
"""
cubelist = rebadge_forecasts_as_latest_cycle(cubelist)
# Take all the realizations from all the input cube and
# put in one array
all_realizations = [
cube.coord("realization").points for cube in cubelist
]
all_realizations = np.concatenate(all_realizations)
# Find unique realizations
unique_realizations = np.unique(all_realizations)
# If we have fewer unique realizations than total realizations we have
# duplicate realizations so we rebadge all realizations in the cubelist
if len(unique_realizations) < len(all_realizations):
first_realization = 0
for cube in cubelist:
n_realization = len(cube.coord("realization").points)
cube.coord("realization").points = np.arange(
first_realization,
first_realization + n_realization,
dtype=np.int32)
first_realization = first_realization + n_realization
# slice over realization to deal with cases where direct concatenation
# would result in a non-monotonic coordinate
lagged_ensemble = concatenate_cubes(
cubelist,
master_coord="realization",
coords_to_slice_over=["realization"])
return lagged_ensemble
def process(self, cubelist):
"""
Take an input cubelist containing forecasts from different cycles and
merges them into a single cube.
The steps taken are:
1. If no cycletime is given then find the latest cycle time from
the input cubes.
2. Update the forecast periods in each input cube to be relative
to the new cycletime.
3. Checks if there are duplicate realization numbers. If a
duplicate is found, renumbers all of the realizations to remove
any duplicates.
4. Merge cubes into one cube, removing any metadata that
doesn't match.
"""
if self.cycletime is None:
cycletime = find_latest_cycletime(cubelist)
else:
cycletime = cycletime_to_datetime(self.cycletime)
cubelist = unify_forecast_reference_time(cubelist, cycletime)
# Take all the realizations from all the input cube and
# put in one array
all_realizations = [
cube.coord("realization").points for cube in cubelist
]
all_realizations = np.concatenate(all_realizations)
# Find unique realizations
unique_realizations = np.unique(all_realizations)
# If we have fewer unique realizations than total realizations we have
# duplicate realizations so we rebadge all realizations in the cubelist
if len(unique_realizations) < len(all_realizations):
first_realization = 0
for cube in cubelist:
n_realization = len(cube.coord("realization").points)
cube.coord("realization").points = np.arange(
first_realization, first_realization + n_realization)
first_realization = first_realization + n_realization
# slice over realization to deal with cases where direct concatenation
# would result in a non-monotonic coordinate
lagged_ensemble = concatenate_cubes(
cubelist,
master_coord="realization",
coords_to_slice_over=["realization"])
return lagged_ensemble
def test_cubelist_type_and_data(self):
"""
Test that the utility returns an iris.cube.Cube with the expected
resulting data, if a CubeList containing non-identical cubes
(different values for the time coordinate) is passed in as the input.
"""
cube = self.cube.copy()
cube.transpose([1, 0, 2, 3])
expected_result = (
np.vstack([cube.data, cube.data]).transpose([1, 0, 2, 3]))
cubelist = iris.cube.CubeList([self.cube, self.later_cube])
result = concatenate_cubes(
cubelist, coords_to_slice_over=["realization"])
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(expected_result, result.data)
def test_cubelist_slice_over_time_only(self):
"""
Test that the utility returns an iris.cube.Cube with the expected
time coordinate, if a CubeList containing cubes with different
timesteps is passed in as the input.
"""
expected_time_points = [
self.cube.coord("time").points[0],
self.later_cube.coord("time").points[0]]
cubelist = iris.cube.CubeList([self.cube, self.later_cube])
result = concatenate_cubes(
cubelist, coords_to_slice_over=["time"])
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(
result.coord("time").points, expected_time_points)
def test_cubelist_different_var_names(self):
"""
Test that the utility returns an iris.cube.Cube, if a CubeList
containing non-identical cubes is passed in as the input.
"""
cube1 = self.cube.copy()
cube2 = self.cube.copy()
cube2.coord("time").points = np.float64(412230.0)
cube1.coord("time").var_name = "time_0"
cube2.coord("time").var_name = "time_1"
cubelist = iris.cube.CubeList([cube1, cube2])
result = concatenate_cubes(cubelist)
self.assertIsInstance(result, Cube)
def test_cubelist_slice_over_time_only(self):
"""
Test that the utility returns an iris.cube.Cube with the expected
time coordinate, if a CubeList containing cubes with different
timesteps is passed in as the input.
"""
cube1 = self.cube.copy()
cube2 = self.cube.copy()
cube2.coord("time").points = np.array([412230.0], dtype=np.float64)
cubelist = iris.cube.CubeList([cube1, cube2])
result = concatenate_cubes(cubelist, coords_to_slice_over=["time"])
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(
result.coord("time").points, [412227.0, 412230.0])
def process(self, cube):
"""
Apply the weighted blend for each point in the given coordinate.
Args:
cube : iris.cube.Cube
Cube to blend.
Returns:
cube: iris.cube.Cube
The processed cube, with the same coordinates as the input
cube. The points in one coordinate will be blended with the
adjacent points based on a triangular weighting function of the
specified width.
"""
# We need to correct all the coordinates associated with the dimension
# we are collapsing over, so find the relevant coordinates now.
dimension_to_collapse = cube.coord_dims(self.coord)
coords_to_correct = cube.coords(dimensions=dimension_to_collapse)
coords_to_correct = [coord.name() for coord in coords_to_correct]
# We will also need to correct the bounds on these coordinates,
# as bounds will be added when the blending happens, so add bounds if
# it doesn't have some already.
for coord in coords_to_correct:
if not cube.coord(coord).has_bounds():
cube.coord(coord).guess_bounds()
# Set up a plugin to calculate the triangular weights.
WeightsPlugin = ChooseDefaultWeightsTriangular(
self.width, units=self.parameter_units)
# Set up the blending function, based on whether weighted blending or
# maximum probabilities are needed.
BlendingPlugin = WeightedBlendAcrossWholeDimension(self.coord,
self.mode)
result = iris.cube.CubeList([])
# Loop over each point in the coordinate we are blending over, and
# calculate a new weighted average for it.
for cube_slice in cube.slices_over(self.coord):
point = cube_slice.coord(self.coord).points[0]
weights = WeightsPlugin.process(cube, self.coord, point)
blended_cube = BlendingPlugin.process(cube, weights)
self.correct_collapsed_coordinates(cube_slice, blended_cube,
coords_to_correct)
result.append(blended_cube)
result = concatenate_cubes(result)
return result
def apply_params_entry(self):
"""
Wrapping function to calculate the forecast predictor and forecast
variance prior to applying coefficients to the current forecast.
Returns:
(tuple) : tuple containing:
**calibrated_forecast_predictor** (CubeList):
CubeList containing both the calibrated version of the
ensemble predictor, either the ensemble mean/members.
**calibrated_forecast_variance** (CubeList):
CubeList containing both the calibrated version of the
ensemble variance, either the ensemble mean/members.
**calibrated_forecast_coefficients** (CubeList):
CubeList containing both the coefficients for calibrating
the ensemble.
"""
# Ensure predictor_of_mean_flag is valid.
check_predictor_of_mean_flag(self.predictor_of_mean_flag)
rename_coordinate(
self.current_forecast, "ensemble_member_id", "realization")
current_forecast_cubes = concatenate_cubes(
self.current_forecast)
if self.predictor_of_mean_flag.lower() in ["mean"]:
forecast_predictors = current_forecast_cubes.collapsed(
"realization", iris.analysis.MEAN)
elif self.predictor_of_mean_flag.lower() in ["members"]:
forecast_predictors = current_forecast_cubes
forecast_vars = current_forecast_cubes.collapsed(
"realization", iris.analysis.VARIANCE)
(calibrated_forecast_predictor, calibrated_forecast_var,
calibrated_forecast_coefficients) = self._apply_params(
forecast_predictors, forecast_vars, self.optimised_coeffs,
self.coeff_names, self.predictor_of_mean_flag)
return (calibrated_forecast_predictor,
calibrated_forecast_var,
calibrated_forecast_coefficients)
def test_cubelist_with_forecast_reference_time_only(self):
"""
Test that the utility returns an iris.cube.Cube with the expected
resulting data, if a CubeList containing cubes with different
forecast_reference_time coordinates is passed in as the input.
This makes sure that the forecast_reference_time from the input cubes
is maintained within the output cube, after concatenation.
"""
self.later_cube.coord("forecast_reference_time").points = (
self.later_cube.coord("forecast_reference_time").points + 3600)
expected_frt_points = [
self.cube.coord("forecast_reference_time").points[0],
self.later_cube.coord("forecast_reference_time").points[0]]
cubelist = iris.cube.CubeList([self.cube, self.later_cube])
result = concatenate_cubes(
cubelist, coordinates_for_association=["forecast_reference_time"])
self.assertArrayAlmostEqual(
result.coord("forecast_reference_time").points,
expected_frt_points)
def test_cubelist_type_and_data(self):
"""
Test that the utility returns an iris.cube.Cube with the expected
resulting data, if a CubeList containing non-identical cubes
(different values for the time coordinate) is passed in as the input.
"""
cube1 = self.cube.copy()
cube2 = self.cube.copy()
cube3 = self.cube.copy()
cube3.transpose([1, 0, 2, 3])
expected_result = np.vstack([cube3.data, cube3.data])
cube2.coord("time").points = np.array([412230.0], dtype=np.float64)
cubelist = iris.cube.CubeList([cube1, cube2])
result = concatenate_cubes(cubelist)
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(expected_result, result.data)
def _create_historic_forecasts(cube, number_of_days=5):
"""
Function to create a set of pseudo historic forecast cubes, based on the
input cube, and assuming that there will be one forecast per day at the
same hour of the day.
"""
historic_forecasts = CubeList([])
no_of_hours_in_day = 24
time_range = np.linspace(
no_of_hours_in_day, no_of_hours_in_day*number_of_days,
num=number_of_days, endpoint=True)
for index in time_range:
temp_cube = cube.copy()
temp_cube.coord("forecast_reference_time").points = (
temp_cube.coord("forecast_reference_time").points - index)
temp_cube.coord("time").points = temp_cube.coord("time").points - index
temp_cube.data -= 2
historic_forecasts.append(temp_cube)
historic_forecast = concatenate_cubes(historic_forecasts)
return historic_forecast
