def test_model_id_attr_mismatch(self):
"""Test that when a model ID attribute string is specified that does
not match the model ID attribute key name on both cubes to be merged,
an error is thrown"""
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_matching_model_config")
msg = "Cannot create model ID coordinate"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(self.non_mo_cubelist)
def test_model_id_attr_mismatch_one_cube(self):
"""Test that when a model ID attribute string is specified that only
matches the model ID attribute key name on one of the cubes to be
merged, an error is thrown"""
self.non_mo_cubelist[1].attributes.pop("non_mo_model_config")
self.non_mo_cubelist[1].attributes[
"non_matching_model_config"] = "non_uk_det"
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_matching_model_config")
msg = "Cannot create model ID coordinate"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(self.non_mo_cubelist)
def test_record_run_no_model_attr(self):
"""Test recording the source runs without a model attribute."""
plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration",
record_run_attr="mosg__model_run",
)
self.cube_ukv.attributes.pop("mosg__model_configuration")
msg = "Failure to record run information"
with self.assertRaisesRegex(Exception, msg):
plugin.process([self.cube_ukv, self.cube_enuk])
def test_dict(self):
"""Test dictionary option for model blending with non-equal weights"""
data = np.ones((3, 3, 3), dtype=np.float32)
thresholds = np.array([276, 277, 278], dtype=np.float32)
ukv_cube = set_up_probability_cube(data,
thresholds,
time=dt(2018, 9, 10, 7),
frt=dt(2018, 9, 10, 1),
standard_grid_metadata="uk_det")
enukx_cube = set_up_probability_cube(data,
thresholds,
time=dt(2018, 9, 10, 7),
frt=dt(2018, 9, 10, 1),
standard_grid_metadata="uk_ens")
merger = MergeCubesForWeightedBlending(
"model_id",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration")
cube = merger.process([ukv_cube, enukx_cube])
plugin = WeightAndBlend("model_id",
"dict",
weighting_coord="forecast_period",
wts_dict=MODEL_WEIGHTS)
# at 6 hours lead time we should have 1/3 UKV and 2/3 MOGREPS-UK,
# according to the dictionary weights specified above
weights = plugin._calculate_blending_weights(cube)
self.assertArrayEqual(
weights.coord("model_configuration").points, ["uk_det", "uk_ens"])
self.assertArrayAlmostEqual(weights.data,
np.array([0.3333333, 0.6666667]))
def test_non_mo_model_id(self):
"""Test that a model ID attribute string can be specified when
merging multi model cubes"""
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_mo_model_config")
result = plugin.process(self.non_mo_cubelist)
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(result.coord("model_id").points, [0, 1000])
def setUp(self):
"""Set up cube and plugin"""
cubelist = set_up_masked_cubes()
merger = MergeCubesForWeightedBlending(
"model_id", weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration")
self.cube = merger.process(cubelist)
self.plugin = WeightAndBlend(
"model_id", "dict", weighting_coord="forecast_period",
wts_dict=MODEL_WEIGHTS)
self.initial_weights = (
self.plugin._calculate_blending_weights(self.cube))
def test_record_run(self):
"""Test recording the source runs in a blend record run attribute."""
plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration",
record_run_attr="mosg__model_run",
)
cube = plugin.process(self.cubelist)
self.assertEqual(
cube.attributes["mosg__model_run"],
"uk_det:20151123T0300Z:\nuk_ens:20151123T0000Z:",
)
def test_blend_realizations(self):
"""Test processing works for merging over coordinates that don't
require specific setup"""
data = np.ones((1, 3, 3), dtype=np.float32)
cube1 = set_up_variable_cube(data, realizations=np.array([0]))
cube1 = iris.util.squeeze(cube1)
cube2 = set_up_variable_cube(data, realizations=np.array([1]))
cube2 = iris.util.squeeze(cube2)
plugin = MergeCubesForWeightedBlending("realization")
result = plugin.process([cube1, cube2])
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(result.coord("realization").points, np.array([0, 1]))
self.assertEqual(result[0].metadata, cube1.metadata)
self.assertEqual(result[1].metadata, cube2.metadata)
def test_record_run_existing(self):
"""Test recording blend source runs with existing record attributes."""
plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration",
record_run_attr="mosg__model_run",
)
self.cube_ukv.attributes[
"mosg__model_run"] = "uk_det:20151123T0200Z:\nuk_det:20151123T0300Z:"
cube = plugin.process([self.cube_ukv, self.cube_enuk])
self.assertEqual(
cube.attributes["mosg__model_run"],
"uk_det:20151123T0200Z:\nuk_det:20151123T0300Z:\nuk_ens:20151123T0000Z:",
)
def test_cycle_blend(self):
"""Test merge for blending over forecast_reference_time"""
cube = self.cube_ukv.copy()
cube.coord("forecast_reference_time").points = (
cube.coord("forecast_reference_time").points + 3600)
cube.coord("forecast_period").points = (
cube.coord("forecast_reference_time").points - 3600)
plugin = MergeCubesForWeightedBlending("forecast_reference_time")
result = plugin.process([self.cube_ukv, cube])
self.assertIsInstance(result, iris.cube.Cube)
self.assertIn(result.coord("forecast_reference_time"),
result.coords(dim_coords=True))
# check no model coordinates have been added
with self.assertRaises(iris.exceptions.CoordinateNotFoundError):
result.coord(MODEL_BLEND_COORD)
with self.assertRaises(iris.exceptions.CoordinateNotFoundError):
result.coord(MODEL_NAME_COORD)
def test_blend_coord_ascending(self):
"""Test the order of the output blend coordinate is always ascending,
independent of the input cube order"""
frt = self.cube_ukv.coord("forecast_reference_time").points[0]
fp = self.cube_ukv.coord("forecast_period").points[0]
cube1 = self.cube_ukv.copy()
cube1.coord("forecast_reference_time").points = [frt + 3600]
cube1.coord("forecast_period").points = [fp - 3600]
cube2 = self.cube_ukv.copy()
cube2.coord("forecast_reference_time").points = [frt + 7200]
cube2.coord("forecast_period").points = [fp - 7200]
# input unordered cubes; expect ordered output
expected_points = np.array([frt, frt + 3600, frt + 7200],
dtype=np.int64)
plugin = MergeCubesForWeightedBlending("forecast_reference_time")
result = plugin.process([cube1, self.cube_ukv, cube2])
self.assertArrayEqual(
result.coord("forecast_reference_time").points, expected_points)
def main(argv=None):
"""Load in arguments and ensure they are set correctly.
Then load in the data to blend and calculate default weights
using the method chosen before carrying out the blending."""
parser = ArgParser(
description='Calculate the default weights to apply in weighted '
'blending plugins using the ChooseDefaultWeightsLinear or '
'ChooseDefaultWeightsNonLinear plugins. Then apply these '
'weights to the dataset using the BasicWeightedAverage plugin.'
' Required for ChooseDefaultWeightsLinear: y0val and ynval.'
' Required for ChooseDefaultWeightsNonLinear: cval.'
' Required for ChooseWeightsLinear with dict: wts_dict.')
parser.add_argument('--wts_calc_method',
metavar='WEIGHTS_CALCULATION_METHOD',
choices=['linear', 'nonlinear', 'dict'],
default='linear',
help='Method to use to calculate '
'weights used in blending. "linear" (default): '
'calculate linearly varying blending weights. '
'"nonlinear": calculate blending weights that decrease'
' exponentially with increasing blending coordinate. '
'"dict": calculate weights using a dictionary passed '
'in as a command line argument.')
parser.add_argument('coordinate',
type=str,
metavar='COORDINATE_TO_AVERAGE_OVER',
help='The coordinate over which the blending '
'will be applied.')
parser.add_argument('--coordinate_unit',
metavar='UNIT_STRING',
default='hours since 1970-01-01 00:00:00',
help='Units for blending coordinate. Default= '
'hours since 1970-01-01 00:00:00')
parser.add_argument('--calendar',
metavar='CALENDAR',
help='Calendar for time coordinate. Default=gregorian')
parser.add_argument('--cycletime',
metavar='CYCLETIME',
type=str,
help='The forecast reference time to be used after '
'blending has been applied, in the format '
'YYYYMMDDTHHMMZ. If not provided, the blended file '
'will take the latest available forecast reference '
'time from the input cubes supplied.')
parser.add_argument('--model_id_attr',
metavar='MODEL_ID_ATTR',
type=str,
default="mosg__model_configuration",
help='The name of the netCDF file attribute to be '
'used to identify the source model for '
'multi-model blends. Default assumes Met Office '
'model metadata. Must be present on all input '
'files if blending over models.')
parser.add_argument('--spatial_weights_from_mask',
action='store_true',
default=False,
help='If set this option will result in the generation'
' of spatially varying weights based on the'
' masks of the data we are blending. The'
' one dimensional weights are first calculated '
' using the chosen weights calculation method,'
' but the weights will then be adjusted spatially'
' based on where there is masked data in the data'
' we are blending. The spatial weights are'
' calculated using the'
' SpatiallyVaryingWeightsFromMask plugin.')
parser.add_argument('weighting_mode',
metavar='WEIGHTED_BLEND_MODE',
choices=['weighted_mean', 'weighted_maximum'],
help='The method used in the weighted blend. '
'"weighted_mean": calculate a normal weighted'
' mean across the coordinate. '
'"weighted_maximum": multiplies the values in the'
' coordinate by the weights, and then takes the'
' maximum.')
parser.add_argument('input_filepaths',
metavar='INPUT_FILES',
nargs="+",
help='Paths to input files to be blended.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
spatial = parser.add_argument_group(
'Spatial weights from mask options',
'Options for calculating the spatial weights using the '
'SpatiallyVaryingWeightsFromMask plugin.')
spatial.add_argument('--fuzzy_length',
metavar='FUZZY_LENGTH',
type=float,
default=20000,
help='When calculating spatially varying weights we'
' can smooth the weights so that areas close to'
' areas that are masked have lower weights than'
' those further away. This fuzzy length controls'
' the scale over which the weights are smoothed.'
' The fuzzy length is in terms of m, the'
' default is 20km. This distance is then'
' converted into a number of grid squares,'
' which does not have to be an integer. Assumes'
' the grid spacing is the same in the x and y'
' directions, and raises an error if this is not'
' true. See SpatiallyVaryingWeightsFromMask for'
' more detail.')
linear = parser.add_argument_group(
'linear weights options', 'Options for the linear weights '
'calculation in '
'ChooseDefaultWeightsLinear')
linear.add_argument('--y0val',
metavar='LINEAR_STARTING_POINT',
type=float,
help='The relative value of the weighting start point '
'(lowest value of blend coord) for choosing default '
'linear weights. This must be a positive float or 0.')
linear.add_argument('--ynval',
metavar='LINEAR_END_POINT',
type=float,
help='The relative value of the weighting '
'end point (highest value of blend coord) for choosing'
' default linear weights. This must be a positive '
'float or 0. Note that if blending over forecast '
'reference time, ynval >= y0val would normally be '
'expected (to give greater weight to the more recent '
'forecast).')
nonlinear = parser.add_argument_group(
'nonlinear weights options', 'Options for the non-linear '
'weights calculation in '
'ChooseDefaultWeightsNonLinear')
nonlinear.add_argument('--cval',
metavar='NON_LINEAR_FACTOR',
type=float,
help='Factor used to determine how skewed the '
'non linear weights will be. '
'A value of 1 implies equal weighting. If not '
'set, a default value of cval=0.85 is set.')
wts_dict = parser.add_argument_group(
'dict weights options', 'Options for linear weights to be '
'calculated based on parameters '
'read from a json file dict')
wts_dict.add_argument('--wts_dict',
metavar='WEIGHTS_DICTIONARY',
help='Path to json file containing dictionary from '
'which to calculate blending weights. Dictionary '
'format is as specified in the improver.blending.'
'weights.ChooseWeightsLinear plugin.')
wts_dict.add_argument('--weighting_coord',
metavar='WEIGHTING_COORD',
default='forecast_period',
help='Name of '
'coordinate over which linear weights should be '
'scaled. This coordinate must be avilable in the '
'weights dictionary.')
args = parser.parse_args(args=argv)
# if the linear weights method is called with non-linear args or vice
# versa, exit with error
if (args.wts_calc_method == "linear") and args.cval:
parser.wrong_args_error('cval', 'linear')
if ((args.wts_calc_method == "nonlinear")
and np.any([args.y0val, args.ynval])):
parser.wrong_args_error('y0val, ynval', 'non-linear')
if (args.wts_calc_method == "dict") and not args.wts_dict:
parser.error('Dictionary is required if --wts_calc_method="dict"')
# set blending coordinate units
if "time" in args.coordinate:
coord_unit = Unit(args.coordinate_unit, args.calendar)
elif args.coordinate_unit != 'hours since 1970-01-01 00:00:00.':
coord_unit = args.coordinate_unit
else:
coord_unit = 'no_unit'
# For blending across models, only blending across "model_id" is directly
# supported. This is because the blending coordinate must be sortable, in
# order to ensure that the data cube and the weights cube have coordinates
# in the same order for blending. Whilst the model_configuration is
# sortable itself, as it is associated with model_id, which is the
# dimension coordinate, sorting the model_configuration coordinate can
# result in the model_id coordinate becoming non-monotonic. As dimension
# coordinates must be monotonic, this leads to the model_id coordinate
# being demoted to an auxiliary coordinate. Therefore, for simplicity
# model_id is used as the blending coordinate, instead of
# model_configuration.
# TODO: Support model_configuration as a blending coordinate directly.
if args.coordinate == "model_configuration":
blend_coord = "model_id"
dict_coord = "model_configuration"
else:
blend_coord = args.coordinate
dict_coord = args.coordinate
# load cubes to be blended
cubelist = load_cubelist(args.input_filepaths)
# determine whether or not to equalise forecast periods for model
# blending weights calculation
weighting_coord = (args.weighting_coord
if args.weighting_coord else "forecast_period")
# prepare cubes for weighted blending
merger = MergeCubesForWeightedBlending(blend_coord,
weighting_coord=weighting_coord,
model_id_attr=args.model_id_attr)
cube = merger.process(cubelist, cycletime=args.cycletime)
# if the coord for blending does not exist or has only one value,
# update metadata only
coord_names = [coord.name() for coord in cube.coords()]
if (blend_coord not in coord_names) or (len(
cube.coord(blend_coord).points) == 1):
result = cube.copy()
conform_metadata(result, cube, blend_coord, cycletime=args.cycletime)
# raise a warning if this happened because the blend coordinate
# doesn't exist
if blend_coord not in coord_names:
warnings.warn('Blend coordinate {} is not present on input '
'data'.format(blend_coord))
# otherwise, calculate weights and blend across specified dimension
else:
weights = calculate_blending_weights(
cube,
blend_coord,
args.wts_calc_method,
wts_dict=args.wts_dict,
weighting_coord=args.weighting_coord,
coord_unit=coord_unit,
y0val=args.y0val,
ynval=args.ynval,
cval=args.cval,
dict_coord=dict_coord)
if args.spatial_weights_from_mask:
check_if_grid_is_equal_area(cube)
grid_cells_x, _ = convert_distance_into_number_of_grid_cells(
cube, args.fuzzy_length, int_grid_cells=False)
SpatialWeightsPlugin = SpatiallyVaryingWeightsFromMask(
grid_cells_x)
weights = SpatialWeightsPlugin.process(cube, weights, blend_coord)
# blend across specified dimension
BlendingPlugin = WeightedBlendAcrossWholeDimension(
blend_coord, args.weighting_mode, cycletime=args.cycletime)
result = BlendingPlugin.process(cube, weights=weights)
save_netcdf(result, args.output_filepath)
class Test_process(IrisTest):
"""Test the process method"""
def setUp(self):
"""Set up some probability cubes from different models"""
data = np.array(
[
0.9 * np.ones((3, 3)), 0.5 * np.ones((3, 3)), 0.1 * np.ones(
(3, 3))
],
dtype=np.float32,
)
thresholds = np.array([273.0, 275.0, 277.0], dtype=np.float32)
time_point = dt(2015, 11, 23, 7)
time_bounds = [dt(2015, 11, 23, 4), time_point]
# set up a MOGREPS-UK cube with 7 hour forecast period
self.cube_enuk = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata="uk_ens",
time=time_point,
frt=dt(2015, 11, 23, 0),
time_bounds=time_bounds,
)
# set up a UKV cube with 4 hour forecast period
self.cube_ukv = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata="uk_det",
time=time_point,
frt=dt(2015, 11, 23, 3),
time_bounds=time_bounds,
)
self.cubelist = iris.cube.CubeList([self.cube_enuk, self.cube_ukv])
# set up a nowcast cube
self.cube_nowcast = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata="nc_det",
time=time_point,
frt=dt(2015, 11, 23, 3, 15),
time_bounds=time_bounds,
)
# set up some non-UK test cubes
cube_non_mo_ens = self.cube_enuk.copy()
cube_non_mo_ens.attributes.pop("mosg__model_configuration")
cube_non_mo_ens.attributes["non_mo_model_config"] = "non_uk_ens"
cube_non_mo_det = self.cube_ukv.copy()
cube_non_mo_det.attributes.pop("mosg__model_configuration")
cube_non_mo_det.attributes["non_mo_model_config"] = "non_uk_det"
self.non_mo_cubelist = iris.cube.CubeList(
[cube_non_mo_ens, cube_non_mo_det])
# set up plugin for multi-model blending weighted by forecast period
self.plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration",
)
def test_multi_model_merge(self):
"""Test models merge OK and have expected model coordinates"""
result = self.plugin.process(self.cubelist)
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(
result.coord(MODEL_BLEND_COORD).points, [0, 1000])
self.assertArrayEqual(
result.coord(MODEL_NAME_COORD).points, ["uk_ens", "uk_det"])
def test_time_coords(self):
"""Test merged cube has scalar time coordinates if weighting models
by forecast period"""
result = self.plugin.process(self.cubelist)
# test resulting cube has single 4 hour (shorter) forecast period
self.assertEqual(result.coord("forecast_period").points, [4 * 3600])
# check time and frt points are also consistent with the UKV input cube
self.assertEqual(
result.coord("time").points,
self.cube_ukv.coord("time").points)
self.assertEqual(
result.coord("forecast_reference_time").points,
self.cube_ukv.coord("forecast_reference_time").points,
)
def test_cycle_blend(self):
"""Test merge for blending over forecast_reference_time"""
cube = self.cube_ukv.copy()
cube.coord("forecast_reference_time").points = (
cube.coord("forecast_reference_time").points + 3600)
cube.coord("forecast_period").points = (
cube.coord("forecast_reference_time").points - 3600)
plugin = MergeCubesForWeightedBlending("forecast_reference_time")
result = plugin.process([self.cube_ukv, cube])
self.assertIsInstance(result, iris.cube.Cube)
self.assertIn(result.coord("forecast_reference_time"),
result.coords(dim_coords=True))
# check no model coordinates have been added
with self.assertRaises(iris.exceptions.CoordinateNotFoundError):
result.coord(MODEL_BLEND_COORD)
with self.assertRaises(iris.exceptions.CoordinateNotFoundError):
result.coord(MODEL_NAME_COORD)
def test_blend_coord_ascending(self):
"""Test the order of the output blend coordinate is always ascending,
independent of the input cube order"""
frt = self.cube_ukv.coord("forecast_reference_time").points[0]
fp = self.cube_ukv.coord("forecast_period").points[0]
cube1 = self.cube_ukv.copy()
cube1.coord("forecast_reference_time").points = [frt + 3600]
cube1.coord("forecast_period").points = [fp - 3600]
cube2 = self.cube_ukv.copy()
cube2.coord("forecast_reference_time").points = [frt + 7200]
cube2.coord("forecast_period").points = [fp - 7200]
# input unordered cubes; expect ordered output
expected_points = np.array([frt, frt + 3600, frt + 7200],
dtype=np.int64)
plugin = MergeCubesForWeightedBlending("forecast_reference_time")
result = plugin.process([cube1, self.cube_ukv, cube2])
self.assertArrayEqual(
result.coord("forecast_reference_time").points, expected_points)
def test_cycletime(self):
"""Test merged cube has updated forecast reference time and forecast
period if specified using the 'cycletime' argument"""
result = self.plugin.process(self.cubelist, cycletime="20151123T0600Z")
# test resulting cube has forecast period consistent with cycletime
self.assertEqual(result.coord("forecast_period").points, [3600])
self.assertEqual(
result.coord("forecast_reference_time").points,
self.cube_ukv.coord("forecast_reference_time").points + 3 * 3600,
)
# check validity time is unchanged
self.assertEqual(
result.coord("time").points,
self.cube_ukv.coord("time").points)
def test_non_mo_model_id(self):
"""Test that a model ID attribute string can be specified when
merging multi model cubes"""
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_mo_model_config")
result = plugin.process(self.non_mo_cubelist)
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(
result.coord(MODEL_BLEND_COORD).points, [0, 1000])
def test_model_id_attr_mismatch(self):
"""Test that when a model ID attribute string is specified that does
not match the model ID attribute key name on both cubes to be merged,
an error is thrown"""
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_matching_model_config")
msg = "Cannot create model ID coordinate"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(self.non_mo_cubelist)
def test_model_id_attr_mismatch_one_cube(self):
"""Test that when a model ID attribute string is specified that only
matches the model ID attribute key name on one of the cubes to be
merged, an error is thrown"""
self.non_mo_cubelist[1].attributes.pop("non_mo_model_config")
self.non_mo_cubelist[1].attributes[
"non_matching_model_config"] = "non_uk_det"
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_matching_model_config")
msg = "Cannot create model ID coordinate"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(self.non_mo_cubelist)
def test_time_bounds_mismatch(self):
"""Test failure for cycle blending when time bounds ranges are not
matched (ie cycle blending different "accumulation periods")"""
cube2 = self.cube_ukv.copy()
cube2.coord("forecast_reference_time").points = (
cube2.coord("forecast_reference_time").points + 3600)
cube2.coord("time").bounds = [
cube2.coord("time").bounds[0, 0] + 3600,
cube2.coord("time").bounds[0, 1],
]
cube2.coord("forecast_period").bounds = [
cube2.coord("forecast_period").bounds[0, 0] + 3600,
cube2.coord("forecast_period").bounds[0, 1],
]
msg = "Cube with mismatching time bounds ranges cannot be blended"
with self.assertRaisesRegex(ValueError, msg):
MergeCubesForWeightedBlending("forecast_reference_time").process(
[self.cube_ukv, cube2])
def test_blend_coord_not_present(self):
"""Test exception when blend coord is not present on inputs"""
msg = "realization coordinate is not present on all input cubes"
with self.assertRaisesRegex(ValueError, msg):
MergeCubesForWeightedBlending("realization").process(self.cubelist)
def test_blend_realizations(self):
"""Test processing works for merging over coordinates that don't
require specific setup"""
data = np.ones((1, 3, 3), dtype=np.float32)
cube1 = set_up_variable_cube(data, realizations=np.array([0]))
cube1 = iris.util.squeeze(cube1)
cube2 = set_up_variable_cube(data, realizations=np.array([1]))
cube2 = iris.util.squeeze(cube2)
plugin = MergeCubesForWeightedBlending("realization")
result = plugin.process([cube1, cube2])
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(
result.coord("realization").points, np.array([0, 1]))
self.assertEqual(result[0].metadata, cube1.metadata)
self.assertEqual(result[1].metadata, cube2.metadata)
def test_handling_blend_time(self):
"""Test merging works with mismatched and / or missing blend time
coordinates"""
blend_time_ukv = self.cube_ukv.coord("forecast_reference_time").copy()
blend_time_ukv.rename("blend_time")
self.cube_ukv.add_aux_coord(blend_time_ukv)
blend_time_enuk = self.cube_enuk.coord(
"forecast_reference_time").copy()
blend_time_enuk.rename("blend_time")
self.cube_enuk.add_aux_coord(blend_time_enuk)
plugin = MergeCubesForWeightedBlending(
"model_id", model_id_attr="mosg__model_configuration")
result = plugin([self.cube_nowcast, self.cube_ukv, self.cube_enuk])
self.assertNotIn("blend_time", get_coord_names(result))
def test_forecast_coord_deprecation(self):
"""Test merging works if some (but not all) inputs have previously been cycle
blended"""
for cube in [self.cube_ukv, self.cube_enuk]:
for coord in ["forecast_period", "forecast_reference_time"]:
cube.coord(coord).attributes.update(
{"deprecation_message": "blah"})
plugin = MergeCubesForWeightedBlending(
"model_id", model_id_attr="mosg__model_configuration")
result = plugin([self.cube_nowcast, self.cube_ukv, self.cube_enuk])
for coord in ["forecast_period", "forecast_reference_time"]:
self.assertNotIn("deprecation_message",
result.coord(coord).attributes)
def process(self,
cubelist,
cycletime=None,
model_id_attr=None,
spatial_weights=False,
fuzzy_length=20000):
"""
Merge a cubelist, calculate appropriate blend weights and compute the
weighted mean. Returns a single cube collapsed over the dimension
given by self.blend_coord.
Args:
cubelist (iris.cube.CubeList):
List of cubes to be merged and blended
Kwargs:
cycletime (str):
Forecast reference time to use for output cubes, in the format
YYYYMMDDTHHMMZ. If not set, the latest of the input cube
forecast reference times is used.
model_id_attr (str):
Name of the attribute by which to identify the source model and
construct "model" coordinates for blending.
spatial_weights (bool):
If true, calculate spatial weights.
fuzzy_length (float):
Distance (in metres) over which to smooth spatial weights.
Default is 20 km.
"""
# Prepare cubes for weighted blending, including creating model_id and
# model_configuration coordinates for multi-model blending. The merged
# cube has a monotonically ascending blend coordinate. Plugin raises an
# error if blend_coord is not present on all input cubes.
merger = MergeCubesForWeightedBlending(
self.blend_coord,
weighting_coord=self.weighting_coord,
model_id_attr=model_id_attr)
cube = merger.process(cubelist, cycletime=cycletime)
# if blend_coord has only one value, or is not present (case where only
# one model has been provided for a model blend) update metadata only
coord_names = [coord.name() for coord in cube.coords()]
if (self.blend_coord not in coord_names
or len(cube.coord(self.blend_coord).points) == 1):
result = cube.copy()
conform_metadata(result,
cube,
self.blend_coord,
cycletime=cycletime)
# otherwise, calculate weights and blend across specified dimension
else:
# set up special treatment for model blending
if "model" in self.blend_coord:
self.blend_coord = "model_id"
# calculate blend weights
weights = self._calculate_blending_weights(cube)
if spatial_weights:
weights = self._update_spatial_weights(cube, weights,
fuzzy_length)
# blend across specified dimension
BlendingPlugin = WeightedBlendAcrossWholeDimension(
self.blend_coord, cycletime=cycletime)
result = BlendingPlugin.process(cube, weights=weights)
return result
class Test_process(IrisTest):
"""Test the process method"""
def setUp(self):
"""Set up some probability cubes from different models"""
data = np.array([
0.9 * np.ones((3, 3)), 0.5 * np.ones((3, 3)), 0.1 * np.ones((3, 3))
],
dtype=np.float32)
thresholds = np.array([273., 275., 277.], dtype=np.float32)
time_point = dt(2015, 11, 23, 7)
time_bounds = [dt(2015, 11, 23, 4), time_point]
# set up a MOGREPS-UK cube with 7 hour forecast period
self.cube_enuk = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata='uk_ens',
time=time_point,
frt=dt(2015, 11, 23, 0),
time_bounds=time_bounds)
# set up a UKV cube with 4 hour forecast period
self.cube_ukv = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata='uk_det',
time=time_point,
frt=dt(2015, 11, 23, 3),
time_bounds=time_bounds)
self.cubelist = iris.cube.CubeList([self.cube_enuk, self.cube_ukv])
# set up some non-UK test cubes
cube_non_mo_ens = self.cube_enuk.copy()
cube_non_mo_ens.attributes.pop("mosg__model_configuration")
cube_non_mo_ens.attributes['non_mo_model_config'] = 'non_uk_ens'
cube_non_mo_det = self.cube_ukv.copy()
cube_non_mo_det.attributes.pop("mosg__model_configuration")
cube_non_mo_det.attributes['non_mo_model_config'] = 'non_uk_det'
self.non_mo_cubelist = iris.cube.CubeList(
[cube_non_mo_ens, cube_non_mo_det])
# set up plugin for multi-model blending weighted by forecast period
self.plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration")
def test_basic(self):
"""Test single cube is returned unmodified"""
cube = self.cube_enuk.copy()
result = self.plugin.process(cube)
self.assertArrayAlmostEqual(result.data, self.cube_enuk.data)
self.assertEqual(result.metadata, self.cube_enuk.metadata)
def test_single_item_list(self):
"""Test cube from single item list is returned unmodified"""
cubelist = iris.cube.CubeList([self.cube_enuk.copy()])
result = self.plugin.process(cubelist)
self.assertArrayAlmostEqual(result.data, self.cube_enuk.data)
self.assertEqual(result.metadata, self.cube_enuk.metadata)
def test_multi_model_merge(self):
"""Test models merge OK and have expected model coordinates"""
result = self.plugin.process(self.cubelist)
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(result.coord("model_id").points, [0, 1000])
self.assertArrayEqual(
result.coord("model_configuration").points, ["uk_ens", "uk_det"])
def test_time_coords(self):
"""Test merged cube has scalar time coordinates if weighting models
by forecast period"""
result = self.plugin.process(self.cubelist)
# test resulting cube has single 4 hour (shorter) forecast period
self.assertEqual(result.coord("forecast_period").points, [4 * 3600])
# check time and frt points are also consistent with the UKV input cube
self.assertEqual(
result.coord("time").points,
self.cube_ukv.coord("time").points)
self.assertEqual(
result.coord("forecast_reference_time").points,
self.cube_ukv.coord("forecast_reference_time").points)
def test_cycle_blend(self):
"""Test merge for blending over forecast_reference_time"""
cube = self.cube_ukv.copy()
cube.coord("forecast_reference_time").points = (
cube.coord("forecast_reference_time").points + 3600)
cube.coord("forecast_period").points = (
cube.coord("forecast_reference_time").points - 3600)
plugin = MergeCubesForWeightedBlending("forecast_reference_time")
result = plugin.process([self.cube_ukv, cube])
self.assertIsInstance(result, iris.cube.Cube)
self.assertIn(result.coord("forecast_reference_time"),
result.coords(dim_coords=True))
# check forecast period coordinate has been removed
with self.assertRaises(iris.exceptions.CoordinateNotFoundError):
result.coord("forecast_period")
# check no model coordinates have been added
with self.assertRaises(iris.exceptions.CoordinateNotFoundError):
result.coord("model_id")
with self.assertRaises(iris.exceptions.CoordinateNotFoundError):
result.coord("model_configuration")
def test_cycletime(self):
"""Test merged cube has updated forecast reference time and forecast
period if specified using the 'cycletime' argument"""
result = self.plugin.process(self.cubelist, cycletime="20151123T0600Z")
# test resulting cube has forecast period consistent with cycletime
self.assertEqual(result.coord("forecast_period").points, [3600])
self.assertEqual(
result.coord("forecast_reference_time").points,
self.cube_ukv.coord("forecast_reference_time").points + 3 * 3600)
# check validity time is unchanged
self.assertEqual(
result.coord("time").points,
self.cube_ukv.coord("time").points)
def test_non_mo_model_id(self):
"""Test that a model ID attribute string can be specified when
merging multi model cubes"""
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_mo_model_config")
result = plugin.process(self.non_mo_cubelist)
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(result.coord("model_id").points, [0, 1000])
def test_model_id_attr_mismatch(self):
"""Test that when a model ID attribute string is specified that does
not match the model ID attribute key name on both cubes to be merged,
an error is thrown"""
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_matching_model_config")
msg = "Cannot create model ID coordinate"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(self.non_mo_cubelist)
def test_model_id_attr_mismatch_one_cube(self):
"""Test that when a model ID attribute string is specified that only
matches the model ID attribute key name on one of the cubes to be
merged, an error is thrown"""
self.non_mo_cubelist[1].attributes.pop("non_mo_model_config")
self.non_mo_cubelist[1].attributes[
"non_matching_model_config"] = "non_uk_det"
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_matching_model_config")
msg = "Cannot create model ID coordinate"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(self.non_mo_cubelist)
def test_time_bounds_mismatch(self):
"""Test failure for cycle blending when time bounds ranges are not
matched (ie cycle blending different "accumulation periods")"""
cube2 = self.cube_ukv.copy()
cube2.coord("forecast_reference_time").points = (
cube2.coord("forecast_reference_time").points + 3600)
cube2.coord("time").bounds = [
cube2.coord("time").bounds[0, 0] + 3600,
cube2.coord("time").bounds[0, 1]
]
cube2.coord("forecast_period").bounds = [
cube2.coord("forecast_period").bounds[0, 0] + 3600,
cube2.coord("forecast_period").bounds[0, 1]
]
msg = "Cube with mismatching time bounds ranges cannot be blended"
with self.assertRaisesRegex(ValueError, msg):
MergeCubesForWeightedBlending("forecast_reference_time").process(
[self.cube_ukv, cube2])
def test_blend_coord_not_present(self):
"""Test exception when blend coord is not present on inputs"""
msg = "realization coordinate is not present on all input cubes"
with self.assertRaisesRegex(ValueError, msg):
MergeCubesForWeightedBlending("realization").process(self.cubelist)
def test_blend_realizations(self):
"""Test processing works for merging over coordinates that don't
require specific setup"""
data = np.ones((1, 3, 3), dtype=np.float32)
cube1 = set_up_variable_cube(data, realizations=np.array([0]))
cube1 = iris.util.squeeze(cube1)
cube2 = set_up_variable_cube(data, realizations=np.array([1]))
cube2 = iris.util.squeeze(cube2)
plugin = MergeCubesForWeightedBlending("realization")
result = plugin.process([cube1, cube2])
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(
result.coord("realization").points, np.array([0, 1]))
self.assertEqual(result[0].metadata, cube1.metadata)
self.assertEqual(result[1].metadata, cube2.metadata)
