def test_dict(self):
"""Test dictionary option for model blending with non-equal weights"""
data = np.ones((3, 3, 3), dtype=np.float32)
thresholds = np.array([276, 277, 278], dtype=np.float32)
ukv_cube = set_up_probability_cube(data,
thresholds,
time=dt(2018, 9, 10, 7),
frt=dt(2018, 9, 10, 1),
standard_grid_metadata="uk_det")
enukx_cube = set_up_probability_cube(data,
thresholds,
time=dt(2018, 9, 10, 7),
frt=dt(2018, 9, 10, 1),
standard_grid_metadata="uk_ens")
merger = MergeCubesForWeightedBlending(
"model_id",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration")
cube = merger.process([ukv_cube, enukx_cube])
plugin = WeightAndBlend("model_id",
"dict",
weighting_coord="forecast_period",
wts_dict=MODEL_WEIGHTS)
# at 6 hours lead time we should have 1/3 UKV and 2/3 MOGREPS-UK,
# according to the dictionary weights specified above
weights = plugin._calculate_blending_weights(cube)
self.assertArrayEqual(
weights.coord("model_configuration").points, ["uk_det", "uk_ens"])
self.assertArrayAlmostEqual(weights.data,
np.array([0.3333333, 0.6666667]))
def setUp(self):
"""Set up some probability cubes from different models"""
data = np.array(
[0.9 * np.ones((3, 3)), 0.5 * np.ones((3, 3)), 0.1 * np.ones((3, 3))],
dtype=np.float32,
)
thresholds = np.array([273.0, 275.0, 277.0], dtype=np.float32)
time_point = dt(2015, 11, 23, 7)
# set up a MOGREPS-UK cube with 7 hour forecast period
self.cube_enuk = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata="uk_ens",
time=time_point,
frt=dt(2015, 11, 23, 0),
)
# set up a UKV cube with 4 hour forecast period
self.cube_ukv = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata="uk_det",
time=time_point,
frt=dt(2015, 11, 23, 3),
)
self.cubelist = iris.cube.CubeList([self.cube_enuk, self.cube_ukv])
self.plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration",
)
def setUp(self):
"""Set up a list of cubes from different models with some probability
data in them."""
# make a cube with a forecast reference time and period labelled as
# coming from the UKV
data = np.full((3, 3), 0.6, dtype=np.float32)
self.ukv_cube = set_up_variable_cube(
data,
name='probability_of_air_temperature_above_threshold',
units='1',
time=dt(2017, 1, 10, 3, 0),
frt=dt(2017, 1, 9, 23, 0),
standard_grid_metadata='uk_det')
# make a cube labelled as coming from MOGREPS-UK, with a different
# forecast reference time from the UKV cube
self.enuk_cube = set_up_variable_cube(
data,
name='probability_of_air_temperature_above_threshold',
units='1',
time=dt(2017, 1, 10, 3, 0),
frt=dt(2017, 1, 10, 0, 0),
standard_grid_metadata='uk_ens')
# make a cube list and merged cube containing the two model cubes, for
# use in defining reference coordinates for tests below
self.cubelist = iris.cube.CubeList([self.ukv_cube, self.enuk_cube])
# set up a plugin for multi-model blending
self.plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration")
def test_remove_fp(self):
"""Test function removes forecast_period coord if blending over
forecast_reference_time"""
plugin = MergeCubesForWeightedBlending("forecast_reference_time")
plugin._rationalise_blend_time_coords(self.cubelist)
for cube in self.cubelist:
self.assertTrue("forecast_period" not in cube.coords())
def test_null_cubes_have_fp(self):
"""Test function does nothing if blending over forecast_reference_time
where a forecast period coordinate exists"""
reference_cubelist = self.cubelist.copy()
plugin = MergeCubesForWeightedBlending("forecast_reference_time")
plugin._rationalise_blend_time_coords(self.cubelist)
self.assertEqual(self.cubelist, reference_cubelist)
def test_optional_args(self):
"""Test model ID and weighting coordinate setting"""
plugin = MergeCubesForWeightedBlending(
"model_id",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration")
self.assertEqual(plugin.weighting_coord, "forecast_period")
self.assertEqual(plugin.model_id_attr, "mosg__model_configuration")
def test_non_mo_model_id(self):
"""Test that a model ID attribute string can be specified when
merging multi model cubes"""
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_mo_model_config")
result = plugin.process(self.non_mo_cubelist)
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(result.coord("model_id").points, [0, 1000])
def setUp(self):
"""Set up some probability cubes from different models"""
data = np.array(
[0.9 * np.ones((3, 3)), 0.5 * np.ones((3, 3)), 0.1 * np.ones((3, 3))],
dtype=np.float32,
)
thresholds = np.array([273.0, 275.0, 277.0], dtype=np.float32)
time_point = dt(2015, 11, 23, 7)
time_bounds = [dt(2015, 11, 23, 4), time_point]
# set up a MOGREPS-UK cube with 7 hour forecast period
self.cube_enuk = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata="uk_ens",
time=time_point,
frt=dt(2015, 11, 23, 0),
time_bounds=time_bounds,
)
# set up a UKV cube with 4 hour forecast period
self.cube_ukv = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata="uk_det",
time=time_point,
frt=dt(2015, 11, 23, 3),
time_bounds=time_bounds,
)
self.cubelist = iris.cube.CubeList([self.cube_enuk, self.cube_ukv])
# set up a nowcast cube
self.cube_nowcast = set_up_probability_cube(
data.copy(),
thresholds,
standard_grid_metadata="nc_det",
time=time_point,
frt=dt(2015, 11, 23, 3, 15),
time_bounds=time_bounds,
)
# set up some non-UK test cubes
cube_non_mo_ens = self.cube_enuk.copy()
cube_non_mo_ens.attributes.pop("mosg__model_configuration")
cube_non_mo_ens.attributes["non_mo_model_config"] = "non_uk_ens"
cube_non_mo_det = self.cube_ukv.copy()
cube_non_mo_det.attributes.pop("mosg__model_configuration")
cube_non_mo_det.attributes["non_mo_model_config"] = "non_uk_det"
self.non_mo_cubelist = iris.cube.CubeList([cube_non_mo_ens, cube_non_mo_det])
# set up plugin for multi-model blending weighted by forecast period
self.plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration",
)
def test_model_id_attr_mismatch(self):
"""Test that when a model ID attribute string is specified that does
not match the model ID attribute key name on both cubes to be merged,
an error is thrown"""
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_matching_model_config")
msg = "Cannot create model ID coordinate"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(self.non_mo_cubelist)
def test_warning_unnecessary_model_id_attr(self, warning_list=None):
"""Test warning if model_id_attr is set for non-model blending"""
warning_msg = "model_id_attr not required"
plugin = MergeCubesForWeightedBlending(
"realization", model_id_attr="mosg__model_configuration")
self.assertTrue(
any(item.category == UserWarning for item in warning_list))
self.assertTrue(any(warning_msg in str(item) for item in warning_list))
self.assertIsNone(plugin.model_id_attr)
def test_record_run_no_model_attr(self):
"""Test recording the source runs without a model attribute."""
plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration",
record_run_attr="mosg__model_run",
)
self.cube_ukv.attributes.pop("mosg__model_configuration")
msg = "Failure to record run information"
with self.assertRaisesRegex(Exception, msg):
plugin.process([self.cube_ukv, self.cube_enuk])
def setUp(self):
"""Set up cube and plugin"""
cubelist = set_up_masked_cubes()
merger = MergeCubesForWeightedBlending(
"model_id", weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration")
self.cube = merger.process(cubelist)
self.plugin = WeightAndBlend(
"model_id", "dict", weighting_coord="forecast_period",
wts_dict=MODEL_WEIGHTS)
self.initial_weights = (
self.plugin._calculate_blending_weights(self.cube))
def test_model_id_attr_mismatch_one_cube(self):
"""Test that when a model ID attribute string is specified that only
matches the model ID attribute key name on one of the cubes to be
merged, an error is thrown"""
self.non_mo_cubelist[1].attributes.pop("non_mo_model_config")
self.non_mo_cubelist[1].attributes[
"non_matching_model_config"] = "non_uk_det"
plugin = MergeCubesForWeightedBlending(
"model", model_id_attr="non_matching_model_config")
msg = "Cannot create model ID coordinate"
with self.assertRaisesRegex(ValueError, msg):
plugin.process(self.non_mo_cubelist)
def test_record_run(self):
"""Test recording the source runs in a blend record run attribute."""
plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration",
record_run_attr="mosg__model_run",
)
cube = plugin.process(self.cubelist)
self.assertEqual(
cube.attributes["mosg__model_run"],
"uk_det:20151123T0300Z:\nuk_ens:20151123T0000Z:",
)
def test_forecast_coord_deprecation(self):
"""Test merging works if some (but not all) inputs have previously been cycle
blended"""
for cube in [self.cube_ukv, self.cube_enuk]:
for coord in ["forecast_period", "forecast_reference_time"]:
cube.coord(coord).attributes.update({"deprecation_message": "blah"})
plugin = MergeCubesForWeightedBlending(
"model_id", model_id_attr="mosg__model_configuration"
)
result = plugin([self.cube_nowcast, self.cube_ukv, self.cube_enuk])
for coord in ["forecast_period", "forecast_reference_time"]:
self.assertNotIn("deprecation_message", result.coord(coord).attributes)
def test_blend_realizations(self):
"""Test processing works for merging over coordinates that don't
require specific setup"""
data = np.ones((1, 3, 3), dtype=np.float32)
cube1 = set_up_variable_cube(data, realizations=np.array([0]))
cube1 = iris.util.squeeze(cube1)
cube2 = set_up_variable_cube(data, realizations=np.array([1]))
cube2 = iris.util.squeeze(cube2)
plugin = MergeCubesForWeightedBlending("realization")
result = plugin.process([cube1, cube2])
self.assertIsInstance(result, iris.cube.Cube)
self.assertArrayEqual(result.coord("realization").points, np.array([0, 1]))
self.assertEqual(result[0].metadata, cube1.metadata)
self.assertEqual(result[1].metadata, cube2.metadata)
def test_record_run_existing(self):
"""Test recording blend source runs with existing record attributes."""
plugin = MergeCubesForWeightedBlending(
"model",
weighting_coord="forecast_period",
model_id_attr="mosg__model_configuration",
record_run_attr="mosg__model_run",
)
self.cube_ukv.attributes[
"mosg__model_run"] = "uk_det:20151123T0200Z:\nuk_det:20151123T0300Z:"
cube = plugin.process([self.cube_ukv, self.cube_enuk])
self.assertEqual(
cube.attributes["mosg__model_run"],
"uk_det:20151123T0200Z:\nuk_det:20151123T0300Z:\nuk_ens:20151123T0000Z:",
)
def test_handling_blend_time(self):
"""Test merging works with mismatched and / or missing blend time
coordinates"""
blend_time_ukv = self.cube_ukv.coord("forecast_reference_time").copy()
blend_time_ukv.rename("blend_time")
self.cube_ukv.add_aux_coord(blend_time_ukv)
blend_time_enuk = self.cube_enuk.coord(
"forecast_reference_time").copy()
blend_time_enuk.rename("blend_time")
self.cube_enuk.add_aux_coord(blend_time_enuk)
plugin = MergeCubesForWeightedBlending(
"model_id", model_id_attr="mosg__model_configuration")
result = plugin([self.cube_nowcast, self.cube_ukv, self.cube_enuk])
self.assertNotIn("blend_time", get_coord_names(result))
def test_cycle_blend(self):
"""Test merge for blending over forecast_reference_time"""
cube = self.cube_ukv.copy()
cube.coord("forecast_reference_time").points = (
cube.coord("forecast_reference_time").points + 3600)
cube.coord("forecast_period").points = (
cube.coord("forecast_reference_time").points - 3600)
plugin = MergeCubesForWeightedBlending("forecast_reference_time")
result = plugin.process([self.cube_ukv, cube])
self.assertIsInstance(result, iris.cube.Cube)
self.assertIn(result.coord("forecast_reference_time"),
result.coords(dim_coords=True))
# check no model coordinates have been added
with self.assertRaises(iris.exceptions.CoordinateNotFoundError):
result.coord(MODEL_BLEND_COORD)
with self.assertRaises(iris.exceptions.CoordinateNotFoundError):
result.coord(MODEL_NAME_COORD)
def test_time_bounds_mismatch(self):
"""Test failure for cycle blending when time bounds ranges are not
matched (ie cycle blending different "accumulation periods")"""
cube2 = self.cube_ukv.copy()
cube2.coord("forecast_reference_time").points = (
cube2.coord("forecast_reference_time").points + 3600)
cube2.coord("time").bounds = [
cube2.coord("time").bounds[0, 0] + 3600,
cube2.coord("time").bounds[0, 1],
]
cube2.coord("forecast_period").bounds = [
cube2.coord("forecast_period").bounds[0, 0] + 3600,
cube2.coord("forecast_period").bounds[0, 1],
]
msg = "Cube with mismatching time bounds ranges cannot be blended"
with self.assertRaisesRegex(ValueError, msg):
MergeCubesForWeightedBlending("forecast_reference_time").process(
[self.cube_ukv, cube2])
def test_blend_coord_ascending(self):
"""Test the order of the output blend coordinate is always ascending,
independent of the input cube order"""
frt = self.cube_ukv.coord("forecast_reference_time").points[0]
fp = self.cube_ukv.coord("forecast_period").points[0]
cube1 = self.cube_ukv.copy()
cube1.coord("forecast_reference_time").points = [frt + 3600]
cube1.coord("forecast_period").points = [fp - 3600]
cube2 = self.cube_ukv.copy()
cube2.coord("forecast_reference_time").points = [frt + 7200]
cube2.coord("forecast_period").points = [fp - 7200]
# input unordered cubes; expect ordered output
expected_points = np.array([frt, frt + 3600, frt + 7200],
dtype=np.int64)
plugin = MergeCubesForWeightedBlending("forecast_reference_time")
result = plugin.process([cube1, self.cube_ukv, cube2])
self.assertArrayEqual(
result.coord("forecast_reference_time").points, expected_points)
def process(
self,
cubelist: Union[List[Cube], CubeList],
cycletime: Optional[str] = None,
model_id_attr: Optional[str] = None,
spatial_weights: bool = False,
fuzzy_length: float = 20000,
attributes_dict: Optional[Dict[str, str]] = None,
) -> Cube:
"""
Merge a cubelist, calculate appropriate blend weights and compute the
weighted mean. Returns a single cube collapsed over the dimension
given by self.blend_coord.
Args:
cubelist:
List of cubes to be merged and blended
cycletime:
The forecast reference time to be used after blending has been
applied, in the format YYYYMMDDTHHMMZ. If not provided, the
blended file takes the latest available forecast reference time
from the input datasets supplied.
model_id_attr:
The name of the dataset attribute to be used to identify the source
model when blending data from different models.
spatial_weights:
If True, this option will result in the generation of spatially
varying weights based on the masks of the data we are blending.
The one dimensional weights are first calculated using the chosen
weights calculation method, but the weights will then be adjusted
spatially based on where there is masked data in the data we are
blending. The spatial weights are calculated using the
SpatiallyVaryingWeightsFromMask plugin.
fuzzy_length:
When calculating spatially varying weights we can smooth the
weights so that areas close to areas that are masked have lower
weights than those further away. This fuzzy length controls the
scale over which the weights are smoothed. The fuzzy length is in
terms of m, the default is 20km. This distance is then converted
into a number of grid squares, which does not have to be an
integer. Assumes the grid spacing is the same in the x and y
directions and raises an error if this is not true. See
SpatiallyVaryingWeightsFromMask for more details.
attributes_dict:
Dictionary describing required changes to attributes after blending
Returns:
Cube of blended data.
Warns:
UserWarning: If blending masked data without spatial weights.
This has not been fully tested.
"""
# Prepare cubes for weighted blending, including creating custom metadata
# for multi-model blending. The merged cube has a monotonically ascending
# blend coordinate. Plugin raises an error if blend_coord is not present on
# all input cubes.
merger = MergeCubesForWeightedBlending(
self.blend_coord,
weighting_coord=self.weighting_coord,
model_id_attr=model_id_attr,
)
cube = merger(cubelist, cycletime=cycletime)
if "model" in self.blend_coord:
self.blend_coord = copy(MODEL_BLEND_COORD)
coords_to_remove = get_coords_to_remove(cube, self.blend_coord)
if len(cube.coord(self.blend_coord).points) > 1:
weights = self._calculate_blending_weights(cube)
cube, weights = self._remove_zero_weighted_slices(cube, weights)
# Deal with case of only one input cube or non-zero-weighted slice
if len(cube.coord(self.blend_coord).points) == 1:
result = cube
else:
if spatial_weights:
weights = self._update_spatial_weights(cube, weights,
fuzzy_length)
elif np.ma.is_masked(cube.data):
# Raise warning if blending masked arrays using non-spatial weights.
warnings.warn(
"Blending masked data without spatial weights has not been"
" fully tested.")
# Blend across specified dimension
BlendingPlugin = WeightedBlendAcrossWholeDimension(
self.blend_coord)
result = BlendingPlugin(cube, weights=weights)
# Remove custom metadata and and update time-type coordinates. Remove
# non-time-type coordinate that were previously associated with the blend
# dimension (coords_to_remove). Add user-specified and standard blend
# attributes.
update_blended_metadata(
result,
self.blend_coord,
coords_to_remove=coords_to_remove,
cycletime=cycletime,
attributes_dict=attributes_dict,
model_id_attr=model_id_attr,
)
return result
def main(argv=None):
"""Load in arguments and ensure they are set correctly.
Then load in the data to blend and calculate default weights
using the method chosen before carrying out the blending."""
parser = ArgParser(
description='Calculate the default weights to apply in weighted '
'blending plugins using the ChooseDefaultWeightsLinear or '
'ChooseDefaultWeightsNonLinear plugins. Then apply these '
'weights to the dataset using the BasicWeightedAverage plugin.'
' Required for ChooseDefaultWeightsLinear: y0val and ynval.'
' Required for ChooseDefaultWeightsNonLinear: cval.'
' Required for ChooseWeightsLinear with dict: wts_dict.')
parser.add_argument('--wts_calc_method',
metavar='WEIGHTS_CALCULATION_METHOD',
choices=['linear', 'nonlinear', 'dict'],
default='linear',
help='Method to use to calculate '
'weights used in blending. "linear" (default): '
'calculate linearly varying blending weights. '
'"nonlinear": calculate blending weights that decrease'
' exponentially with increasing blending coordinate. '
'"dict": calculate weights using a dictionary passed '
'in as a command line argument.')
parser.add_argument('coordinate',
type=str,
metavar='COORDINATE_TO_AVERAGE_OVER',
help='The coordinate over which the blending '
'will be applied.')
parser.add_argument('--coordinate_unit',
metavar='UNIT_STRING',
default='hours since 1970-01-01 00:00:00',
help='Units for blending coordinate. Default= '
'hours since 1970-01-01 00:00:00')
parser.add_argument('--calendar',
metavar='CALENDAR',
help='Calendar for time coordinate. Default=gregorian')
parser.add_argument('--cycletime',
metavar='CYCLETIME',
type=str,
help='The forecast reference time to be used after '
'blending has been applied, in the format '
'YYYYMMDDTHHMMZ. If not provided, the blended file '
'will take the latest available forecast reference '
'time from the input cubes supplied.')
parser.add_argument('--model_id_attr',
metavar='MODEL_ID_ATTR',
type=str,
default="mosg__model_configuration",
help='The name of the netCDF file attribute to be '
'used to identify the source model for '
'multi-model blends. Default assumes Met Office '
'model metadata. Must be present on all input '
'files if blending over models.')
parser.add_argument('--spatial_weights_from_mask',
action='store_true',
default=False,
help='If set this option will result in the generation'
' of spatially varying weights based on the'
' masks of the data we are blending. The'
' one dimensional weights are first calculated '
' using the chosen weights calculation method,'
' but the weights will then be adjusted spatially'
' based on where there is masked data in the data'
' we are blending. The spatial weights are'
' calculated using the'
' SpatiallyVaryingWeightsFromMask plugin.')
parser.add_argument('weighting_mode',
metavar='WEIGHTED_BLEND_MODE',
choices=['weighted_mean', 'weighted_maximum'],
help='The method used in the weighted blend. '
'"weighted_mean": calculate a normal weighted'
' mean across the coordinate. '
'"weighted_maximum": multiplies the values in the'
' coordinate by the weights, and then takes the'
' maximum.')
parser.add_argument('input_filepaths',
metavar='INPUT_FILES',
nargs="+",
help='Paths to input files to be blended.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
spatial = parser.add_argument_group(
'Spatial weights from mask options',
'Options for calculating the spatial weights using the '
'SpatiallyVaryingWeightsFromMask plugin.')
spatial.add_argument('--fuzzy_length',
metavar='FUZZY_LENGTH',
type=float,
default=20000,
help='When calculating spatially varying weights we'
' can smooth the weights so that areas close to'
' areas that are masked have lower weights than'
' those further away. This fuzzy length controls'
' the scale over which the weights are smoothed.'
' The fuzzy length is in terms of m, the'
' default is 20km. This distance is then'
' converted into a number of grid squares,'
' which does not have to be an integer. Assumes'
' the grid spacing is the same in the x and y'
' directions, and raises an error if this is not'
' true. See SpatiallyVaryingWeightsFromMask for'
' more detail.')
linear = parser.add_argument_group(
'linear weights options', 'Options for the linear weights '
'calculation in '
'ChooseDefaultWeightsLinear')
linear.add_argument('--y0val',
metavar='LINEAR_STARTING_POINT',
type=float,
help='The relative value of the weighting start point '
'(lowest value of blend coord) for choosing default '
'linear weights. This must be a positive float or 0.')
linear.add_argument('--ynval',
metavar='LINEAR_END_POINT',
type=float,
help='The relative value of the weighting '
'end point (highest value of blend coord) for choosing'
' default linear weights. This must be a positive '
'float or 0. Note that if blending over forecast '
'reference time, ynval >= y0val would normally be '
'expected (to give greater weight to the more recent '
'forecast).')
nonlinear = parser.add_argument_group(
'nonlinear weights options', 'Options for the non-linear '
'weights calculation in '
'ChooseDefaultWeightsNonLinear')
nonlinear.add_argument('--cval',
metavar='NON_LINEAR_FACTOR',
type=float,
help='Factor used to determine how skewed the '
'non linear weights will be. '
'A value of 1 implies equal weighting. If not '
'set, a default value of cval=0.85 is set.')
wts_dict = parser.add_argument_group(
'dict weights options', 'Options for linear weights to be '
'calculated based on parameters '
'read from a json file dict')
wts_dict.add_argument('--wts_dict',
metavar='WEIGHTS_DICTIONARY',
help='Path to json file containing dictionary from '
'which to calculate blending weights. Dictionary '
'format is as specified in the improver.blending.'
'weights.ChooseWeightsLinear plugin.')
wts_dict.add_argument('--weighting_coord',
metavar='WEIGHTING_COORD',
default='forecast_period',
help='Name of '
'coordinate over which linear weights should be '
'scaled. This coordinate must be avilable in the '
'weights dictionary.')
args = parser.parse_args(args=argv)
# if the linear weights method is called with non-linear args or vice
# versa, exit with error
if (args.wts_calc_method == "linear") and args.cval:
parser.wrong_args_error('cval', 'linear')
if ((args.wts_calc_method == "nonlinear")
and np.any([args.y0val, args.ynval])):
parser.wrong_args_error('y0val, ynval', 'non-linear')
if (args.wts_calc_method == "dict") and not args.wts_dict:
parser.error('Dictionary is required if --wts_calc_method="dict"')
# set blending coordinate units
if "time" in args.coordinate:
coord_unit = Unit(args.coordinate_unit, args.calendar)
elif args.coordinate_unit != 'hours since 1970-01-01 00:00:00.':
coord_unit = args.coordinate_unit
else:
coord_unit = 'no_unit'
# For blending across models, only blending across "model_id" is directly
# supported. This is because the blending coordinate must be sortable, in
# order to ensure that the data cube and the weights cube have coordinates
# in the same order for blending. Whilst the model_configuration is
# sortable itself, as it is associated with model_id, which is the
# dimension coordinate, sorting the model_configuration coordinate can
# result in the model_id coordinate becoming non-monotonic. As dimension
# coordinates must be monotonic, this leads to the model_id coordinate
# being demoted to an auxiliary coordinate. Therefore, for simplicity
# model_id is used as the blending coordinate, instead of
# model_configuration.
# TODO: Support model_configuration as a blending coordinate directly.
if args.coordinate == "model_configuration":
blend_coord = "model_id"
dict_coord = "model_configuration"
else:
blend_coord = args.coordinate
dict_coord = args.coordinate
# load cubes to be blended
cubelist = load_cubelist(args.input_filepaths)
# determine whether or not to equalise forecast periods for model
# blending weights calculation
weighting_coord = (args.weighting_coord
if args.weighting_coord else "forecast_period")
# prepare cubes for weighted blending
merger = MergeCubesForWeightedBlending(blend_coord,
weighting_coord=weighting_coord,
model_id_attr=args.model_id_attr)
cube = merger.process(cubelist, cycletime=args.cycletime)
# if the coord for blending does not exist or has only one value,
# update metadata only
coord_names = [coord.name() for coord in cube.coords()]
if (blend_coord not in coord_names) or (len(
cube.coord(blend_coord).points) == 1):
result = cube.copy()
conform_metadata(result, cube, blend_coord, cycletime=args.cycletime)
# raise a warning if this happened because the blend coordinate
# doesn't exist
if blend_coord not in coord_names:
warnings.warn('Blend coordinate {} is not present on input '
'data'.format(blend_coord))
# otherwise, calculate weights and blend across specified dimension
else:
weights = calculate_blending_weights(
cube,
blend_coord,
args.wts_calc_method,
wts_dict=args.wts_dict,
weighting_coord=args.weighting_coord,
coord_unit=coord_unit,
y0val=args.y0val,
ynval=args.ynval,
cval=args.cval,
dict_coord=dict_coord)
if args.spatial_weights_from_mask:
check_if_grid_is_equal_area(cube)
grid_cells_x, _ = convert_distance_into_number_of_grid_cells(
cube, args.fuzzy_length, int_grid_cells=False)
SpatialWeightsPlugin = SpatiallyVaryingWeightsFromMask(
grid_cells_x)
weights = SpatialWeightsPlugin.process(cube, weights, blend_coord)
# blend across specified dimension
BlendingPlugin = WeightedBlendAcrossWholeDimension(
blend_coord, args.weighting_mode, cycletime=args.cycletime)
result = BlendingPlugin.process(cube, weights=weights)
save_netcdf(result, args.output_filepath)
def process(
self,
cubelist,
cycletime=None,
model_id_attr=None,
spatial_weights=False,
fuzzy_length=20000,
attributes_dict=None,
):
"""
Merge a cubelist, calculate appropriate blend weights and compute the
weighted mean. Returns a single cube collapsed over the dimension
given by self.blend_coord.
Args:
cubelist (iris.cube.CubeList):
List of cubes to be merged and blended
cycletime (str):
Forecast reference time to use for output cubes, in the format
YYYYMMDDTHHMMZ. If not set, the latest of the input cube
forecast reference times is used.
model_id_attr (str):
Name of the attribute by which to identify the source model and
construct "model" coordinates for blending.
spatial_weights (bool):
If true, calculate spatial weights.
fuzzy_length (float):
Distance (in metres) over which to smooth spatial weights.
Default is 20 km.
attributes_dict (dict or None):
Changes to cube attributes to be applied after blending
Returns:
iris.cube.Cube:
Cube of blended data.
Warns:
UserWarning: If blending masked data without spatial weights.
This has not been fully tested.
"""
# Prepare cubes for weighted blending, including creating custom metadata
# for multi-model blending. The merged cube has a monotonically ascending
# blend coordinate. Plugin raises an error if blend_coord is not present on
# all input cubes.
merger = MergeCubesForWeightedBlending(
self.blend_coord,
weighting_coord=self.weighting_coord,
model_id_attr=model_id_attr,
)
cube = merger(cubelist, cycletime=cycletime)
if "model" in self.blend_coord:
self.blend_coord = copy(MODEL_BLEND_COORD)
coords_to_remove = get_coords_to_remove(cube, self.blend_coord)
if len(cube.coord(self.blend_coord).points) > 1:
weights = self._calculate_blending_weights(cube)
cube, weights = self._remove_zero_weighted_slices(cube, weights)
# Deal with case of only one input cube or non-zero-weighted slice
if len(cube.coord(self.blend_coord).points) == 1:
result = cube
else:
if spatial_weights:
weights = self._update_spatial_weights(cube, weights,
fuzzy_length)
elif np.ma.is_masked(cube.data):
# Raise warning if blending masked arrays using non-spatial weights.
warnings.warn(
"Blending masked data without spatial weights has not been"
" fully tested.")
# Blend across specified dimension
BlendingPlugin = WeightedBlendAcrossWholeDimension(
self.blend_coord)
result = BlendingPlugin(cube, weights=weights)
# Remove custom metadata and and update time-type coordinates. Remove
# non-time-type coordinate that were previously associated with the blend
# dimension (coords_to_remove). Add user-specified and standard blend
# attributes.
update_blended_metadata(
result,
self.blend_coord,
coords_to_remove=coords_to_remove,
cycletime=cycletime,
attributes_dict=attributes_dict,
model_id_attr=model_id_attr,
)
return result
def test_error_missing_model_id_attr(self):
"""Test exception is raised if blending over model with no identifying
attribute"""
msg = "model_id_attr required to blend over model_id"
with self.assertRaisesRegex(ValueError, msg):
MergeCubesForWeightedBlending("model_id")
def test_basic(self):
"""Test default initialisation"""
plugin = MergeCubesForWeightedBlending("realization")
self.assertEqual(plugin.blend_coord, "realization")
self.assertIsNone(plugin.weighting_coord)
self.assertIsNone(plugin.model_id_attr)
def test_blend_coord_not_present(self):
"""Test exception when blend coord is not present on inputs"""
msg = "realization coordinate is not present on all input cubes"
with self.assertRaisesRegex(ValueError, msg):
MergeCubesForWeightedBlending("realization").process(self.cubelist)
def test_null_irrelevant_coord(self):
"""Test function does nothing if not given a relevant coord"""
reference_cubelist = self.cubelist.copy()
plugin = MergeCubesForWeightedBlending("realization")
plugin._rationalise_blend_time_coords(self.cubelist)
self.assertEqual(self.cubelist, reference_cubelist)
def process(
self,
cubelist,
cycletime=None,
model_id_attr=None,
spatial_weights=False,
fuzzy_length=20000,
attributes_dict=None,
):
"""
Merge a cubelist, calculate appropriate blend weights and compute the
weighted mean. Returns a single cube collapsed over the dimension
given by self.blend_coord.
Args:
cubelist (iris.cube.CubeList):
List of cubes to be merged and blended
cycletime (str):
Forecast reference time to use for output cubes, in the format
YYYYMMDDTHHMMZ. If not set, the latest of the input cube
forecast reference times is used.
model_id_attr (str):
Name of the attribute by which to identify the source model and
construct "model" coordinates for blending.
spatial_weights (bool):
If true, calculate spatial weights.
fuzzy_length (float):
Distance (in metres) over which to smooth spatial weights.
Default is 20 km.
attributes_dict (dict or None):
Changes to cube attributes to be applied after blending
Warns:
UserWarning: If blending masked data without spatial weights.
This has not been fully tested.
"""
# Prepare cubes for weighted blending, including creating model_id and
# model_configuration coordinates for multi-model blending. The merged
# cube has a monotonically ascending blend coordinate. Plugin raises an
# error if blend_coord is not present on all input cubes.
merger = MergeCubesForWeightedBlending(
self.blend_coord,
weighting_coord=self.weighting_coord,
model_id_attr=model_id_attr,
)
cube = merger(cubelist, cycletime=cycletime)
# if blend_coord has only one value (for example cycle blending with
# only one cycle available), or is not present (case where only
# one model has been provided for a model blend), update attributes
# and ensure that the forecast reference time on the returned cube
# is set to the current IMPROVER processing cycle.
coord_names = [coord.name() for coord in cube.coords()]
if (self.blend_coord not in coord_names
or len(cube.coord(self.blend_coord).points) == 1):
result = cube.copy()
if attributes_dict is not None:
amend_attributes(result, attributes_dict)
(result, ) = rebadge_forecasts_as_latest_cycle([result], cycletime)
# otherwise, calculate weights and blend across specified dimension
else:
# set up special treatment for model blending
if "model" in self.blend_coord:
self.blend_coord = "model_id"
# calculate blend weights
weights = self._calculate_blending_weights(cube)
if spatial_weights:
weights = self._update_spatial_weights(cube, weights,
fuzzy_length)
elif np.ma.is_masked(cube.data):
# Raise warning if blending masked arrays using non-spatial weights.
warnings.warn(
"Blending masked data without spatial weights has not been"
" fully tested.")
# blend across specified dimension
BlendingPlugin = WeightedBlendAcrossWholeDimension(
self.blend_coord)
result = BlendingPlugin(
cube,
weights=weights,
cycletime=cycletime,
attributes_dict=attributes_dict,
)
return result
def process(self,
cubelist,
cycletime=None,
model_id_attr=None,
spatial_weights=False,
fuzzy_length=20000):
"""
Merge a cubelist, calculate appropriate blend weights and compute the
weighted mean. Returns a single cube collapsed over the dimension
given by self.blend_coord.
Args:
cubelist (iris.cube.CubeList):
List of cubes to be merged and blended
Kwargs:
cycletime (str):
Forecast reference time to use for output cubes, in the format
YYYYMMDDTHHMMZ. If not set, the latest of the input cube
forecast reference times is used.
model_id_attr (str):
Name of the attribute by which to identify the source model and
construct "model" coordinates for blending.
spatial_weights (bool):
If true, calculate spatial weights.
fuzzy_length (float):
Distance (in metres) over which to smooth spatial weights.
Default is 20 km.
"""
# Prepare cubes for weighted blending, including creating model_id and
# model_configuration coordinates for multi-model blending. The merged
# cube has a monotonically ascending blend coordinate. Plugin raises an
# error if blend_coord is not present on all input cubes.
merger = MergeCubesForWeightedBlending(
self.blend_coord,
weighting_coord=self.weighting_coord,
model_id_attr=model_id_attr)
cube = merger.process(cubelist, cycletime=cycletime)
# if blend_coord has only one value, or is not present (case where only
# one model has been provided for a model blend) update metadata only
coord_names = [coord.name() for coord in cube.coords()]
if (self.blend_coord not in coord_names
or len(cube.coord(self.blend_coord).points) == 1):
result = cube.copy()
conform_metadata(result,
cube,
self.blend_coord,
cycletime=cycletime)
# otherwise, calculate weights and blend across specified dimension
else:
# set up special treatment for model blending
if "model" in self.blend_coord:
self.blend_coord = "model_id"
# calculate blend weights
weights = self._calculate_blending_weights(cube)
if spatial_weights:
weights = self._update_spatial_weights(cube, weights,
fuzzy_length)
# blend across specified dimension
BlendingPlugin = WeightedBlendAcrossWholeDimension(
self.blend_coord, cycletime=cycletime)
result = BlendingPlugin.process(cube, weights=weights)
return result