def test_error_range_greater_than_domain_size(self):
"""Test correct exception raised when the distance is larger than the
corner-to-corner distance of the domain."""
distance = 42500.0
msg = "Distance of 42500.0m exceeds max domain distance of "
with self.assertRaisesRegex(ValueError, msg):
convert_distance_into_number_of_grid_cells(self.cube, distance)
def test_single_point_range_negative(self):
"""Test behaviour with a non-zero point with negative range."""
distance = -1.0 * self.DISTANCE
msg = "distance of -6100.0m gives a negative cell extent"
with self.assertRaisesRegexp(ValueError, msg):
convert_distance_into_number_of_grid_cells(
self.cube, distance, self.MAX_DISTANCE_IN_GRID_CELLS)
def test_single_point_lat_long(self):
"""Test behaviour for a single grid cell on lat long grid."""
cube = set_up_cube_lat_long()
msg = "Invalid grid: projection_x/y coords required"
with self.assertRaisesRegexp(ValueError, msg):
convert_distance_into_number_of_grid_cells(
cube, self.DISTANCE, self.MAX_DISTANCE_IN_GRID_CELLS)
def test_single_point_range_0(self):
"""Test behaviour with a non-zero point with zero range."""
distance = 5
msg = "Distance of 5m gives zero cell extent"
with self.assertRaisesRegexp(ValueError, msg):
convert_distance_into_number_of_grid_cells(
self.cube, distance, self.MAX_DISTANCE_IN_GRID_CELLS)
def test_single_point_range_greater_than_domain(self):
"""Test correct exception raised when the distance is larger than the
corner-to-corner distance of the domain."""
distance = 42500.0
msg = "Distance of 42500.0m exceeds max domain distance of "
with self.assertRaisesRegexp(ValueError, msg):
convert_distance_into_number_of_grid_cells(
self.cube, distance, self.MAX_DISTANCE_IN_GRID_CELLS)
def test_single_point_range_lots(self):
"""Test behaviour with a non-zero point with unhandleable range."""
distance = 40000.0
max_distance_in_grid_cells = 10
msg = "distance of 40000.0m exceeds maximum grid cell extent"
with self.assertRaisesRegexp(ValueError, msg):
convert_distance_into_number_of_grid_cells(
self.cube, distance, max_distance_in_grid_cells)
def test_error_outside_maximum_distance(self):
"""Test behaviour with a non-zero point with unhandleable range."""
distance = 40000.0
max_distance_in_grid_cells = 10
msg = "Distance of 40000.0m exceeds maximum permitted"
with self.assertRaisesRegex(ValueError, msg):
convert_distance_into_number_of_grid_cells(
self.cube,
distance,
max_distance_in_grid_cells=max_distance_in_grid_cells)
def test_basic_distance_to_grid_cells_km_grid(self):
"""Test the distance-to-grid-cell conversion, grid in km."""
self.cube.coord("projection_x_coordinate").convert_units("kilometres")
self.cube.coord("projection_y_coordinate").convert_units("kilometres")
result = convert_distance_into_number_of_grid_cells(
self.cube, self.DISTANCE, self.MAX_DISTANCE_IN_GRID_CELLS)
self.assertEqual(result, (3, 3))
def test_basic_distance_to_grid_cells(self):
"""Test the distance in metres to grid cell conversion."""
result = convert_distance_into_number_of_grid_cells(
self.cube,
self.DISTANCE,
max_distance_in_grid_cells=self.MAX_DISTANCE_IN_GRID_CELLS)
self.assertEqual(result, (3, 3))
def run(self, cube, radius, mask_cube=None):
"""
Call the methods required to apply a square neighbourhood
method to a cube.
The steps undertaken are:
1. Set up cubes by determining, if the arrays are masked.
2. Pad the input array with a halo and then calculate the neighbourhood
of the haloed array.
3. Remove the halo from the neighbourhooded array and deal with a mask,
if required.
Args:
cube (iris.cube.Cube):
Cube containing the array to which the square neighbourhood
will be applied.
radius (float):
Radius in metres for use in specifying the number of
grid cells used to create a square neighbourhood.
mask_cube (iris.cube.Cube):
Cube containing the array to be used as a mask.
Returns:
neighbourhood_averaged_cube (iris.cube.Cube):
Cube containing the smoothed field after the square
neighbourhood method has been applied.
"""
# If the data is masked, the mask will be processed as well as the
# original_data * mask array.
original_attributes = cube.attributes
original_methods = cube.cell_methods
grid_cells_x = (convert_distance_into_number_of_grid_cells(
cube, radius, max_distance_in_grid_cells=MAX_RADIUS_IN_GRID_CELLS))
grid_cells_y = grid_cells_x
result_slices = iris.cube.CubeList()
for cube_slice in cube.slices(
[cube.coord(axis='y'), cube.coord(axis='x')]):
(cube_slice, mask,
nan_array) = (self.set_up_cubes_to_be_neighbourhooded(
cube_slice, mask_cube))
neighbourhood_averaged_cube = (
self._pad_and_calculate_neighbourhood(cube_slice, mask,
grid_cells_x,
grid_cells_y))
neighbourhood_averaged_cube = (self._remove_padding_and_mask(
neighbourhood_averaged_cube, cube_slice, mask, grid_cells_x,
grid_cells_y))
neighbourhood_averaged_cube.data[nan_array.astype(bool)] = np.nan
result_slices.append(neighbourhood_averaged_cube)
neighbourhood_averaged_cube = result_slices.merge_cube()
neighbourhood_averaged_cube.cell_methods = original_methods
neighbourhood_averaged_cube.attributes = original_attributes
neighbourhood_averaged_cube = check_cube_coordinates(
cube, neighbourhood_averaged_cube)
return neighbourhood_averaged_cube
def test_max_distance(self):
"""
Test the distance in metres to grid cell conversion within a maximum
distance in grid cells.
"""
result = convert_distance_into_number_of_grid_cells(
self.cube, self.DISTANCE, max_distance_in_grid_cells=50)
self.assertEqual(result, 3)
def remove_cube_halo(cube, halo_radius):
"""
Remove halo of halo_radius from a cube.
This function converts the halo radius into
the number of grid points in the x and y coordinate
that need to be removed. It then calls remove_halo_from_cube
which only acts on a cube with x and y coordinates so we
need to slice the cube and them merge the cube back together
ensuring the resulting cube has the same dimension coordinates.
Args:
cube (iris.cube.Cube):
Cube on extended grid
halo_radius (float):
Size of border to remove, in metres
Returns:
iris.cube.Cube:
New cube with the halo removed.
"""
halo_size_x = convert_distance_into_number_of_grid_cells(cube,
halo_radius,
axis='x')
halo_size_y = convert_distance_into_number_of_grid_cells(cube,
halo_radius,
axis='y')
result_slices = iris.cube.CubeList()
for cube_slice in cube.slices([cube.coord(axis='y'),
cube.coord(axis='x')]):
cube_halo = remove_halo_from_cube(cube_slice, halo_size_x, halo_size_y)
result_slices.append(cube_halo)
result = result_slices.merge_cube()
# re-promote any scalar dimensions lost in slice / merge
req_dims = [coord.name() for coord in cube.coords(dim_coords=True)]
present_dims = [coord.name() for coord in result.coords(dim_coords=True)]
for coord in req_dims:
if coord not in present_dims:
result = iris.util.new_axis(result, coord)
# re-order (needed if scalar dimensions have been re-added)
enforce_coordinate_ordering(result, req_dims)
return result
def test_distance_to_grid_cells_other_axis(self):
"""Test the distance in metres to grid cell conversion along the
y-axis."""
self.cube.coord(
axis='y').points = 0.5 * self.cube.coord(axis='y').points
result = convert_distance_into_number_of_grid_cells(self.cube,
self.DISTANCE,
axis='y')
self.assertEqual(result, 6)
def test_basic_distance_to_grid_cells_different_max_distance(self):
"""
Test the distance in metres to grid cell conversion for an alternative
max distance in grid cells.
"""
max_distance_in_grid_cells = 50
result = convert_distance_into_number_of_grid_cells(
self.cube, self.DISTANCE, max_distance_in_grid_cells)
self.assertEqual(result, (3, 3))
def run(self, cube, radius, mask_cube=None):
"""
Call the methods required to apply a square neighbourhood
method to a cube.
The steps undertaken are:
1. Set up cubes by determining, if the arrays are masked.
2. Pad the input array with a halo and then calculate the neighbourhood
of the haloed array.
3. Remove the halo from the neighbourhooded array and deal with a mask,
if required.
Args:
cube (Iris.cube.Cube):
Cube containing the array to which the square neighbourhood
will be applied.
radius (Float):
Radius in metres for use in specifying the number of
grid cells used to create a square neighbourhood.
Keyword Args:
mask_cube (Iris.cube.Cube):
Cube containing the array to be used as a mask.
Returns:
neighbourhood_averaged_cube (Iris.cube.Cube):
Cube containing the smoothed field after the square
neighbourhood method has been applied.
"""
# If the data is masked, the mask will be processed as well as the
# original_data * mask array.
original_attributes = cube.attributes
original_methods = cube.cell_methods
grid_cells_x, grid_cells_y = (
convert_distance_into_number_of_grid_cells(
cube, radius, MAX_RADIUS_IN_GRID_CELLS))
cubes_to_sum = (self._set_up_cubes_to_be_neighbourhooded(
cube, mask_cube))
neighbourhood_averaged_cubes = (self._pad_and_calculate_neighbourhood(
cubes_to_sum, grid_cells_x, grid_cells_y))
neighbourhood_averaged_cube = (self._remove_padding_and_mask(
neighbourhood_averaged_cubes, cubes_to_sum, cube.name(),
grid_cells_x, grid_cells_y))
neighbourhood_averaged_cube.cell_methods = original_methods
neighbourhood_averaged_cube.attributes = original_attributes
neighbourhood_averaged_cube = check_cube_coordinates(
cube, neighbourhood_averaged_cube)
return neighbourhood_averaged_cube
def remove_cube_halo(cube, halo_radius):
"""
Remove halo of halo_radius from a cube.
This function converts the halo radius into
the number of grid points in the x and y coordinate
that need to be removed. It then calls remove_halo_from_cube
which only acts on a cube with x and y coordinates so we
need to slice the cube and them merge the cube back together
ensuring the resulting cube has the same dimension coordinates.
Args:
cube (iris.cube.Cube):
Cube on extended grid
halo_radius (float):
Size of border to remove, in metres
Returns:
result (iris.cube.Cube):
New cube with the halo removed.
"""
halo_size_x, halo_size_y = convert_distance_into_number_of_grid_cells(
cube, halo_radius)
result_slices = iris.cube.CubeList()
for cube_slice in cube.slices([cube.coord(axis='y'),
cube.coord(axis='x')]):
cube_halo = remove_halo_from_cube(cube_slice,
halo_size_x,
halo_size_y)
result_slices.append(cube_halo)
result = result_slices.merge_cube()
req_coords = []
for coord in cube.coords(dim_coords=True):
req_coords.append(coord.name())
result = enforce_coordinate_ordering(
result, req_coords, promote_scalar=True)
return result
def _update_spatial_weights(self, cube, weights, fuzzy_length):
"""
Update weights using spatial information
Args:
cube (iris.cube.Cube):
Cube of input data to be blended
weights (iris.cube.Cube):
Initial 1D cube of weights scaled by self.weighting_coord
fuzzy_length (float):
Distance (in metres) over which to smooth weights at domain
boundaries
Returns:
weights (iris.cube.Cube):
Updated 3D cube of spatially-varying weights
"""
check_if_grid_is_equal_area(cube)
grid_cells_x, _ = convert_distance_into_number_of_grid_cells(
cube, fuzzy_length, int_grid_cells=False)
SpatialWeightsPlugin = SpatiallyVaryingWeightsFromMask(grid_cells_x)
weights = SpatialWeightsPlugin.process(cube, weights, self.blend_coord)
return weights
def test_basic_no_limit(self):
"""Test the distance in metres to grid cell conversion still works when
the maximum distance limit is not explicitly set."""
result = convert_distance_into_number_of_grid_cells(
self.cube, self.DISTANCE)
self.assertEqual(result, (3, 3))
def main(argv=None):
"""Load in arguments and ensure they are set correctly.
Then load in the data to blend and calculate default weights
using the method chosen before carrying out the blending."""
parser = ArgParser(
description='Calculate the default weights to apply in weighted '
'blending plugins using the ChooseDefaultWeightsLinear or '
'ChooseDefaultWeightsNonLinear plugins. Then apply these '
'weights to the dataset using the BasicWeightedAverage plugin.'
' Required for ChooseDefaultWeightsLinear: y0val and ynval.'
' Required for ChooseDefaultWeightsNonLinear: cval.'
' Required for ChooseWeightsLinear with dict: wts_dict.')
parser.add_argument('--wts_calc_method',
metavar='WEIGHTS_CALCULATION_METHOD',
choices=['linear', 'nonlinear', 'dict'],
default='linear',
help='Method to use to calculate '
'weights used in blending. "linear" (default): '
'calculate linearly varying blending weights. '
'"nonlinear": calculate blending weights that decrease'
' exponentially with increasing blending coordinate. '
'"dict": calculate weights using a dictionary passed '
'in as a command line argument.')
parser.add_argument('coordinate',
type=str,
metavar='COORDINATE_TO_AVERAGE_OVER',
help='The coordinate over which the blending '
'will be applied.')
parser.add_argument('--coordinate_unit',
metavar='UNIT_STRING',
default='hours since 1970-01-01 00:00:00',
help='Units for blending coordinate. Default= '
'hours since 1970-01-01 00:00:00')
parser.add_argument('--calendar',
metavar='CALENDAR',
help='Calendar for time coordinate. Default=gregorian')
parser.add_argument('--cycletime',
metavar='CYCLETIME',
type=str,
help='The forecast reference time to be used after '
'blending has been applied, in the format '
'YYYYMMDDTHHMMZ. If not provided, the blended file '
'will take the latest available forecast reference '
'time from the input cubes supplied.')
parser.add_argument('--model_id_attr',
metavar='MODEL_ID_ATTR',
type=str,
default="mosg__model_configuration",
help='The name of the netCDF file attribute to be '
'used to identify the source model for '
'multi-model blends. Default assumes Met Office '
'model metadata. Must be present on all input '
'files if blending over models.')
parser.add_argument('--spatial_weights_from_mask',
action='store_true',
default=False,
help='If set this option will result in the generation'
' of spatially varying weights based on the'
' masks of the data we are blending. The'
' one dimensional weights are first calculated '
' using the chosen weights calculation method,'
' but the weights will then be adjusted spatially'
' based on where there is masked data in the data'
' we are blending. The spatial weights are'
' calculated using the'
' SpatiallyVaryingWeightsFromMask plugin.')
parser.add_argument('weighting_mode',
metavar='WEIGHTED_BLEND_MODE',
choices=['weighted_mean', 'weighted_maximum'],
help='The method used in the weighted blend. '
'"weighted_mean": calculate a normal weighted'
' mean across the coordinate. '
'"weighted_maximum": multiplies the values in the'
' coordinate by the weights, and then takes the'
' maximum.')
parser.add_argument('input_filepaths',
metavar='INPUT_FILES',
nargs="+",
help='Paths to input files to be blended.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
spatial = parser.add_argument_group(
'Spatial weights from mask options',
'Options for calculating the spatial weights using the '
'SpatiallyVaryingWeightsFromMask plugin.')
spatial.add_argument('--fuzzy_length',
metavar='FUZZY_LENGTH',
type=float,
default=20000,
help='When calculating spatially varying weights we'
' can smooth the weights so that areas close to'
' areas that are masked have lower weights than'
' those further away. This fuzzy length controls'
' the scale over which the weights are smoothed.'
' The fuzzy length is in terms of m, the'
' default is 20km. This distance is then'
' converted into a number of grid squares,'
' which does not have to be an integer. Assumes'
' the grid spacing is the same in the x and y'
' directions, and raises an error if this is not'
' true. See SpatiallyVaryingWeightsFromMask for'
' more detail.')
linear = parser.add_argument_group(
'linear weights options', 'Options for the linear weights '
'calculation in '
'ChooseDefaultWeightsLinear')
linear.add_argument('--y0val',
metavar='LINEAR_STARTING_POINT',
type=float,
help='The relative value of the weighting start point '
'(lowest value of blend coord) for choosing default '
'linear weights. This must be a positive float or 0.')
linear.add_argument('--ynval',
metavar='LINEAR_END_POINT',
type=float,
help='The relative value of the weighting '
'end point (highest value of blend coord) for choosing'
' default linear weights. This must be a positive '
'float or 0. Note that if blending over forecast '
'reference time, ynval >= y0val would normally be '
'expected (to give greater weight to the more recent '
'forecast).')
nonlinear = parser.add_argument_group(
'nonlinear weights options', 'Options for the non-linear '
'weights calculation in '
'ChooseDefaultWeightsNonLinear')
nonlinear.add_argument('--cval',
metavar='NON_LINEAR_FACTOR',
type=float,
help='Factor used to determine how skewed the '
'non linear weights will be. '
'A value of 1 implies equal weighting. If not '
'set, a default value of cval=0.85 is set.')
wts_dict = parser.add_argument_group(
'dict weights options', 'Options for linear weights to be '
'calculated based on parameters '
'read from a json file dict')
wts_dict.add_argument('--wts_dict',
metavar='WEIGHTS_DICTIONARY',
help='Path to json file containing dictionary from '
'which to calculate blending weights. Dictionary '
'format is as specified in the improver.blending.'
'weights.ChooseWeightsLinear plugin.')
wts_dict.add_argument('--weighting_coord',
metavar='WEIGHTING_COORD',
default='forecast_period',
help='Name of '
'coordinate over which linear weights should be '
'scaled. This coordinate must be avilable in the '
'weights dictionary.')
args = parser.parse_args(args=argv)
# if the linear weights method is called with non-linear args or vice
# versa, exit with error
if (args.wts_calc_method == "linear") and args.cval:
parser.wrong_args_error('cval', 'linear')
if ((args.wts_calc_method == "nonlinear")
and np.any([args.y0val, args.ynval])):
parser.wrong_args_error('y0val, ynval', 'non-linear')
if (args.wts_calc_method == "dict") and not args.wts_dict:
parser.error('Dictionary is required if --wts_calc_method="dict"')
# set blending coordinate units
if "time" in args.coordinate:
coord_unit = Unit(args.coordinate_unit, args.calendar)
elif args.coordinate_unit != 'hours since 1970-01-01 00:00:00.':
coord_unit = args.coordinate_unit
else:
coord_unit = 'no_unit'
# For blending across models, only blending across "model_id" is directly
# supported. This is because the blending coordinate must be sortable, in
# order to ensure that the data cube and the weights cube have coordinates
# in the same order for blending. Whilst the model_configuration is
# sortable itself, as it is associated with model_id, which is the
# dimension coordinate, sorting the model_configuration coordinate can
# result in the model_id coordinate becoming non-monotonic. As dimension
# coordinates must be monotonic, this leads to the model_id coordinate
# being demoted to an auxiliary coordinate. Therefore, for simplicity
# model_id is used as the blending coordinate, instead of
# model_configuration.
# TODO: Support model_configuration as a blending coordinate directly.
if args.coordinate == "model_configuration":
blend_coord = "model_id"
dict_coord = "model_configuration"
else:
blend_coord = args.coordinate
dict_coord = args.coordinate
# load cubes to be blended
cubelist = load_cubelist(args.input_filepaths)
# determine whether or not to equalise forecast periods for model
# blending weights calculation
weighting_coord = (args.weighting_coord
if args.weighting_coord else "forecast_period")
# prepare cubes for weighted blending
merger = MergeCubesForWeightedBlending(blend_coord,
weighting_coord=weighting_coord,
model_id_attr=args.model_id_attr)
cube = merger.process(cubelist, cycletime=args.cycletime)
# if the coord for blending does not exist or has only one value,
# update metadata only
coord_names = [coord.name() for coord in cube.coords()]
if (blend_coord not in coord_names) or (len(
cube.coord(blend_coord).points) == 1):
result = cube.copy()
conform_metadata(result, cube, blend_coord, cycletime=args.cycletime)
# raise a warning if this happened because the blend coordinate
# doesn't exist
if blend_coord not in coord_names:
warnings.warn('Blend coordinate {} is not present on input '
'data'.format(blend_coord))
# otherwise, calculate weights and blend across specified dimension
else:
weights = calculate_blending_weights(
cube,
blend_coord,
args.wts_calc_method,
wts_dict=args.wts_dict,
weighting_coord=args.weighting_coord,
coord_unit=coord_unit,
y0val=args.y0val,
ynval=args.ynval,
cval=args.cval,
dict_coord=dict_coord)
if args.spatial_weights_from_mask:
check_if_grid_is_equal_area(cube)
grid_cells_x, _ = convert_distance_into_number_of_grid_cells(
cube, args.fuzzy_length, int_grid_cells=False)
SpatialWeightsPlugin = SpatiallyVaryingWeightsFromMask(
grid_cells_x)
weights = SpatialWeightsPlugin.process(cube, weights, blend_coord)
# blend across specified dimension
BlendingPlugin = WeightedBlendAcrossWholeDimension(
blend_coord, args.weighting_mode, cycletime=args.cycletime)
result = BlendingPlugin.process(cube, weights=weights)
save_netcdf(result, args.output_filepath)
def test_basic_distance_to_grid_cells(self):
"""Test the distance in metres to grid cell conversion along the
x-axis (default)."""
result = convert_distance_into_number_of_grid_cells(
self.cube, self.DISTANCE)
self.assertEqual(result, 3)
def test_basic_distance_to_grid_cells_float(self):
"""Test the distance in metres to grid cell conversion."""
result = convert_distance_into_number_of_grid_cells(
self.cube, self.DISTANCE, int_grid_cells=False)
self.assertEqual(result, 3.05)
def test_error_zero_grid_cell_range(self):
"""Test behaviour with a non-zero point with zero range."""
distance = 5
msg = "Distance of 5m gives zero cell extent"
with self.assertRaisesRegex(ValueError, msg):
convert_distance_into_number_of_grid_cells(self.cube, distance)
def test_error_negative_distance(self):
"""Test behaviour with a non-zero point with negative range."""
distance = -1.0 * self.DISTANCE
msg = "Please specify a positive distance in metres"
with self.assertRaisesRegex(ValueError, msg):
convert_distance_into_number_of_grid_cells(self.cube, distance)
