def shape_weights(cube: Cube, weights: Cube) -> ndarray:
"""
The function shapes weights to match the diagnostic cube. A cube of
weights that vary across the blending coordinate will be broadcast to
match the complete multidimensional cube shape. A multidimensional cube
of weights will be checked to ensure that the coordinate names match
between the two cubes. If they match the order will be enforced and
then the shape will be checked. If the shapes match the weights will be
returned as an array.
Args:
cube:
The data cube on which a coordinate is being blended.
weights:
Cube of blending weights.
Returns:
An array of weights that matches the cube data shape.
Raises:
ValueError: If weights cube coordinates do not match the diagnostic
cube in the case of a multidimensional weights cube.
ValueError: If weights cube shape is not broadcastable to the data
cube shape.
"""
# Check that a multidimensional weights cube has coordinates that match
# the diagnostic cube. Checking names only to not to be too exacting.
weight_dims = get_dim_coord_names(weights)
cube_dims = get_dim_coord_names(cube)
if set(weight_dims) == set(cube_dims):
enforce_coordinate_ordering(weights, cube_dims)
weights_array = weights.data.astype(FLOAT_DTYPE)
else:
# Map array of weights to shape of cube to collapse.
dim_map = []
dim_coords = [coord.name() for coord in weights.dim_coords]
# Loop through dim coords in weights cube and find the dim the
# coord relates to in the cube we are collapsing.
for dim_coord in dim_coords:
try:
dim_map.append(cube.coord_dims(dim_coord)[0])
except CoordinateNotFoundError:
message = (
"{} is a coordinate on the weights cube but it is not "
"found on the cube we are trying to collapse.")
raise ValueError(message.format(dim_coord))
try:
weights_array = iris.util.broadcast_to_shape(
np.array(weights.data, dtype=FLOAT_DTYPE),
cube.shape,
tuple(dim_map),
)
except ValueError:
msg = ("Weights cube is not a compatible shape with the"
" data cube. Weights: {}, Diagnostic: {}".format(
weights.shape, cube.shape))
raise ValueError(msg)
return weights_array
def test_preserves_dimension_order(self):
"""Test order of original cube dimensions is preserved on subsetting"""
self.uk_cube.transpose([1, 0])
expected_dims = get_dim_coord_names(self.uk_cube)
result = subset_data(self.uk_cube, grid_spec=self.grid_spec)
result_dims = get_dim_coord_names(result)
self.assertSequenceEqual(result_dims, expected_dims)
def _create_template_slice(self, cube_to_collapse):
"""
Create a template cube from a slice of the cube we are collapsing.
The slice will be over blend_coord, y and x and will remove any other
dimensions. This means that the resulting spatial weights won't vary in
any other dimension other than the blend_coord. If the mask does
vary in another dimension an error is raised.
Args:
cube_to_collapse (iris.cube.Cube):
The cube that will be collapsed along the blend_coord
using the spatial weights generated using this plugin. Must
be masked where there is invalid data.
Returns:
iris.cube.Cube:
A cube with dimensions blend_coord, y, x, on which to shape the
output weights cube.
Raises:
ValueError: if the blend coordinate is associated with more than
one dimension on the cube to collapse, or no dimension
ValueError: if the mask on cube_to_collapse varies along a
dimension other than the dimension associated with
blend_coord.
"""
self.blend_coord = find_blend_dim_coord(cube_to_collapse,
self.blend_coord)
# Find original dim coords in input cube
original_dim_coords = get_dim_coord_names(cube_to_collapse)
# Slice over required coords
x_coord = cube_to_collapse.coord(axis="x").name()
y_coord = cube_to_collapse.coord(axis="y").name()
coords_to_slice_over = [self.blend_coord, y_coord, x_coord]
if original_dim_coords == coords_to_slice_over:
return cube_to_collapse
# Check mask does not vary over additional dimensions
slices = cube_to_collapse.slices(coords_to_slice_over)
first_slice = next(slices)
if np.ma.is_masked(first_slice.data):
first_mask = first_slice.data.mask
for cube_slice in slices:
if not np.all(cube_slice.data.mask == first_mask):
message = (
"The mask on the input cube can only vary along the "
"blend_coord, differences in the mask were found "
"along another dimension")
raise ValueError(message)
# Remove non-spatial non-blend dimensions, returning a 3D template cube without
# additional scalar coordinates (eg realization)
for coord in original_dim_coords:
if coord not in coords_to_slice_over:
first_slice.remove_coord(coord)
# Return slice template
return first_slice
def test_single_percentile(self):
"""Test dimensions of output at median only"""
collapse_coord = ["realization"]
plugin = PercentileConverter(collapse_coord, percentiles=[50])
result = plugin.process(self.cube)
result_coords = get_coord_names(result)
self.assertNotIn("realization", result_coords)
self.assertIn("percentile", result_coords)
self.assertNotIn("percentile", get_dim_coord_names(result))
def _create_output_cube(self, template, data, points, bounds):
"""
Populates a template cube with data from the integration
Args:
template (iris.cube.Cube):
Copy of upper or lower bounds cube, based on direction of
integration
data (list or numpy.ndarray):
Integrated data
points (list or numpy.ndarray):
Points values for the integrated coordinate. These will not
match the template cube if any slices were skipped in the
integration, and therefore are used to slice the template cube
to match the data array.
bounds (list or numpy.ndarray):
Bounds values for the integrated coordinate
Returns:
iris.cube.Cube
"""
# extract required slices from template cube
template = template.extract(
iris.Constraint(coord_values={
self.coord_name_to_integrate: lambda x: x in points
}))
# re-promote integrated coord to dimension coord if need be
aux_coord_names = [coord.name() for coord in template.aux_coords]
if self.coord_name_to_integrate in aux_coord_names:
template = iris.util.new_axis(template,
self.coord_name_to_integrate)
# order dimensions on the template cube so that the integrated
# coordinate is first (as this is the leading dimension on the
# data array)
enforce_coordinate_ordering(template, self.coord_name_to_integrate)
# generate appropriate metadata for new cube
attributes = generate_mandatory_attributes([template])
coord_dtype = template.coord(self.coord_name_to_integrate).dtype
name, units = self._generate_output_name_and_units()
# create new cube from template
integrated_cube = create_new_diagnostic_cube(name,
units,
template,
attributes,
data=np.array(data))
integrated_cube.coord(self.coord_name_to_integrate).bounds = np.array(
bounds).astype(coord_dtype)
# re-order cube to match dimensions of input cube
ordered_dimensions = get_dim_coord_names(self.input_cube)
enforce_coordinate_ordering(integrated_cube, ordered_dimensions)
return integrated_cube
def test_null_percentiles(self):
"""Test effect of "neutral" emos coefficients in percentile space
(this is small but non-zero due to limited sampling of the
distribution)"""
result = ApplyEMOS()(self.percentiles, self.coefficients, realizations_count=3)
self.assertIn("percentile", get_dim_coord_names(result))
self.assertArrayAlmostEqual(result.data, self.null_percentiles_expected)
self.assertAlmostEqual(
np.mean(result.data), self.null_percentiles_expected_mean
)
def remove_cube_halo(cube, halo_radius):
"""
Remove halo of halo_radius from a cube.
This function converts the halo radius into
the number of grid points in the x and y coordinate
that need to be removed. It then calls remove_halo_from_cube
which only acts on a cube with x and y coordinates so we
need to slice the cube and them merge the cube back together
ensuring the resulting cube has the same dimension coordinates.
Args:
cube (iris.cube.Cube):
Cube on extended grid
halo_radius (float):
Size of border to remove, in metres
Returns:
iris.cube.Cube:
New cube with the halo removed.
"""
halo_size_x = distance_to_number_of_grid_cells(cube, halo_radius, axis="x")
halo_size_y = distance_to_number_of_grid_cells(cube, halo_radius, axis="y")
result_slices = iris.cube.CubeList()
for cube_slice in cube.slices([cube.coord(axis="y"),
cube.coord(axis="x")]):
cube_halo = remove_halo_from_cube(cube_slice, halo_size_x, halo_size_y)
result_slices.append(cube_halo)
result = result_slices.merge_cube()
# re-promote any scalar dimensions lost in slice / merge
req_dims = get_dim_coord_names(cube)
present_dims = get_dim_coord_names(result)
for coord in req_dims:
if coord not in present_dims:
result = iris.util.new_axis(result, coord)
# re-order (needed if scalar dimensions have been re-added)
enforce_coordinate_ordering(result, req_dims)
return result
def test_null_realizations(self):
"""Test effect of "neutral" emos coefficients in realization space"""
expected_mean = np.mean(self.realizations.data)
expected_data = np.array([
np.full((3, 3), 10.433333),
np.full((3, 3), 10.670206),
np.full((3, 3), 10.196461),
])
result = ApplyEMOS()(self.realizations, self.coefficients)
self.assertIn("realization", get_dim_coord_names(result))
self.assertArrayAlmostEqual(result.data, expected_data)
self.assertAlmostEqual(np.mean(result.data), expected_mean)
def test_null_percentiles_truncnorm_standard_shape_parameters(self):
"""Test effect of "neutral" emos coefficients in percentile space
(this is small but non-zero due to limited sampling of the
distribution) for the truncated normal distribution."""
coefficients = iris.cube.CubeList([])
for cube in self.coefficients:
cube.attributes["distribution"] = "truncnorm"
cube.attributes["shape_parameters"] = np.array([0, np.inf], np.float32)
coefficients.append(cube)
result = ApplyEMOS()(self.percentiles, coefficients, realizations_count=3)
self.assertIn("percentile", get_dim_coord_names(result))
self.assertArrayAlmostEqual(result.data, self.null_percentiles_expected)
self.assertAlmostEqual(
np.mean(result.data), self.null_percentiles_expected_mean
)
def test_null_percentiles(self):
"""Test effect of "neutral" emos coefficients in percentile space
(this is small but non-zero due to limited sampling of the
distribution)"""
expected_mean = np.mean(self.percentiles.data)
expected_data = np.array([
np.full((3, 3), 10.265101),
np.full((3, 3), 10.4),
np.full((3, 3), 10.534898),
])
result = ApplyEMOS()(self.percentiles,
self.coefficients,
realizations_count=3)
self.assertIn("percentile", get_dim_coord_names(result))
self.assertArrayAlmostEqual(result.data, expected_data)
self.assertAlmostEqual(np.mean(result.data), expected_mean)
def thin_cube(cube: Cube, thinning_dict: Dict[str, int]) -> Cube:
"""
Thin the coordinate by taking every X points, defined in the thinning dict
as {coordinate: X}
Args:
cube:
The cube containing the coordinates to be thinned.
thinning_dict:
A dictionary of coordinate and the step value, i.e. a step of 2
will skip every other point
Returns:
A cube with thinned coordinates.
"""
coord_names = get_dim_coord_names(cube)
slices = [slice(None, None, None)] * len(coord_names)
for key, val in thinning_dict.items():
slices[coord_names.index(key)] = slice(None, None, val)
return cube[tuple(slices)]
def test_null_percentiles_truncnorm_alternative_shape_parameters(self):
"""Test effect of "neutral" emos coefficients in percentile space
(this is small but non-zero due to limited sampling of the
distribution) for the truncated normal distribution with alternative
shape parameters to show the truncnorm distribution having an effect."""
coefficients = iris.cube.CubeList([])
for cube in self.coefficients:
cube.attributes["distribution"] = "truncnorm"
cube.attributes["shape_parameters"] = np.array([10, np.inf], np.float32)
coefficients.append(cube)
expected_mean = np.mean(self.percentiles.data)
expected_data = np.array(
[
np.full((3, 3), 10.275656),
np.full((3, 3), 10.405704),
np.full((3, 3), 10.5385),
]
)
result = ApplyEMOS()(self.percentiles, coefficients, realizations_count=3)
self.assertIn("percentile", get_dim_coord_names(result))
self.assertArrayAlmostEqual(result.data, expected_data)
self.assertNotAlmostEqual(np.mean(result.data), expected_mean)
def process(self, cube, weights=None):
"""Calculate weighted blend across the chosen coord, for either
probabilistic or percentile data. If there is a percentile
coordinate on the cube, it will blend using the
PercentileBlendingAggregator but the percentile coordinate must
have at least two points.
Args:
cube (iris.cube.Cube):
Cube to blend across the coord.
weights (iris.cube.Cube):
Cube of blending weights. This will have 1 or 3 dimensions,
corresponding either to blend dimension on the input cube with or
without and additional 2 spatial dimensions. If None, the input cube
is blended with equal weights across the blending dimension.
Returns:
iris.cube.Cube:
Containing the weighted blend across the chosen coordinate (typically
forecast reference time or model).
Raises:
TypeError : If the first argument not a cube.
CoordinateNotFoundError : If coordinate to be collapsed not found
in cube.
CoordinateNotFoundError : If coordinate to be collapsed not found
in provided weights cube.
ValueError : If coordinate to be collapsed is not a dimension.
"""
if not isinstance(cube, iris.cube.Cube):
msg = ("The first argument must be an instance of iris.cube.Cube "
"but is {}.".format(type(cube)))
raise TypeError(msg)
if not cube.coords(self.blend_coord):
msg = "Coordinate to be collapsed not found in cube."
raise CoordinateNotFoundError(msg)
output_dims = get_dim_coord_names(
next(cube.slices_over(self.blend_coord)))
self.blend_coord = find_blend_dim_coord(cube, self.blend_coord)
# Ensure input cube and weights cube are ordered equivalently along
# blending coordinate.
cube = sort_coord_in_cube(cube, self.blend_coord)
if weights is not None:
if not weights.coords(self.blend_coord):
msg = "Coordinate to be collapsed not found in weights cube."
raise CoordinateNotFoundError(msg)
weights = sort_coord_in_cube(weights, self.blend_coord)
# Check that the time coordinate is single valued if required.
self.check_compatible_time_points(cube)
# Do blending and update metadata
if self.check_percentile_coord(cube):
enforce_coordinate_ordering(cube, [self.blend_coord, "percentile"])
result = self.percentile_weighted_mean(cube, weights)
else:
enforce_coordinate_ordering(cube, [self.blend_coord])
result = self.weighted_mean(cube, weights)
# Reorder resulting dimensions to match input
enforce_coordinate_ordering(result, output_dims)
return result
def process(self,
temperature,
orography,
land_sea_mask,
model_id_attr=None):
"""Calculates the lapse rate from the temperature and orography cubes.
Args:
temperature (iris.cube.Cube):
Cube of air temperatures (K).
orography (iris.cube.Cube):
Cube containing orography data (metres)
land_sea_mask (iris.cube.Cube):
Cube containing a binary land-sea mask. True for land-points
and False for Sea.
model_id_attr (str):
Name of the attribute used to identify the source model for
blending. This is inherited from the input temperature cube.
Returns:
iris.cube.Cube:
Cube containing lapse rate (K m-1)
Raises
------
TypeError: If input cubes are not cubes
ValueError: If input cubes are the wrong units.
"""
if not isinstance(temperature, iris.cube.Cube):
msg = "Temperature input is not a cube, but {}"
raise TypeError(msg.format(type(temperature)))
if not isinstance(orography, iris.cube.Cube):
msg = "Orography input is not a cube, but {}"
raise TypeError(msg.format(type(orography)))
if not isinstance(land_sea_mask, iris.cube.Cube):
msg = "Land/Sea mask input is not a cube, but {}"
raise TypeError(msg.format(type(land_sea_mask)))
# Converts cube units.
temperature_cube = temperature.copy()
temperature_cube.convert_units("K")
orography.convert_units("metres")
# Extract x/y co-ordinates.
x_coord = temperature_cube.coord(axis="x").name()
y_coord = temperature_cube.coord(axis="y").name()
# Extract orography and land/sea mask data.
orography_data = next(orography.slices([y_coord, x_coord])).data
land_sea_mask_data = next(land_sea_mask.slices([y_coord,
x_coord])).data
# Fill sea points with NaN values.
orography_data = np.where(land_sea_mask_data, orography_data, np.nan)
# Create list of arrays over "realization" coordinate
has_realization_dimension = False
original_dimension_order = None
if temperature_cube.coords("realization", dim_coords=True):
original_dimension_order = get_dim_coord_names(temperature_cube)
enforce_coordinate_ordering(temperature_cube, "realization")
temp_data_slices = temperature_cube.data
has_realization_dimension = True
else:
temp_data_slices = [temperature_cube.data]
# Calculate lapse rate for each realization
lapse_rate_data = []
for temperature_data in temp_data_slices:
lapse_rate_array = self._generate_lapse_rate_array(
temperature_data, orography_data, land_sea_mask_data)
lapse_rate_data.append(lapse_rate_array)
lapse_rate_data = np.array(lapse_rate_data)
if not has_realization_dimension:
lapse_rate_data = np.squeeze(lapse_rate_data)
attributes = generate_mandatory_attributes([temperature],
model_id_attr=model_id_attr)
lapse_rate_cube = create_new_diagnostic_cube(
"air_temperature_lapse_rate",
"K m-1",
temperature_cube,
attributes,
data=lapse_rate_data,
)
if original_dimension_order:
enforce_coordinate_ordering(lapse_rate_cube,
original_dimension_order)
return lapse_rate_cube
def test_single_threshold(self):
"""Test a cube with one threshold correctly stores this as a scalar
coordinate"""
result = set_up_probability_cube(self.data[1:2], self.thresholds[1:2])
dim_coords = get_dim_coord_names(result)
self.assertNotIn("air_temperature", dim_coords)
def test_single_percentile(self):
"""Test a cube with one percentile correctly stores this as a scalar
coordinate"""
result = set_up_percentile_cube(self.data[1:2], self.percentiles[1:2])
dim_coords = get_dim_coord_names(result)
self.assertNotIn("percentile", dim_coords)
def subset_data(
cube: Cube,
grid_spec: Optional[Dict[str, Dict[str, int]]] = None,
site_list: Optional[List] = None,
) -> Cube:
"""Extract a spatial cutout or subset of sites from data
to generate suite reference outputs.
Args:
cube:
Input dataset
grid_spec:
Dictionary containing bounding grid points and an integer "thinning
factor" for each of UK and global grid, to create cutouts. Eg a
"thinning factor" of 10 would mean every 10th point being taken for
the cutout. The expected dictionary has keys that are spatial coordinate
names, with values that are dictionaries with "min", "max" and "thin" keys.
site_list:
List of WMO site IDs to extract. These IDs must match the type and format
of the "wmo_id" coordinate on the input spot cube.
Returns:
Subset of input cube as specified by input constraints
Raises:
ValueError:
If site_list is not provided for a spot data cube
ValueError:
If the spot data cube does not contain any of the required sites
ValueError:
If grid_spec is not provided for a gridded cube
ValueError:
If grid_spec does not contain entries for the spatial coordinates on
the input gridded data
ValueError:
If the grid_spec provided does not overlap with the cube domain
"""
if cube.coords("spot_index"):
if site_list is None:
raise ValueError("site_list required to extract from spot data")
constraint = Constraint(
coord_values={"wmo_id": lambda x: x in site_list})
result = cube.extract(constraint)
if result is None:
raise ValueError(
f"Cube does not contain any of the required sites: {site_list}"
)
else:
if grid_spec is None:
raise ValueError("grid_spec required to extract from gridded data")
x_coord = cube.coord(axis="x").name()
y_coord = cube.coord(axis="y").name()
for coord in [y_coord, x_coord]:
if coord not in grid_spec:
raise ValueError(
f"Cube coordinates {y_coord}, {x_coord} are not present within "
f"{grid_spec.keys()}")
def _create_cutout(cube, grid_spec):
"""Given a gridded data cube and boundary limits for cutout dimensions,
create cutout. Expects cube on either lat-lon or equal area grid.
"""
x_coord = cube.coord(axis="x").name()
y_coord = cube.coord(axis="y").name()
xmin = grid_spec[x_coord]["min"]
xmax = grid_spec[x_coord]["max"]
ymin = grid_spec[y_coord]["min"]
ymax = grid_spec[y_coord]["max"]
# need to use cube intersection for circular coordinates (longitude)
if x_coord == "longitude":
lat_constraint = Constraint(
latitude=lambda y: ymin <= y.point <= ymax)
cutout = cube.extract(lat_constraint)
if cutout is None:
return cutout
cutout = cutout.intersection(longitude=(xmin, xmax),
ignore_bounds=True)
# intersection creates a new coordinate with default datatype - we
# therefore need to re-cast to meet the IMPROVER standard
cutout.coord("longitude").points = cutout.coord(
"longitude").points.astype(FLOAT_DTYPE)
if cutout.coord("longitude").bounds is not None:
cutout.coord("longitude").bounds = cutout.coord(
"longitude").bounds.astype(FLOAT_DTYPE)
else:
x_constraint = Constraint(
projection_x_coordinate=lambda x: xmin <= x.point <= xmax)
y_constraint = Constraint(
projection_y_coordinate=lambda y: ymin <= y.point <= ymax)
cutout = cube.extract(x_constraint & y_constraint)
return cutout
cutout = _create_cutout(cube, grid_spec)
if cutout is None:
raise ValueError(
"Cube domain does not overlap with cutout specified:\n"
f"{x_coord}: {grid_spec[x_coord]}, {y_coord}: {grid_spec[y_coord]}"
)
original_coords = get_dim_coord_names(cutout)
thin_x = grid_spec[x_coord]["thin"]
thin_y = grid_spec[y_coord]["thin"]
result_list = CubeList()
try:
for subcube in cutout.slices([y_coord, x_coord]):
result_list.append(subcube[::thin_y, ::thin_x])
except ValueError as cause:
# error is raised if X or Y coordinate are single-valued (non-dimensional)
if "iterator" in str(cause) and "dimension" in str(cause):
raise ValueError(
"Function does not support single point extraction")
else:
raise
result = result_list.merge_cube()
enforce_coordinate_ordering(result, original_coords)
return result
