def test_defaults(self):
"""Test default arguments produce cube with expected dimensions
and metadata"""
result = set_up_variable_cube(self.data)
# check type, data and attributes
self.assertIsInstance(result, iris.cube.Cube)
self.assertEqual(result.standard_name, "air_temperature")
self.assertEqual(result.name(), "air_temperature")
self.assertEqual(result.units, "K")
self.assertArrayAlmostEqual(result.data, self.data)
self.assertEqual(result.attributes, {})
# check dimension coordinates
self.assertEqual(result.coord_dims("latitude"), (0, ))
self.assertEqual(result.coord_dims("longitude"), (1, ))
# check scalar time coordinates
for time_coord in ["time", "forecast_reference_time"]:
self.assertEqual(result.coord(time_coord).dtype, np.int64)
self.assertEqual(result.coord("forecast_period").dtype, np.int32)
expected_time = datetime(2017, 11, 10, 4, 0)
time_point = iris_time_to_datetime(result.coord("time"))[0]
self.assertEqual(time_point, expected_time)
expected_frt = datetime(2017, 11, 10, 0, 0)
frt_point = iris_time_to_datetime(
result.coord("forecast_reference_time"))[0]
self.assertEqual(frt_point, expected_frt)
self.assertEqual(result.coord("forecast_period").units, "seconds")
self.assertEqual(result.coord("forecast_period").points[0], 14400)
check_mandatory_standards(result)
def test_float64_cube_data(self):
"""Test a failure of a cube with 64-bit float data."""
self.cube.data = self.cube.data.astype(np.float64)
msg = ("does not have required dtype.\n"
"Expected: float32, Actual: float64")
with self.assertRaisesRegex(ValueError, msg):
check_mandatory_standards(self.cube)
def test_defaults(self):
"""Test default arguments produce cube with expected dimensions
and metadata"""
result = set_up_percentile_cube(self.data, self.percentiles)
perc_coord = result.coord("percentile")
self.assertArrayEqual(perc_coord.points, self.percentiles)
self.assertEqual(perc_coord.units, "%")
check_mandatory_standards(result)
def test_float64_cube_coord_points(self):
"""Test a failure of a cube with 64-bit float coord points."""
self.cube.coord("projection_x_coordinate").points = self.cube.coord(
"projection_x_coordinate").points.astype(np.float64)
msg = ("does not have required dtype.\n"
"Expected: float32, Actual \\(points\\): float64")
with self.assertRaisesRegex(ValueError, msg):
check_mandatory_standards(self.cube)
def assert_metadata_ok(output_cube):
"""Checks that the meta-data of output_cube are as expected"""
assert isinstance(output_cube, Cube)
assert output_cube.dtype == np.float32
assert list(output_cube.coord_dims("time")) == [
n for n, in [output_cube.coord_dims(c) for c in ["latitude", "longitude"]]
]
assert output_cube.coord("time").dtype == np.int64
check_mandatory_standards(output_cube)
def test_float64_cube_coord_bounds(self):
"""Test a failure of a cube with 64-bit float coord bounds."""
x_coord = self.cube.coord("projection_x_coordinate")
x_coord.bounds = np.array([(point - 10.0, point + 10.0)
for point in x_coord.points],
dtype=np.float64)
msg = ("does not have required dtype.\n"
"Expected: float32, "
"Actual \\(points\\): float32, "
"Actual \\(bounds\\): float64")
with self.assertRaisesRegex(ValueError, msg):
check_mandatory_standards(self.cube)
def test_basic(self):
"""Test addition of a leading height coordinate"""
result = add_coordinate(
self.input_cube, self.height_points, 'height',
coord_units=self.height_unit)
self.assertIsInstance(result, iris.cube.Cube)
self.assertSequenceEqual(result.shape, (10, 3, 4))
self.assertEqual(result.coord_dims('height'), (0,))
self.assertArrayAlmostEqual(
result.coord('height').points, self.height_points)
self.assertEqual(result.coord('height').dtype, np.float32)
self.assertEqual(result.coord('height').units, self.height_unit)
check_mandatory_standards(result)
def forecast_period_coord(
cube: Cube, force_lead_time_calculation: bool = False
) -> Coord:
"""
Return the lead time coordinate (forecast_period) from a cube, either by
reading an existing forecast_period coordinate, or by calculating the
difference between time and forecast_reference_time.
Args:
cube:
Cube from which the lead times will be determined.
force_lead_time_calculation:
Force the lead time to be calculated from the
forecast_reference_time and the time coordinate, even if
the forecast_period coordinate exists. Default is False.
Returns:
New forecast_period coord. A DimCoord is returned if the
forecast_period coord is already present in the cube as a
DimCoord and this coord does not need changing, otherwise
it will be an AuxCoord.
"""
create_dim_coord = False
if cube.coords("forecast_period"):
if isinstance(cube.coord("forecast_period"), DimCoord):
create_dim_coord = True
if cube.coords("forecast_period") and not force_lead_time_calculation:
result_coord = cube.coord("forecast_period").copy()
elif cube.coords("time") and cube.coords("forecast_reference_time"):
# Cube must adhere to mandatory standards for safe time calculations
check_mandatory_standards(cube)
# Try to calculate forecast period from forecast reference time and
# time coordinates
result_coord = _calculate_forecast_period(
cube.coord("time"),
cube.coord("forecast_reference_time"),
dim_coord=create_dim_coord,
)
else:
msg = (
"The forecast period coordinate is not available within {}."
"The time coordinate and forecast_reference_time "
"coordinate were also not available for calculating "
"the forecast_period.".format(cube)
)
raise CoordinateNotFoundError(msg)
return result_coord
def test_conformant_cubes(self):
"""Test conformant data, percentile and probability cubes all pass
(no error is thrown and cube is not changed)"""
cubelist = [
self.cube, self.probability_cube, self.percentile_cube]
for cube in cubelist:
result = cube.copy()
check_mandatory_standards(result)
# The following statement renders each cube into an XML string
# describing all aspects of the cube (including a checksum of the
# data) to verify that nothing has been changed anywhere on the
# cube.
self.assertStringEqual(CubeList([cube]).xml(checksum=True),
CubeList([result]).xml(checksum=True))
def test_defaults(self):
"""Test default arguments produce cube with expected dimensions
and metadata"""
result = set_up_probability_cube(self.data, self.thresholds)
thresh_coord = find_threshold_coordinate(result)
self.assertEqual(result.name(),
"probability_of_air_temperature_above_threshold")
self.assertEqual(result.units, "1")
self.assertArrayEqual(thresh_coord.points, self.thresholds)
self.assertEqual(thresh_coord.name(), "air_temperature")
self.assertEqual(thresh_coord.var_name, "threshold")
self.assertEqual(thresh_coord.units, "K")
self.assertEqual(len(thresh_coord.attributes), 1)
self.assertEqual(thresh_coord.attributes["spp__relative_to_threshold"],
"above")
check_mandatory_standards(result)
def _check_metadata(cube):
"""
Checks cube metadata that needs to be correct to guarantee data integrity
Args:
cube (iris.cube.Cube):
Cube to be checked
Raises:
ValueError: if time coordinates do not have the required datatypes
and units; needed because values may be wrong
ValueError: if numerical datatypes are other than 32-bit (except
where specified); needed because values may be wrong
ValueError: if cube dataset has unknown units; because this may cause
misinterpretation on "load"
"""
check_mandatory_standards(cube)
if cf_units.Unit(cube.units).is_unknown():
raise ValueError("{} has unknown units".format(cube.name()))
def test_multiple_errors(self):
"""Test a list of errors is correctly caught and re-raised"""
self.percentile_cube.coord(
"percentile").points = self.percentile_cube.coord(
"percentile").points.astype(np.float64)
self.percentile_cube.coord("forecast_period").convert_units("minutes")
self.percentile_cube.coord(
"forecast_period").points = self.percentile_cube.coord(
"forecast_period").points.astype(np.int64)
msg = ("percentile of type .*DimCoord.* "
"does not have required dtype.\n"
"Expected: float32, Actual \\(points\\): float64\n"
"forecast_period of type .*DimCoord.* "
"does not have required dtype.\n"
"Expected: int32, Actual \\(points\\): int64\n"
"forecast_period of type .*DimCoord.* "
"does not have required units.\n"
"Expected: seconds, Actual: minutes")
with self.assertRaisesRegex(ValueError, msg):
check_mandatory_standards(self.percentile_cube)
def run(self, cube: Cube) -> None:
"""Populates self-consistent interpreted parameters, or raises collated errors
describing (as far as posible) how the metadata are a) not self-consistent,
and / or b) not consistent with the Met Office IMPROVER standard.
Although every effort has been made to return as much information as possible,
collated errors may not be complete if the issue is fundamental. The developer
is advised to rerun this tool after each fix, until no further problems are
raised.
"""
# 1) Interpret diagnostic and type-specific metadata, including cell methods
if cube.name() in ANCILLARIES:
self.field_type = self.ANCIL
self.diagnostic = cube.name()
if cube.cell_methods:
self.errors.append(f"Unexpected cell methods {cube.cell_methods}")
elif cube.name() in SPECIAL_CASES:
self.field_type = self.diagnostic = cube.name()
if cube.name() == "weather_code":
for cm in cube.cell_methods:
if cm == WXCODE_MODE_CM and cube.name() in WXCODE_NAMES:
pass
else:
self.errors.append(
f"Unexpected cell methods {cube.cell_methods}"
)
elif cube.name() == "wind_from_direction":
if cube.cell_methods:
expected = CellMethod(method="mean", coords="realization")
if len(cube.cell_methods) > 1 or cube.cell_methods[0] != expected:
self.errors.append(
f"Unexpected cell methods {cube.cell_methods}"
)
else:
self.unhandled = True
return
else:
if "probability" in cube.name() and "threshold" in cube.name():
self.field_type = self.PROB
self.check_probability_cube_metadata(cube)
else:
self.diagnostic = cube.name()
try:
perc_coord = find_percentile_coordinate(cube)
except CoordinateNotFoundError:
coords = get_coord_names(cube)
if any(
[cube.coord(coord).var_name == "threshold" for coord in coords]
):
self.field_type = self.PROB
self.check_probability_cube_metadata(cube)
else:
self.field_type = self.DIAG
else:
self.field_type = self.PERC
if perc_coord.name() != PERC_COORD:
self.errors.append(
f"Percentile coordinate should have name {PERC_COORD}, "
f"has {perc_coord.name()}"
)
if perc_coord.units != "%":
self.errors.append(
"Percentile coordinate should have units of %, "
f"has {perc_coord.units}"
)
self.check_cell_methods(cube)
# 2) Interpret model and blend information from cube attributes
self.check_attributes(cube.attributes)
# 3) Check whether expected coordinates are present
coords = get_coord_names(cube)
if "spot_index" in coords:
self.check_spot_data(cube, coords)
if self.field_type == self.ANCIL:
# there is no definitive standard for time coordinates on static ancillaries
pass
elif cube.coords("time_in_local_timezone"):
# For data on local timezones, the time coordinate will match the horizontal
# dimensions and there will be no forecast period.
expected_coords = set(LOCAL_TIME_COORDS + UNBLENDED_TIME_COORDS)
expected_coords.discard("forecast_period")
self._check_coords_present(coords, expected_coords)
self._check_coords_are_horizontal(cube, ["time"])
elif self.blended:
self._check_coords_present(coords, BLENDED_TIME_COORDS)
else:
self._check_coords_present(coords, UNBLENDED_TIME_COORDS)
# 4) Check points are equal to upper bounds for bounded time coordinates
for coord in ["time", "forecast_period"]:
if coord in get_coord_names(cube):
self._check_coord_bounds(cube, coord)
# 5) Check datatypes on data and coordinates
try:
check_mandatory_standards(cube)
except ValueError as cause:
self.errors.append(str(cause))
# 6) Check multiple realizations only exist for ensemble models
if self.field_type == self.DIAG:
try:
realization_coord = cube.coord("realization")
except CoordinateNotFoundError:
pass
else:
model_id = cube.attributes.get(self.model_id_attr, "ens")
if "ens" not in model_id and len(realization_coord.points) > 1:
self.errors.append(
f"Deterministic model should not have {len(realization_coord.points)} "
"realizations"
)
# 7) Raise collated errors if present
if self.errors:
raise ValueError("\n".join(self.errors))
def set_up_variable_cube(data,
name='air_temperature',
units='K',
spatial_grid='latlon',
time=datetime(2017, 11, 10, 4, 0),
time_bounds=None,
frt=datetime(2017, 11, 10, 0, 0),
realizations=None,
include_scalar_coords=None,
attributes=None,
standard_grid_metadata=None):
"""
Set up a cube containing a single variable field with:
- x/y spatial dimensions (equal area or lat / lon)
- optional leading "realization" dimension
- "time", "forecast_reference_time" and "forecast_period" scalar coords
- option to specify additional scalar coordinates
- configurable attributes
Args:
data (numpy.ndarray):
2D (y-x ordered) or 3D (realization-y-x ordered) array of data
to put into the cube.
name (str):
Variable name (standard / long)
units (str):
Variable units
spatial_grid (str):
What type of x/y coordinate values to use. Permitted values are
"latlon" or "equalarea".
time (datetime.datetime):
Single cube validity time
time_bounds (tuple or list of datetime.datetime instances):
Lower and upper bound on time point, if required
frt (datetime.datetime):
Single cube forecast reference time
realizations (list or numpy.ndarray):
List of forecast realizations. If not present, taken from the
leading dimension of the input data array (if 3D).
include_scalar_coords (list):
List of iris.coords.DimCoord or AuxCoord instances of length 1.
attributes (dict):
Optional cube attributes.
standard_grid_metadata (str):
Recognised mosg__model_configuration for which to set up Met
Office standard grid attributes. Should be 'uk_det', 'uk_ens',
'gl_det' or 'gl_ens'.
"""
# construct spatial dimension coordimates
ypoints = data.shape[-2]
xpoints = data.shape[-1]
y_coord, x_coord = construct_xy_coords(ypoints, xpoints, spatial_grid)
# construct realization dimension for 3D data, and dim_coords list
ndims = len(data.shape)
if ndims == 3:
if realizations is not None:
if len(realizations) != data.shape[0]:
raise ValueError(
'Cannot generate {} realizations from data of shape '
'{}'.format(len(realizations), data.shape))
realizations = np.array(realizations)
if issubclass(realizations.dtype.type, np.integer):
# expect integer realizations
realizations = realizations.astype(np.int32)
else:
# option needed for percentile & probability cube setup
realizations = realizations.astype(np.float32)
else:
realizations = np.arange(data.shape[0]).astype(np.int32)
realization_coord = DimCoord(realizations, "realization", units="1")
dim_coords = [(realization_coord, 0), (y_coord, 1), (x_coord, 2)]
elif ndims == 2:
dim_coords = [(y_coord, 0), (x_coord, 1)]
else:
raise ValueError(
'Expected 2 or 3 dimensions on input data: got {}'.format(ndims))
# construct list of aux_coords_and_dims
scalar_coords = construct_scalar_time_coords(time, time_bounds, frt)
if include_scalar_coords is not None:
for coord in include_scalar_coords:
scalar_coords.append((coord, None))
# set up attributes
cube_attrs = {}
if standard_grid_metadata is not None:
cube_attrs.update(MOSG_GRID_DEFINITION[standard_grid_metadata])
if attributes is not None:
cube_attrs.update(attributes)
# create data cube
cube = iris.cube.Cube(data,
units=units,
attributes=cube_attrs,
dim_coords_and_dims=dim_coords,
aux_coords_and_dims=scalar_coords)
cube.rename(name)
# don't allow unit tests to set up invalid cubes
check_mandatory_standards(cube)
return cube
def test_string_coord(self):
"""Test conformant data with a cube with a coord of strings."""
self.cube.add_aux_coord(AuxCoord(["kittens"], long_name="animal"))
check_mandatory_standards(self.cube)
def set_up_variable_cube(
data,
name="air_temperature",
units="K",
spatial_grid="latlon",
time=datetime(2017, 11, 10, 4, 0),
time_bounds=None,
frt=datetime(2017, 11, 10, 0, 0),
realizations=None,
include_scalar_coords=None,
attributes=None,
standard_grid_metadata=None,
grid_spacing=None,
domain_corner=None,
height_levels=None,
pressure=False,
):
"""
Set up a cube containing a single variable field with:
- x/y spatial dimensions (equal area or lat / lon)
- optional leading "realization" dimension
- optional "height" dimension
- "time", "forecast_reference_time" and "forecast_period" scalar coords
- option to specify additional scalar coordinates
- configurable attributes
Args:
data (numpy.ndarray):
2D (y-x ordered) or 3D (realization-y-x ordered) array of data
to put into the cube.
name (Optional[str]):
Variable name (standard / long)
units (Optional[str]):
Variable units
spatial_grid (Optional[str]):
What type of x/y coordinate values to use. Permitted values are
"latlon" or "equalarea".
time (Optional[datetime.datetime]):
Single cube validity time
time_bounds (Optional[Sequence[datetime.datetime]]):
Lower and upper bound on time point, if required
frt (Optional[datetime.datetime]):
Single cube forecast reference time
realizations (Optional[List[numpy.ndarray]]):
List of forecast realizations. If not present, taken from the
leading dimension of the input data array (if 3D).
include_scalar_coords (Optional[List[iris.coords.DimCoord] or List[iris.coords.AuxCoord]]):
List of iris.coords.DimCoord or AuxCoord instances of length 1.
attributes (Optional[Dict[Any]]):
Optional cube attributes.
standard_grid_metadata (Optional[str]):
Recognised mosg__model_configuration for which to set up Met
Office standard grid attributes. Should be 'uk_det', 'uk_ens',
'gl_det' or 'gl_ens'.
grid_spacing (Optional[float]):
Grid resolution (degrees for latlon or metres for equalarea).
domain_corner (Optional[Tuple[float, float]]):
Bottom left corner of grid domain (y,x) (degrees for latlon or metres for equalarea).
height_levels (Optional[List[float]]):
List of height levels in metres or pressure levels in Pa.
pressure (Optional[bool]):
Flag to indicate whether the height levels are specified as pressure, in Pa. If False, use height in metres.
Returns:
iris.cube.Cube:
Cube containing a single variable field
"""
# construct spatial dimension coordimates
ypoints = data.shape[-2]
xpoints = data.shape[-1]
y_coord, x_coord = construct_yx_coords(
ypoints,
xpoints,
spatial_grid,
grid_spacing=grid_spacing,
domain_corner=domain_corner,
)
dim_coords = _construct_dimension_coords(
data, y_coord, x_coord, realizations, height_levels, pressure
)
# construct list of aux_coords_and_dims
scalar_coords = construct_scalar_time_coords(time, time_bounds, frt)
if include_scalar_coords is not None:
for coord in include_scalar_coords:
scalar_coords.append((coord, None))
# set up attributes
cube_attrs = {}
if standard_grid_metadata is not None:
cube_attrs.update(MOSG_GRID_DEFINITION[standard_grid_metadata])
if attributes is not None:
cube_attrs.update(attributes)
# create data cube
cube = iris.cube.Cube(
data,
units=units,
attributes=cube_attrs,
dim_coords_and_dims=dim_coords,
aux_coords_and_dims=scalar_coords,
)
cube.rename(name)
# don't allow unit tests to set up invalid cubes
check_mandatory_standards(cube)
return cube
def test_int64_cube_data(self):
"""Test conformant data with a cube with 64-bit integer data."""
self.cube.data = self.cube.data.astype(np.int64)
check_mandatory_standards(self.cube)
def _check_inputs(self, cubes):
"""Check the inputs prior to calculating the accumulations.
Args:
cubes: iris.cube.CubeList
Cube list of precipitation rates that will be checked for their
appropriateness in calculating the requested accumulations.
The timesteps between the cubes in this cubelist are
expected to be regular.
Returns:
(tuple): tuple containing:
**cubes** (iris.cube.CubeList):
Modified version of the input cube list of precipitation
rates that have had the units of the coordinates and
cube data enforced. The cube list has also been sorted by
time.
**time_interval** (float):
Interval between the timesteps from the input cubelist.
Raises:
ValueError: The input rates cubes must be at regularly spaced
time intervals.
ValueError: The accumulation period is less than the time interval
between the rates cubes.
ValueError: The specified accumulation period is not cleanly
divisible by the time interval.
"""
# Standardise inputs to expected units
standardised_cubes = []
for cube in cubes:
check_mandatory_standards(cube)
new_cube = cube.copy()
new_cube.convert_units("m s-1")
standardised_cubes.append(new_cube)
cubes = standardised_cubes
# Sort cubes into time order and calculate intervals.
cubes, times = self.sort_cubes_by_time(cubes)
try:
(time_interval, ) = np.unique(np.diff(times,
axis=0)).astype(np.int32)
except ValueError:
msg = ("Accumulation is designed to work with "
"rates cubes at regular time intervals. Cubes "
"provided are unevenly spaced in time; time intervals are "
"{}.".format(np.diff(times, axis=0)))
raise ValueError(msg)
if self.accumulation_period is None:
# If no accumulation period is specified, assume that the input
# cubes will be used to construct a single accumulation period.
(self.accumulation_period,
) = cubes[-1].coord("forecast_period").points
# Ensure that the accumulation period is int32.
self.accumulation_period = np.int32(self.accumulation_period)
fraction, integral = np.modf(self.accumulation_period / time_interval)
# Check whether the accumulation period is less than the time_interval
# i.e. the integral is equal to zero. In this case, the rates cubes
# are too widely spaced to compute the requested accumulation period.
if integral == 0:
msg = (
"The accumulation_period is less than the time interval "
"between the rates cubes. The rates cubes provided are "
"therefore insufficient for computing the accumulation period "
"requested. accumulation period specified: {}, "
"time interval specified: {}".format(self.accumulation_period,
time_interval))
raise ValueError(msg)
# Ensure the accumulation period is cleanly divisible by the time
# interval.
if fraction != 0:
msg = ("The specified accumulation period ({}) is not divisible "
"by the time intervals between rates cubes ({}). As "
"a result it is not possible to calculate the desired "
"total accumulation period.".format(
self.accumulation_period, time_interval))
raise ValueError(msg)
if self.forecast_periods is None:
# If no forecast periods are specified, then the accumulation
# periods calculated will end at the forecast period from
# each of the input cubes.
self.forecast_periods = [
cube.coord("forecast_period").points for cube in cubes
if cube.coord("forecast_period").points >= time_interval
]
# Check whether any forecast periods are less than the accumulation
# period. This is expected if the accumulation period is e.g. 1 hour,
# however, the forecast periods are e.g. [15, 30, 45] minutes.
# In this case, the forecast periods are filtered, so that only
# complete accumulation periods will be calculated.
if any(self.forecast_periods < self.accumulation_period):
forecast_periods = [
fp for fp in self.forecast_periods
if fp >= self.accumulation_period
]
self.forecast_periods = forecast_periods
return cubes, time_interval
