def test_data_datatype_fail(self):
"""Test error is raised when enforce=False"""
self.percentile_cube.data = (self.percentile_cube.data.astype(
np.float64))
msg = "does not conform"
with self.assertRaisesRegex(ValueError, msg):
cube_units.enforce_units_and_dtypes(self.percentile_cube,
enforce=False)
def test_coord_units_fail(self):
"""Test error is raised when enforce=False"""
self.probability_cube.coord('air_temperature').convert_units(
'Fahrenheit')
msg = "does not conform"
with self.assertRaisesRegex(ValueError, msg):
cube_units.enforce_units_and_dtypes(self.probability_cube,
enforce=False)
def test_quantity_unavailable(self):
"""Test error raised if the named quantity is not listed in the
dictionary standard"""
self.data_cube.rename("number_of_fish")
self.data_cube.units = "1"
msg = "Name 'number_of_fish' is not uniquely defined in units.py"
with self.assertRaisesRegex(KeyError, msg):
cube_units.enforce_units_and_dtypes(self.data_cube)
def test_coord_datatype_fail(self):
"""Test error is raised when enforce=False"""
self.percentile_cube.coord('percentile').points = (
self.percentile_cube.coord('percentile').points.astype(np.int32))
msg = "does not conform"
with self.assertRaisesRegex(ValueError, msg):
cube_units.enforce_units_and_dtypes(self.percentile_cube,
enforce=False)
def save_netcdf(cubelist, filename):
"""Save the input Cube or CubeList as a NetCDF file.
Uses the functionality provided by iris.fileformats.netcdf.save with
local_keys to record non-global attributes as data attributes rather than
global attributes.
Args:
cubelist (iris.cube.Cube or iris.cube.CubeList):
Cube or list of cubes to be saved
filename (str):
Filename to save input cube(s)
Raises:
warning if cubelist contains cubes of varying dimensions.
"""
if isinstance(cubelist, iris.cube.Cube):
cubelist = [cubelist]
for cube in cubelist:
_order_cell_methods(cube)
enforce_units_and_dtypes(cube, enforce=False)
# If all xy slices are the same shape, use this to determine
# the chunksize for the netCDF (eg. 1, 1, 970, 1042)
chunksizes = None
if len(set([cube.shape[:2] for cube in cubelist])) == 1:
cube = cubelist[0]
if cube.ndim >= 2:
xy_chunksizes = [cube.shape[-2], cube.shape[-1]]
chunksizes = tuple([1] * (cube.ndim - 2) + xy_chunksizes)
else:
msg = ("Chunksize not set as cubelist "
"contains cubes of varying dimensions")
warnings.warn(msg)
global_keys = [
'title', 'um_version', 'grid_id', 'source', 'Conventions',
'mosg__grid_type', 'mosg__model_configuration', 'mosg__grid_domain',
'mosg__grid_version', 'institution', 'history', 'bald__isPrefixedBy'
]
local_keys = {
key
for cube in cubelist for key in cube.attributes.keys()
if key not in global_keys
}
cubelist = _append_metadata_cube(cubelist, global_keys)
iris.fileformats.netcdf.save(cubelist,
filename,
local_keys=local_keys,
complevel=1,
shuffle=True,
zlib=True,
chunksizes=chunksizes)
def test_multiple_errors(self):
"""Test a list of errors is correctly caught and re-raised"""
self.data_cube.convert_units('Fahrenheit')
self.probability_cube.coord('air_temperature').convert_units('degC')
msg = ("The following errors were raised during processing:\n"
"air_temperature with units Fahrenheit and datatype float32 "
"does not conform to expected standard \\(units K, datatype "
"\\<class 'numpy.float32'\\>\\)\n"
"air_temperature with units degC and datatype float32 "
"does not conform to expected standard \\(units K, datatype "
"\\<class 'numpy.float32'\\>\\)\n")
with self.assertRaisesRegex(ValueError, msg):
cube_units.enforce_units_and_dtypes(
[self.data_cube, self.probability_cube], enforce=False)
def test_data_units_enforce(self):
"""Test units are changed on the returned cube and the input cube is
unmodified"""
self.data_cube.convert_units('Fahrenheit')
result, = cube_units.enforce_units_and_dtypes(self.data_cube)
self.assertEqual(result.units, 'K')
self.assertEqual(self.data_cube.units, 'Fahrenheit')
def test_data_datatype_enforce(self):
"""Test dataset datatypes are enforced"""
self.data_cube.data = self.data_cube.data.astype(np.float64)
result, = cube_units.enforce_units_and_dtypes(self.data_cube)
self.assertEqual(result.dtype, np.float32)
# check input is unchanged
self.assertEqual(self.data_cube.dtype, np.float64)
def test_coord_units_enforce(self):
"""Test coordinate units are enforced and the input cube is
unmodified"""
test_coord = 'projection_x_coordinate'
self.data_cube.coord(test_coord).convert_units('km')
result, = cube_units.enforce_units_and_dtypes(self.data_cube)
self.assertEqual(self.data_cube.coord(test_coord).units, 'km')
self.assertEqual(result.coord(test_coord).units, 'm')
def test_coord_datatype_enforce(self):
"""Test coordinate datatypes are enforced (using substring processing)
"""
test_coord = 'forecast_reference_time'
self.data_cube.coord(test_coord).points = (
self.data_cube.coord(test_coord).points.astype(np.float64))
result, = cube_units.enforce_units_and_dtypes(self.data_cube)
self.assertEqual(result.coord(test_coord).dtype, np.int64)
# check input is unchanged
self.assertEqual(self.data_cube.coord(test_coord).dtype, np.float64)
def test_conformant_cubes(self):
"""Test conformant data, percentile and probability cubes are all
passed when enforce=False (ie set to fail on non-conformance)"""
cubelist = [
self.data_cube, self.probability_cube, self.percentile_cube
]
result = cube_units.enforce_units_and_dtypes(cubelist, enforce=False)
self.assertIsInstance(result, iris.cube.CubeList)
for cube, ref in zip(result, cubelist):
self.assertArrayAlmostEqual(cube.data, ref.data)
self.assertEqual(cube.metadata, ref.metadata)
def test_coordinates_correctly_identified(self):
"""Test all coordinates in a heterogeneous cube list are identified and
corrected"""
self.percentile_cube.coord('percentile').points = (
self.percentile_cube.coord('percentile').points.astype(np.int32))
self.probability_cube.coord('air_temperature').convert_units(
'Fahrenheit')
result = cube_units.enforce_units_and_dtypes(
[self.percentile_cube, self.probability_cube])
self.assertEqual(result[0].coord('percentile').dtype, np.float32)
self.assertEqual(result[1].coord('air_temperature').units, 'K')
def test_subset_of_coordinates(self):
"""Test function can enforce on a selected subset of coordinates and
leave all others unchanged"""
self.percentile_cube.coord('percentile').points = (
self.percentile_cube.coord('percentile').points.astype(np.int32))
self.percentile_cube.coord('time').convert_units(
'hours since 1970-01-01 00:00:00')
self.probability_cube.coord('air_temperature').convert_units(
'Fahrenheit')
self.probability_cube.coord('forecast_period').convert_units('h')
result = cube_units.enforce_units_and_dtypes(
[self.percentile_cube, self.probability_cube],
coords=["time", "forecast_period"])
self.assertEqual(result[0].coord('percentile').dtype, np.int32)
self.assertEqual(result[0].coord('time').units,
'seconds since 1970-01-01 00:00:00')
self.assertEqual(result[1].coord('air_temperature').units,
'Fahrenheit')
self.assertEqual(result[1].coord('forecast_period').units, 's')
def test_cube_input(self):
"""Test function behaves sensibly with a single cube"""
result = cube_units.enforce_units_and_dtypes(self.data_cube)
self.assertIsInstance(result, iris.cube.CubeList)
self.assertArrayAlmostEqual(result[0].data, self.data_cube.data)
self.assertEqual(result[0].metadata, self.data_cube.metadata)
def test_basic(self):
"""Test function returns a CubeList"""
cubelist = [self.data_cube]
result = cube_units.enforce_units_and_dtypes(cubelist)
self.assertIsInstance(result, iris.cube.CubeList)
def _check_inputs(self, cubes):
"""Check the inputs prior to calculating the accumulations.
Args:
cubes: iris.cube.CubeList
Cube list of precipitation rates that will be checked for their
appropriateness in calculating the requested accumulations.
The timesteps between the cubes in this cubelist are
expected to be regular.
Returns:
(tuple): tuple containing
**cubes** (iris.cube.CubeList):
Modified version of the input cube list of precipitation
rates that have had the units of the coordinates and
cube data enforced. The cube list has also been sorted by
time.
**time_interval** (float):
Interval between the timesteps from the input cubelist.
Raises:
ValueError: The input rates cubes must be at regularly spaced
time intervals.
ValueError: The accumulation period is less than the time interval
between the rates cubes.
ValueError: The specified accumulation period is not cleanly
divisible by the time interval.
"""
# Standardise inputs to expected units
cubes = enforce_units_and_dtypes(
cubes,
coords=['time', 'forecast_reference_time', 'forecast_period'])
# Sort cubes into time order and calculate intervals.
cubes, times = self.sort_cubes_by_time(cubes)
try:
time_interval, = np.unique(np.diff(times, axis=0)).astype(np.int32)
except ValueError:
msg = ("Accumulation is designed to work with "
"rates cubes at regular time intervals. Cubes "
"provided are unevenly spaced in time; time intervals are "
"{}.".format(np.diff(times, axis=0)))
raise ValueError(msg)
if self.accumulation_period is None:
# If no accumulation period is specified, assume that the input
# cubes will be used to construct a single accumulation period.
self.accumulation_period, = (
cubes[-1].coord("forecast_period").points)
# Ensure that the accumulation period is int32.
self.accumulation_period = np.int32(self.accumulation_period)
fraction, integral = np.modf(self.accumulation_period / time_interval)
# Check whether the accumulation period is less than the time_interval
# i.e. the integral is equal to zero. In this case, the rates cubes
# are too widely spaced to compute the requested accumulation period.
if integral == 0:
msg = (
"The accumulation_period is less than the time interval "
"between the rates cubes. The rates cubes provided are "
"therefore insufficient for computing the accumulation period "
"requested. accumulation period specified: {}, "
"time interval specified: {}".format(self.accumulation_period,
time_interval))
raise ValueError(msg)
# Ensure the accumulation period is cleanly divisible by the time
# interval.
if fraction != 0:
msg = ("The specified accumulation period ({}) is not divisible "
"by the time intervals between rates cubes ({}). As "
"a result it is not possible to calculate the desired "
"total accumulation period.".format(
self.accumulation_period, time_interval))
raise ValueError(msg)
if self.forecast_periods is None:
# If no forecast periods are specified, then the accumulation
# periods calculated will end at the forecast period from
# each of the input cubes.
self.forecast_periods = [
cube.coord("forecast_period").points for cube in cubes
if cube.coord("forecast_period").points >= time_interval
]
# Check whether any forecast periods are less than the accumulation
# period. This is expected if the accumulation period is e.g. 1 hour,
# however, the forecast periods are e.g. [15, 30, 45] minutes.
# In this case, the forecast periods are filtered, so that only
# complete accumulation periods will be calculated.
if any(self.forecast_periods < self.accumulation_period):
forecast_periods = [
fp for fp in self.forecast_periods
if fp >= self.accumulation_period
]
self.forecast_periods = forecast_periods
return cubes, time_interval
def test_data_units_fail(self):
"""Test error is raised when enforce=False"""
self.data_cube.convert_units('Fahrenheit')
msg = "does not conform"
with self.assertRaisesRegex(ValueError, msg):
cube_units.enforce_units_and_dtypes(self.data_cube, enforce=False)
