def test_exception_raised(self):
"""Test that a CoordinateNotFoundError exception is raised if the
forecast_period, or the time and forecast_reference_time,
are not present.
"""
cube = set_up_cube()
msg = "The forecast period coordinate is not available"
with self.assertRaisesRegexp(CoordinateNotFoundError, msg):
forecast_period_coord(cube)
def setUp(self):
"""Set up a UK deterministic cube for testing."""
self.cycletime = datetime.datetime(2017, 1, 10, 6, 0)
cube_uk_det = set_up_variable_cube(np.full((4, 4),
273.15,
dtype=np.float32),
time=self.cycletime,
frt=datetime.datetime(
2017, 1, 10, 3, 0))
cube_uk_det.remove_coord("forecast_period")
# set up forecast periods of 6, 8 and 10 hours
time_points = [1484038800, 1484046000, 1484053200]
cube_uk_det = add_coordinate(
cube_uk_det,
time_points,
"time",
dtype=np.int64,
coord_units="seconds since 1970-01-01 00:00:00")
fp_coord = forecast_period_coord(cube_uk_det)
cube_uk_det.add_aux_coord(fp_coord, data_dims=0)
self.cube_uk_det = add_coordinate(cube_uk_det, [1000], "model_id")
self.cube_uk_det.add_aux_coord(
iris.coords.AuxCoord(["uk_det"], long_name="model_configuration"))
def test_negative_forecast_periods_warning(self, warning_list=None):
"""Test that a warning is raised if the point within the
time coordinate is prior to the point within the
forecast_reference_time, and therefore the forecast_period values that
have been generated are negative.
"""
cube = add_forecast_reference_time_and_forecast_period(set_up_cube())
cube.remove_coord("forecast_period")
cube.coord("forecast_reference_time").points = 402295.0
cube.coord("time").points = 402192.5
msg = "The values for the time"
forecast_period_coord(cube)
self.assertTrue(len(warning_list) == 1)
self.assertTrue(
any(item.category == UserWarning for item in warning_list))
self.assertTrue(msg in str(warning_list[0]))
def test_negative_forecast_periods_warning(self, warning_list=None):
"""Test that a warning is raised if the point within the
time coordinate is prior to the point within the
forecast_reference_time, and therefore the forecast_period values that
have been generated are negative.
"""
cube = set_up_variable_cube(np.ones((3, 3), dtype=np.float32))
cube.remove_coord("forecast_period")
# default cube has a 4 hour forecast period, so add 5 hours to frt
cube.coord("forecast_reference_time").points = (
cube.coord("forecast_reference_time").points + 5 * 3600)
warning_msg = "The values for the time"
forecast_period_coord(cube)
self.assertTrue(
any(item.category == UserWarning for item in warning_list))
self.assertTrue(any(warning_msg in str(item) for item in warning_list))
def update_time_and_forecast_period(cube, increment):
"""Updates time and forecast period points on an existing cube by a given
increment (in units of time)"""
cube.coord("time").points = cube.coord("time").points + increment
forecast_period = forecast_period_coord(
cube, force_lead_time_calculation=True)
cube.replace_coord(forecast_period)
return cube
def test_check_coordinate_without_forecast_period(self):
"""Test that the data within the coord is as expected with the
expected units, when the input cube has a time coordinate and a
forecast_reference_time coordinate.
"""
fp_coord = self.cube.coord("forecast_period").copy()
expected_result = fp_coord
self.cube.remove_coord("forecast_period")
result = forecast_period_coord(self.cube)
self.assertEqual(result, expected_result)
def test_check_coordinate(self):
"""Test that the data within the coord is as expected with the
expected units, when the input cube has a forecast_period coordinate.
"""
fp_coord = self.cube.coord("forecast_period").copy()
expected_points = fp_coord.points
expected_units = str(fp_coord.units)
result = forecast_period_coord(self.cube)
self.assertArrayEqual(result.points, expected_points)
self.assertEqual(str(result.units), expected_units)
def test_check_time_unit_conversion(self):
"""Test that the data within the coord is as expected with the
expected units, when the input cube has a time coordinate with units
other than the usual units of seconds since 1970-01-01 00:00:00.
"""
expected_result = self.cube.coord("forecast_period")
self.cube.coord("time").convert_units(
"hours since 1970-01-01 00:00:00")
result = forecast_period_coord(self.cube,
force_lead_time_calculation=True)
self.assertEqual(result, expected_result)
def test_check_coordinate_force_lead_time_calculation(self):
"""Test that the data within the coord is as expected with the
expected units, when the input cube has a forecast_period coordinate.
"""
cube = add_forecast_reference_time_and_forecast_period(set_up_cube())
fp_coord = cube.coord("forecast_period").copy()
fp_coord.convert_units("seconds")
expected_points = fp_coord.points
expected_units = str(fp_coord.units)
result = forecast_period_coord(cube, force_lead_time_calculation=True)
self.assertArrayAlmostEqual(result.points, expected_points)
self.assertEqual(result.units, expected_units)
def test_check_coordinate_without_forecast_period(self):
"""Test that the data within the coord is as expected with the
expected units, when the input cube has a time coordinate and a
forecast_reference_time coordinate.
"""
cube = add_forecast_reference_time_and_forecast_period(set_up_cube())
fp_coord = cube.coord("forecast_period").copy()
fp_coord.convert_units("seconds")
expected_result = fp_coord
cube.remove_coord("forecast_period")
result = forecast_period_coord(cube)
self.assertEqual(result, expected_result)
def test_check_time_unit_conversion(self):
"""Test that the data within the coord is as expected with the
expected units, when the input cube has a time coordinate with units
other than the usual units of hours since 1970-01-01 00:00:00.
"""
cube = add_forecast_reference_time_and_forecast_period(set_up_cube())
fp_coord = cube.coord("forecast_period").copy()
fp_coord.convert_units("seconds")
expected_result = fp_coord
cube.coord("time").convert_units("seconds since 1970-01-01 00:00:00")
result = forecast_period_coord(cube, force_lead_time_calculation=True)
self.assertEqual(result, expected_result)
def test_check_coordinate_in_hours_force_lead_time_calculation(self):
"""Test that the data within the coord is as expected with the
expected units, when the input cube has a forecast_period coordinate.
"""
fp_coord = self.cube.coord("forecast_period").copy()
fp_coord.convert_units("hours")
expected_points = fp_coord.points
expected_units = str(fp_coord.units)
result = forecast_period_coord(self.cube,
force_lead_time_calculation=True,
result_units=fp_coord.units)
self.assertArrayEqual(result.points, expected_points)
self.assertEqual(result.units, expected_units)
def test_check_time_unit_has_bounds(self):
"""Test that the forecast_period coord has bounds if time has bounds.
"""
cube = set_up_variable_cube(
np.ones((3, 3), dtype=np.float32),
time=datetime.datetime(2018, 3, 12, 20, 0),
frt=datetime.datetime(2018, 3, 12, 15, 0),
time_bounds=[datetime.datetime(2018, 3, 12, 19, 0),
datetime.datetime(2018, 3, 12, 20, 0)])
expected_result = cube.coord("forecast_period").copy()
expected_result.bounds = [[14400, 18000]]
result = forecast_period_coord(cube, force_lead_time_calculation=True)
self.assertEqual(result, expected_result)
def test_cubelist_input(self):
"""Test when supplying a cubelist as input containing cubes
representing UK deterministic and UK ensemble model configuration
and unifying the forecast_reference_time, so that both model
configurations have a common forecast_reference_time."""
cube_uk_ens = set_up_variable_cube(np.full((3, 4, 4),
273.15,
dtype=np.float32),
time=self.cycletime,
frt=datetime.datetime(
2017, 1, 10, 4, 0))
cube_uk_ens.remove_coord("forecast_period")
# set up forecast periods of 5, 7 and 9 hours
time_points = [1484031600, 1484038800, 1484046000]
cube_uk_ens = add_coordinate(
cube_uk_ens,
time_points,
"time",
dtype=np.int64,
coord_units="seconds since 1970-01-01 00:00:00")
fp_coord = forecast_period_coord(cube_uk_ens)
cube_uk_ens.add_aux_coord(fp_coord, data_dims=0)
expected_uk_det = self.cube_uk_det.copy()
frt_units = expected_uk_det.coord('forecast_reference_time').units
frt_points = [
np.round(frt_units.date2num(self.cycletime)).astype(np.int64)
]
expected_uk_det.coord("forecast_reference_time").points = frt_points
expected_uk_det.coord("forecast_period").points = (
np.array([3, 5, 7]) * 3600)
expected_uk_ens = cube_uk_ens.copy()
expected_uk_ens.coord("forecast_reference_time").points = frt_points
expected_uk_ens.coord("forecast_period").points = (
np.array([1, 3, 5]) * 3600)
expected = iris.cube.CubeList([expected_uk_det, expected_uk_ens])
cubes = iris.cube.CubeList([self.cube_uk_det, cube_uk_ens])
result = unify_forecast_reference_time(cubes, self.cycletime)
self.assertIsInstance(result, iris.cube.CubeList)
self.assertEqual(result, expected)
def test_basic(self):
"""Test that an iris.coord.DimCoord is returned."""
result = forecast_period_coord(self.cube)
self.assertIsInstance(result, iris.coords.DimCoord)
def test_basic_AuxCoord(self):
"""Test that an iris.coord.AuxCoord is returned."""
self.cube.remove_coord('forecast_period')
result = forecast_period_coord(self.cube,
force_lead_time_calculation=True)
self.assertIsInstance(result, iris.coords.AuxCoord)
def conform_metadata(cube,
cube_orig,
coord,
cycletime=None,
coords_for_bounds_removal=None):
"""Ensure that the metadata conforms after blending together across
the chosen coordinate.
The metadata adjustments are:
- Forecast reference time: If a cycletime is not present, the
most recent available forecast_reference_time is used.
If a cycletime is present, the cycletime is used as the
forecast_reference_time instead.
- Forecast period: If a forecast_period coordinate is present,
and cycletime is not present, the lowest forecast_period is
used. If a forecast_period coordinate is present, and the
cycletime is present, forecast_periods are forceably calculated
from the time and forecast_reference_time coordinate. This is
because, if the cycletime is present, then the
forecast_reference_time will also have been just re-calculated, so
the forecast_period coordinate needs to be reset to match the
newly calculated forecast_reference_time.
- Forecast reference time and time: If forecast_reference_time and
time coordinates are present, then a forecast_period coordinate is
calculated and added to the cube.
- Model_id, model_realization and realization coordinates are removed.
- Remove bounds from the scalar coordinates, if the coordinates
are specified within the coords_for_bounds_removal argument.
Args:
cube (iris.cube.Cube):
Cube containing the metadata to be adjusted.
cube_orig (iris.cube.Cube):
Cube containing metadata that may be useful for adjusting
metadata on the `cube` variable.
coord (str):
Coordinate that has been blended. This allows specific metadata
changes to be limited to whichever coordinate is being blended.
Keyword Args:
cycletime (str):
The cycletime in a YYYYMMDDTHHMMZ format e.g. 20171122T0100Z.
coords_for_bounds_removal (None or list):
List of coordinates that are scalar and should have their bounds
removed.
Returns:
cube (iris.cube.Cube):
Cube containing the adjusted metadata.
"""
if coord in ["forecast_reference_time", "model"]:
if cube.coords("forecast_reference_time"):
if cycletime is None:
new_cycletime = (np.max(
cube_orig.coord("forecast_reference_time").points))
else:
cycletime_units = (
cube_orig.coord("forecast_reference_time").units.origin)
cycletime_calendar = (
cube.coord("forecast_reference_time").units.calendar)
new_cycletime = cycletime_to_number(
cycletime,
time_unit=cycletime_units,
calendar=cycletime_calendar)
# Preserve the data type to avoid converting ints to floats.
fr_type = cube.coord("forecast_reference_time").dtype
new_cycletime = np.round(new_cycletime).astype(fr_type)
cube.coord("forecast_reference_time").points = new_cycletime
cube.coord("forecast_reference_time").bounds = None
if cube.coords("forecast_period"):
forecast_period = (forecast_period_coord(
cube, force_lead_time_calculation=True))
forecast_period.bounds = None
forecast_period.convert_units(cube.coord("forecast_period").units)
forecast_period.var_name = cube.coord("forecast_period").var_name
cube.replace_coord(forecast_period)
elif cube.coords("forecast_reference_time") and cube.coords("time"):
forecast_period = (forecast_period_coord(cube))
ndim = cube.coord_dims("time")
cube.add_aux_coord(forecast_period, data_dims=ndim)
for coord in ["model_id", "model_realization", "realization"]:
if cube.coords(coord) and cube.coord(coord).shape == (1, ):
cube.remove_coord(coord)
if coords_for_bounds_removal is None:
coords_for_bounds_removal = []
for coord in cube.coords():
if coord.name() in coords_for_bounds_removal:
if coord.shape == (1, ) and coord.has_bounds():
coord.bounds = None
return cube
def test_basic_AuxCoord(self):
"""Test that an iris.coord.AuxCoord is returned."""
cube = add_forecast_reference_time_and_forecast_period(set_up_cube())
cube.remove_coord('forecast_period')
result = forecast_period_coord(cube, force_lead_time_calculation=True)
self.assertIsInstance(result, iris.coords.AuxCoord)
def test_basic(self):
"""Test that an iris.coord.DimCoord is returned."""
cube = add_forecast_reference_time_and_forecast_period(set_up_cube())
result = forecast_period_coord(cube)
self.assertIsInstance(result, iris.coords.DimCoord)
def process(self, cube, mask_cube=None):
"""
Supply neighbourhood processing method, in order to smooth the
input cube.
Args:
cube (Iris.cube.Cube):
Cube to apply a neighbourhood processing method to, in order to
generate a smoother field.
Keyword Args:
mask_cube (Iris.cube.Cube):
Cube containing the array to be used as a mask.
Returns:
cube (Iris.cube.Cube):
Cube after applying a neighbourhood processing method, so that
the resulting field is smoothed.
"""
if (not getattr(self.neighbourhood_method, "run", None)
or not callable(self.neighbourhood_method.run)):
msg = ("{} is not valid as a neighbourhood_method. "
"Please choose a valid neighbourhood_method with a "
"run method.".format(self.neighbourhood_method))
raise ValueError(msg)
# Check if a dimensional realization coordinate exists. If so, the
# cube is sliced, so that it becomes a scalar coordinate.
try:
cube.coord('realization', dim_coords=True)
except iris.exceptions.CoordinateNotFoundError:
slices_over_realization = [cube]
else:
slices_over_realization = cube.slices_over("realization")
if 'source_realizations' in cube.attributes:
msg = ("Realizations and attribute source_realizations "
"should not both be set in input cube")
raise ValueError(msg)
if np.isnan(cube.data).any():
raise ValueError("Error: NaN detected in input cube data")
cubes_real = []
for cube_realization in slices_over_realization:
if self.lead_times is None:
cube_new = self.neighbourhood_method.run(cube_realization,
self.radii,
mask_cube=mask_cube)
else:
# Interpolate to find the radius at each required lead time.
fp_coord = forecast_period_coord(cube_realization)
fp_coord.convert_units("hours")
required_radii = self._find_radii(
cube_lead_times=fp_coord.points)
cubes_time = iris.cube.CubeList([])
# Find the number of grid cells required for creating the
# neighbourhood, and then apply the neighbourhood
# processing method to smooth the field.
for cube_slice, radius in (zip(
cube_realization.slices_over("time"), required_radii)):
cube_slice = self.neighbourhood_method.run(
cube_slice, radius, mask_cube=mask_cube)
cubes_time.append(cube_slice)
if len(cubes_time) > 1:
cube_new = concatenate_cubes(cubes_time,
coords_to_slice_over=["time"])
else:
cube_new = cubes_time[0]
cubes_real.append(cube_new)
if len(cubes_real) > 1:
combined_cube = concatenate_cubes(
cubes_real, coords_to_slice_over=["realization"])
else:
combined_cube = cubes_real[0]
# Promote dimensional coordinates that used to be present.
exception_coordinates = (find_dimension_coordinate_mismatch(
cube, combined_cube, two_way_mismatch=False))
combined_cube = check_cube_coordinates(
cube, combined_cube, exception_coordinates=exception_coordinates)
return combined_cube
def conform_metadata(cube, cube_orig, coord, cycletime=None):
"""Ensure that the metadata conforms after blending together across
the chosen coordinate.
The metadata adjustments are:
- Forecast reference time: If a cycletime is not present, the
most recent available forecast_reference_time is used.
If a cycletime is present, the cycletime is used as the
forecast_reference_time instead.
- Forecast period: If a forecast_period coordinate is present,
and cycletime is not present, the lowest forecast_period is
used. If a forecast_period coordinate is present, and the
cycletime is present, forecast_periods are forceably re-calculated
from the time and forecast_reference_time coordinates so that their
values are self-consistent.
- Forecast reference time and time: If forecast_reference_time and
time coordinates are present, then a forecast_period coordinate is
calculated and added to the cube.
- Model_id, model_realization and realization coordinates are removed.
- A title attribute is added to the cube if none is found. Otherwise
the attributes are unchanged.
Args:
cube (iris.cube.Cube):
Cube containing the metadata to be adjusted.
cube_orig (iris.cube.Cube):
Cube containing metadata that may be useful for adjusting
metadata on the `cube` variable.
coord (str):
Coordinate that has been blended. This allows specific metadata
changes to be limited to whichever coordinate is being blended.
cycletime (str):
The cycletime in a YYYYMMDDTHHMMZ format e.g. 20171122T0100Z.
Returns:
cube (iris.cube.Cube):
Cube containing the adjusted metadata.
"""
# unify time coordinates for cycle and grid (model) blends
if coord in ["forecast_reference_time", "model_id"]:
# if cycle blending, update forecast reference time and remove bounds
if cube.coords("forecast_reference_time"):
if cycletime is None:
new_cycletime = (np.max(
cube_orig.coord("forecast_reference_time").points))
else:
cycletime_units = (
cube_orig.coord("forecast_reference_time").units.origin)
cycletime_calendar = (
cube.coord("forecast_reference_time").units.calendar)
new_cycletime = cycletime_to_number(
cycletime,
time_unit=cycletime_units,
calendar=cycletime_calendar)
# Preserve the data type to avoid converting ints to floats.
frt_type = cube.coord("forecast_reference_time").dtype
new_cycletime = np.round(new_cycletime).astype(frt_type)
cube.coord("forecast_reference_time").points = new_cycletime
cube.coord("forecast_reference_time").bounds = None
# recalculate forecast period coordinate
if cube.coords("forecast_period"):
forecast_period = forecast_period_coord(
cube, force_lead_time_calculation=True)
forecast_period.convert_units(cube.coord("forecast_period").units)
forecast_period.var_name = cube.coord("forecast_period").var_name
cube.replace_coord(forecast_period)
elif cube.coords("forecast_reference_time") and cube.coords("time"):
forecast_period = forecast_period_coord(cube)
ndim = cube.coord_dims("time")
cube.add_aux_coord(forecast_period, data_dims=ndim)
# update blended cube attributes
if "title" not in cube.attributes.keys():
cube.attributes["title"] = "IMPROVER Model Forecast"
# remove appropriate scalar coordinates
for crd in ["model_id", "model_configuration", "realization"]:
if cube.coords(crd) and cube.coord(crd).shape == (1, ):
cube.remove_coord(crd)
return cube
def add_coordinate(incube,
coord_points,
coord_name,
coord_units=None,
dtype=np.float32,
order=None,
is_datetime=False):
"""
Function to duplicate a sample cube with an additional coordinate to create
a cubelist. The cubelist is merged to create a single cube, which can be
reordered to place the new coordinate in the required position.
Args:
incube (iris.cube.Cube):
Cube to be duplicated.
coord_points (list):
Values for the coordinate.
coord_name (str):
Long name of the coordinate to be added.
Kwargs:
coord_units (str):
Coordinate unit required.
dtype (type):
Datatype for coordinate points.
order (list):
Optional list of integers to reorder the dimensions on the new
merged cube. For example, if the new coordinate is required to
be in position 1 on a 4D cube, use order=[1, 0, 2, 3] to swap the
new coordinate position with that of the original leading
coordinate.
is_datetime (bool):
If "true", the leading coordinate points have been given as a
list of datetime objects and need converting. In this case the
"coord_units" argument is overridden and the time points provided
in seconds. The "dtype" argument is overridden and set to int64.
Returns:
iris.cube.Cube:
Cube containing an additional dimension coordinate.
"""
# if the coordinate already exists as a scalar coordinate, remove it
cube = incube.copy()
try:
cube.remove_coord(coord_name)
except CoordinateNotFoundError:
pass
# if new coordinate points are provided as datetimes, convert to seconds
if is_datetime:
coord_units = TIME_UNIT
dtype = np.int64
new_coord_points = []
for val in coord_points:
time_point_seconds = np.round(date2num(val, TIME_UNIT,
CALENDAR)).astype(np.int64)
new_coord_points.append(time_point_seconds)
coord_points = new_coord_points
cubes = iris.cube.CubeList([])
for val in coord_points:
temp_cube = cube.copy()
temp_cube.add_aux_coord(
DimCoord(np.array([val], dtype=dtype),
long_name=coord_name,
units=coord_units))
# recalculate forecast period if time or frt have been updated
if is_datetime and "time" in coord_name:
forecast_period = forecast_period_coord(
temp_cube, force_lead_time_calculation=True)
try:
temp_cube.replace_coord(forecast_period)
except CoordinateNotFoundError:
temp_cube.add_aux_coord(forecast_period)
cubes.append(temp_cube)
new_cube = cubes.merge_cube()
if order is not None:
new_cube.transpose(order)
return new_cube
