def process(self, cube, mask_cube):
"""
1. Iterate over the chosen coordinate within the mask_cube and apply
the mask at each iteration to the cube that is to be neighbourhood
processed.
2. Concatenate the cubes from each iteration together to create a
single cube.
Args:
cube (Iris.cube.Cube):
Cube containing the array to which the square neighbourhood
will be applied.
mask_cube (Iris.cube.Cube):
Cube containing the array to be used as a mask.
Returns:
concatenated_cube (Iris.cube.Cube):
Cube containing the smoothed field after the square
neighbourhood method has been applied when applying masking
for each point along the coord_for_masking coordinate.
The resulting cube is concatenated so that the dimension
coordinates match the input cube.
"""
yname = cube.coord(axis='y').name()
xname = cube.coord(axis='x').name()
result_slices = iris.cube.CubeList([])
# Take 2D slices of the input cube for memory issues.
for x_y_slice in cube.slices([yname, xname]):
cube_slices = iris.cube.CubeList([])
# Apply each mask in in mask_cube to the 2D input slice.
for cube_slice in mask_cube.slices_over(self.coord_for_masking):
output_cube = NeighbourhoodProcessing(
self.neighbourhood_method, self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
sum_or_fraction=self.sum_or_fraction, re_mask=self.re_mask
).process(x_y_slice, mask_cube=cube_slice)
coord_object = cube_slice.coord(self.coord_for_masking).copy()
output_cube.add_aux_coord(coord_object)
output_cube = iris.util.new_axis(
output_cube, self.coord_for_masking)
cube_slices.append(output_cube)
concatenated_cube = cube_slices.concatenate_cube()
exception_coordinates = (
find_dimension_coordinate_mismatch(
x_y_slice, concatenated_cube, two_way_mismatch=False))
concatenated_cube = check_cube_coordinates(
x_y_slice, concatenated_cube,
exception_coordinates=exception_coordinates)
result_slices.append(concatenated_cube)
result = result_slices.merge_cube()
exception_coordinates = (
find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False))
result = check_cube_coordinates(
cube, result,
exception_coordinates=exception_coordinates)
return result
def run(self, cube, radius):
"""
Method to apply a circular kernel to the data within the input cube in
order to derive percentiles over the kernel.
Args:
cube : Iris.cube.Cube
Cube containing array to apply processing to.
radius : Float
Radius in metres for use in specifying the number of
grid cells used to create a circular neighbourhood.
Returns:
result : Iris.cube.Cube
Cube containing the percentile fields.
Has percentile as an added dimension.
"""
# Check that the cube has an equal area grid.
check_if_grid_is_equal_area(cube)
# Take data array and identify X and Y axes indices
ranges_tuple = convert_distance_into_number_of_grid_cells(
cube, radius, MAX_RADIUS_IN_GRID_CELLS)
ranges_xy = np.array(ranges_tuple)
kernel = circular_kernel(ranges_xy, ranges_tuple, weighted_mode=False)
# Loop over each 2D slice to reduce memory demand and derive
# percentiles on the kernel. Will return an extra dimension.
pctcubelist = iris.cube.CubeList()
for slice_2d in cube.slices(['projection_y_coordinate',
'projection_x_coordinate']):
pctcubelist.append(
self.pad_and_unpad_cube(slice_2d, kernel))
result = pctcubelist.merge_cube()
exception_coordinates = (
find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False))
result = (
check_cube_coordinates(
cube, result, exception_coordinates=exception_coordinates))
# Arrange cube, so that the coordinate order is:
# realization, percentile, other coordinates.
required_order = []
if result.coords("realization"):
if result.coords("realization", dimensions=[]):
result = iris.util.new_axis(result, "realization")
required_order.append(result.coord_dims("realization")[0])
if result.coords("percentiles_over_neighbourhood"):
required_order.append(
result.coord_dims("percentiles_over_neighbourhood")[0])
other_coords = []
for coord in result.dim_coords:
if coord.name() not in ["realization",
"percentiles_over_neighbourhood"]:
other_coords.append(result.coord_dims(coord.name())[0])
required_order.extend(other_coords)
result.transpose(required_order)
return result
def test_basic(self):
"""Test that the method returns the expected cube type with coords"""
result = self.plugin.process(
CubeList([self.fg_cube, self.ltng_cube, self.precip_cube]))
self.assertIsInstance(result, Cube)
# We expect the threshold coordinate to have been removed.
self.assertCountEqual(
find_dimension_coordinate_mismatch(result, self.precip_cube),
['threshold'])
self.assertTrue(result.name() == 'probability_of_lightning')
def test_two_way_mismatch(self):
"""Test when finding a two-way mismatch, when the first and second
cube contain different coordinates."""
first_cube = self.cube.copy()
second_cube = next(self.cube.slices_over("realization")).copy()
second_cube.remove_coord("realization")
second_cube = add_coordinate(second_cube, [10, 20], "height", "m")
result = find_dimension_coordinate_mismatch(first_cube, second_cube)
self.assertIsInstance(result, list)
self.assertListEqual(result, ["height", "realization"])
def test_two_way_mismatch(self):
"""Test when finding a two-way mismatch, when the first and second
cube contain different coordinates."""
cube = set_up_cube()
first_cube = cube.copy()
first_cube.remove_coord("time")
second_cube = cube.copy()
second_cube.remove_coord("realization")
result = find_dimension_coordinate_mismatch(first_cube, second_cube)
self.assertIsInstance(result, list)
self.assertListEqual(result, ["time", "realization"])
def test_mismatch_in_first_cube(self):
"""Test when finding a one-way mismatch, so that the second cube has
a missing coordinate. This returns an empty list."""
cube = set_up_cube()
first_cube = cube.copy()
second_cube = cube.copy()
second_cube.remove_coord("time")
result = find_dimension_coordinate_mismatch(
first_cube, second_cube, two_way_mismatch=False)
self.assertIsInstance(result, list)
self.assertFalse(result)
def test_mismatch_in_second_cube(self):
"""Test when finding a one-way mismatch, so that the first cube has
a missing coordinate. This returns a list with the missing coordinate
name.l"""
cube = set_up_cube()
first_cube = cube.copy()
first_cube.remove_coord("time")
second_cube = cube.copy()
result = find_dimension_coordinate_mismatch(
first_cube, second_cube, two_way_mismatch=False)
self.assertIsInstance(result, list)
self.assertListEqual(result, ["time"])
def test_mismatch_in_second_cube(self):
"""Test when finding a one-way mismatch, so that the first cube has
a missing coordinate. This returns a list with the missing coordinate
name."""
first_cube = next(self.cube.slices_over("realization")).copy()
first_cube.remove_coord("realization")
second_cube = self.cube.copy()
result = find_dimension_coordinate_mismatch(first_cube,
second_cube,
two_way_mismatch=False)
self.assertIsInstance(result, list)
self.assertListEqual(result, ["realization"])
def process(self, cube: Cube) -> Cube:
"""
Method to apply a circular kernel to the data within the input cube in
order to derive percentiles over the kernel.
Args:
cube:
Cube containing array to apply processing to.
Returns:
Cube containing the percentile fields.
Has percentile as an added dimension.
"""
super().process(cube)
if np.ma.is_masked(cube.data):
msg = ("The use of masked input cubes is not yet implemented in"
" the GeneratePercentilesFromANeighbourhood plugin.")
raise NotImplementedError(msg)
# Check that the cube has an equal area grid.
check_if_grid_is_equal_area(cube)
# Take data array and identify X and Y axes indices
grid_cell = distance_to_number_of_grid_cells(cube, self.radius)
check_radius_against_distance(cube, self.radius)
kernel = circular_kernel(grid_cell, weighted_mode=False)
# Loop over each 2D slice to reduce memory demand and derive
# percentiles on the kernel. Will return an extra dimension.
pctcubelist = iris.cube.CubeList()
for slice_2d in cube.slices(
["projection_y_coordinate", "projection_x_coordinate"]):
pctcubelist.append(self.pad_and_unpad_cube(slice_2d, kernel))
result = pctcubelist.merge_cube()
exception_coordinates = find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False)
result = check_cube_coordinates(
cube, result, exception_coordinates=exception_coordinates)
# Arrange cube, so that the coordinate order is:
# realization, percentile, other coordinates.
required_order = []
if result.coords("realization", dim_coords=True):
required_order.append(result.coord_dims("realization")[0])
if result.coords("percentile", dim_coords=True):
required_order.append(result.coord_dims("percentile")[0])
other_coords = []
for coord in result.dim_coords:
if coord.name() not in ["realization", "percentile"]:
other_coords.append(result.coord_dims(coord.name())[0])
required_order.extend(other_coords)
result.transpose(required_order)
return result
def test_basic(self):
"""Test that the method returns the expected cube type with coords"""
result = self.plugin(
CubeList([self.fg_cube, self.ltng_cube, self.precip_cube]))
self.assertIsInstance(result, Cube)
# We expect the threshold coordinate to have been removed.
threshold_coord = find_threshold_coordinate(self.precip_cube).name()
self.assertCountEqual(
find_dimension_coordinate_mismatch(result, self.precip_cube),
[threshold_coord],
)
self.assertEqual(result.name(),
"probability_of_rate_of_lightning_above_threshold")
self.assertEqual(result.units, "1")
def test_no_mismatch(self):
"""Test if there is no mismatch between the dimension coordinates."""
cube = set_up_cube()
result = find_dimension_coordinate_mismatch(cube, cube)
self.assertIsInstance(result, list)
self.assertFalse(result)
def process(self, cube: Cube, mask_cube: Optional[Cube] = None) -> Cube:
"""
Supply neighbourhood processing method, in order to smooth the
input cube.
Args:
cube:
Cube to apply a neighbourhood processing method to, in order to
generate a smoother field.
mask_cube:
Cube containing the array to be used as a mask.
Returns:
Cube after applying a neighbourhood processing method, so that
the resulting field is smoothed.
"""
if not getattr(self.neighbourhood_method, "run", None) or not callable(
self.neighbourhood_method.run):
msg = ("{} is not valid as a neighbourhood_method. "
"Please choose a valid neighbourhood_method with a "
"run method.".format(self.neighbourhood_method))
raise ValueError(msg)
# Check if a dimensional realization coordinate exists. If so, the
# cube is sliced, so that it becomes a scalar coordinate.
try:
cube.coord("realization", dim_coords=True)
except iris.exceptions.CoordinateNotFoundError:
slices_over_realization = [cube]
else:
slices_over_realization = cube.slices_over("realization")
if np.isnan(cube.data).any():
raise ValueError("Error: NaN detected in input cube data")
cubes_real = []
for cube_realization in slices_over_realization:
if self.lead_times is None:
cube_new = self.neighbourhood_method.run(cube_realization,
self.radii,
mask_cube=mask_cube)
else:
# Interpolate to find the radius at each required lead time.
fp_coord = forecast_period_coord(cube_realization)
fp_coord.convert_units("hours")
required_radii = self._find_radii(
cube_lead_times=fp_coord.points)
cubes_time = iris.cube.CubeList([])
# Find the number of grid cells required for creating the
# neighbourhood, and then apply the neighbourhood
# processing method to smooth the field.
for cube_slice, radius in zip(
cube_realization.slices_over("time"), required_radii):
cube_slice = self.neighbourhood_method.run(
cube_slice, radius, mask_cube=mask_cube)
cubes_time.append(cube_slice)
cube_new = MergeCubes()(cubes_time)
cubes_real.append(cube_new)
if len(cubes_real) > 1:
combined_cube = MergeCubes()(cubes_real,
slice_over_realization=True)
else:
combined_cube = cubes_real[0]
# Promote dimensional coordinates that used to be present.
exception_coordinates = find_dimension_coordinate_mismatch(
cube, combined_cube, two_way_mismatch=False)
combined_cube = check_cube_coordinates(
cube, combined_cube, exception_coordinates=exception_coordinates)
return combined_cube
def run(self,
cube: Cube,
radius: float,
mask_cube: Optional[Cube] = None) -> Cube:
"""
Method to apply a circular kernel to the data within the input cube in
order to derive percentiles over the kernel.
Args:
cube:
Cube containing array to apply processing to.
radius:
Radius in metres for use in specifying the number of
grid cells used to create a circular neighbourhood.
mask_cube:
Cube containing the array to be used as a mask.
Returns:
Cube containing the percentile fields.
Has percentile as an added dimension.
"""
if mask_cube is not None:
msg = ("The use of a mask cube with a circular kernel is not "
"yet implemented.")
raise NotImplementedError(msg)
# Check that the cube has an equal area grid.
check_if_grid_is_equal_area(cube)
# Take data array and identify X and Y axes indices
grid_cell = distance_to_number_of_grid_cells(cube, radius)
check_radius_against_distance(cube, radius)
ranges_xy = np.array((grid_cell, grid_cell))
kernel = circular_kernel(ranges_xy, grid_cell, weighted_mode=False)
# Loop over each 2D slice to reduce memory demand and derive
# percentiles on the kernel. Will return an extra dimension.
pctcubelist = iris.cube.CubeList()
for slice_2d in cube.slices(
["projection_y_coordinate", "projection_x_coordinate"]):
pctcubelist.append(self.pad_and_unpad_cube(slice_2d, kernel))
result = pctcubelist.merge_cube()
exception_coordinates = find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False)
result = check_cube_coordinates(
cube, result, exception_coordinates=exception_coordinates)
# Arrange cube, so that the coordinate order is:
# realization, percentile, other coordinates.
required_order = []
if result.coords("realization"):
if result.coords("realization", dimensions=[]):
result = iris.util.new_axis(result, "realization")
required_order.append(result.coord_dims("realization")[0])
if result.coords("percentile"):
required_order.append(result.coord_dims("percentile")[0])
other_coords = []
for coord in result.dim_coords:
if coord.name() not in ["realization", "percentile"]:
other_coords.append(result.coord_dims(coord.name())[0])
required_order.extend(other_coords)
result.transpose(required_order)
return result
def process(self, cube, mask_cube):
"""
1. Iterate over the chosen coordinate within the mask_cube and apply
the mask at each iteration to the cube that is to be neighbourhood
processed.
2. Concatenate the cubes from each iteration together to create a
single cube.
Args:
cube (iris.cube.Cube):
Cube containing the array to which the square neighbourhood
will be applied.
mask_cube (iris.cube.Cube):
Cube containing the array to be used as a mask.
Returns:
iris.cube.Cube:
Cube containing the smoothed field after the square
neighbourhood method has been applied when applying masking
for each point along the coord_for_masking coordinate.
The resulting cube is concatenated so that the dimension
coordinates match the input cube.
"""
yname = cube.coord(axis="y").name()
xname = cube.coord(axis="x").name()
result_slices = iris.cube.CubeList([])
if self.collapse_weights is None:
collapse_plugin = None
else:
collapse_plugin = CollapseMaskedNeighbourhoodCoordinate(
self.coord_for_masking, self.collapse_weights)
# Take 2D slices of the input cube for memory issues.
prev_x_y_slice = None
for x_y_slice in cube.slices([yname, xname]):
if prev_x_y_slice is not None and np.array_equal(
prev_x_y_slice.data, x_y_slice.data):
# Use same result as last time!
prev_result = result_slices[-1].copy()
for coord in x_y_slice.coords(dim_coords=False):
prev_result.coord(coord).points = coord.points.copy()
result_slices.append(prev_result)
continue
prev_x_y_slice = x_y_slice
cube_slices = iris.cube.CubeList([])
plugin = NeighbourhoodProcessing(
self.neighbourhood_method,
self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
sum_or_fraction=self.sum_or_fraction,
re_mask=self.re_mask,
)
# Apply each mask in in mask_cube to the 2D input slice.
for cube_slice in mask_cube.slices_over(self.coord_for_masking):
output_cube = plugin(x_y_slice, mask_cube=cube_slice)
coord_object = cube_slice.coord(self.coord_for_masking).copy()
output_cube.add_aux_coord(coord_object)
output_cube = iris.util.new_axis(output_cube,
self.coord_for_masking)
cube_slices.append(output_cube)
concatenated_cube = cube_slices.concatenate_cube()
exception_coordinates = find_dimension_coordinate_mismatch(
x_y_slice, concatenated_cube, two_way_mismatch=False)
concatenated_cube = check_cube_coordinates(
x_y_slice,
concatenated_cube,
exception_coordinates=exception_coordinates,
)
if collapse_plugin:
concatenated_cube = collapse_plugin(concatenated_cube)
result_slices.append(concatenated_cube)
result = result_slices.merge_cube()
exception_coordinates = find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False)
result = check_cube_coordinates(
cube, result, exception_coordinates=exception_coordinates)
return result
def process(self, cube: Cube, mask_cube: Cube) -> Cube:
"""
Apply neighbourhood processing with a mask to the input cube,
collapsing the coord_for_masking if collapse_weights have been provided.
Args:
cube:
Cube containing the array to which the square neighbourhood
will be applied.
mask_cube:
Cube containing the array to be used as a mask. The data in
this array is not an instance of numpy.ma.MaskedArray. Any sea
points that should be ignored are set to zeros in every layer
of the mask_cube.
Returns:
Cube containing the smoothed field after the square
neighbourhood method has been applied when applying masking
for each point along the coord_for_masking coordinate.
The resulting cube is concatenated so that the dimension
coordinates match the input cube.
"""
plugin = NeighbourhoodProcessing(
self.neighbourhood_method,
self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
sum_only=self.sum_only,
re_mask=self.re_mask,
)
yname = cube.coord(axis="y").name()
xname = cube.coord(axis="x").name()
result_slices = iris.cube.CubeList([])
# Take 2D slices of the input cube for memory issues.
prev_x_y_slice = None
for x_y_slice in cube.slices([yname, xname]):
if prev_x_y_slice is not None and np.array_equal(
prev_x_y_slice.data, x_y_slice.data):
# Use same result as last time!
prev_result = result_slices[-1].copy()
for coord in x_y_slice.coords(dim_coords=False):
prev_result.coord(coord).points = coord.points.copy()
result_slices.append(prev_result)
continue
prev_x_y_slice = x_y_slice
cube_slices = iris.cube.CubeList([])
# Apply each mask in in mask_cube to the 2D input slice.
for mask_slice in mask_cube.slices_over(self.coord_for_masking):
output_cube = plugin(x_y_slice, mask_cube=mask_slice)
coord_object = mask_slice.coord(self.coord_for_masking).copy()
output_cube.add_aux_coord(coord_object)
output_cube = iris.util.new_axis(output_cube,
self.coord_for_masking)
cube_slices.append(output_cube)
concatenated_cube = cube_slices.concatenate_cube()
if self.collapse_weights is not None:
concatenated_cube = self.collapse_mask_coord(concatenated_cube)
result_slices.append(concatenated_cube)
result = result_slices.merge_cube()
# Promote any single value dimension coordinates if they were
# dimension on the input cube.
exception_coordinates = find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False)
result = check_cube_coordinates(
cube, result, exception_coordinates=exception_coordinates)
return result
def process(self, cube):
"""
Supply neighbourhood processing method, in order to smooth the
input cube.
Parameters
----------
cube : Iris.cube.Cube
Cube to apply a neighbourhood processing method to, in order to
generate a smoother field.
Returns
-------
cube : Iris.cube.Cube
Cube after applying a neighbourhood processing method, so that the
resulting field is smoothed.
"""
if (not getattr(self.neighbourhood_method, "run", None)
or not callable(self.neighbourhood_method.run)):
msg = ("{} is not valid as a neighbourhood_method. "
"Please choose a valid neighbourhood_method with a "
"run method.".format(self.neighbourhood_method))
raise ValueError(msg)
# Check if the realization coordinate exists. If there are multiple
# values for the realization, then an exception is raised. Otherwise,
# the cube is sliced, so that the realization becomes a scalar
# coordinate.
try:
realiz_coord = cube.coord('realization')
except iris.exceptions.CoordinateNotFoundError:
if 'source_realizations' in cube.attributes:
num_ens = len(cube.attributes['source_realizations'])
else:
num_ens = 1.0
slices_over_realization = [cube]
else:
num_ens = len(realiz_coord.points)
slices_over_realization = cube.slices_over("realization")
if 'source_realizations' in cube.attributes:
msg = ("Realizations and attribute source_realizations "
"should not both be set in input cube")
raise ValueError(msg)
if np.isnan(cube.data).any():
raise ValueError("Error: NaN detected in input cube data")
cubelist = iris.cube.CubeList([])
for cube_realization in slices_over_realization:
if self.lead_times is None:
radius = self._find_radii(num_ens)
cube_new = self.neighbourhood_method.run(
cube_realization, radius)
else:
cube_lead_times = (find_required_lead_times(cube_realization))
# Interpolate to find the radius at each required lead time.
required_radii = (self._find_radii(
num_ens, cube_lead_times=cube_lead_times))
cubes = iris.cube.CubeList([])
# Find the number of grid cells required for creating the
# neighbourhood, and then apply the neighbourhood
# processing method to smooth the field.
for cube_slice, radius in (zip(
cube_realization.slices_over("time"), required_radii)):
cube_slice = self.neighbourhood_method.run(
cube_slice, radius)
cube_slice = iris.util.new_axis(cube_slice, "time")
cubes.append(cube_slice)
cube_new = concatenate_cubes(cubes,
coords_to_slice_over=["time"])
if cube_new.coords("realization", dim_coords=False):
cube_new = iris.util.new_axis(cube_new, "realization")
cubelist.append(cube_new)
combined_cube = cubelist.concatenate_cube()
# Promote dimensional coordinates that have been demoted to scalars.
exception_coordinates = (find_dimension_coordinate_mismatch(
cube, combined_cube, two_way_mismatch=False))
combined_cube = check_cube_coordinates(
cube, combined_cube, exception_coordinates=exception_coordinates)
return combined_cube
