def test_with_multiple_realizations(request, cube_with_realizations, land_fixture):
"""Test for multiple realizations, so that multiple
iterations will be required within the process method."""
cube = cube_with_realizations
land = request.getfixturevalue(land_fixture) if land_fixture else None
expected = np.array(
[
[
[0.0, 0.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
],
[
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 0.0],
],
]
)
cube.data[0, 2, 1] = 1.0
cube.data[1, 1, 3] = 1.0
result = OccurrenceWithinVicinity(radius=RADIUS, land_mask_cube=land)(cube)
assert isinstance(result, Cube)
assert np.allclose(result.data, expected)
def test_two_radii_provided_exception(cube, radius):
"""Test an exception is raised if both radius and grid_point_radius are
provided as arguments."""
with pytest.raises(
ValueError, match="Only one of radius or grid_point_radius should be set"
):
OccurrenceWithinVicinity(radius=radius, grid_point_radius=2)
def test_with_multiple_times(request, cube_with_realizations, land_fixture):
"""Test for multiple times, so that multiple
iterations will be required within the process method."""
cube = cube_with_realizations
land = request.getfixturevalue(land_fixture) if land_fixture else None
expected = np.array(
[
[
[0.0, 0.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
],
[
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 0.0],
],
]
)
cube = cube[0]
cube = add_coordinate(cube, TIMESTEPS, "time", is_datetime=True,)
cube.data[0, 2, 1] = 1.0
cube.data[1, 1, 3] = 1.0
orig_shape = cube.data.shape
result = OccurrenceWithinVicinity(radius=RADIUS, land_mask_cube=land)(cube)
assert isinstance(result, Cube)
assert result.data.shape == orig_shape
assert np.allclose(result.data, expected)
def process(self, cube):
"""
Identify the probability of having a phenomenon occur within a
vicinity.
The steps for this are as follows:
1. Calculate the occurrence of a phenomenon within a defined vicinity.
2. If the cube contains a realization dimension coordinate, find the
mean.
3. Compute neighbourhood processing.
Args:
cube : Iris.cube.Cube
A cube that has been thresholded.
Returns:
cube : Iris.cube.Cube
A cube containing neighbourhood probabilities to represent the
probability of an occurrence within the vicinity given a
pre-defined spatial uncertainty.
"""
cube = OccurrenceWithinVicinity(self.distance).process(cube)
if cube.coord_dims('realization'):
cube = cube.collapsed('realization', iris.analysis.MEAN)
cube = NeighbourhoodProcessing(self.neighbourhood_method, self.radii,
self.lead_times, self.unweighted_mode,
self.ens_factor).process(cube)
return cube
def process(cube: cli.inputcube, vicinity: cli.comma_separated_list = None):
"""Module to apply vicinity processing to data.
Calculate the maximum value within a vicinity radius about each point
in each x-y slice of the input cube.
If working with thresholded data, using this CLI to calculate a
neighbourhood maximum ensemble probability, the vicinity process must
be applied prior to averaging across the ensemble, i.e. collapsing the
realization coordinate. A typical chain might look like:
threshold --> vicinity --> realization collapse
Note that the threshold CLI can be used to perform this series of steps
in a single call without intermediate output.
Users should ensure they do not inadvertently apply vicinity processing
twice, once within the threshold CLI and then again using this CLI.
Args:
cube (iris.cube.Cube):
A cube containing data to which a vicinity is to be applied.
vicinity (list of float / int):
List of distances in metres used to define the vicinities within
which to search for an occurrence. Each vicinity provided will
lead to a different gridded field.
Returns:
iris.cube.Cube:
Cube with the vicinity processed data.
"""
from improver.utilities.spatial import OccurrenceWithinVicinity
vicinity = [float(x) for x in vicinity]
return OccurrenceWithinVicinity(radii=vicinity).process(cube)
def test_masked_data(self):
"""Test masked values are ignored in OccurrenceWithinVicinity."""
expected = np.array([
[1.0, 1.0, 1.0, 0.0, 10.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
])
data = np.zeros((1, 1, 5, 5))
data[0, 0, 0, 1] = 1.0
data[0, 0, 2, 3] = 1.0
data[0, 0, 0, 4] = 10.0
mask = np.zeros((1, 1, 5, 5))
mask[0, 0, 0, 4] = 1
masked_data = np.ma.array(data, mask=mask)
cube = set_up_cube(
masked_data,
"lwe_precipitation_rate",
"m s-1",
y_dimension_values=self.grid_values,
x_dimension_values=self.grid_values,
)
cube = cube[0, 0, :, :]
result = OccurrenceWithinVicinity(
self.distance).maximum_within_vicinity(cube)
self.assertIsInstance(result, Cube)
self.assertIsInstance(result.data, np.ma.core.MaskedArray)
self.assertArrayAlmostEqual(result.data.data, expected)
self.assertArrayAlmostEqual(result.data.mask, mask[0, 0, :, :])
def find_max_in_nbhood_orography(self, orography_cube):
"""
Find the maximum value of the orography in the neighbourhood around
each grid point. If self.grid_point_radius is zero, the orography is used
without neighbourhooding.
Args:
orography_cube (iris.cube.Cube):
The cube containing a single 2 dimensional array of orography
data
Returns:
iris.cube.Cube:
The cube containing the maximum in the grid_point_radius neighbourhood
of the orography data or the orography data itself if the radius is zero
"""
if self.grid_point_radius >= 1:
radius_in_metres = number_of_grid_cells_to_distance(
orography_cube, self.grid_point_radius
)
max_in_nbhood_orog = OccurrenceWithinVicinity(radius_in_metres)(
orography_cube
)
return max_in_nbhood_orog
else:
return orography_cube.copy()
def test_with_multiple_realizations(self):
"""Test for multiple realizations, so that multiple
iterations will be required within the process method."""
expected = np.array([
[[
[0.0, 0.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
]],
[[
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 0.0],
]],
])
data = np.zeros((2, 1, 4, 4))
data[0, 0, 2, 1] = 1.0
data[1, 0, 1, 3] = 1.0
cube = set_up_cube(data,
"lwe_precipitation_rate",
"m s-1",
realizations=np.array([0, 1]))
result = OccurrenceWithinVicinity(self.distance)(cube)
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(result.data, expected)
def test_with_multiple_times(self):
"""Test for multiple times, so that multiple
iterations will be required within the process method."""
expected = np.array([[
[
[0.0, 0.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
],
[
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 0.0],
],
]])
data = np.zeros((1, 2, 4, 4))
data[0, 0, 2, 1] = 1.0
data[0, 1, 1, 3] = 1.0
cube = set_up_cube(
data,
"lwe_precipitation_rate",
"m s-1",
timesteps=np.array([402192.5, 402195.5]),
)
orig_shape = cube.data.shape
result = OccurrenceWithinVicinity(self.distance)(cube)
self.assertIsInstance(result, Cube)
self.assertEqual(result.data.shape, orig_shape)
self.assertArrayAlmostEqual(result.data, expected)
def test_with_multiple_times(self):
"""Test for multiple times, so that multiple
iterations will be required within the process method."""
expected = np.array([
[
[0.0, 0.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
],
[
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 0.0],
],
])
cube = self.cube[0]
cube = add_coordinate(
cube,
self.timesteps,
"time",
is_datetime=True,
)
cube.data[0, 2, 1] = 1.0
cube.data[1, 1, 3] = 1.0
orig_shape = cube.data.shape
result = OccurrenceWithinVicinity(self.distance)(cube)
self.assertIsInstance(result, Cube)
self.assertEqual(result.data.shape, orig_shape)
self.assertArrayAlmostEqual(result.data, expected)
def test_coordinate_order_with_multiple_realizations_and_times(
cube_with_realizations, kwargs):
"""Test the output coordinate order for input cubes with multiple
realizations and times using multiple vicinity radii."""
cube = cube_with_realizations
cube = add_coordinate(
cube,
TIMESTEPS,
"time",
order=[1, 0, 2, 3],
is_datetime=True,
)
# Add the expected radius_of_vicinity coordinate dimension size
orig_shape = list(cube.data.copy().shape)
orig_shape.insert(-2, len(list(kwargs.values())[0]))
# Add the expected radius_of_vicinity coordinate dimension name
orig_order = [crd.name() for crd in cube.coords(dim_coords=True)]
orig_order.insert(-2, "radius_of_vicinity")
result = OccurrenceWithinVicinity(**kwargs)(cube)
result_order = [crd.name() for crd in result.coords(dim_coords=True)]
assert list(result.data.shape) == orig_shape
assert result_order == orig_order
def test_basic(self):
"""Test for binary events to determine where there is an occurrence
within the vicinity."""
expected = np.array([
[1.0, 1.0, 1.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
])
data = np.zeros((1, 1, 5, 5))
data[0, 0, 0, 1] = 1.0
data[0, 0, 2, 3] = 1.0
cube = set_up_cube(
data,
"lwe_precipitation_rate",
"m s-1",
y_dimension_values=self.grid_values,
x_dimension_values=self.grid_values,
)
cube = cube[0, 0, :, :]
result = OccurrenceWithinVicinity(
self.distance).maximum_within_vicinity(cube)
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(result.data, expected)
def test_negative_radii_provided_exception(cube, kwargs):
"""Test an exception is raised if the radius provided in either form is
a negative value."""
expected = "Vicinity processing requires only positive vicinity radii"
with pytest.raises(ValueError, match=expected):
OccurrenceWithinVicinity(**kwargs).get_grid_point_radius(cube)
def test_with_invalid_land_mask_name(land_mask_cube):
"""Test that a mis-named land mask is rejected correctly."""
bad_mask_cube = land_mask_cube.copy()
bad_mask_cube.rename("kittens")
with pytest.raises(
ValueError,
match="Expected land_mask_cube to be called land_binary_mask, not kittens",
):
OccurrenceWithinVicinity(radius=RADIUS, land_mask_cube=bad_mask_cube)
def test_no_radii_provided_exception(cube, kwargs):
"""Test an exception is raised if neither radii nor grid_point_radii are
set to non-zero values."""
expected = ("Vicinity processing requires that one of radii or "
"grid_point_radii should be set to a non-zero value")
with pytest.raises(ValueError, match=expected):
OccurrenceWithinVicinity(**kwargs)
def test_zero_radius(request, fixture_name, keyword, value):
"""Test that if a zero radius / grid point radius, or no radius / grid point
radius is provided, the input data is returned unchanged. This test uses
both an equal area and latlon grid as a 0 radius can be applied to both."""
cube = request.getfixturevalue(fixture_name)
kwargs = {keyword: value}
expected = cube.data.copy()
result = OccurrenceWithinVicinity(**kwargs).maximum_within_vicinity(cube)
assert np.allclose(result.data, expected)
def test_basic(cube, binary_expected, kwargs):
"""Test for binary events to determine where there is an occurrence
within the vicinity."""
cube.data[0, 1] = 1.0
cube.data[2, 3] = 1.0
result = OccurrenceWithinVicinity(**kwargs).maximum_within_vicinity(cube)
assert isinstance(result, Cube)
assert np.allclose(result.data, binary_expected)
def test_two_radii_types_provided_exception(cube, radius):
"""Test an exception is raised if both radii and grid_point_radii are
provided as non-zero arguments."""
expected = ("Vicinity processing requires that only one of radii or "
"grid_point_radii should be set")
with pytest.raises(ValueError, match=expected):
OccurrenceWithinVicinity(radii=[radius], grid_point_radii=2)
def test_basic_latlon(latlon_cube, binary_expected):
"""Test for occurrence in vicinity calculation on a lat-lon (non equal
area) grid using a grid_point_radius."""
latlon_cube.data[0, 1] = 1.0
latlon_cube.data[2, 3] = 1.0
result = OccurrenceWithinVicinity(
grid_point_radii=[GRID_POINT_RADIUS]).process(latlon_cube)
assert isinstance(result, Cube)
assert np.allclose(result.data, binary_expected)
def test_with_invalid_land_mask_coords(cube, land_mask_cube):
"""Test that a spatially mis-matched land mask is rejected correctly."""
bad_mask_cube = land_mask_cube.copy()
bad_points = np.array(bad_mask_cube.coord(axis="x").points)
bad_points[0] += 1
bad_mask_cube.coord(axis="x").points = bad_points
with pytest.raises(
ValueError,
match="Supplied cube do not have the same spatial coordinates and land mask",
):
OccurrenceWithinVicinity(radius=RADIUS, land_mask_cube=bad_mask_cube)(cube)
def test_no_realization_or_time(self):
"""Test for no realizations and no times, so that the iterations
will not require slicing cubes within the process method."""
expected = np.array([[0., 0., 0., 0.], [1., 1., 1., 0.],
[1., 1., 1., 0.], [1., 1., 1., 0.]])
data = np.zeros((1, 1, 4, 4))
data[0, 0, 2, 1] = 1.0
cube = set_up_cube(data,
"lwe_precipitation_rate",
"m s-1",
realizations=np.array([0]))
cube = iris.util.squeeze(cube)
result = OccurrenceWithinVicinity(self.distance).process(cube)
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(result.data, expected)
def test_fuzzy(cube):
"""Test for non-binary events to determine where there is an occurrence
within the vicinity."""
expected = np.array([
[1.0, 1.0, 1.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 0.5, 0.5],
[0.0, 0.0, 0.5, 0.5, 0.5],
[0.0, 0.0, 0.5, 0.5, 0.5],
[0.0, 0.0, 0.0, 0.0, 0.0],
])
cube.data[0, 1] = 1.0
cube.data[2, 3] = 0.5
result = OccurrenceWithinVicinity(radii=[RADIUS]).process(cube)
assert isinstance(result, Cube)
assert np.allclose(result.data, expected)
def test_basic(cube):
"""Test for binary events to determine where there is an occurrence
within the vicinity."""
expected = np.array([
[1.0, 1.0, 1.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0, 1.0],
[0.0, 0.0, 1.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
])
cube.data[0, 1] = 1.0
cube.data[2, 3] = 1.0
result = OccurrenceWithinVicinity(DISTANCE).maximum_within_vicinity(cube)
assert isinstance(result, Cube)
assert np.allclose(result.data, expected)
def test_different_distance(cube, kwargs):
"""Test for binary events to determine where there is an occurrence
within the vicinity for an alternative distance."""
expected = np.array([
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
[0.0, 1.0, 1.0, 1.0, 1.0],
[0.0, 1.0, 1.0, 1.0, 1.0],
])
cube.data[0, 1] = 1.0
cube.data[2, 3] = 1.0
result = OccurrenceWithinVicinity(**kwargs).process(cube)
assert isinstance(result, Cube)
assert np.allclose(result.data, expected)
def process(self, cube):
"""
Identify the probability of having a phenomenon occur within a
vicinity.
The steps for this are as follows:
1. Calculate the occurrence of a phenomenon within
a defined vicinity.
2. If the cube contains a realization dimension coordinate,
find the mean.
3. Compute neighbourhood processing.
Args:
cube (iris.cube.Cube):
A cube that has been thresholded.
Returns:
cube (iris.cube.Cube):
A cube containing neighbourhood probabilities to represent the
probability of an occurrence within the vicinity given a
pre-defined spatial uncertainty.
"""
cube = OccurrenceWithinVicinity(self.distance).process(cube)
try:
if cube.coord_dims('realization'):
ens_members = cube.coord('realization').points
# BUG in iris: collapsed returns a masked cube regardless of
# input status. If input is not masked, output mask does not
# match data. Fix is to re-cast output to an unmasked array.
cube_is_masked = isinstance(cube.data, np.ma.MaskedArray)
cube = cube.collapsed('realization', iris.analysis.MEAN)
if not cube_is_masked:
cube.data = np.array(cube.data)
cube.remove_coord('realization')
cube.attributes['source_realizations'] = ens_members
except iris.exceptions.CoordinateNotFoundError:
pass
cube = NeighbourhoodProcessing(
self.neighbourhood_method,
self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
ens_factor=self.ens_factor).process(cube)
cube.rename(cube.name() + '_in_vicinity')
return cube
def test_no_realization_or_time(self):
"""Test for no realizations and no times, so that the iterations
will not require slicing cubes within the process method."""
expected = np.array([
[0.0, 0.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
[1.0, 1.0, 1.0, 0.0],
])
cube = self.cube[0]
cube.data[2, 1] = 1.0
orig_shape = cube.data.shape
result = OccurrenceWithinVicinity(self.distance)(cube)
self.assertIsInstance(result, Cube)
self.assertEqual(result.data.shape, orig_shape)
self.assertArrayAlmostEqual(result.data, expected)
def test_fuzzy(self):
"""Test for non-binary events to determine where there is an occurrence
within the vicinity."""
expected = np.array([
[1.0, 1.0, 1.0, 0.0, 0.0],
[1.0, 1.0, 1.0, 0.5, 0.5],
[0.0, 0.0, 0.5, 0.5, 0.5],
[0.0, 0.0, 0.5, 0.5, 0.5],
[0.0, 0.0, 0.0, 0.0, 0.0],
])
self.cube.data[0, 1] = 1.0
self.cube.data[2, 3] = 0.5
result = OccurrenceWithinVicinity(
self.distance).maximum_within_vicinity(self.cube)
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(result.data, expected)
def test_different_distance(self):
"""Test for binary events to determine where there is an occurrence
within the vicinity for an alternative distance."""
expected = np.array([
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
[0.0, 1.0, 1.0, 1.0, 1.0],
[0.0, 1.0, 1.0, 1.0, 1.0],
])
self.cube.data[0, 1] = 1.0
self.cube.data[2, 3] = 1.0
distance = 4000.0
result = OccurrenceWithinVicinity(distance).maximum_within_vicinity(
self.cube)
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(result.data, expected)
def test_metadata(request, cube, kwargs, expected_coord):
"""Test that the metadata on the cube reflects the data it contains
following the application of vicinity processing.
Parameterisation tests this using a radius defined as a distance or as
a number of grid points."""
expected_coord = request.getfixturevalue(expected_coord)
plugin = OccurrenceWithinVicinity(**kwargs)
# repeat the test with the same plugin instance to ensure self variables
# have not been modified.
for _ in range(2):
result = plugin.process(cube)
assert isinstance(result, Cube)
assert result.coord("radius_of_vicinity") == expected_coord
assert "in_vicinity" in result.name()
def test_with_land_mask(cube, land_mask_cube):
"""Test that a land mask is used correctly."""
expected = np.array([
[1.0, 1.0, 1.0, 10.0, 10.0],
[1.0, 1.0, 1.0, 10.0, 10.0],
[0.0, 0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
])
cube.data[0, 1] = 1.0 # would not cross mask
cube.data[2, 3] = 1.0 # would cross mask
cube.data[0, 4] = 10.0 # would not cross mask
result = OccurrenceWithinVicinity(
radii=[RADIUS], land_mask_cube=land_mask_cube).process(cube)
assert isinstance(result, Cube)
assert ~isinstance(result.data, np.ma.core.MaskedArray)
assert np.allclose(result.data, expected)
