def process(self, cube, mask_cube):
"""
1. Iterate over the chosen coordinate within the mask_cube and apply
the mask at each iteration to the cube that is to be neighbourhood
processed.
2. Concatenate the cubes from each iteration together to create a
single cube.
Args:
cube (Iris.cube.Cube):
Cube containing the array to which the square neighbourhood
will be applied.
mask_cube (Iris.cube.Cube):
Cube containing the array to be used as a mask.
Returns:
concatenated_cube (Iris.cube.Cube):
Cube containing the smoothed field after the square
neighbourhood method has been applied when applying masking
for each point along the coord_for_masking coordinate.
The resulting cube is concatenated so that the dimension
coordinates match the input cube.
"""
yname = cube.coord(axis='y').name()
xname = cube.coord(axis='x').name()
result_slices = iris.cube.CubeList([])
# Take 2D slices of the input cube for memory issues.
for x_y_slice in cube.slices([yname, xname]):
cube_slices = iris.cube.CubeList([])
# Apply each mask in in mask_cube to the 2D input slice.
for cube_slice in mask_cube.slices_over(self.coord_for_masking):
output_cube = NeighbourhoodProcessing(
self.neighbourhood_method, self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
sum_or_fraction=self.sum_or_fraction, re_mask=self.re_mask
).process(x_y_slice, mask_cube=cube_slice)
coord_object = cube_slice.coord(self.coord_for_masking).copy()
output_cube.add_aux_coord(coord_object)
output_cube = iris.util.new_axis(
output_cube, self.coord_for_masking)
cube_slices.append(output_cube)
concatenated_cube = cube_slices.concatenate_cube()
exception_coordinates = (
find_dimension_coordinate_mismatch(
x_y_slice, concatenated_cube, two_way_mismatch=False))
concatenated_cube = check_cube_coordinates(
x_y_slice, concatenated_cube,
exception_coordinates=exception_coordinates)
result_slices.append(concatenated_cube)
result = result_slices.merge_cube()
exception_coordinates = (
find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False))
result = check_cube_coordinates(
cube, result,
exception_coordinates=exception_coordinates)
return result
def _generate_mask(self):
"""
Generates a boolean mask of areas NOT to calculate orographic
enhancement. Criteria for calculating orographic enhancement are that
all of the following are true:
- 3x3 mean topography height >= threshold (20 m)
- Relative humidity (fraction) >= threshold (0.8)
- v dot grad z (wind x topography gradient) >= threshold (0.0005)
The mask is therefore "True" if any of these conditions are false.
Returns:
mask (numpy.ndarray):
Boolean mask - where True, set orographic enhancement to a
default zero value
"""
# calculate mean 3x3 (square nbhood) orography heights
radius = convert_number_of_grid_cells_into_distance(self.topography, 1)
topo_nbhood = NeighbourhoodProcessing('square',
radius).process(self.topography)
topo_nbhood.convert_units('m')
# create mask
mask = np.full(topo_nbhood.shape, False, dtype=bool)
mask = np.where(topo_nbhood.data < self.orog_thresh_m, True, mask)
mask = np.where(self.humidity.data < self.rh_thresh_ratio, True, mask)
mask = np.where(abs(self.vgradz) < self.vgradz_thresh_ms, True, mask)
return mask
def test_external_mask_square(self):
"""Test the _calculate_neighbourhood method when an external mask is
passed in and re-masking is applied."""
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data_for_masked_tests,
mask=self.mask)
self.assertArrayAlmostEqual(result.data, self.expected_array)
self.assertArrayAlmostEqual(result.mask, self.expected_mask)
def test_masked_array_re_mask_true_square(self):
"""Test the _calculate_neighbourhood method when masked data is
passed in and re-masking is applied."""
input_data = np.ma.masked_where(self.mask == 0,
self.data_for_masked_tests)
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(input_data)
self.assertArrayAlmostEqual(result.data, self.expected_array)
self.assertArrayAlmostEqual(result.mask, self.expected_mask)
def test_basic_float_cube_lead_times_is_none(self):
"""Test _find_radii returns a float with the correct value."""
neighbourhood_method = "circular"
ens_factor = 0.8
num_ens = 2.0
radius = 6300
plugin = NBHood(neighbourhood_method, radius, ens_factor=ens_factor)
result = plugin._find_radii(num_ens)
expected_result = 3563.8181771801998
self.assertIsInstance(result, float)
self.assertAlmostEquals(result, expected_result)
def test_masked_array_re_mask_false(self):
"""Test the _calculate_neighbourhood method when masked data is
passed in and re-masking is not applied."""
input_data = np.ma.masked_where(self.mask == 0,
self.data_for_masked_tests)
plugin = NeighbourhoodProcessing("square", self.RADIUS, re_mask=False)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(input_data)
self.assertArrayAlmostEqual(result, self.expected_array)
with self.assertRaises(AttributeError):
result.mask
def test_basic_square(self):
"""Test the _calculate_neighbourhood method with a square neighbourhood."""
expected_array = np.array([
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 8 / 9, 8 / 9, 8 / 9, 1.0],
[1.0, 8 / 9, 8 / 9, 8 / 9, 1.0],
[1.0, 8 / 9, 8 / 9, 8 / 9, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
])
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result, expected_array)
def test_basic_circular(self):
"""Test the _calculate_neighbourhood method with a circular neighbourhood."""
expected_array = np.array([
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 0.8, 1.0, 1.0],
[1.0, 0.8, 0.8, 0.8, 1.0],
[1.0, 1.0, 0.8, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
])
plugin = NeighbourhoodProcessing("circular", self.RADIUS)
plugin.kernel = self.circular_kernel
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result.data, expected_array)
def test_basic_square_sum(self):
"""Test the _calculate_neighbourhood method calculating a sum in
a square neighbourhood."""
expected_array = np.array([
[4.0, 6.0, 6.0, 6.0, 4.0],
[6.0, 8.0, 8.0, 8.0, 6.0],
[6.0, 8.0, 8.0, 8.0, 6.0],
[6.0, 8.0, 8.0, 8.0, 6.0],
[4.0, 6.0, 6.0, 6.0, 4.0],
])
plugin = NeighbourhoodProcessing("square", self.RADIUS, sum_only=True)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result, expected_array)
def process(self, cube):
"""
Identify the probability of having a phenomenon occur within a
vicinity.
The steps for this are as follows:
1. Calculate the occurrence of a phenomenon within a defined vicinity.
2. If the cube contains a realization dimension coordinate, find the
mean.
3. Compute neighbourhood processing.
Args:
cube : Iris.cube.Cube
A cube that has been thresholded.
Returns:
cube : Iris.cube.Cube
A cube containing neighbourhood probabilities to represent the
probability of an occurrence within the vicinity given a
pre-defined spatial uncertainty.
"""
cube = OccurrenceWithinVicinity(self.distance).process(cube)
if cube.coord_dims('realization'):
cube = cube.collapsed('realization', iris.analysis.MEAN)
cube = NeighbourhoodProcessing(self.neighbourhood_method, self.radii,
self.lead_times, self.unweighted_mode,
self.ens_factor).process(cube)
return cube
def test_basic_array_cube_lead_times_an_array(self):
"""Test _find_radii returns an array with the correct values."""
neighbourhood_method = "circular"
ens_factor = 0.9
num_ens = 2.0
fp_points = np.array([2, 3, 4])
radii = [10000, 20000, 30000]
lead_times = [2, 3, 4]
plugin = NBHood(neighbourhood_method,
radii,
lead_times=lead_times,
ens_factor=ens_factor)
result = plugin._find_radii(num_ens, cube_lead_times=fp_points)
expected_result = np.array([6363.961031, 12727.922061, 19091.883092])
self.assertIsInstance(result, np.ndarray)
self.assertArrayAlmostEqual(result, expected_result)
def test_interpolation(self):
"""Test that interpolation is working as expected in _find_radii."""
fp_points = np.array([2, 3, 4])
neighbourhood_method = "circular"
ens_factor = 0.8
num_ens = 4.0
fp_points = np.array([2, 3, 4])
radii = [10000, 30000]
lead_times = [2, 4]
plugin = NBHood(neighbourhood_method,
radii,
lead_times=lead_times,
ens_factor=ens_factor)
result = plugin._find_radii(num_ens, cube_lead_times=fp_points)
expected_result = np.array([4000., 8000., 12000.])
self.assertArrayAlmostEqual(result, expected_result)
def test_source_realizations(self):
"""Test when the array has source_realization attribute."""
member_list = [0, 1, 2, 3]
cube = (set_up_cube_with_no_realizations(
source_realizations=member_list))
radii = 15000
ens_factor = 0.8
neighbourhood_method = "circular"
plugin = NBHood(neighbourhood_method, radii, ens_factor=ens_factor)
result = plugin.process(cube)
self.assertIsInstance(result, Cube)
expected = np.ones([1, 16, 16])
expected[0, 6:9, 6:9] = ([0.91666667, 0.875,
0.91666667], [0.875, 0.83333333, 0.875],
[0.91666667, 0.875, 0.91666667])
self.assertArrayAlmostEqual(result.data, expected)
def test_radii_varying_with_lead_time(self):
"""
Test that a cube is returned when the radius varies with lead time.
"""
cube = set_up_cube(num_time_points=3)
iris.util.promote_aux_coord_to_dim_coord(cube, "time")
time_points = cube.coord("time").points
fp_points = [2, 3, 4]
cube = add_forecast_reference_time_and_forecast_period(
cube, time_point=time_points, fp_point=fp_points)
radii = [10000, 20000, 30000]
lead_times = [2, 3, 4]
neighbourhood_method = "circular"
plugin = NBHood(neighbourhood_method, radii, lead_times)
result = plugin.process(cube)
self.assertIsInstance(result, Cube)
def test_basic_circular_sum(self):
"""Test the _calculate_neighbourhood method calculating a sum in
a circular neighbourhood."""
expected_array = np.array([
[5.0, 5.0, 5.0, 5.0, 5.0],
[5.0, 5.0, 4.0, 5.0, 5.0],
[5.0, 4.0, 4.0, 4.0, 5.0],
[5.0, 5.0, 4.0, 5.0, 5.0],
[5.0, 5.0, 5.0, 5.0, 5.0],
])
plugin = NeighbourhoodProcessing("circular",
self.RADIUS,
sum_only=True)
plugin.kernel = self.circular_kernel
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result.data, expected_array)
def test_square_nbhood_with_weighted_mode(self):
"""Test that desired error message is raised, if the neighbourhood
method is square and the weighted_mode option is used."""
radii = 10000
msg = "weighted_mode can only be used if neighbourhood_method is circular"
with self.assertRaisesRegex(ValueError, msg):
NeighbourhoodProcessing("square", radii, weighted_mode=True)
def test_neighbourhood_method_does_not_exist(self):
"""Test that desired error message is raised, if the neighbourhood
method does not exist."""
neighbourhood_method = "nonsense"
radii = 10000
msg = "nonsense is not a valid neighbourhood_method"
with self.assertRaisesRegex(ValueError, msg):
NeighbourhoodProcessing(neighbourhood_method, radii)
def test_masked_array_re_mask_true_circular(self):
"""Test the _calculate_neighbourhood method when masked data is
passed in and re-masking is applied with a circular neighbourhood."""
expected_array = np.array([
[np.nan, 0.5, 0.5, 0.5, 1.0],
[1.0, 1.0, 0.6, 0.5, 0.0],
[np.nan, 1.0, 0.75, 0.4, 0.0],
[np.nan, 1.0, 1.0, 0.5, 0.5],
[np.nan, 1.0, 0.75, 0.5, 0.0],
])
input_data = np.ma.masked_where(self.mask == 0,
self.data_for_masked_tests)
plugin = NeighbourhoodProcessing("circular", self.RADIUS)
plugin.kernel = self.circular_kernel
result = plugin._calculate_neighbourhood(input_data)
self.assertArrayAlmostEqual(result.data, expected_array)
self.assertArrayAlmostEqual(result.mask, self.expected_mask)
def test_radii_varying_with_lead_time_with_interpolation(self):
"""
Test that a cube is returned for the following conditions:
1. The radius varies with lead time.
2. Linear interpolation is required to create values for the radii
which are required but were not specified within the 'radii'
argument.
"""
cube = set_up_cube(num_time_points=3)
iris.util.promote_aux_coord_to_dim_coord(cube, "time")
time_points = cube.coord("time").points
fp_points = [2, 3, 4]
cube = add_forecast_reference_time_and_forecast_period(
cube, time_point=time_points, fp_point=fp_points)
radii = [10000, 30000]
lead_times = [2, 4]
neighbourhood_method = "circular"
plugin = NBHood(neighbourhood_method, radii, lead_times)
result = plugin.process(cube)
self.assertIsInstance(result, Cube)
def test_cube_metadata(self):
"""Test the result has the correct attributes and cell methods"""
neighbourhood_method = "square"
radii = 2000
self.cube.attributes = {"Conventions": "CF-1.5"}
self.cube.add_cell_method(CellMethod("mean", coords="time"))
result = NeighbourhoodProcessing(neighbourhood_method,
radii)(self.cube)
self.assertIsInstance(result, Cube)
self.assertTupleEqual(result.cell_methods, self.cube.cell_methods)
self.assertDictEqual(result.attributes, self.cube.attributes)
def test_complex(self):
"""Test that data containing complex numbers is sensibly processed"""
self.data = self.data.astype(complex)
self.data[1, 3] = 0.5 + 0.5j
self.data[4, 3] = 0.4 + 0.6j
expected_array = np.array([
[
1.0 + 0.0j,
1.0 + 0.0j,
0.91666667 + 0.083333333j,
0.91666667 + 0.083333333j,
0.875 + 0.125j,
],
[
1.0 + 0.0j,
0.88888889 + 0.0j,
0.83333333 + 0.055555556j,
0.83333333 + 0.055555556j,
0.91666667 + 0.083333333j,
],
[
1.0 + 0.0j,
0.88888889 + 0.0j,
0.83333333 + 0.055555556j,
0.83333333 + 0.055555556j,
0.91666667 + 0.083333333j,
],
[
1.0 + 0.0j,
0.88888889 + 0.0j,
0.82222222 + 0.066666667j,
0.82222222 + 0.066666667j,
0.9 + 0.1j,
],
[1.0 + 0.0j, 1.0 + 0.0j, 0.9 + 0.1j, 0.9 + 0.1j, 0.85 + 0.15j],
])
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result, expected_array)
def test_radii_varying_with_lead_time_check_data(self):
"""
Test that the expected data is produced when the radius
varies with lead time.
"""
cube = set_up_cube(zero_point_indices=((0, 0, 7, 7), (
0,
1,
7,
7,
), (0, 2, 7, 7)),
num_time_points=3)
expected = np.ones_like(cube.data)
expected[0, 0, 6:9, 6:9] = ([0.91666667, 0.875,
0.91666667], [0.875, 0.83333333, 0.875],
[0.91666667, 0.875, 0.91666667])
expected[0, 1, 5:10, 5:10] = SINGLE_POINT_RANGE_3_CENTROID
expected[0, 2, 4:11, 4:11] = ([
1, 0.9925, 0.985, 0.9825, 0.985, 0.9925, 1
], [0.9925, 0.98, 0.9725, 0.97, 0.9725, 0.98,
0.9925], [0.985, 0.9725, 0.965, 0.9625, 0.965, 0.9725, 0.985], [
0.9825, 0.97, 0.9625, 0.96, 0.9625, 0.97, 0.9825
], [0.985, 0.9725, 0.965, 0.9625, 0.965, 0.9725,
0.985], [0.9925, 0.98, 0.9725, 0.97, 0.9725, 0.98,
0.9925], [1, 0.9925, 0.985, 0.9825, 0.985, 0.9925, 1])
iris.util.promote_aux_coord_to_dim_coord(cube, "time")
time_points = cube.coord("time").points
fp_points = [2, 3, 4]
cube = add_forecast_reference_time_and_forecast_period(
cube, time_point=time_points, fp_point=fp_points)
radii = [6000, 8000, 10000]
lead_times = [2, 3, 4]
neighbourhood_method = "circular"
plugin = NBHood(neighbourhood_method, radii, lead_times)
result = plugin.process(cube)
self.assertArrayAlmostEqual(result.data, expected)
def __init__(self, backup_method='neighbourhood'):
"""Initialise class."""
self.backup_methods = ['first_realization', 'neighbourhood']
self.backup_method = backup_method
if self.backup_method not in self.backup_methods:
msg = ('Invalid option for keyword backup_method '
'({})'.format(self.backup_method))
raise ValueError(msg)
# Any points where the r-values are below the threshold is regarded as
# containing ambigous data.
self.r_thresh = 0.01
# Creates cubelists to hold data.
self.wdir_cube_list = iris.cube.CubeList()
self.r_vals_cube_list = iris.cube.CubeList()
self.confidence_measure_cube_list = iris.cube.CubeList()
# Radius used in neighbourhood plugin as determined in IMPRO-491
self.nb_radius = 6000. # metres
# Initialise neighbourhood plugin ready for use
self.nbhood = NeighbourhoodProcessing('square',
self.nb_radius,
weighted_mode=False)
def test_external_mask_with_masked_data_square(self):
"""Test the _calculate_neighbourhood method when masked data is
passed in and an external mask is passed in and re-masking is applied."""
mask = np.array([
[1, 0, 1, 1, 0],
[1, 1, 1, 1, 0],
[1, 0, 1, 1, 1],
[1, 0, 1, 1, 0],
[1, 0, 1, 1, 0],
])
external_mask = np.array([
[0, 1, 1, 1, 1],
[0, 1, 1, 1, 1],
[0, 1, 1, 1, 1],
[0, 1, 1, 1, 1],
[0, 1, 1, 1, 1],
])
self.data = np.ma.masked_where(mask == 0, self.data_for_masked_tests)
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data, external_mask)
self.assertArrayAlmostEqual(result.data, self.expected_array)
self.assertArrayAlmostEqual(result.mask, self.expected_mask)
def process(self, cube):
"""
Identify the probability of having a phenomenon occur within a
vicinity.
The steps for this are as follows:
1. Calculate the occurrence of a phenomenon within
a defined vicinity.
2. If the cube contains a realization dimension coordinate,
find the mean.
3. Compute neighbourhood processing.
Args:
cube (iris.cube.Cube):
A cube that has been thresholded.
Returns:
cube (iris.cube.Cube):
A cube containing neighbourhood probabilities to represent the
probability of an occurrence within the vicinity given a
pre-defined spatial uncertainty.
"""
cube = OccurrenceWithinVicinity(self.distance).process(cube)
try:
if cube.coord_dims('realization'):
ens_members = cube.coord('realization').points
# BUG in iris: collapsed returns a masked cube regardless of
# input status. If input is not masked, output mask does not
# match data. Fix is to re-cast output to an unmasked array.
cube_is_masked = isinstance(cube.data, np.ma.MaskedArray)
cube = cube.collapsed('realization', iris.analysis.MEAN)
if not cube_is_masked:
cube.data = np.array(cube.data)
cube.remove_coord('realization')
cube.attributes['source_realizations'] = ens_members
except iris.exceptions.CoordinateNotFoundError:
pass
cube = NeighbourhoodProcessing(
self.neighbourhood_method,
self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
ens_factor=self.ens_factor).process(cube)
cube.rename(cube.name() + '_in_vicinity')
return cube
def test_square_neighbourhood(self):
"""Test that the square neighbourhood processing is successful."""
nbhood_result = np.array([
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 0.88888889, 0.88888889, 0.88888889, 1.0],
[1.0, 0.88888889, 0.88888889, 0.88888889, 1.0],
[1.0, 0.88888889, 0.88888889, 0.88888889, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
])
expected = np.broadcast_to(nbhood_result, (3, 5, 5))
neighbourhood_method = "square"
radii = 2000
result = NeighbourhoodProcessing(neighbourhood_method,
radii)(self.cube)
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(result.data, expected)
self.assertTupleEqual(result.cell_methods, self.cube.cell_methods)
self.assertDictEqual(result.attributes, self.cube.attributes)
def process(self, cube):
"""
Identify the probability of having a phenomenon occur within a
vicinity.
The steps for this are as follows:
1. Calculate the occurrence of a phenomenon within
a defined vicinity.
2. If the cube contains a realization dimension coordinate,
find the mean.
3. Compute neighbourhood processing.
Args:
cube (iris.cube.Cube):
A cube that has been thresholded.
Returns:
cube (iris.cube.Cube):
A cube containing neighbourhood probabilities to represent the
probability of an occurrence within the vicinity given a
pre-defined spatial uncertainty.
"""
cube = OccurrenceWithinVicinity(self.distance).process(cube)
try:
if cube.coord_dims('realization'):
ens_members = cube.coord('realization').points
cube = cube.collapsed('realization', iris.analysis.MEAN)
cube.remove_coord('realization')
cube.attributes['source_realizations'] = ens_members
except iris.exceptions.CoordinateNotFoundError:
pass
cube = NeighbourhoodProcessing(
self.neighbourhood_method, self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
ens_factor=self.ens_factor).process(cube)
cube.rename(cube.name() + '_in_vicinity')
return cube
def process(
cube: cli.inputcube,
mask: cli.inputcube,
weights: cli.inputcube = None,
*,
neighbourhood_shape="square",
radii: cli.comma_separated_list,
lead_times: cli.comma_separated_list = None,
area_sum=False,
):
""" Module to process land and sea separately before combining them.
Neighbourhood the input dataset over two distinct regions of land and sea.
If performed as a single level neighbourhood, a land-sea mask should be
provided. If instead topographic_zone neighbourhooding is being employed,
the mask should be one of topographic zones. In the latter case a weights
array is also needed to collapse the topographic_zone coordinate. These
weights are created with the improver generate-topography-bands-weights
CLI and should be made using a land-sea mask, which will then be employed
within this code to draw the distinction between the two surface types.
Args:
cube (iris.cube.Cube):
A cube to be processed.
mask (iris.cube.Cube):
A cube containing either a mask of topographic zones over land or
a land-sea mask. If this is a land-sea mask, land points should be
set to one and sea points set to zero.
weights (iris.cube.Cube):
A cube containing the weights which are used for collapsing the
dimension gained through masking. These weights must have been
created using a land-sea mask. (Optional).
neighbourhood_shape (str):
Name of the neighbourhood method to use.
Options: "circular", "square".
Default: "square".
radii (list of float):
The radius or a list of radii in metres of the neighbourhood to
apply.
If it is a list, it must be the same length as lead_times, which
defines at which lead time to use which nbhood radius. The radius
will be interpolated for intermediate lead times.
lead_times (list of int):
The lead times in hours that correspond to the radii to be used.
If lead_times are set, radii must be a list the same length as
lead_times. Lead times must be given as integer values.
area_sum (bool):
Return sum rather than fraction over the neighbourhood area.
Returns:
(tuple): tuple containing:
**result** (iris.cube.Cube):
A cube of the processed data.
Raises:
ValueError:
If the topographic zone mask has the attribute
topographic_zones_include_seapoints.
IOError:
if a weights cube isn't given and a topographic_zone mask is given.
ValueError:
If the weights cube has the attribute
topographic_zones_include_seapoints.
RuntimeError:
If lead times are not None and has a different length to radii.
TypeError:
A weights cube has been provided but no topographic zone.
"""
import numpy as np
from improver.nbhood.nbhood import NeighbourhoodProcessing
from improver.nbhood.use_nbhood import ApplyNeighbourhoodProcessingWithAMask
masking_coordinate = None
if any(
"topographic_zone" in coord.name() for coord in mask.coords(dim_coords=True)
):
if mask.attributes["topographic_zones_include_seapoints"] == "True":
raise ValueError(
"The topographic zones mask cube must have been "
"masked to exclude sea points, but "
"topographic_zones_include_seapoints = True"
)
if not weights:
raise TypeError(
"A weights cube must be provided if using a mask "
"of topographic zones to collapse the resulting "
"vertical dimension."
)
if weights.attributes["topographic_zones_include_seapoints"] == "True":
raise ValueError(
"The weights cube must be masked to exclude sea "
"points, but topographic_zones_include_seapoints "
"= True"
)
masking_coordinate = "topographic_zone"
land_sea_mask = weights[0].copy(data=weights[0].data.mask)
land_sea_mask.rename("land_binary_mask")
land_sea_mask.remove_coord(masking_coordinate)
# Create land and sea masks in IMPROVER format (inverse of
# numpy standard) 1 - include this region, 0 - exclude this region.
land_only = land_sea_mask.copy(
data=np.logical_not(land_sea_mask.data).astype(int)
)
sea_only = land_sea_mask.copy(data=land_sea_mask.data.astype(int))
else:
if weights is not None:
raise TypeError("A weights cube has been provided but will not be " "used")
land_sea_mask = mask
# In this case the land is set to 1 and the sea is set to 0 in the
# input mask.
sea_only = land_sea_mask.copy(
data=np.logical_not(land_sea_mask.data).astype(int)
)
land_only = land_sea_mask.copy(data=land_sea_mask.data.astype(int))
if lead_times is None:
radius_or_radii = float(radii[0])
else:
if len(radii) != len(lead_times):
raise RuntimeError(
"If leadtimes are supplied, it must be a list"
" of equal length to a list of radii."
)
radius_or_radii = [float(x) for x in radii]
lead_times = [int(x) for x in lead_times]
# Section for neighbourhood processing land points.
if land_only.data.max() > 0.0:
if masking_coordinate is None:
result_land = NeighbourhoodProcessing(
neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
sum_only=area_sum,
re_mask=True,
)(cube, land_only)
else:
result_land = ApplyNeighbourhoodProcessingWithAMask(
masking_coordinate,
neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
collapse_weights=weights,
sum_only=area_sum,
)(cube, mask)
result = result_land
# Section for neighbourhood processing sea points.
if sea_only.data.max() > 0.0:
result_sea = NeighbourhoodProcessing(
neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
sum_only=area_sum,
re_mask=True,
)(cube, sea_only)
result = result_sea
# Section for combining land and sea points following land and sea points
# being neighbourhood processed individually.
if sea_only.data.max() > 0.0 and land_only.data.max() > 0.0:
# Recombine cubes to be a single output.
combined_data = result_land.data.filled(0) + result_sea.data.filled(0)
result = result_land.copy(data=combined_data)
return result
def _calculate_convective_ratio(self, cubelist, threshold_list):
"""
Calculate the convective ratio by:
1. Apply neighbourhood processing to cubes that have been thresholded
using an upper and lower threshold.
2. Calculate the convective ratio by:
higher_threshold_cube / lower_threshold_cube.
For example, the higher_threshold might be 5 mm/hr, whilst the
lower_threshold might be 0.1 mm/hr.
The convective ratio can have the following values:
* A non-zero fractional value, indicating that both the higher
and lower thresholds were exceeded.
* A zero value, if the lower threshold was exceeded, whilst the
higher threshold was not exceeded.
* A NaN value (np.nan), if neither the higher or lower thresholds
were exceeded, such that the convective ratio was 0/0.
Args:
cube (iris.cube.CubeList):
Cubelist containing cubes from which the convective ratio
will be calculated. The cube should have been thresholded,
so that values within cube.data are between 0.0 and 1.0.
threshold_list (list):
The list of thresholds.
Returns:
numpy.ndarray:
Array of convective ratio.
Raises:
ValueError: If a value of infinity or a value greater than 1.0
are found within the convective ratio.
"""
neighbourhooded_cube_dict = {}
for cube, threshold in zip(cubelist, threshold_list):
neighbourhooded_cube = NeighbourhoodProcessing(
self.neighbourhood_method,
self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
)(cube)
neighbourhooded_cube_dict[threshold] = neighbourhooded_cube
# Ignore runtime warnings from divide by 0 errors.
with np.errstate(invalid="ignore", divide="ignore"):
convective_ratio = np.divide(
neighbourhooded_cube_dict[self.higher_threshold].data,
neighbourhooded_cube_dict[self.lower_threshold].data,
)
infinity_condition = np.sum(np.isinf(convective_ratio)) > 0.0
with np.errstate(invalid="ignore"):
greater_than_1_condition = np.sum(convective_ratio > 1.0) > 0.0
if infinity_condition or greater_than_1_condition:
if infinity_condition:
start_msg = ("A value of infinity was found for the "
"convective ratio: {}.").format(convective_ratio)
elif greater_than_1_condition:
start_msg = ("A value of greater than 1.0 was found for the "
"convective ratio: {}.").format(convective_ratio)
msg = ("{}\nThis value is not plausible as the fraction above the "
"higher threshold must be less than the fraction "
"above the lower threshold.").format(start_msg)
raise ValueError(msg)
return convective_ratio
def process(cube,
mask,
radius=None,
radii_by_lead_time=None,
weights=None,
sum_or_fraction="fraction",
return_intermediate=False):
""" Module to process land and sea separately before combining them.
Neighbourhood the input dataset over two distinct regions of land and sea.
If performed as a single level neighbourhood, a land-sea mask should be
provided. If instead topographic_zone neighbourhooding is being employed,
the mask should be one of topographic zones. In the latter case a weights
array is also needed to collapse the topographic_zone coordinate. These
weights are created with the improver generate-topography-bands-weights
CLI and should be made using a land-sea mask, which will then be employed
within this code to draw the distinction between the two surface types.
Args:
cube (iris.cube.Cube):
A cube to be processed.
mask (iris.cube.Cube):
A cube containing either a mask of topographic zones over land or
a land-sea mask.
radius (float):
The radius in metres of the neighbourhood to apply.
Rounded up to convert into integer number of grid points east and
north, based on the characteristic spacing at the zero indices of
the cube projection-x and y coordinates.
Default is None.
radii_by_lead_time (list):
A list with the radius in metres at [0] and the lead_time at [1]
Lead time is a List of lead times or forecast periods, at which
the radii within 'radii' are defined. The lead times are expected
in hours.
Default is None
weights (iris.cube.Cube):
A cube containing the weights which are used for collapsing the
dimension gained through masking. These weights must have been
created using a land-sea mask.
Default is None.
sum_or_fraction (str):
The neighbourhood output can either be in the form of a sum of the
neighbourhood, or a fraction calculated by dividing the sum of the
neighbourhood by the neighbourhood area.
Default is 'fraction'
return_intermediate (bool):
If True will return a cube with results following topographic
masked neighbourhood processing of land points and prior to
collapsing the topographic_zone coordinate. If no topographic
masked neighbourhooding occurs, there will be no intermediate cube
and a warning.
Default is False.
Returns:
(tuple): tuple containing:
**result** (iris.cube.Cube):
A cube of the processed data.
**intermediate_cube** (iris.cube.Cube or None):
A cube of the intermediate data, before collapsing.
Raises:
ValueError:
If the topographic zone mask has the attribute
topographic_zones_include_seapoints.
IOError:
if a weights cube isn't given and a topographic_zone mask is given.
ValueError:
If the weights cube has the attribute
topographic_zones_include_seapoints.
Warns:
warning:
A weights cube has been provided but no topographic zone.
"""
masking_coordinate = intermediate_cube = None
if any([
'topographic_zone' in coord.name()
for coord in mask.coords(dim_coords=True)
]):
if mask.attributes['topographic_zones_include_seapoints'] == 'True':
raise ValueError('The topographic zones mask cube must have been '
'masked to exclude sea points, but '
'topographic_zones_include_seapoints = True')
if not weights:
raise TypeError('A weights cube must be provided if using a mask '
'of topographic zones to collapse the resulting '
'vertical dimension.')
if weights.attributes['topographic_zones_include_seapoints'] == 'True':
raise ValueError('The weights cube must be masked to exclude sea '
'points, but topographic_zones_include_seapoints '
'= True')
masking_coordinate = 'topographic_zone'
landmask = weights[0].copy(data=weights[0].data.mask)
landmask.rename('land_binary_mask')
landmask.remove_coord(masking_coordinate)
# Create land and sea masks in IMPROVER format (inverse of
# numpy standard) 1 - include this region, 0 - exclude this region.
land_only = landmask.copy(
data=np.logical_not(landmask.data).astype(int))
sea_only = landmask.copy(data=landmask.data.astype(int))
else:
if weights is None:
warnings.warn('A weights cube has been provided but will not be '
'used as there is no topographic zone coordinate '
'to collapse.')
landmask = mask
# In this case the land is set to 1 and the sea is set to 0 in the
# input mask.
sea_only = landmask.copy(
data=np.logical_not(landmask.data).astype(int))
land_only = landmask.copy(data=landmask.data.astype(int))
radius_or_radii, lead_times = radius_or_radii_and_lead(
radius, radii_by_lead_time)
if return_intermediate is not None and masking_coordinate is None:
warnings.warn('No topographic_zone coordinate found, so no '
'intermediate file will be saved.')
# Section for neighbourhood processing land points.
if land_only.data.max() > 0.0:
if masking_coordinate is None:
result_land = NeighbourhoodProcessing(
'square',
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=sum_or_fraction,
re_mask=True).process(cube, land_only)
else:
result_land = ApplyNeighbourhoodProcessingWithAMask(
masking_coordinate,
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=sum_or_fraction,
re_mask=False).process(cube, mask)
if return_intermediate:
intermediate_cube = result_land.copy()
# Collapse the masking coordinate.
result_land = CollapseMaskedNeighbourhoodCoordinate(
masking_coordinate, weights=weights).process(result_land)
result = result_land
# Section for neighbourhood processing sea points.
if sea_only.data.max() > 0.0:
result_sea = NeighbourhoodProcessing('square',
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=sum_or_fraction,
re_mask=True).process(
cube, sea_only)
result = result_sea
# Section for combining land and sea points following land and sea points
# being neighbourhood processed individually.
if sea_only.data.max() > 0.0 and land_only.data.max() > 0.0:
# Recombine cubes to be a single output.
combined_data = result_land.data.filled(0) + result_sea.data.filled(0)
result = result_land.copy(data=combined_data)
return result, intermediate_cube
