def process(self, cube):
"""
Identify the probability of having a phenomenon occur within a
vicinity.
The steps for this are as follows:
1. Calculate the occurrence of a phenomenon within a defined vicinity.
2. If the cube contains a realization dimension coordinate, find the
mean.
3. Compute neighbourhood processing.
Args:
cube : Iris.cube.Cube
A cube that has been thresholded.
Returns:
cube : Iris.cube.Cube
A cube containing neighbourhood probabilities to represent the
probability of an occurrence within the vicinity given a
pre-defined spatial uncertainty.
"""
cube = OccurrenceWithinVicinity(self.distance).process(cube)
if cube.coord_dims('realization'):
cube = cube.collapsed('realization', iris.analysis.MEAN)
cube = NeighbourhoodProcessing(self.neighbourhood_method, self.radii,
self.lead_times, self.unweighted_mode,
self.ens_factor).process(cube)
return cube
def test_square_nbhood_with_weighted_mode(self):
"""Test that desired error message is raised, if the neighbourhood
method is square and the weighted_mode option is used."""
radii = 10000
msg = "weighted_mode can only be used if neighbourhood_method is circular"
with self.assertRaisesRegex(ValueError, msg):
NeighbourhoodProcessing("square", radii, weighted_mode=True)
def process(self, cube, mask_cube):
"""
1. Iterate over the chosen coordinate within the mask_cube and apply
the mask at each iteration to the cube that is to be neighbourhood
processed.
2. Concatenate the cubes from each iteration together to create a
single cube.
Args:
cube (Iris.cube.Cube):
Cube containing the array to which the square neighbourhood
will be applied.
mask_cube (Iris.cube.Cube):
Cube containing the array to be used as a mask.
Returns:
concatenated_cube (Iris.cube.Cube):
Cube containing the smoothed field after the square
neighbourhood method has been applied when applying masking
for each point along the coord_for_masking coordinate.
The resulting cube is concatenated so that the dimension
coordinates match the input cube.
"""
yname = cube.coord(axis='y').name()
xname = cube.coord(axis='x').name()
result_slices = iris.cube.CubeList([])
# Take 2D slices of the input cube for memory issues.
for x_y_slice in cube.slices([yname, xname]):
cube_slices = iris.cube.CubeList([])
# Apply each mask in in mask_cube to the 2D input slice.
for cube_slice in mask_cube.slices_over(self.coord_for_masking):
output_cube = NeighbourhoodProcessing(
self.neighbourhood_method, self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
sum_or_fraction=self.sum_or_fraction, re_mask=self.re_mask
).process(x_y_slice, mask_cube=cube_slice)
coord_object = cube_slice.coord(self.coord_for_masking).copy()
output_cube.add_aux_coord(coord_object)
output_cube = iris.util.new_axis(
output_cube, self.coord_for_masking)
cube_slices.append(output_cube)
concatenated_cube = cube_slices.concatenate_cube()
exception_coordinates = (
find_dimension_coordinate_mismatch(
x_y_slice, concatenated_cube, two_way_mismatch=False))
concatenated_cube = check_cube_coordinates(
x_y_slice, concatenated_cube,
exception_coordinates=exception_coordinates)
result_slices.append(concatenated_cube)
result = result_slices.merge_cube()
exception_coordinates = (
find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False))
result = check_cube_coordinates(
cube, result,
exception_coordinates=exception_coordinates)
return result
def _generate_mask(self):
"""
Generates a boolean mask of areas NOT to calculate orographic
enhancement. Criteria for calculating orographic enhancement are that
all of the following are true:
- 3x3 mean topography height >= threshold (20 m)
- Relative humidity (fraction) >= threshold (0.8)
- v dot grad z (wind x topography gradient) >= threshold (0.0005)
The mask is therefore "True" if any of these conditions are false.
Returns:
mask (numpy.ndarray):
Boolean mask - where True, set orographic enhancement to a
default zero value
"""
# calculate mean 3x3 (square nbhood) orography heights
radius = convert_number_of_grid_cells_into_distance(self.topography, 1)
topo_nbhood = NeighbourhoodProcessing('square',
radius).process(self.topography)
topo_nbhood.convert_units('m')
# create mask
mask = np.full(topo_nbhood.shape, False, dtype=bool)
mask = np.where(topo_nbhood.data < self.orog_thresh_m, True, mask)
mask = np.where(self.humidity.data < self.rh_thresh_ratio, True, mask)
mask = np.where(abs(self.vgradz) < self.vgradz_thresh_ms, True, mask)
return mask
def test_neighbourhood_method_does_not_exist(self):
"""Test that desired error message is raised, if the neighbourhood
method does not exist."""
neighbourhood_method = "nonsense"
radii = 10000
msg = "nonsense is not a valid neighbourhood_method"
with self.assertRaisesRegex(ValueError, msg):
NeighbourhoodProcessing(neighbourhood_method, radii)
def test_external_mask_square(self):
"""Test the _calculate_neighbourhood method when an external mask is
passed in and re-masking is applied."""
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data_for_masked_tests,
mask=self.mask)
self.assertArrayAlmostEqual(result.data, self.expected_array)
self.assertArrayAlmostEqual(result.mask, self.expected_mask)
def test_cube_metadata(self):
"""Test the result has the correct attributes and cell methods"""
neighbourhood_method = "square"
radii = 2000
self.cube.attributes = {"Conventions": "CF-1.5"}
self.cube.add_cell_method(CellMethod("mean", coords="time"))
result = NeighbourhoodProcessing(neighbourhood_method,
radii)(self.cube)
self.assertIsInstance(result, Cube)
self.assertTupleEqual(result.cell_methods, self.cube.cell_methods)
self.assertDictEqual(result.attributes, self.cube.attributes)
def test_masked_array_re_mask_true_square(self):
"""Test the _calculate_neighbourhood method when masked data is
passed in and re-masking is applied."""
input_data = np.ma.masked_where(self.mask == 0,
self.data_for_masked_tests)
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(input_data)
self.assertArrayAlmostEqual(result.data, self.expected_array)
self.assertArrayAlmostEqual(result.mask, self.expected_mask)
def test_masked_array_re_mask_false(self):
"""Test the _calculate_neighbourhood method when masked data is
passed in and re-masking is not applied."""
input_data = np.ma.masked_where(self.mask == 0,
self.data_for_masked_tests)
plugin = NeighbourhoodProcessing("square", self.RADIUS, re_mask=False)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(input_data)
self.assertArrayAlmostEqual(result, self.expected_array)
with self.assertRaises(AttributeError):
result.mask
def test_basic_circular(self):
"""Test the _calculate_neighbourhood method with a circular neighbourhood."""
expected_array = np.array([
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 0.8, 1.0, 1.0],
[1.0, 0.8, 0.8, 0.8, 1.0],
[1.0, 1.0, 0.8, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
])
plugin = NeighbourhoodProcessing("circular", self.RADIUS)
plugin.kernel = self.circular_kernel
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result.data, expected_array)
def test_basic_square(self):
"""Test the _calculate_neighbourhood method with a square neighbourhood."""
expected_array = np.array([
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 8 / 9, 8 / 9, 8 / 9, 1.0],
[1.0, 8 / 9, 8 / 9, 8 / 9, 1.0],
[1.0, 8 / 9, 8 / 9, 8 / 9, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
])
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result, expected_array)
def test_basic_square_sum(self):
"""Test the _calculate_neighbourhood method calculating a sum in
a square neighbourhood."""
expected_array = np.array([
[4.0, 6.0, 6.0, 6.0, 4.0],
[6.0, 8.0, 8.0, 8.0, 6.0],
[6.0, 8.0, 8.0, 8.0, 6.0],
[6.0, 8.0, 8.0, 8.0, 6.0],
[4.0, 6.0, 6.0, 6.0, 4.0],
])
plugin = NeighbourhoodProcessing("square", self.RADIUS, sum_only=True)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result, expected_array)
def process(self, cube):
"""
Identify the probability of having a phenomenon occur within a
vicinity.
The steps for this are as follows:
1. Calculate the occurrence of a phenomenon within
a defined vicinity.
2. If the cube contains a realization dimension coordinate,
find the mean.
3. Compute neighbourhood processing.
Args:
cube (iris.cube.Cube):
A cube that has been thresholded.
Returns:
cube (iris.cube.Cube):
A cube containing neighbourhood probabilities to represent the
probability of an occurrence within the vicinity given a
pre-defined spatial uncertainty.
"""
cube = OccurrenceWithinVicinity(self.distance).process(cube)
try:
if cube.coord_dims('realization'):
ens_members = cube.coord('realization').points
# BUG in iris: collapsed returns a masked cube regardless of
# input status. If input is not masked, output mask does not
# match data. Fix is to re-cast output to an unmasked array.
cube_is_masked = isinstance(cube.data, np.ma.MaskedArray)
cube = cube.collapsed('realization', iris.analysis.MEAN)
if not cube_is_masked:
cube.data = np.array(cube.data)
cube.remove_coord('realization')
cube.attributes['source_realizations'] = ens_members
except iris.exceptions.CoordinateNotFoundError:
pass
cube = NeighbourhoodProcessing(
self.neighbourhood_method,
self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
ens_factor=self.ens_factor).process(cube)
cube.rename(cube.name() + '_in_vicinity')
return cube
def test_basic_circular_sum(self):
"""Test the _calculate_neighbourhood method calculating a sum in
a circular neighbourhood."""
expected_array = np.array([
[5.0, 5.0, 5.0, 5.0, 5.0],
[5.0, 5.0, 4.0, 5.0, 5.0],
[5.0, 4.0, 4.0, 4.0, 5.0],
[5.0, 5.0, 4.0, 5.0, 5.0],
[5.0, 5.0, 5.0, 5.0, 5.0],
])
plugin = NeighbourhoodProcessing("circular",
self.RADIUS,
sum_only=True)
plugin.kernel = self.circular_kernel
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result.data, expected_array)
def test_square_neighbourhood(self):
"""Test that the square neighbourhood processing is successful."""
nbhood_result = np.array([
[1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 0.88888889, 0.88888889, 0.88888889, 1.0],
[1.0, 0.88888889, 0.88888889, 0.88888889, 1.0],
[1.0, 0.88888889, 0.88888889, 0.88888889, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0],
])
expected = np.broadcast_to(nbhood_result, (3, 5, 5))
neighbourhood_method = "square"
radii = 2000
result = NeighbourhoodProcessing(neighbourhood_method,
radii)(self.cube)
self.assertIsInstance(result, Cube)
self.assertArrayAlmostEqual(result.data, expected)
self.assertTupleEqual(result.cell_methods, self.cube.cell_methods)
self.assertDictEqual(result.attributes, self.cube.attributes)
def test_masked_array_re_mask_true_circular(self):
"""Test the _calculate_neighbourhood method when masked data is
passed in and re-masking is applied with a circular neighbourhood."""
expected_array = np.array([
[np.nan, 0.5, 0.5, 0.5, 1.0],
[1.0, 1.0, 0.6, 0.5, 0.0],
[np.nan, 1.0, 0.75, 0.4, 0.0],
[np.nan, 1.0, 1.0, 0.5, 0.5],
[np.nan, 1.0, 0.75, 0.5, 0.0],
])
input_data = np.ma.masked_where(self.mask == 0,
self.data_for_masked_tests)
plugin = NeighbourhoodProcessing("circular", self.RADIUS)
plugin.kernel = self.circular_kernel
result = plugin._calculate_neighbourhood(input_data)
self.assertArrayAlmostEqual(result.data, expected_array)
self.assertArrayAlmostEqual(result.mask, self.expected_mask)
def test_complex(self):
"""Test that data containing complex numbers is sensibly processed"""
self.data = self.data.astype(complex)
self.data[1, 3] = 0.5 + 0.5j
self.data[4, 3] = 0.4 + 0.6j
expected_array = np.array([
[
1.0 + 0.0j,
1.0 + 0.0j,
0.91666667 + 0.083333333j,
0.91666667 + 0.083333333j,
0.875 + 0.125j,
],
[
1.0 + 0.0j,
0.88888889 + 0.0j,
0.83333333 + 0.055555556j,
0.83333333 + 0.055555556j,
0.91666667 + 0.083333333j,
],
[
1.0 + 0.0j,
0.88888889 + 0.0j,
0.83333333 + 0.055555556j,
0.83333333 + 0.055555556j,
0.91666667 + 0.083333333j,
],
[
1.0 + 0.0j,
0.88888889 + 0.0j,
0.82222222 + 0.066666667j,
0.82222222 + 0.066666667j,
0.9 + 0.1j,
],
[1.0 + 0.0j, 1.0 + 0.0j, 0.9 + 0.1j, 0.9 + 0.1j, 0.85 + 0.15j],
])
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data)
self.assertArrayAlmostEqual(result, expected_array)
def process(self, cube):
"""
Identify the probability of having a phenomenon occur within a
vicinity.
The steps for this are as follows:
1. Calculate the occurrence of a phenomenon within
a defined vicinity.
2. If the cube contains a realization dimension coordinate,
find the mean.
3. Compute neighbourhood processing.
Args:
cube (iris.cube.Cube):
A cube that has been thresholded.
Returns:
cube (iris.cube.Cube):
A cube containing neighbourhood probabilities to represent the
probability of an occurrence within the vicinity given a
pre-defined spatial uncertainty.
"""
cube = OccurrenceWithinVicinity(self.distance).process(cube)
try:
if cube.coord_dims('realization'):
ens_members = cube.coord('realization').points
cube = cube.collapsed('realization', iris.analysis.MEAN)
cube.remove_coord('realization')
cube.attributes['source_realizations'] = ens_members
except iris.exceptions.CoordinateNotFoundError:
pass
cube = NeighbourhoodProcessing(
self.neighbourhood_method, self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
ens_factor=self.ens_factor).process(cube)
cube.rename(cube.name() + '_in_vicinity')
return cube
def __init__(self, backup_method: str = "neighbourhood") -> None:
"""Initialise class."""
self.backup_methods = ["first_realization", "neighbourhood"]
self.backup_method = backup_method
if self.backup_method not in self.backup_methods:
msg = "Invalid option for keyword backup_method " "({})".format(
self.backup_method)
raise ValueError(msg)
# Any points where the r-values are below the threshold is regarded as
# containing ambigous data.
self.r_thresh = 0.01
# Creates cubelists to hold data.
self.wdir_cube_list = iris.cube.CubeList()
self.r_vals_cube_list = iris.cube.CubeList()
# Radius used in neighbourhood plugin as determined in IMPRO-491
self.nb_radius = 6000.0 # metres
# Initialise neighbourhood plugin ready for use
self.nbhood = NeighbourhoodProcessing("square",
self.nb_radius,
weighted_mode=False)
def test_external_mask_with_masked_data_square(self):
"""Test the _calculate_neighbourhood method when masked data is
passed in and an external mask is passed in and re-masking is applied."""
mask = np.array([
[1, 0, 1, 1, 0],
[1, 1, 1, 1, 0],
[1, 0, 1, 1, 1],
[1, 0, 1, 1, 0],
[1, 0, 1, 1, 0],
])
external_mask = np.array([
[0, 1, 1, 1, 1],
[0, 1, 1, 1, 1],
[0, 1, 1, 1, 1],
[0, 1, 1, 1, 1],
[0, 1, 1, 1, 1],
])
self.data = np.ma.masked_where(mask == 0, self.data_for_masked_tests)
plugin = NeighbourhoodProcessing("square", self.RADIUS)
plugin.nb_size = self.nbhood_size
result = plugin._calculate_neighbourhood(self.data, external_mask)
self.assertArrayAlmostEqual(result.data, self.expected_array)
self.assertArrayAlmostEqual(result.mask, self.expected_mask)
def _calculate_convective_ratio(self, cubelist, threshold_list):
"""
Calculate the convective ratio by:
1. Apply neighbourhood processing to cubes that have been thresholded
using an upper and lower threshold.
2. Calculate the convective ratio by:
higher_threshold_cube / lower_threshold_cube.
For example, the higher_threshold might be 5 mm/hr, whilst the
lower_threshold might be 0.1 mm/hr.
The convective ratio can have the following values:
* A non-zero fractional value, indicating that both the higher
and lower thresholds were exceeded.
* A zero value, if the lower threshold was exceeded, whilst the
higher threshold was not exceeded.
* A NaN value (np.nan), if neither the higher or lower thresholds
were exceeded, such that the convective ratio was 0/0.
Args:
cube (iris.cube.CubeList):
Cubelist containing cubes from which the convective ratio
will be calculated. The cube should have been thresholded,
so that values within cube.data are between 0.0 and 1.0.
threshold_list (list):
The list of thresholds.
Returns:
numpy.ndarray:
Array of convective ratio.
Raises:
ValueError: If a value of infinity or a value greater than 1.0
are found within the convective ratio.
"""
neighbourhooded_cube_dict = {}
for cube, threshold in zip(cubelist, threshold_list):
neighbourhooded_cube = NeighbourhoodProcessing(
self.neighbourhood_method,
self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
)(cube)
neighbourhooded_cube_dict[threshold] = neighbourhooded_cube
# Ignore runtime warnings from divide by 0 errors.
with np.errstate(invalid="ignore", divide="ignore"):
convective_ratio = np.divide(
neighbourhooded_cube_dict[self.higher_threshold].data,
neighbourhooded_cube_dict[self.lower_threshold].data,
)
infinity_condition = np.sum(np.isinf(convective_ratio)) > 0.0
with np.errstate(invalid="ignore"):
greater_than_1_condition = np.sum(convective_ratio > 1.0) > 0.0
if infinity_condition or greater_than_1_condition:
if infinity_condition:
start_msg = ("A value of infinity was found for the "
"convective ratio: {}.").format(convective_ratio)
elif greater_than_1_condition:
start_msg = ("A value of greater than 1.0 was found for the "
"convective ratio: {}.").format(convective_ratio)
msg = ("{}\nThis value is not plausible as the fraction above the "
"higher threshold must be less than the fraction "
"above the lower threshold.").format(start_msg)
raise ValueError(msg)
return convective_ratio
def process(
cube: cli.inputcube,
mask: cli.inputcube,
weights: cli.inputcube = None,
*,
neighbourhood_shape="square",
radii: cli.comma_separated_list,
lead_times: cli.comma_separated_list = None,
area_sum=False,
):
""" Module to process land and sea separately before combining them.
Neighbourhood the input dataset over two distinct regions of land and sea.
If performed as a single level neighbourhood, a land-sea mask should be
provided. If instead topographic_zone neighbourhooding is being employed,
the mask should be one of topographic zones. In the latter case a weights
array is also needed to collapse the topographic_zone coordinate. These
weights are created with the improver generate-topography-bands-weights
CLI and should be made using a land-sea mask, which will then be employed
within this code to draw the distinction between the two surface types.
Args:
cube (iris.cube.Cube):
A cube to be processed.
mask (iris.cube.Cube):
A cube containing either a mask of topographic zones over land or
a land-sea mask. If this is a land-sea mask, land points should be
set to one and sea points set to zero.
weights (iris.cube.Cube):
A cube containing the weights which are used for collapsing the
dimension gained through masking. These weights must have been
created using a land-sea mask. (Optional).
neighbourhood_shape (str):
Name of the neighbourhood method to use.
Options: "circular", "square".
Default: "square".
radii (list of float):
The radius or a list of radii in metres of the neighbourhood to
apply.
If it is a list, it must be the same length as lead_times, which
defines at which lead time to use which nbhood radius. The radius
will be interpolated for intermediate lead times.
lead_times (list of int):
The lead times in hours that correspond to the radii to be used.
If lead_times are set, radii must be a list the same length as
lead_times. Lead times must be given as integer values.
area_sum (bool):
Return sum rather than fraction over the neighbourhood area.
Returns:
(tuple): tuple containing:
**result** (iris.cube.Cube):
A cube of the processed data.
Raises:
ValueError:
If the topographic zone mask has the attribute
topographic_zones_include_seapoints.
IOError:
if a weights cube isn't given and a topographic_zone mask is given.
ValueError:
If the weights cube has the attribute
topographic_zones_include_seapoints.
RuntimeError:
If lead times are not None and has a different length to radii.
TypeError:
A weights cube has been provided but no topographic zone.
"""
import numpy as np
from improver.nbhood.nbhood import NeighbourhoodProcessing
from improver.nbhood.use_nbhood import ApplyNeighbourhoodProcessingWithAMask
masking_coordinate = None
if any(
"topographic_zone" in coord.name() for coord in mask.coords(dim_coords=True)
):
if mask.attributes["topographic_zones_include_seapoints"] == "True":
raise ValueError(
"The topographic zones mask cube must have been "
"masked to exclude sea points, but "
"topographic_zones_include_seapoints = True"
)
if not weights:
raise TypeError(
"A weights cube must be provided if using a mask "
"of topographic zones to collapse the resulting "
"vertical dimension."
)
if weights.attributes["topographic_zones_include_seapoints"] == "True":
raise ValueError(
"The weights cube must be masked to exclude sea "
"points, but topographic_zones_include_seapoints "
"= True"
)
masking_coordinate = "topographic_zone"
land_sea_mask = weights[0].copy(data=weights[0].data.mask)
land_sea_mask.rename("land_binary_mask")
land_sea_mask.remove_coord(masking_coordinate)
# Create land and sea masks in IMPROVER format (inverse of
# numpy standard) 1 - include this region, 0 - exclude this region.
land_only = land_sea_mask.copy(
data=np.logical_not(land_sea_mask.data).astype(int)
)
sea_only = land_sea_mask.copy(data=land_sea_mask.data.astype(int))
else:
if weights is not None:
raise TypeError("A weights cube has been provided but will not be " "used")
land_sea_mask = mask
# In this case the land is set to 1 and the sea is set to 0 in the
# input mask.
sea_only = land_sea_mask.copy(
data=np.logical_not(land_sea_mask.data).astype(int)
)
land_only = land_sea_mask.copy(data=land_sea_mask.data.astype(int))
if lead_times is None:
radius_or_radii = float(radii[0])
else:
if len(radii) != len(lead_times):
raise RuntimeError(
"If leadtimes are supplied, it must be a list"
" of equal length to a list of radii."
)
radius_or_radii = [float(x) for x in radii]
lead_times = [int(x) for x in lead_times]
# Section for neighbourhood processing land points.
if land_only.data.max() > 0.0:
if masking_coordinate is None:
result_land = NeighbourhoodProcessing(
neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
sum_only=area_sum,
re_mask=True,
)(cube, land_only)
else:
result_land = ApplyNeighbourhoodProcessingWithAMask(
masking_coordinate,
neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
collapse_weights=weights,
sum_only=area_sum,
)(cube, mask)
result = result_land
# Section for neighbourhood processing sea points.
if sea_only.data.max() > 0.0:
result_sea = NeighbourhoodProcessing(
neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
sum_only=area_sum,
re_mask=True,
)(cube, sea_only)
result = result_sea
# Section for combining land and sea points following land and sea points
# being neighbourhood processed individually.
if sea_only.data.max() > 0.0 and land_only.data.max() > 0.0:
# Recombine cubes to be a single output.
combined_data = result_land.data.filled(0) + result_sea.data.filled(0)
result = result_land.copy(data=combined_data)
return result
def process(cube,
mask,
radius=None,
radii_by_lead_time=None,
weights=None,
sum_or_fraction="fraction",
return_intermediate=False):
""" Module to process land and sea separately before combining them.
Neighbourhood the input dataset over two distinct regions of land and sea.
If performed as a single level neighbourhood, a land-sea mask should be
provided. If instead topographic_zone neighbourhooding is being employed,
the mask should be one of topographic zones. In the latter case a weights
array is also needed to collapse the topographic_zone coordinate. These
weights are created with the improver generate-topography-bands-weights
CLI and should be made using a land-sea mask, which will then be employed
within this code to draw the distinction between the two surface types.
Args:
cube (iris.cube.Cube):
A cube to be processed.
mask (iris.cube.Cube):
A cube containing either a mask of topographic zones over land or
a land-sea mask.
radius (float):
The radius in metres of the neighbourhood to apply.
Rounded up to convert into integer number of grid points east and
north, based on the characteristic spacing at the zero indices of
the cube projection-x and y coordinates.
Default is None.
radii_by_lead_time (list):
A list with the radius in metres at [0] and the lead_time at [1]
Lead time is a List of lead times or forecast periods, at which
the radii within 'radii' are defined. The lead times are expected
in hours.
Default is None
weights (iris.cube.Cube):
A cube containing the weights which are used for collapsing the
dimension gained through masking. These weights must have been
created using a land-sea mask.
Default is None.
sum_or_fraction (str):
The neighbourhood output can either be in the form of a sum of the
neighbourhood, or a fraction calculated by dividing the sum of the
neighbourhood by the neighbourhood area.
Default is 'fraction'
return_intermediate (bool):
If True will return a cube with results following topographic
masked neighbourhood processing of land points and prior to
collapsing the topographic_zone coordinate. If no topographic
masked neighbourhooding occurs, there will be no intermediate cube
and a warning.
Default is False.
Returns:
(tuple): tuple containing:
**result** (iris.cube.Cube):
A cube of the processed data.
**intermediate_cube** (iris.cube.Cube or None):
A cube of the intermediate data, before collapsing.
Raises:
ValueError:
If the topographic zone mask has the attribute
topographic_zones_include_seapoints.
IOError:
if a weights cube isn't given and a topographic_zone mask is given.
ValueError:
If the weights cube has the attribute
topographic_zones_include_seapoints.
Warns:
warning:
A weights cube has been provided but no topographic zone.
"""
masking_coordinate = intermediate_cube = None
if any([
'topographic_zone' in coord.name()
for coord in mask.coords(dim_coords=True)
]):
if mask.attributes['topographic_zones_include_seapoints'] == 'True':
raise ValueError('The topographic zones mask cube must have been '
'masked to exclude sea points, but '
'topographic_zones_include_seapoints = True')
if not weights:
raise TypeError('A weights cube must be provided if using a mask '
'of topographic zones to collapse the resulting '
'vertical dimension.')
if weights.attributes['topographic_zones_include_seapoints'] == 'True':
raise ValueError('The weights cube must be masked to exclude sea '
'points, but topographic_zones_include_seapoints '
'= True')
masking_coordinate = 'topographic_zone'
landmask = weights[0].copy(data=weights[0].data.mask)
landmask.rename('land_binary_mask')
landmask.remove_coord(masking_coordinate)
# Create land and sea masks in IMPROVER format (inverse of
# numpy standard) 1 - include this region, 0 - exclude this region.
land_only = landmask.copy(
data=np.logical_not(landmask.data).astype(int))
sea_only = landmask.copy(data=landmask.data.astype(int))
else:
if weights is None:
warnings.warn('A weights cube has been provided but will not be '
'used as there is no topographic zone coordinate '
'to collapse.')
landmask = mask
# In this case the land is set to 1 and the sea is set to 0 in the
# input mask.
sea_only = landmask.copy(
data=np.logical_not(landmask.data).astype(int))
land_only = landmask.copy(data=landmask.data.astype(int))
radius_or_radii, lead_times = radius_or_radii_and_lead(
radius, radii_by_lead_time)
if return_intermediate is not None and masking_coordinate is None:
warnings.warn('No topographic_zone coordinate found, so no '
'intermediate file will be saved.')
# Section for neighbourhood processing land points.
if land_only.data.max() > 0.0:
if masking_coordinate is None:
result_land = NeighbourhoodProcessing(
'square',
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=sum_or_fraction,
re_mask=True).process(cube, land_only)
else:
result_land = ApplyNeighbourhoodProcessingWithAMask(
masking_coordinate,
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=sum_or_fraction,
re_mask=False).process(cube, mask)
if return_intermediate:
intermediate_cube = result_land.copy()
# Collapse the masking coordinate.
result_land = CollapseMaskedNeighbourhoodCoordinate(
masking_coordinate, weights=weights).process(result_land)
result = result_land
# Section for neighbourhood processing sea points.
if sea_only.data.max() > 0.0:
result_sea = NeighbourhoodProcessing('square',
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=sum_or_fraction,
re_mask=True).process(
cube, sea_only)
result = result_sea
# Section for combining land and sea points following land and sea points
# being neighbourhood processed individually.
if sea_only.data.max() > 0.0 and land_only.data.max() > 0.0:
# Recombine cubes to be a single output.
combined_data = result_land.data.filled(0) + result_sea.data.filled(0)
result = result_land.copy(data=combined_data)
return result, intermediate_cube
def process(self, cube, mask_cube):
"""
1. Iterate over the chosen coordinate within the mask_cube and apply
the mask at each iteration to the cube that is to be neighbourhood
processed.
2. Concatenate the cubes from each iteration together to create a
single cube.
Args:
cube (iris.cube.Cube):
Cube containing the array to which the square neighbourhood
will be applied.
mask_cube (iris.cube.Cube):
Cube containing the array to be used as a mask.
Returns:
iris.cube.Cube:
Cube containing the smoothed field after the square
neighbourhood method has been applied when applying masking
for each point along the coord_for_masking coordinate.
The resulting cube is concatenated so that the dimension
coordinates match the input cube.
"""
yname = cube.coord(axis="y").name()
xname = cube.coord(axis="x").name()
result_slices = iris.cube.CubeList([])
if self.collapse_weights is None:
collapse_plugin = None
else:
collapse_plugin = CollapseMaskedNeighbourhoodCoordinate(
self.coord_for_masking, self.collapse_weights)
# Take 2D slices of the input cube for memory issues.
prev_x_y_slice = None
for x_y_slice in cube.slices([yname, xname]):
if prev_x_y_slice is not None and np.array_equal(
prev_x_y_slice.data, x_y_slice.data):
# Use same result as last time!
prev_result = result_slices[-1].copy()
for coord in x_y_slice.coords(dim_coords=False):
prev_result.coord(coord).points = coord.points.copy()
result_slices.append(prev_result)
continue
prev_x_y_slice = x_y_slice
cube_slices = iris.cube.CubeList([])
plugin = NeighbourhoodProcessing(
self.neighbourhood_method,
self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
sum_or_fraction=self.sum_or_fraction,
re_mask=self.re_mask,
)
# Apply each mask in in mask_cube to the 2D input slice.
for cube_slice in mask_cube.slices_over(self.coord_for_masking):
output_cube = plugin(x_y_slice, mask_cube=cube_slice)
coord_object = cube_slice.coord(self.coord_for_masking).copy()
output_cube.add_aux_coord(coord_object)
output_cube = iris.util.new_axis(output_cube,
self.coord_for_masking)
cube_slices.append(output_cube)
concatenated_cube = cube_slices.concatenate_cube()
exception_coordinates = find_dimension_coordinate_mismatch(
x_y_slice, concatenated_cube, two_way_mismatch=False)
concatenated_cube = check_cube_coordinates(
x_y_slice,
concatenated_cube,
exception_coordinates=exception_coordinates,
)
if collapse_plugin:
concatenated_cube = collapse_plugin(concatenated_cube)
result_slices.append(concatenated_cube)
result = result_slices.merge_cube()
exception_coordinates = find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False)
result = check_cube_coordinates(
cube, result, exception_coordinates=exception_coordinates)
return result
def main(argv=None):
"""Load in arguments and get going."""
parser = ArgParser(
description='Apply the requested neighbourhood method via '
'the NeighbourhoodProcessing plugin to a file '
'whose data can be loaded as a single iris.cube.Cube.')
parser.add_argument(
'neighbourhood_output',
metavar='NEIGHBOURHOOD_OUTPUT',
help='The form of the results generated using neighbourhood '
'processing. If "probabilities" is selected, the mean '
'probability within a neighbourhood is calculated. If '
'"percentiles" is selected, then the percentiles are calculated '
'within a neighbourhood. Calculating percentiles from a '
'neighbourhood is only supported for a circular neighbourhood. '
'Options: "probabilities", "percentiles".')
parser.add_argument('neighbourhood_shape',
metavar='NEIGHBOURHOOD_SHAPE',
choices=["circular", "square"],
help='The shape of the neighbourhood to apply in '
'neighbourhood processing. Only a "circular" '
'neighbourhood shape is applicable for '
'calculating "percentiles" output. '
'Options: "circular", "square".')
group = parser.add_mutually_exclusive_group()
group.add_argument('--radius',
metavar='RADIUS',
type=float,
help='The radius (in m) for neighbourhood processing.')
group.add_argument('--radii-by-lead-time',
metavar=('RADII_BY_LEAD_TIME', 'LEAD_TIME_IN_HOURS'),
nargs=2,
help='The radii for neighbourhood processing '
'and the associated lead times at which the radii are '
'valid. The radii are in metres whilst the lead time '
'has units of hours. The radii and lead times are '
'expected as individual comma-separated lists with '
'the list of radii given first followed by a list of '
'lead times to indicate at what lead time each radii '
'should be used. For example: 10000,12000,14000 1,2,3 '
'where a lead time of 1 hour uses a radius of 10000m, '
'a lead time of 2 hours uses a radius of 12000m, etc.')
parser.add_argument('--degrees_as_complex',
action='store_true',
default=False,
help='Set this flag to process angles,'
' eg wind directions, as complex numbers. Not '
'compatible with circular kernel, percentiles or '
'recursive filter.')
parser.add_argument('--weighted_mode',
action='store_true',
default=False,
help='For neighbourhood processing using a circular '
'kernel, setting the weighted_mode indicates the '
'weighting decreases with radius. '
'If weighted_mode is not set, a constant '
'weighting is assumed. weighted_mode is only '
'applicable for calculating "probability" '
'neighbourhood output.')
parser.add_argument('--sum_or_fraction',
default="fraction",
choices=["sum", "fraction"],
help='The neighbourhood output can either be in the '
'form of a sum of the neighbourhood, or a '
'fraction calculated by dividing the sum of the '
'neighbourhood by the neighbourhood area. '
'"fraction" is the default option.')
parser.add_argument('--re_mask',
action='store_true',
help='If re_mask is set (i.e. True), the original '
'un-neighbourhood processed mask is applied to '
'mask out the neighbourhood processed dataset. '
'If not set, re_mask defaults to False and the '
'original un-neighbourhood processed mask is '
'not applied. Therefore, the neighbourhood '
'processing may result in values being present '
'in areas that were originally masked. ')
parser.add_argument('--percentiles',
metavar='PERCENTILES',
default=DEFAULT_PERCENTILES,
nargs='+',
type=float,
help='Calculate values at the specified percentiles '
'from the neighbourhood surrounding each grid '
'point.')
parser.add_argument('input_filepath',
metavar='INPUT_FILE',
help='A path to an input NetCDF file to be processed.')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
parser.add_argument('--input_mask_filepath',
metavar='INPUT_MASK_FILE',
help='A path to an input mask NetCDF file to be '
'used to mask the input file. '
'This is currently only supported for square '
'neighbourhoods. The data should contain 1 for '
'usable points and 0 for discarded points, e.g. '
'a land-mask.')
parser.add_argument('--halo_radius',
metavar='HALO_RADIUS',
default=None,
type=float,
help='radius in metres of excess halo to clip.'
' Used where a larger'
' grid was defined than the standard grid'
' and we want to clip the grid back to the'
' standard grid e.g. for global data'
' regridded to UK area. Default=None')
parser.add_argument('--apply-recursive-filter',
action='store_true',
default=False,
help='Option to apply the recursive filter to a '
'square neighbourhooded output dataset, '
'converting it into a Gaussian-like kernel or '
'smoothing over short distances. '
'The filter uses an alpha '
'parameter (0 < alpha < 1) to control what '
'proportion of the probability is passed onto '
'the next grid-square in the x and y directions. '
'The alpha parameter can be set on a grid-square '
'by grid-square basis for the x and y directions '
'separately (using two arrays of alpha '
'parameters of the same dimensionality as the '
'domain). Alternatively a single alpha value can '
'be set for each of the x and y directions. These'
' methods can be mixed, e.g. an array for the x '
'direction and a float for the y direction and '
'vice versa. The recursive filter cannot be '
'applied to a circular kernel')
parser.add_argument('--input_filepath_alphas_x_cube',
metavar='ALPHAS_X_FILE',
help='A path to a NetCDF file describing the alpha '
'factors to be used for smoothing in the x '
'direction when applying the recursive filter')
parser.add_argument('--input_filepath_alphas_y_cube',
metavar='ALPHAS_Y_FILE',
help='A path to a NetCDF file describing the alpha '
'factors to be used for smoothing in the y '
'direction when applying the recursive filter')
parser.add_argument('--alpha_x',
metavar='ALPHA_X',
default=None,
type=float,
help='A single alpha factor (0 < alpha_x < 1) to be '
'applied to every grid square in the x '
'direction when applying the recursive filter')
parser.add_argument('--alpha_y',
metavar='ALPHA_Y',
default=None,
type=float,
help='A single alpha factor (0 < alpha_y < 1) to be '
'applied to every grid square in the y '
'direction when applying the recursive filter.')
parser.add_argument('--iterations',
metavar='ITERATIONS',
default=1,
type=int,
help='Number of times to apply the filter, default=1 '
'(typically < 5)')
args = parser.parse_args(args=argv)
if (args.neighbourhood_output == "percentiles"
and args.neighbourhood_shape == "square"):
parser.wrong_args_error('square', 'neighbourhood_shape')
if (args.neighbourhood_output == "percentiles" and args.weighted_mode):
parser.wrong_args_error('weighted_mode',
'neighbourhood_shape=percentiles')
if (args.neighbourhood_output == "probabilities"
and args.percentiles != DEFAULT_PERCENTILES):
parser.wrong_args_error('percentiles',
'neighbourhood_shape=probabilities')
if (args.input_mask_filepath and args.neighbourhood_shape == "circular"):
parser.wrong_args_error('neighbourhood_shape=circular',
'input_mask_filepath')
if args.degrees_as_complex:
if args.neighbourhood_output == "percentiles":
parser.error('Cannot generate percentiles from complex numbers')
if args.neighbourhood_shape == "circular":
parser.error('Cannot process complex numbers with circular '
'neighbourhoods')
if args.apply_recursive_filter:
parser.error('Cannot process complex numbers with recursive '
'filter')
cube = load_cube(args.input_filepath)
if args.degrees_as_complex:
# convert cube data into complex numbers
cube.data = WindDirection.deg_to_complex(cube.data)
if args.radius:
radius_or_radii = args.radius
lead_times = None
elif args.radii_by_lead_time:
radius_or_radii = args.radii_by_lead_time[0].split(",")
lead_times = args.radii_by_lead_time[1].split(",")
if args.input_mask_filepath:
mask_cube = load_cube(args.input_mask_filepath)
else:
mask_cube = None
if args.neighbourhood_output == "probabilities":
result = (NeighbourhoodProcessing(args.neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
weighted_mode=args.weighted_mode,
sum_or_fraction=args.sum_or_fraction,
re_mask=args.re_mask).process(
cube, mask_cube=mask_cube))
elif args.neighbourhood_output == "percentiles":
result = (GeneratePercentilesFromANeighbourhood(
args.neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
percentiles=args.percentiles).process(cube))
# If the '--apply-recursive-filter' option has been specified in the
# input command, pass the neighbourhooded 'result' cube obtained above
# through the recursive-filter plugin before saving the output.
# The recursive filter is only applicable to square neighbourhoods.
if args.neighbourhood_shape == 'square' and args.apply_recursive_filter:
alphas_x_cube = None
alphas_y_cube = None
if args.input_filepath_alphas_x_cube is not None:
alphas_x_cube = load_cube(args.input_filepath_alphas_x_cube)
if args.input_filepath_alphas_y_cube is not None:
alphas_y_cube = load_cube(args.input_filepath_alphas_y_cube)
result = RecursiveFilter(alpha_x=args.alpha_x,
alpha_y=args.alpha_y,
iterations=args.iterations,
re_mask=args.re_mask).process(
result,
alphas_x=alphas_x_cube,
alphas_y=alphas_y_cube,
mask_cube=mask_cube)
elif args.neighbourhood_shape == 'circular' and \
args.apply_recursive_filter:
raise ValueError('Recursive filter option is not applicable to '
'circular neighbourhoods. ')
if args.degrees_as_complex:
# convert neighbourhooded cube back to degrees
result.data = WindDirection.complex_to_deg(result.data)
if args.halo_radius is not None:
result = remove_cube_halo(result, args.halo_radius)
save_netcdf(result, args.output_filepath)
def process(self, cube: Cube, mask_cube: Cube) -> Cube:
"""
Apply neighbourhood processing with a mask to the input cube,
collapsing the coord_for_masking if collapse_weights have been provided.
Args:
cube:
Cube containing the array to which the square neighbourhood
will be applied.
mask_cube:
Cube containing the array to be used as a mask. The data in
this array is not an instance of numpy.ma.MaskedArray. Any sea
points that should be ignored are set to zeros in every layer
of the mask_cube.
Returns:
Cube containing the smoothed field after the square
neighbourhood method has been applied when applying masking
for each point along the coord_for_masking coordinate.
The resulting cube is concatenated so that the dimension
coordinates match the input cube.
"""
plugin = NeighbourhoodProcessing(
self.neighbourhood_method,
self.radii,
lead_times=self.lead_times,
weighted_mode=self.weighted_mode,
sum_only=self.sum_only,
re_mask=self.re_mask,
)
yname = cube.coord(axis="y").name()
xname = cube.coord(axis="x").name()
result_slices = iris.cube.CubeList([])
# Take 2D slices of the input cube for memory issues.
prev_x_y_slice = None
for x_y_slice in cube.slices([yname, xname]):
if prev_x_y_slice is not None and np.array_equal(
prev_x_y_slice.data, x_y_slice.data):
# Use same result as last time!
prev_result = result_slices[-1].copy()
for coord in x_y_slice.coords(dim_coords=False):
prev_result.coord(coord).points = coord.points.copy()
result_slices.append(prev_result)
continue
prev_x_y_slice = x_y_slice
cube_slices = iris.cube.CubeList([])
# Apply each mask in in mask_cube to the 2D input slice.
for mask_slice in mask_cube.slices_over(self.coord_for_masking):
output_cube = plugin(x_y_slice, mask_cube=mask_slice)
coord_object = mask_slice.coord(self.coord_for_masking).copy()
output_cube.add_aux_coord(coord_object)
output_cube = iris.util.new_axis(output_cube,
self.coord_for_masking)
cube_slices.append(output_cube)
concatenated_cube = cube_slices.concatenate_cube()
if self.collapse_weights is not None:
concatenated_cube = self.collapse_mask_coord(concatenated_cube)
result_slices.append(concatenated_cube)
result = result_slices.merge_cube()
# Promote any single value dimension coordinates if they were
# dimension on the input cube.
exception_coordinates = find_dimension_coordinate_mismatch(
cube, result, two_way_mismatch=False)
result = check_cube_coordinates(
cube, result, exception_coordinates=exception_coordinates)
return result
def main(argv=None):
"""Load in arguments for applying neighbourhood processing when using a
mask."""
parser = ArgParser(
description='Neighbourhood the input dataset over two distinct regions'
' of land and sea. If performed as a single level neighbourhood, a '
'land-sea mask should be provided. If instead topographic_zone '
'neighbourhooding is being employed, the mask should be one of '
'topographic zones. In the latter case a weights array is also needed'
' to collapse the topographic_zone coordinate. These weights are '
'created with the improver generate-topography-bands-weights CLI and '
'should be made using a land-sea mask, which will then be employed '
'within this code to draw the distinction between the two surface '
'types.')
parser.add_argument('input_filepath',
metavar='INPUT_FILE',
help='A path to an input NetCDF file to be processed.')
parser.add_argument('input_mask_filepath',
metavar='INPUT_MASK',
help=('A path to an input NetCDF file containing '
'either a mask of topographic zones over land '
'or a land-sea mask.'))
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed NetCDF.')
mask_group = parser.add_argument_group(
'Collapse weights - required if using a topographic zones mask')
mask_group.add_argument('--weights_for_collapsing_dim',
metavar='WEIGHTS',
default=None,
help='A path to an weights NetCDF file containing '
'the weights which are used for collapsing the '
'dimension gained through masking. These weights '
'must have been created using a land-sea mask.')
radius_group = parser.add_argument_group(
'Neighbourhooding Radius - Set only one of the options')
group = radius_group.add_mutually_exclusive_group()
group.add_argument('--radius',
metavar='RADIUS',
type=float,
help='The radius (in m) for neighbourhood processing.')
group.add_argument('--radii-by-lead-time',
metavar=('RADII_BY_LEAD_TIME', 'LEAD_TIME_IN_HOURS'),
nargs=2,
help='The radii for neighbourhood processing '
'and the associated lead times at which the radii are '
'valid. The radii are in metres whilst the lead time '
'has units of hours. The radii and lead times are '
'expected as individual comma-separated lists with '
'the list of radii given first followed by a list of '
'lead times to indicate at what lead time each radii '
'should be used. For example: 10000,12000,14000 1,2,3 '
'where a lead time of 1 hour uses a radius of 10000m, '
'a lead time of 2 hours uses a radius of 12000m, etc.')
parser.add_argument('--sum_or_fraction',
default="fraction",
choices=["sum", "fraction"],
help='The neighbourhood output can either be in the '
'form of a sum of the neighbourhood, or a '
'fraction calculated by dividing the sum of the '
'neighbourhood by the neighbourhood area. '
'"fraction" is the default option.')
parser.add_argument('--intermediate_filepath',
default=None,
help='Intermediate filepath for results following '
'topographic masked neighbourhood processing of '
'land points and prior to collapsing the '
'topographic_zone coordinate. Intermediate files '
'will not be produced if no topographic masked '
'neighbourhood processing occurs.')
args = parser.parse_args(args=argv)
cube = load_cube(args.input_filepath)
mask = load_cube(args.input_mask_filepath, no_lazy_load=True)
masking_coordinate = None
if any([
'topographic_zone' in coord.name()
for coord in mask.coords(dim_coords=True)
]):
if mask.attributes['topographic_zones_include_seapoints'] == 'True':
raise ValueError('The topographic zones mask cube must have been '
'masked to exclude sea points, but '
'topographic_zones_include_seapoints = True')
if not args.weights_for_collapsing_dim:
raise IOError('A weights cube must be provided if using a mask '
'of topographic zones to collapse the resulting '
'vertical dimension.')
weights = load_cube(args.weights_for_collapsing_dim, no_lazy_load=True)
if weights.attributes['topographic_zones_include_seapoints'] == 'True':
raise ValueError('The weights cube must be masked to exclude sea '
'points, but topographic_zones_include_seapoints '
'= True')
masking_coordinate = 'topographic_zone'
landmask = weights[0].copy(data=weights[0].data.mask)
landmask.rename('land_binary_mask')
landmask.remove_coord(masking_coordinate)
# Create land and sea masks in IMPROVER format (inverse of
# numpy standard) 1 - include this region, 0 - exclude this region.
land_only = landmask.copy(
data=np.logical_not(landmask.data).astype(int))
sea_only = landmask.copy(data=landmask.data.astype(int))
else:
if args.weights_for_collapsing_dim:
warnings.warn('A weights cube has been provided but will not be '
'used as there is no topographic zone coordinate '
'to collapse.')
landmask = mask
# In this case the land is set to 1 and the sea is set to 0 in the
# input mask.
sea_only = landmask.copy(
data=np.logical_not(landmask.data).astype(int))
land_only = landmask.copy(data=landmask.data.astype(int))
if args.radius:
radius_or_radii = args.radius
lead_times = None
elif args.radii_by_lead_time:
radius_or_radii = args.radii_by_lead_time[0].split(",")
lead_times = args.radii_by_lead_time[1].split(",")
if args.intermediate_filepath is not None and masking_coordinate is None:
msg = ('No topographic_zone coordinate found, so no intermediate file '
'will be saved.')
warnings.warn(msg)
# Section for neighbourhood processing land points.
if land_only.data.max() > 0.0:
if masking_coordinate is not None:
result_land = ApplyNeighbourhoodProcessingWithAMask(
masking_coordinate,
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=args.sum_or_fraction,
re_mask=False).process(cube, mask)
else:
result_land = NeighbourhoodProcessing(
'square',
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=args.sum_or_fraction,
re_mask=True).process(cube, land_only)
if masking_coordinate is not None:
if args.intermediate_filepath is not None:
save_netcdf(result_land, args.intermediate_filepath)
# Collapse the masking coordinate.
result_land = CollapseMaskedNeighbourhoodCoordinate(
masking_coordinate, weights=weights).process(result_land)
result = result_land
# Section for neighbourhood processing sea points.
if sea_only.data.max() > 0.0:
result_sea = NeighbourhoodProcessing(
'square',
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=args.sum_or_fraction,
re_mask=True).process(cube, sea_only)
result = result_sea
# Section for combining land and sea points following land and sea points
# being neighbourhood processed individually.
if sea_only.data.max() > 0.0 and land_only.data.max() > 0.0:
# Recombine cubes to be a single output.
combined_data = result_land.data.filled(0) + result_sea.data.filled(0)
result = result_land.copy(data=combined_data)
save_netcdf(result, args.output_filepath)
def process(cube: cli.inputcube,
mask: cli.inputcube,
weights: cli.inputcube = None,
*,
radii: cli.comma_separated_list,
lead_times: cli.comma_separated_list = None,
area_sum=False,
return_intermediate=False):
""" Module to process land and sea separately before combining them.
Neighbourhood the input dataset over two distinct regions of land and sea.
If performed as a single level neighbourhood, a land-sea mask should be
provided. If instead topographic_zone neighbourhooding is being employed,
the mask should be one of topographic zones. In the latter case a weights
array is also needed to collapse the topographic_zone coordinate. These
weights are created with the improver generate-topography-bands-weights
CLI and should be made using a land-sea mask, which will then be employed
within this code to draw the distinction between the two surface types.
Args:
cube (iris.cube.Cube):
A cube to be processed.
mask (iris.cube.Cube):
A cube containing either a mask of topographic zones over land or
a land-sea mask.
weights (iris.cube.Cube):
A cube containing the weights which are used for collapsing the
dimension gained through masking. These weights must have been
created using a land-sea mask. (Optional).
radii (list of float):
The radius or a list of radii in metres of the neighbourhood to
apply.
If it is a list, it must be the same length as lead_times, which
defines at which lead time to use which nbhood radius. The radius
will be interpolated for intermediate lead times.
lead_times (list of int):
The lead times in hours that correspond to the radii to be used.
If lead_times are set, radii must be a list the same length as
lead_times. Lead times must be given as integer values.
area_sum (bool):
Return sum rather than fraction over the neighbourhood area.
return_intermediate (bool):
Include this option to return a cube with results following
topographic masked neighbourhood processing of land points and
prior to collapsing the topographic_zone coordinate. If no
topographic masked neighbourhooding occurs, there will be no
intermediate cube and a warning.
Returns:
(tuple): tuple containing:
**result** (iris.cube.Cube):
A cube of the processed data.
**intermediate_cube** (iris.cube.Cube):
A cube of the intermediate data, before collapsing.
Raises:
ValueError:
If the topographic zone mask has the attribute
topographic_zones_include_seapoints.
IOError:
if a weights cube isn't given and a topographic_zone mask is given.
ValueError:
If the weights cube has the attribute
topographic_zones_include_seapoints.
RuntimeError:
If lead times are not None and has a different length to radii.
TypeError:
A weights cube has been provided but no topographic zone.
"""
import warnings
import numpy as np
from improver.nbhood.nbhood import NeighbourhoodProcessing
from improver.nbhood.use_nbhood import (
ApplyNeighbourhoodProcessingWithAMask,
CollapseMaskedNeighbourhoodCoordinate)
sum_or_fraction = 'sum' if area_sum else 'fraction'
masking_coordinate = intermediate_cube = None
if any([
'topographic_zone' in coord.name()
for coord in mask.coords(dim_coords=True)
]):
if mask.attributes['topographic_zones_include_seapoints'] == 'True':
raise ValueError('The topographic zones mask cube must have been '
'masked to exclude sea points, but '
'topographic_zones_include_seapoints = True')
if not weights:
raise TypeError('A weights cube must be provided if using a mask '
'of topographic zones to collapse the resulting '
'vertical dimension.')
if weights.attributes['topographic_zones_include_seapoints'] == 'True':
raise ValueError('The weights cube must be masked to exclude sea '
'points, but topographic_zones_include_seapoints '
'= True')
masking_coordinate = 'topographic_zone'
land_sea_mask = weights[0].copy(data=weights[0].data.mask)
land_sea_mask.rename('land_binary_mask')
land_sea_mask.remove_coord(masking_coordinate)
# Create land and sea masks in IMPROVER format (inverse of
# numpy standard) 1 - include this region, 0 - exclude this region.
land_only = land_sea_mask.copy(
data=np.logical_not(land_sea_mask.data).astype(int))
sea_only = land_sea_mask.copy(data=land_sea_mask.data.astype(int))
else:
if weights is not None:
raise TypeError('A weights cube has been provided but will not be '
'used')
land_sea_mask = mask
# In this case the land is set to 1 and the sea is set to 0 in the
# input mask.
sea_only = land_sea_mask.copy(
data=np.logical_not(land_sea_mask.data).astype(int))
land_only = land_sea_mask.copy(data=land_sea_mask.data.astype(int))
if lead_times is None:
radius_or_radii = float(radii[0])
else:
if len(radii) != len(lead_times):
raise RuntimeError("If leadtimes are supplied, it must be a list"
" of equal length to a list of radii.")
radius_or_radii = [float(x) for x in radii]
lead_times = [int(x) for x in lead_times]
if return_intermediate is not None and masking_coordinate is None:
warnings.warn('No topographic_zone coordinate found, so no '
'intermediate file will be saved.')
# Section for neighbourhood processing land points.
if land_only.data.max() > 0.0:
if masking_coordinate is None:
result_land = NeighbourhoodProcessing(
'square',
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=sum_or_fraction,
re_mask=True).process(cube, land_only)
else:
result_land = ApplyNeighbourhoodProcessingWithAMask(
masking_coordinate,
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=sum_or_fraction,
re_mask=False).process(cube, mask)
if return_intermediate:
intermediate_cube = result_land.copy()
# Collapse the masking coordinate.
result_land = CollapseMaskedNeighbourhoodCoordinate(
masking_coordinate, weights=weights).process(result_land)
result = result_land
# Section for neighbourhood processing sea points.
if sea_only.data.max() > 0.0:
result_sea = NeighbourhoodProcessing('square',
radius_or_radii,
lead_times=lead_times,
sum_or_fraction=sum_or_fraction,
re_mask=True).process(
cube, sea_only)
result = result_sea
# Section for combining land and sea points following land and sea points
# being neighbourhood processed individually.
if sea_only.data.max() > 0.0 and land_only.data.max() > 0.0:
# Recombine cubes to be a single output.
combined_data = result_land.data.filled(0) + result_sea.data.filled(0)
result = result_land.copy(data=combined_data)
return result, intermediate_cube
def process(
cube: cli.inputcube,
mask: cli.inputcube = None,
*,
neighbourhood_output,
neighbourhood_shape="square",
radii: cli.comma_separated_list,
lead_times: cli.comma_separated_list = None,
degrees_as_complex=False,
weighted_mode=False,
area_sum=False,
percentiles: cli.comma_separated_list = DEFAULT_PERCENTILES,
halo_radius: float = None,
):
"""Runs neighbourhood processing.
Apply the requested neighbourhood method via the
NeighbourhoodProcessing plugin to a Cube.
Args:
cube (iris.cube.Cube):
The Cube to be processed.
mask (iris.cube.Cube):
A cube to mask the input cube. The data should contain 1 for
usable points and 0 for discarded points.
Can't be used with "percentiles" as neighbourhood_output (Optional)
neighbourhood_output (str):
The form of the results generated using neighbourhood processing.
If "probabilities" is selected, the mean probability with a
neighbourhood is calculated. If "percentiles" is selected, then
the percentiles are calculated with a neighbourhood. Calculating
percentiles from a neighbourhood is only supported for a circular
neighbourhood.
Options: "probabilities", "percentiles".
neighbourhood_shape (str):
Name of the neighbourhood method to use. Only a "circular"
neighbourhood shape is applicable for calculating "percentiles"
output.
Options: "circular", "square".
Default: "square".
radii (list of float):
The radius or a list of radii in metres of the neighbourhood to
apply.
If it is a list, it must be the same length as lead_times, which
defines at which lead time to use which nbhood radius. The radius
will be interpolated for intermediate lead times.
lead_times (list of int):
The lead times in hours that correspond to the radii to be used.
If lead_times are set, radii must be a list the same length as
lead_times.
degrees_as_complex (bool):
Include this option to process angles as complex numbers.
Not compatible with circular kernel or percentiles.
weighted_mode (bool):
Include this option to set the weighting to decrease with radius.
Otherwise a constant weighting is assumed.
weighted_mode is only applicable for calculating "probability"
neighbourhood output using the circular kernel.
area_sum (bool):
Return sum rather than fraction over the neighbourhood area.
percentiles (float):
Calculates value at the specified percentiles from the
neighbourhood surrounding each grid point. This argument has no
effect if the output is probabilities.
halo_radius (float):
Set this radius in metres to define the excess halo to clip. Used
where a larger grid was defined than the standard grid and we want
to clip the grid back to the standard grid. Otherwise no clipping
is applied.
Returns:
iris.cube.Cube:
A processed Cube.
Raises:
RuntimeError:
If weighted_mode is used with the wrong neighbourhood_output.
RuntimeError:
If degree_as_complex is used with
neighbourhood_output='percentiles'.
RuntimeError:
If degree_as_complex is used with neighbourhood_shape='circular'.
"""
from improver.nbhood import radius_by_lead_time
from improver.nbhood.nbhood import (
GeneratePercentilesFromANeighbourhood,
NeighbourhoodProcessing,
)
from improver.utilities.pad_spatial import remove_cube_halo
from improver.wind_calculations.wind_direction import WindDirection
if neighbourhood_output == "percentiles":
if weighted_mode:
raise RuntimeError("weighted_mode cannot be used with"
'neighbourhood_output="percentiles"')
if degrees_as_complex:
raise RuntimeError("Cannot generate percentiles from complex "
"numbers")
if neighbourhood_shape == "circular":
if degrees_as_complex:
raise RuntimeError(
"Cannot process complex numbers with circular neighbourhoods")
if degrees_as_complex:
# convert cube data into complex numbers
cube.data = WindDirection.deg_to_complex(cube.data)
radius_or_radii, lead_times = radius_by_lead_time(radii, lead_times)
if neighbourhood_output == "probabilities":
result = NeighbourhoodProcessing(
neighbourhood_shape,
radius_or_radii,
lead_times=lead_times,
weighted_mode=weighted_mode,
sum_only=area_sum,
re_mask=True,
)(cube, mask_cube=mask)
elif neighbourhood_output == "percentiles":
result = GeneratePercentilesFromANeighbourhood(
radius_or_radii,
lead_times=lead_times,
percentiles=percentiles,
)(cube)
if degrees_as_complex:
# convert neighbourhooded cube back to degrees
result.data = WindDirection.complex_to_deg(result.data)
if halo_radius is not None:
result = remove_cube_halo(result, halo_radius)
return result
def __init__(self, radius: float = 10000.0) -> None:
"""
Initialise class for Nowcast of lightning probability.
Args:
radius:
Radius (metres) over which to neighbourhood process the output
lightning probability. The value supplied applies at T+0
and increases to 2*radius at T+6 hours. The radius is applied
in "process" using the circular neighbourhood plugin.
"""
self.radius = radius
lead_times = [0.0, 6.0]
radii = [self.radius, 2 * self.radius]
self.neighbourhood = NeighbourhoodProcessing("circular",
radii,
lead_times=lead_times)
# pl_dict (dict):
# Lightning probability values to increase first-guess to if
# the lightning_thresholds are exceeded in the nowcast data.
# Dict must have keys 1 and 2 and contain float values.
# The default values are selected to represent lightning risk
# index values of 1 and 2 relating to the key.
self.pl_dict = {1: 1.0, 2: 0.25}
# Lightning-rate threshold for Lightning Risk 1 level
# (dependent on forecast-length)
# Lightning rate thresholds for adjusting the first-guess
# lightning probability (strikes per minute == "min^-1").
# lrt_lev1 must be a function that takes "forecast_period"
# in minutes and returns the lightning rate threshold for
# increasing first-guess lightning probability to risk 1 (LR1).
# This gives a decreasing influence on the extrapolated
# lightning nowcast over forecast_period while retaining an
# influence from the 50 km halo.
self.lrt_lev1 = lambda mins: 0.5 + mins * 2.0 / 360.0
# Lightning-rate threshold for Lightning Risk 2 level
# lrt_lev2 is the lightning rate threshold (as float) for
# increasing first-guess lightning probability to risk 2 (LR2).
self.lrt_lev2 = 0.0
# Set values for handling precipitation rate data
# precipthr (tuple):
# Values for limiting prob(lightning) with prob(precip).
# These are the three prob(precip) thresholds and are designed
# to prevent a large probability of lightning being output if
# the probability of precipitation is very low.
self.precipthr = (0.0, 0.05, 0.1)
# ltngthr (tuple):
# Values for limiting prob(lightning) with prob(precip)
# These are the three prob(lightning) values to scale to.
self.ltngthr = (0.0067, 0.25, 1.0)
# probability thresholds for increasing the prob(lightning)
# phighthresh for heavy precip (>7mm/hr)
# relates to problightning_values[2]
# ptorrthresh for intense precip (>35mm/hr)
# relates to problightning_values[1]
self.phighthresh = 0.4
self.ptorrthresh = 0.2
# ice_scaling (tuple):
# Values for increasing prob(lightning) with VII data.
# These are the three prob(lightning) values to scale to.
self.ice_scaling = (0.1, 0.5, 0.9)
