def test_error_non_coord_units(self):
""" Test error raised if units are provided for a non-coordinate
constraint """
constraint_dict = {"name": "probability_of_precipitation"}
units_dict = {"name": "1"}
with self.assertRaises(CoordinateNotFoundError):
extract_subcube(self.precip_cube, constraint_dict, units_dict)
def test_range_constraint(self):
"""Test that multiple thresholds are extracted correctly when using the
key=[value1:value2] syntax."""
constraints = ["projection_y_coordinate=[1:2]"]
expected = self.precip_cube[:, 1:, :]
result = extract_subcube(self.precip_cube, constraints)
self.assertArrayAlmostEqual(result.data, expected.data)
def test_list_constraints(self):
""" Test that a list of constraints behaves correctly """
constraint_dict = {"threshold": [0.1, 1.0]}
cube = extract_subcube(self.precip_cube, constraint_dict,
self.units_dict)
reference_data = self.precip_cube.data[1:, :, :]
self.assertArrayEqual(cube.data, reference_data)
def test_basic_no_units(self):
""" Test cube extraction for single constraint without units """
constraint_dict = {"name": "probability_of_precipitation"}
cube = extract_subcube(self.precip_cube, constraint_dict)
self.assertIsInstance(cube, iris.cube.Cube)
reference_data = self.precip_cube.data
self.assertArrayEqual(cube.data, reference_data)
def extract_and_check(cube, height_value, units):
"""
Function to attempt to extract a height level.
If no matching level is available an error is raised.
Args:
cube (cube):
Cube to be extracted from and checked it worked.
height_value (float):
The boundary height to be extracted with the input units.
units (str):
The units of the height level to be extracted.
Returns:
iris.cube.Cube:
A cube containing the extracted height level.
Raises:
ValueError: If height level is not found in the input cube.
"""
from improver.utilities.cube_extraction import extract_subcube
# Write constraint in this format so a constraint is constructed that
# is suitable for floating point comparison
height_constraint = [
"height=[{}:{}]".format(height_value - 0.1, height_value + 0.1)
]
cube = extract_subcube(cube, height_constraint, units=[units])
if cube is not None:
return cube
raise ValueError('No data available at height {}{}'.format(
height_value, units))
def process(cube, constraints, units=None):
""" Extract a subset of a single cube.
Extracts subset of data from a single cube, subject to equality-based
constraints.
Using a set of constraints, extract a sub-cube from the provided cube if it
is available.
Args:
cube (iris.cube.Cube):
The Cube from which a sub-cube is extracted
constraints (list):
The constraint(s) to be applied. These must be of the form
"key=value", eg "threshold=1". Scalars, boolean and string
values are supported. Comma-separated lists
e.g. key=[value1,value2,value3]
are supported.
These comma-separated lists can either extract all values
specified in the list or all values specified within a range
e.g. key=[value1:value3].
When a range is specified, this is inclusive of the endpoints of
the range.
units (list):
List of units as strings corresponding to each coordinate in the
list of constraints. One or more "units" may be None and units may
only be associated with coordinate constraints.
Returns:
(iris.cube.Cube):
A single cube matching the input constraints or None. If no
sub-cube is found within the cube that matches the constraints.
"""
return extract_subcube(cube, constraints, units)
def load_and_extract(cube_filepath, height_value, units):
"""
Function to load a cube, attempt to extract a height level.
If no matching level is available an error is raised.
Args:
cube_filepath (str):
Path to the input NetCDF file.
height_value (float):
The boundary height to be extracted with the input units.
units (str):
The units of the height level to be extracted.
Returns:
cube (iris.cube.Cube):
A cube containing the extracted height level.
Raises:
ValueError: If height level is not found in the input cube.
"""
cube = load_cube(cube_filepath)
# Write constraint in this format so a constraint is constructed that
# is suitable for floating point comparison
height_constraint = [
"height=[{}:{}]".format(height_value - 0.1, height_value + 0.1)
]
cube = extract_subcube(cube, height_constraint, units=[units])
if cube is not None:
return cube
raise ValueError('No data available from {} at height {}{}'.format(
cube_filepath, height_value, units))
def test_single_threshold(self):
"""Test that a single threshold is extracted correctly when using the
key=value syntax."""
constraints = ["precipitation_rate=0.03"]
precip_units = ["mm h-1"]
expected = self.precip_cube[0]
result = extract_subcube(self.precip_cube, constraints, units=precip_units)
self.assertArrayAlmostEqual(result.data, expected.data)
def test_multiple_thresholds(self):
"""Test that multiple thresholds are extracted correctly when using the
key=[value1,value2] syntax."""
constraints = ["precipitation_rate=[0.03,0.1]"]
precip_units = ["mm h-1"]
expected = self.precip_cube[:2]
result = extract_subcube(self.precip_cube, constraints, units=precip_units)
self.assertArrayAlmostEqual(result.data, expected.data)
def test_multiple_range_constraints(self):
"""Test that multiple range constraints are extracted correctly when
using the key=[value1:value2] syntax for more than one quantity (i.e.
multiple constraints)."""
constraints = ["precipitation_rate=[0.03:0.1]", "projection_y_coordinate=[1:2]"]
precip_units = ["mm h-1", "m"]
expected = self.precip_cube[0:2, 1:, :]
result = extract_subcube(self.precip_cube, constraints, units=precip_units)
self.assertArrayAlmostEqual(result.data, expected.data)
def test_combination_of_equality_and_range_constraints(self):
"""Test that multiple constraints are extracted correctly when
using a combination of key=[value1,value2] and key=[value1:value2]
syntax."""
constraints = ["precipitation_rate=[0.03,0.1]", "projection_y_coordinate=[1:2]"]
precip_units = ["mm h-1", "m"]
expected = self.precip_cube[0:2, 1:, :]
result = extract_subcube(self.precip_cube, constraints, units=precip_units)
self.assertArrayAlmostEqual(result.data, expected.data)
def test_basic_with_units(self):
""" Test cube extraction for single constraint with units """
constraint_dict = {"threshold": 0.1}
cube = extract_subcube(self.precip_cube, constraint_dict,
self.units_dict)
self.assertIsInstance(cube, iris.cube.Cube)
self.assertEqual(cube.coord("threshold").units, "m s-1")
reference_data = self.precip_cube.data[1, :, :]
self.assertArrayEqual(cube.data, reference_data)
def test_return_none(self):
""" Test function returns None rather than raising an error where
no subcubes match the required constraints, when unit conversion is
required """
constraint_dict = {
"name": "probability_of_precipitation",
"threshold": 5
}
cube = extract_subcube(self.precip_cube, constraint_dict,
self.units_dict)
self.assertFalse(cube)
def test_multiple_constraints_with_units(self):
""" Test behaviour with a list of constraints and units """
constraint_dict = {
"name": "probability_of_precipitation",
"threshold": 0.03
}
cube = extract_subcube(self.precip_cube, constraint_dict,
self.units_dict)
self.assertIsInstance(cube, iris.cube.Cube)
reference_data = self.precip_cube.data[0, :, :]
self.assertArrayEqual(cube.data, reference_data)
def test_single_threshold_use_original_units(self):
"""Test that a single threshold is extracted correctly when using the
key=value syntax without converting the coordinate units back to the
original units."""
constraints = ["precipitation_rate=0.03"]
precip_units = ["mm h-1"]
expected = self.precip_cube[0]
expected.coord("precipitation_rate").convert_units("mm h-1")
result = extract_subcube(self.precip_cube, constraints,
units=precip_units, use_original_units=False)
self.assertArrayAlmostEqual(result.data, expected.data)
self.assertEqual(expected.coord("precipitation_rate"),
result.coord("precipitation_rate"))
def test_thin_global_gridded_cube(self):
""" Subsets a grid from a global grid and thins the data"""
expected_result = np.array([[1.0, 4.0], [17.0, 20.0]])
result = extract_subcube(
self.global_gridded_cube, ["latitude=[42:52:2]", "longitude=[0:7:3]"]
)
self.assertArrayAlmostEqual(result.data, expected_result)
self.assertArrayAlmostEqual(
result.coord("longitude").points, np.array([0.0, 6.0])
)
self.assertArrayAlmostEqual(
result.coord("latitude").points, np.array([45.0, 49.0])
)
def process(
cube: cli.inputcube,
*,
constraints: parameters.multi(min=1),
units: cli.comma_separated_list = None,
ignore_failure=False,
):
""" Extract a subset of a single cube.
Extracts subset of data from a single cube, subject to equality-based
constraints.
Using a set of constraints, extract a sub-cube from the provided cube if it
is available.
Args:
cube (iris.cube.Cube):
The Cube from which a sub-cube is extracted
constraints (list):
The constraint(s) to be applied. These must be of the form
"key=value", eg "threshold=1". Multiple constraints can be provided
by repeating this keyword before each. Scalars, boolean and string
values are supported. Lists of values can be provided
e.g. key=[value1, value2, value3]. Alternatively, ranges can also
be specified e.g. key=[value1:value3].
When a range is specified, this is inclusive of the endpoints of
the range. A range can also be specified with a step value,
e.g. [value1:value2:step].
units (list):
List of units as strings corresponding to each coordinate in the
list of constraints. One or more "units" may be None and units may
only be associated with coordinate constraints. The list should be
entered as a comma separated list without spaces, e.g. mm/hr,K.
ignore_failure (bool):
Option to ignore constraint match failure and return the input
cube.
Returns:
iris.cube.Cube:
A single cube matching the input constraints or None. If no
sub-cube is found within the cube that matches the constraints.
"""
from improver.utilities.cube_extraction import extract_subcube
result = extract_subcube(cube, constraints, units)
if result is None and ignore_failure:
return cube
if result is None:
msg = "Constraint(s) could not be matched in input cube"
raise ValueError(msg)
return result
def main(argv=None):
"""Invoke data extraction."""
parser = ArgParser(description='Extracts subset of data from a single '
'input file, subject to equality-based constraints.')
parser.add_argument('input_file',
metavar='INPUT_FILE',
help="File containing a dataset to extract from.")
parser.add_argument('output_file',
metavar='OUTPUT_FILE',
help="File to write the extracted dataset to.")
parser.add_argument('constraints',
metavar='CONSTRAINTS',
nargs='+',
help='The constraint(s) to be applied. These must be'
' of the form "key=value", eg "threshold=1". Scalars'
', boolean and string values are supported. Comma-'
'separated lists (eg "key=[value1,value2]") are '
'supported. These comma-separated lists can either '
'extract all values specified in the list or '
'all values specified within a range e.g. '
'key=[value1:value2]. When a range is specified, '
'this is inclusive of the endpoints of the range.')
parser.add_argument('--units',
metavar='UNITS',
nargs='+',
default=None,
help='Optional: units of coordinate constraint(s) to '
'be applied, for use when the input coordinate '
'units are not ideal (eg for float equality). If '
'used, this list must match the CONSTRAINTS list in '
'order and length (with null values set to None).')
parser.add_argument('--ignore-failure',
action='store_true',
default=False,
help='Option to ignore constraint match failure and '
'return the input cube.')
args = parser.parse_args(args=argv)
cube = load_cube(args.input_file)
output_cube = extract_subcube(cube, args.constraints, args.units)
if output_cube is None and args.ignore_failure:
save_netcdf(cube, args.output_file)
elif output_cube is None:
msg = ("Constraint(s) could not be matched in input cube")
raise ValueError(msg)
else:
save_netcdf(output_cube, args.output_file)
def test_thin_longitude_global_gridded_cube(self):
""" Subsets a grid from a global grid and thins the data"""
expected_result = np.array(
[
[1.0, 4.0],
[9.0, 12.0],
[17.0, 20.0],
[25.0, 28.0],
[33.0, 36.0],
[41.0, 44.0],
[49.0, 52.0],
],
)
result = extract_subcube(self.global_gridded_cube, ["longitude=[0:7:3]"])
self.assertArrayAlmostEqual(result.data, expected_result)
self.assertArrayAlmostEqual(
result.coord("longitude").points, np.array([0.0, 6.0])
)
self.assertArrayAlmostEqual(
result.coord("latitude").points,
np.array([45.0, 47.0, 49.0, 51.0, 53.0, 55.0, 57.0]),
)
def process(neighbour_cube,
diagnostic_cube,
lapse_rate_cube=None,
apply_lapse_rate_correction=False,
land_constraint=False,
minimum_dz=False,
extract_percentiles=None,
ecc_bounds_warning=False,
metadata_dict=None,
suppress_warnings=False):
"""Module to run spot data extraction.
Extract diagnostic data from gridded fields for spot data sites. It is
possible to apply a temperature lapse rate adjustment to temperature data
that helps to account for differences between the spot site's real altitude
and that of the grid point from which the temperature data is extracted.
Args:
neighbour_cube (iris.cube.Cube):
Cube of spot-data neighbours and the spot site information.
diagnostic_cube (iris.cube.Cube):
Cube containing the diagnostic data to be extracted.
lapse_rate_cube (iris.cube.Cube):
Cube containing temperature lapse rates. If this cube is provided
and a screen temperature cube is being processed, the lapse rates
will be used to adjust the temperature to better represent each
spot's site-altitude.
apply_lapse_rate_correction (bool):
If True, and a lapse rate cube has been provided, extracted
screen temperature will be adjusted to better match the altitude
of the spot site for which they have been extracted.
Default is False.
land_constraint (bool):
If True, the neighbour cube will be interrogated for grid point
neighbours that were identified using a land constraint. This means
that the grid points should be land points except for sites where
none were found within the search radius when the neighbour cube
was created. May be used with minimum_dz.
Default is False.
minimum_dz (bool):
If True, the neighbour cube will be interrogated for grid point
neighbours that were identified using the minimum height
difference constraint. These are grid points that were found to be
the closest in altitude to the spot site within the search radius
defined when the neighbour cube was created. May be used with
land_constraint.
Default is False.
extract_percentiles (list or int):
If set to a percentile value or a list of percentile values,
data corresponding to those percentiles will be returned. For
example [25, 50, 75] will result in the 25th, 50th and 75th
percentiles being returned from a cube of probabilities,
percentiles or realizations.
Note that for percentiles inputs, the desired percentile(s) must
exist in the input cube.
Default is None.
ecc_bounds_warning (bool):
If True, where calculated percentiles are outside the ECC bounds
range, raises a warning rather than an exception.
Default is False.
metadata_dict (dict):
If provided, this dictionary can be used to modify the metadata
of the returned cube.
Default is None.
suppress_warnings (bool):
Suppress warning output. This option should only be used if it
is known that warnings will be generated but they are not required.
Default is None.
Returns:
result (iris.cube.Cube):
The processed cube.
Raises:
ValueError:
If the percentile diagnostic cube does not contain the requested
percentile value.
ValueError:
If the lapse rate cube was provided but the diagnostic being
processed is not air temperature.
ValueError:
If the lapse rate cube provided does not have the name
"air_temperature_lapse_rate"
ValueError:
If the lapse rate cube does not contain a single valued height
coordinate.
Warns:
warning:
If diagnostic cube is not a known probabilistic type.
warning:
If a lapse rate cube was provided, but the height of the
temperature does not match that of the data used.
warning:
If a lapse rate cube was not provided, but the option to apply
the lapse rate correction was enabled.
"""
neighbour_selection_method = NeighbourSelection(
land_constraint=land_constraint,
minimum_dz=minimum_dz).neighbour_finding_method_name()
plugin = SpotExtraction(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(neighbour_cube, diagnostic_cube)
# If a probability or percentile diagnostic cube is provided, extract
# the given percentile if available. This is done after the spot-extraction
# to minimise processing time; usually there are far fewer spot sites than
# grid points.
if extract_percentiles is not None:
try:
perc_coordinate = find_percentile_coordinate(result)
except CoordinateNotFoundError:
if 'probability_of_' in result.name():
result = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=ecc_bounds_warning).process(
result, percentiles=extract_percentiles)
result = iris.util.squeeze(result)
elif result.coords('realization', dim_coords=True):
fast_percentile_method = (False if np.ma.isMaskedArray(
result.data) else True)
result = PercentileConverter(
'realization',
percentiles=extract_percentiles,
fast_percentile_method=fast_percentile_method).process(
result)
else:
msg = ('Diagnostic cube is not a known probabilistic type. '
'The {} percentile could not be extracted. Extracting '
'data from the cube including any leading '
'dimensions.'.format(extract_percentiles))
if not suppress_warnings:
warnings.warn(msg)
else:
constraint = [
'{}={}'.format(perc_coordinate.name(), extract_percentiles)
]
perc_result = extract_subcube(result, constraint)
if perc_result is not None:
result = perc_result
else:
msg = ('The percentile diagnostic cube does not contain the '
'requested percentile value. Requested {}, available '
'{}'.format(extract_percentiles,
perc_coordinate.points))
raise ValueError(msg)
# Check whether a lapse rate cube has been provided and we are dealing with
# temperature data and the lapse-rate option is enabled.
if apply_lapse_rate_correction and lapse_rate_cube:
if not result.name() == "air_temperature":
msg = ("A lapse rate cube was provided, but the diagnostic being "
"processed is not air temperature and cannot be adjusted.")
raise ValueError(msg)
if not lapse_rate_cube.name() == 'air_temperature_lapse_rate':
msg = ("A cube has been provided as a lapse rate cube but does "
"not have the expected name air_temperature_lapse_rate: "
"{}".format(lapse_rate_cube.name()))
raise ValueError(msg)
try:
lapse_rate_height_coord = lapse_rate_cube.coord("height")
except (ValueError, CoordinateNotFoundError):
msg = ("Lapse rate cube does not contain a single valued height "
"coordinate. This is required to ensure it is applied to "
"equivalent temperature data.")
raise ValueError(msg)
# Check the height of the temperature data matches that used to
# calculate the lapse rates. If so, adjust temperatures using the lapse
# rate values.
if diagnostic_cube.coord("height") == lapse_rate_height_coord:
plugin = SpotLapseRateAdjust(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(result, neighbour_cube, lapse_rate_cube)
elif not suppress_warnings:
warnings.warn(
"A lapse rate cube was provided, but the height of the "
"temperature data does not match that of the data used "
"to calculate the lapse rates. As such the temperatures "
"were not adjusted with the lapse rates.")
elif apply_lapse_rate_correction and not lapse_rate_cube:
if not suppress_warnings:
warnings.warn(
"A lapse rate cube was not provided, but the option to "
"apply the lapse rate correction was enabled. No lapse rate "
"correction could be applied.")
# Modify final metadata as described by provided JSON file.
if metadata_dict:
result = amend_metadata(result, **metadata_dict)
# Remove the internal model_grid_hash attribute if present.
result.attributes.pop('model_grid_hash', None)
return result
def process(
neighbour_cube: cli.inputcube,
cube: cli.inputcube,
lapse_rate: cli.inputcube = None,
*,
apply_lapse_rate_correction=False,
land_constraint=False,
similar_altitude=False,
extract_percentiles: cli.comma_separated_list = None,
ignore_ecc_bounds=False,
new_title: str = None,
suppress_warnings=False,
):
"""Module to run spot data extraction.
Extract diagnostic data from gridded fields for spot data sites. It is
possible to apply a temperature lapse rate adjustment to temperature data
that helps to account for differences between the spot site's real altitude
and that of the grid point from which the temperature data is extracted.
Args:
neighbour_cube (iris.cube.Cube):
Cube of spot-data neighbours and the spot site information.
cube (iris.cube.Cube):
Cube containing the diagnostic data to be extracted.
lapse_rate (iris.cube.Cube):
Optional cube containing temperature lapse rates. If this cube is
provided and a screen temperature cube is being processed, the
lapse rates will be used to adjust the temperature to better
represent each spot's site-altitude.
apply_lapse_rate_correction (bool):
Use to apply a lapse-rate correction to screen temperature data so
that the data are a better match the altitude of the spot site for
which they have been extracted.
land_constraint (bool):
Use to select the nearest-with-land-constraint neighbour-selection
method from the neighbour_cube. This means that the grid points
should be land points except for sites where none were found within
the search radius when the neighbour cube was created. May be used
with similar_altitude.
similar_altitude (bool):
Use to select the nearest-with-height-constraint
neighbour-selection method from the neighbour_cube. These are grid
points that were found to be the closest in altitude to the spot
site within the search radius defined when the neighbour cube was
created. May be used with land_constraint.
extract_percentiles (list or int):
If set to a percentile value or a list of percentile values,
data corresponding to those percentiles will be returned. For
example "25, 50, 75" will result in the 25th, 50th and 75th
percentiles being returned from a cube of probabilities,
percentiles or realizations. Deterministic input data will raise
a warning message.
Note that for percentiles inputs, the desired percentile(s) must
exist in the input cube.
ignore_ecc_bounds (bool):
Demotes exceptions where calculated percentiles are outside the ECC
bounds range to warnings.
new_title (str):
New title for the spot-extracted data. If None, this attribute is
removed from the output cube since it has no prescribed standard
and may therefore contain grid information that is no longer
correct after spot-extraction.
suppress_warnings (bool):
Suppress warning output. This option should only be used if it
is known that warnings will be generated but they are not required.
Returns:
iris.cube.Cube:
Cube of spot data.
Raises:
ValueError:
If the percentile diagnostic cube does not contain the requested
percentile value.
ValueError:
If the lapse rate cube was provided but the diagnostic being
processed is not air temperature.
ValueError:
If the lapse rate cube provided does not have the name
"air_temperature_lapse_rate"
ValueError:
If the lapse rate cube does not contain a single valued height
coordinate.
Warns:
warning:
If diagnostic cube is not a known probabilistic type.
warning:
If a lapse rate cube was provided, but the height of the
temperature does not match that of the data used.
warning:
If a lapse rate cube was not provided, but the option to apply
the lapse rate correction was enabled.
"""
import warnings
import iris
import numpy as np
from iris.exceptions import CoordinateNotFoundError
from improver.ensemble_copula_coupling.ensemble_copula_coupling import (
ConvertProbabilitiesToPercentiles, )
from improver.metadata.probabilistic import find_percentile_coordinate
from improver.percentile import PercentileConverter
from improver.spotdata.apply_lapse_rate import SpotLapseRateAdjust
from improver.spotdata.neighbour_finding import NeighbourSelection
from improver.spotdata.spot_extraction import SpotExtraction
from improver.utilities.cube_extraction import extract_subcube
neighbour_selection_method = NeighbourSelection(
land_constraint=land_constraint,
minimum_dz=similar_altitude).neighbour_finding_method_name()
result = SpotExtraction(
neighbour_selection_method=neighbour_selection_method)(
neighbour_cube, cube, new_title=new_title)
# If a probability or percentile diagnostic cube is provided, extract
# the given percentile if available. This is done after the spot-extraction
# to minimise processing time; usually there are far fewer spot sites than
# grid points.
if extract_percentiles:
extract_percentiles = [np.float32(x) for x in extract_percentiles]
try:
perc_coordinate = find_percentile_coordinate(result)
except CoordinateNotFoundError:
if "probability_of_" in result.name():
result = ConvertProbabilitiesToPercentiles(
ecc_bounds_warning=ignore_ecc_bounds)(
result, percentiles=extract_percentiles)
result = iris.util.squeeze(result)
elif result.coords("realization", dim_coords=True):
fast_percentile_method = not np.ma.isMaskedArray(result.data)
result = PercentileConverter(
"realization",
percentiles=extract_percentiles,
fast_percentile_method=fast_percentile_method,
)(result)
else:
msg = ("Diagnostic cube is not a known probabilistic type. "
"The {} percentile could not be extracted. Extracting "
"data from the cube including any leading "
"dimensions.".format(extract_percentiles))
if not suppress_warnings:
warnings.warn(msg)
else:
constraint = [
"{}={}".format(perc_coordinate.name(), extract_percentiles)
]
perc_result = extract_subcube(result, constraint)
if perc_result is not None:
result = perc_result
else:
msg = ("The percentile diagnostic cube does not contain the "
"requested percentile value. Requested {}, available "
"{}".format(extract_percentiles,
perc_coordinate.points))
raise ValueError(msg)
# Check whether a lapse rate cube has been provided and we are dealing with
# temperature data and the lapse-rate option is enabled.
if apply_lapse_rate_correction and lapse_rate:
if not result.name() == "air_temperature":
msg = ("A lapse rate cube was provided, but the diagnostic being "
"processed is not air temperature and cannot be adjusted.")
raise ValueError(msg)
if not lapse_rate.name() == "air_temperature_lapse_rate":
msg = ("A cube has been provided as a lapse rate cube but does "
"not have the expected name air_temperature_lapse_rate: "
"{}".format(lapse_rate.name()))
raise ValueError(msg)
try:
lapse_rate_height_coord = lapse_rate.coord("height")
except (ValueError, CoordinateNotFoundError):
msg = ("Lapse rate cube does not contain a single valued height "
"coordinate. This is required to ensure it is applied to "
"equivalent temperature data.")
raise ValueError(msg)
# Check the height of the temperature data matches that used to
# calculate the lapse rates. If so, adjust temperatures using the lapse
# rate values.
if cube.coord("height") == lapse_rate_height_coord:
plugin = SpotLapseRateAdjust(
neighbour_selection_method=neighbour_selection_method)
result = plugin(result, neighbour_cube, lapse_rate)
elif not suppress_warnings:
warnings.warn(
"A lapse rate cube was provided, but the height of the "
"temperature data does not match that of the data used "
"to calculate the lapse rates. As such the temperatures "
"were not adjusted with the lapse rates.")
elif apply_lapse_rate_correction and not lapse_rate:
if not suppress_warnings:
warnings.warn(
"A lapse rate cube was not provided, but the option to "
"apply the lapse rate correction was enabled. No lapse rate "
"correction could be applied.")
# Remove the internal model_grid_hash attribute if present.
result.attributes.pop("model_grid_hash", None)
return result
def _get_input_cubes(self, input_cubes: CubeList) -> None:
"""
Separates out the rain, sleet, and temperature cubes, checking that:
* No other cubes are present
* Cubes have same dimensions
* Cubes represent the same time quantity (instantaneous or accumulation length)
* Precipitation cube threshold units are compatible
* Precipitation cubes have the same set of thresholds
* A 273.15K (0 Celsius) temperature threshold is available
The temperature cube is also modified if necessary to return probabilties
below threshold values. This data is then thinned to return only the
probabilities of temperature being below the freezing point of water,
0 Celsius.
Args:
input_cubes:
Contains exactly three cubes, a rain rate or accumulation, a
sleet rate or accumulation, and an instantaneous or period
temperature. Accumulations and periods must all represent the
same length of time.
Raises:
ValueError:
If any of the criteria above are not met.
"""
if len(input_cubes) != 3:
raise ValueError(
f"Expected exactly 3 input cubes, found {len(input_cubes)}")
rain_name, sleet_name, temperature_name = self._get_input_cube_names(
input_cubes)
(self.rain, ) = input_cubes.extract(rain_name)
(self.sleet, ) = input_cubes.extract(sleet_name)
(self.temperature, ) = input_cubes.extract(temperature_name)
if not spatial_coords_match([self.rain, self.sleet, self.temperature]):
raise ValueError("Input cubes are not on the same grid")
if (not self.rain.coord("time") == self.sleet.coord("time") ==
self.temperature.coord("time")):
raise ValueError("Input cubes do not have the same time coord")
# Ensure rain and sleet cubes are compatible
rain_threshold = self.rain.coord(var_name="threshold")
sleet_threshold = self.sleet.coord(var_name="threshold")
try:
sleet_threshold.convert_units(rain_threshold.units)
except ValueError:
raise ValueError("Rain and sleet cubes have incompatible units")
if not all(rain_threshold.points == sleet_threshold.points):
raise ValueError(
"Rain and sleet cubes have different threshold values")
# Ensure probabilities relate to temperatures below a threshold
temperature_threshold = self.temperature.coord(var_name="threshold")
self.temperature = to_threshold_inequality(self.temperature,
above=False)
# Simplify the temperature cube to the critical threshold of 273.15K,
# the freezing point of water under typical pressures.
self.temperature = extract_subcube(
self.temperature, [f"{temperature_threshold.name()}=273.15"],
units=["K"])
if self.temperature is None:
raise ValueError(
"No 0 Celsius or equivalent threshold is available "
"in the temperature data")
def main(argv=None):
"""Load in arguments and start spotdata extraction process."""
parser = ArgParser(
description="Extract diagnostic data from gridded fields for spot data"
" sites. It is possible to apply a temperature lapse rate adjustment"
" to temperature data that helps to account for differences between"
" the spot sites real altitude and that of the grid point from which"
" the temperature data is extracted.")
# Input and output files required.
parser.add_argument("neighbour_filepath", metavar="NEIGHBOUR_FILEPATH",
help="Path to a NetCDF file of spot-data neighbours. "
"This file also contains the spot site information.")
parser.add_argument("diagnostic_filepath", metavar="DIAGNOSTIC_FILEPATH",
help="Path to a NetCDF file containing the diagnostic "
"data to be extracted.")
parser.add_argument("temperature_lapse_rate_filepath",
metavar="LAPSE_RATE_FILEPATH", nargs='?',
help="(Optional) Filepath to a NetCDF file containing"
" temperature lapse rates. If this cube is provided,"
" and a screen temperature cube is being processed,"
" the lapse rates will be used to adjust the"
" temperatures to better represent each spot's"
" site-altitude.")
parser.add_argument("output_filepath", metavar="OUTPUT_FILEPATH",
help="The output path for the resulting NetCDF")
parser.add_argument(
"--apply_lapse_rate_correction",
default=False, action="store_true",
help="If the option is set and a lapse rate cube has been "
"provided, extracted screen temperatures will be adjusted to "
"better match the altitude of the spot site for which they have "
"been extracted.")
method_group = parser.add_argument_group(
title="Neighbour finding method",
description="If none of these options are set, the nearest grid point "
"to a spot site will be used without any other constraints.")
method_group.add_argument(
"--land_constraint", default=False, action='store_true',
help="If set the neighbour cube will be interrogated for grid point"
" neighbours that were identified using a land constraint. This means"
" that the grid points should be land points except for sites where"
" none were found within the search radius when the neighbour cube was"
" created. May be used with minimum_dz.")
method_group.add_argument(
"--minimum_dz", default=False, action='store_true',
help="If set the neighbour cube will be interrogated for grid point"
" neighbours that were identified using a minimum height difference"
" constraint. These are grid points that were found to be the closest"
" in altitude to the spot site within the search radius defined when"
" the neighbour cube was created. May be used with land_constraint.")
percentile_group = parser.add_argument_group(
title="Extract percentiles",
description="Extract particular percentiles from probabilistic, "
"percentile, or realization inputs. If deterministic input is "
"provided a warning is raised and all leading dimensions are included "
"in the returned spot-data cube.")
percentile_group.add_argument(
"--extract_percentiles", default=None, nargs='+', type=int,
help="If set to a percentile value or a list of percentile values, "
"data corresponding to those percentiles will be returned. For "
"example setting '--extract_percentiles 25 50 75' will result in the "
"25th, 50th, and 75th percentiles being returned from a cube of "
"probabilities, percentiles, or realizations. Note that for "
"percentile inputs, the desired percentile(s) must exist in the input "
"cube.")
parser.add_argument(
"--ecc_bounds_warning", default=False, action="store_true",
help="If True, where calculated percentiles are outside the ECC "
"bounds range, raise a warning rather than an exception.")
meta_group = parser.add_argument_group("Metadata")
meta_group.add_argument(
"--metadata_json", metavar="METADATA_JSON", default=None,
help="If provided, this JSON file can be used to modify the metadata "
"of the returned netCDF file. Defaults to None.")
output_group = parser.add_argument_group("Suppress Verbose output")
# This CLI may be used to prepare data for verification without knowing the
# form of the input, be it deterministic, realizations or probabilistic.
# A warning is normally raised when attempting to extract a percentile from
# deterministic data as this is not possible; the spot-extraction of the
# entire cube is returned. When preparing data for verification we know
# that we will produce a large number of these warnings when passing in
# deterministic data. This option to suppress warnings is provided to
# reduce the amount of unneeded logging information that is written out.
output_group.add_argument(
"--suppress_warnings", default=False, action="store_true",
help="Suppress warning output. This option should only be used if "
"it is known that warnings will be generated but they are not "
"required.")
args = parser.parse_args(args=argv)
neighbour_cube = load_cube(args.neighbour_filepath)
diagnostic_cube = load_cube(args.diagnostic_filepath)
neighbour_selection_method = NeighbourSelection(
land_constraint=args.land_constraint,
minimum_dz=args.minimum_dz).neighbour_finding_method_name()
plugin = SpotExtraction(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(neighbour_cube, diagnostic_cube)
# If a probability or percentile diagnostic cube is provided, extract
# the given percentile if available. This is done after the spot-extraction
# to minimise processing time; usually there are far fewer spot sites than
# grid points.
if args.extract_percentiles:
try:
perc_coordinate = find_percentile_coordinate(result)
except CoordinateNotFoundError:
if 'probability_of_' in result.name():
result = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=args.ecc_bounds_warning).process(
result, percentiles=args.extract_percentiles)
result = iris.util.squeeze(result)
elif result.coords('realization', dim_coords=True):
fast_percentile_method = (
False if np.ma.isMaskedArray(result.data) else True)
result = PercentileConverter(
'realization', percentiles=args.extract_percentiles,
fast_percentile_method=fast_percentile_method).process(
result)
else:
msg = ('Diagnostic cube is not a known probabilistic type. '
'The {} percentile could not be extracted. Extracting '
'data from the cube including any leading '
'dimensions.'.format(
args.extract_percentiles))
if not args.suppress_warnings:
warnings.warn(msg)
else:
constraint = ['{}={}'.format(perc_coordinate.name(),
args.extract_percentiles)]
perc_result = extract_subcube(result, constraint)
if perc_result is not None:
result = perc_result
else:
msg = ('The percentile diagnostic cube does not contain the '
'requested percentile value. Requested {}, available '
'{}'.format(args.extract_percentiles,
perc_coordinate.points))
raise ValueError(msg)
# Check whether a lapse rate cube has been provided and we are dealing with
# temperature data and the lapse-rate option is enabled.
if (args.temperature_lapse_rate_filepath and
args.apply_lapse_rate_correction):
if not result.name() == "air_temperature":
msg = ("A lapse rate cube was provided, but the diagnostic being "
"processed is not air temperature and cannot be adjusted.")
raise ValueError(msg)
lapse_rate_cube = load_cube(args.temperature_lapse_rate_filepath)
if not lapse_rate_cube.name() == 'air_temperature_lapse_rate':
msg = ("A cube has been provided as a lapse rate cube but does "
"not have the expected name air_temperature_lapse_rate: "
"{}".format(lapse_rate_cube.name()))
raise ValueError(msg)
try:
lapse_rate_height_coord = lapse_rate_cube.coord("height")
except (ValueError, CoordinateNotFoundError):
msg = ("Lapse rate cube does not contain a single valued height "
"coordinate. This is required to ensure it is applied to "
"equivalent temperature data.")
raise ValueError(msg)
# Check the height of the temperature data matches that used to
# calculate the lapse rates. If so, adjust temperatures using the lapse
# rate values.
if diagnostic_cube.coord("height") == lapse_rate_height_coord:
plugin = SpotLapseRateAdjust(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(result, neighbour_cube, lapse_rate_cube)
else:
msg = ("A lapse rate cube was provided, but the height of "
"the temperature data does not match that of the data used "
"to calculate the lapse rates. As such the temperatures "
"were not adjusted with the lapse rates.")
if not args.suppress_warnings:
warnings.warn(msg)
elif (args.apply_lapse_rate_correction and
not args.temperature_lapse_rate_filepath):
msg = ("A lapse rate cube was not provided, but the option to "
"apply the lapse rate correction was enabled. No lapse rate "
"correction could be applied.")
if not args.suppress_warnings:
warnings.warn(msg)
# Modify final metadata as described by provided JSON file.
if args.metadata_json:
with open(args.metadata_json, 'r') as input_file:
metadata_dict = json.load(input_file)
result = amend_metadata(result, **metadata_dict)
# Remove the internal model_grid_hash attribute if present.
result.attributes.pop('model_grid_hash', None)
# Save the spot data cube.
save_netcdf(result, args.output_filepath)
