def test_convert_units(self):
"""Test amend_metadata updates attributes OK. """
changes = "Celsius"
cube = set_up_variable_cube(
np.ones((3, 3), dtype=np.float32), units='K')
result = amend_metadata(cube, units=changes)
self.assertEqual(result.units, "Celsius")
def test_attributes_deleted(self):
"""Test amend_metadata updates attributes OK. """
cube = create_cube_with_threshold()
attributes = {'relative_to_threshold': 'delete'}
result = amend_metadata(cube, 'new_cube_name', np.dtype, None,
attributes)
self.assertFalse('relative_to_threshold' in result.attributes)
def test_basic(self):
"""Test that the function returns a Cube. """
result = amend_metadata(self.cube,
name='new_cube_name',
data_type=np.dtype)
self.assertIsInstance(result, Cube)
self.assertEqual(result.name(), 'new_cube_name')
def test_attributes_deleted(self):
"""Test amend_metadata updates attributes OK. """
attributes = {'attribute_to_update': 'delete'}
result = amend_metadata(
self.cube, name='new_cube_name', data_type=np.dtype,
attributes=attributes)
self.assertFalse('attribute_to_update' in result.attributes)
def test_basic(self):
"""Test that the function returns a Cube. """
cube = create_cube_with_threshold()
result = amend_metadata(cube, 'new_cube_name', np.dtype,
None, None)
self.assertIsInstance(result, Cube)
self.assertEqual(result.name(), 'new_cube_name')
def test_coords_deleted_and_adds(self):
"""Test amend metadata deletes and adds coordinate. """
cube = create_cube_with_threshold()
coords = {'threshold': 'delete', 'new_coord': {'points': [2.0]}}
result = amend_metadata(cube, 'new_cube_name', np.dtype, coords, None)
found_key = 'threshold' in [coord.name() for coord in result.coords()]
self.assertFalse(found_key)
self.assertArrayEqual(
result.coord('new_coord').points, np.array([2.0]))
def process(self,
cube_list,
new_diagnostic_name,
revised_coords=None,
revised_attributes=None):
"""
Create a combined cube.
Args:
cube_list (iris.cube.CubeList):
Cube List contain the cubes to combine.
new_diagnostic_name (str):
New name for the combined diagnostic.
Keyword Args:
revised_coords (dict or None):
Revised coordinates for combined cube.
revised_attributes (dict or None):
Revised attributes for combined cube.
Returns:
result (iris.cube.Cube):
Cube containing the combined data.
"""
if not isinstance(cube_list, iris.cube.CubeList):
msg = ('Expecting data to be an instance of '
'iris.cube.CubeList but is'
' {0:s}.'.format(type(cube_list)))
raise TypeError(msg)
if len(cube_list) < 2:
msg = 'Expecting 2 or more cubes in cube_list'
raise ValueError(msg)
# resulting cube will be based on the first cube.
data_type = cube_list[0].dtype
result = cube_list[0].copy()
for ind in range(1, len(cube_list)):
cube1, cube2 = (resolve_metadata_diff(
result.copy(),
cube_list[ind].copy(),
warnings_on=self.warnings_on))
result = self.combine(cube1, cube2, self.operation)
if self.operation == 'mean':
result = result / len(cube_list)
result = amend_metadata(result,
new_diagnostic_name,
data_type,
revised_coords,
revised_attributes,
warnings_on=self.warnings_on)
return result
def test_attributes_updated_and_added(self):
"""Test amend_metadata updates and adds attributes OK. """
attributes = {'attribute_to_update': 'second_value',
'new_attribute': 'new_value'}
result = amend_metadata(
self.cube, name='new_cube_name', data_type=np.dtype,
attributes=attributes)
self.assertEqual(result.attributes['attribute_to_update'],
'second_value')
self.assertEqual(result.attributes['new_attribute'],
'new_value')
def test_coords_updated(self):
"""Test amend_metadata returns a Cube and updates coord correctly. """
updated_coords = {self.threshold_coord: {'points': [2.0]},
'time': {'points': [402193.5, 402194.5]}}
result = amend_metadata(
self.cube, name='new_cube_name', data_type=np.dtype,
coordinates=updated_coords)
self.assertArrayEqual(result.coord(self.threshold_coord).points,
np.array([2.0]))
self.assertArrayEqual(result.coord('time').points,
np.array([402193.5, 402194.5]))
def test_coords_updated(self):
"""Test amend_metadata returns a Cube and updates coord correctly. """
cube = create_cube_with_threshold()
updated_coords = {'threshold': {'points': [2.0]},
'time': {'points': [402193.5, 402194.5]}}
result = amend_metadata(cube, 'new_cube_name', np.dtype,
updated_coords, None)
self.assertArrayEqual(result.coord('threshold').points,
np.array([2.0]))
self.assertArrayEqual(result.coord('time').points,
np.array([402193.5, 402194.5]))
def test_attributes_updated_and_added(self):
"""Test amend_metadata updates and adds attributes OK. """
cube = create_cube_with_threshold()
attributes = {
'relative_to_threshold': 'between',
'new_attribute': 'new_value'
}
result = amend_metadata(cube, 'new_cube_name', np.dtype, None,
attributes)
self.assertEqual(result.attributes['relative_to_threshold'], 'between')
self.assertEqual(result.attributes['new_attribute'], 'new_value')
def test_attributes_updated_and_added(self):
"""Test amend_metadata updates and adds attributes OK. """
attributes = {
'relative_to_threshold': 'between',
'new_attribute': 'new_value'
}
result = amend_metadata(self.cube,
name='new_cube_name',
data_type=np.dtype,
attributes=attributes)
self.assertEqual(result.attributes['relative_to_threshold'], 'between')
self.assertEqual(result.attributes['new_attribute'], 'new_value')
def test_cell_method_updated_and_added(self):
"""Test amend_metadata updates and adds a cell method. """
cell_methods = {"1": {"action": "add",
"method": "point",
"coords": "time"}}
cm = deepcopy(cell_methods)
cm["1"].pop("action")
expected_cell_method = iris.coords.CellMethod(**cm["1"])
result = amend_metadata(
self.cube, name='new_cube_name', data_type=np.dtype,
cell_methods=cell_methods)
self.assertTrue(expected_cell_method in result.cell_methods)
def test_coords_deleted_and_adds(self):
"""Test amend metadata deletes and adds coordinate. """
coords = {self.threshold_coord: 'delete',
'new_coord': {'points': [2.0]}}
result = amend_metadata(
self.cube, name='new_cube_name', data_type=np.dtype,
coordinates=coords)
found_key = self.threshold_coord in [
coord.name() for coord in result.coords()]
self.assertFalse(found_key)
self.assertArrayEqual(result.coord('new_coord').points,
np.array([2.0]))
def test_cell_method_deleted(self):
"""Test amend_metadata updates attributes OK. """
cell_methods = {"1": {"action": "delete",
"method": "point",
"coords": "time"}}
cm = deepcopy(cell_methods)
cm["1"].pop("action")
cell_method = iris.coords.CellMethod(**cm["1"])
self.cube.cell_methods = (cell_method,)
result = amend_metadata(
self.cube, name='new_cube_name', data_type=np.dtype,
cell_methods=cell_methods)
self.assertEqual(result.cell_methods, ())
def test_warnings_on_works(self, warning_list=None):
"""Test amend_metadata raises warnings """
updated_attributes = {'new_attribute': 'new_value'}
updated_coords = {self.threshold_coord: {'points': [2.0]}}
warning_msg_attr = "Adding or updating attribute"
warning_msg_coord = "Updated coordinate"
result = amend_metadata(
self.cube, name='new_cube_name', data_type=np.dtype,
coordinates=updated_coords, attributes=updated_attributes,
warnings_on=True)
self.assertTrue(any(item.category == UserWarning
for item in warning_list))
self.assertTrue(any(warning_msg_attr in str(item)
for item in warning_list))
self.assertTrue(any(warning_msg_coord in str(item)
for item in warning_list))
self.assertEqual(result.attributes['new_attribute'],
'new_value')
def test_warnings_on_works(self):
"""Test amend_metadata raises warnings """
cube = create_cube_with_threshold()
updated_attributes = {'new_attribute': 'new_value'}
updated_coords = {'threshold': {'points': [2.0]}}
warning_msg_attr = "Adding or updating attribute"
warning_msg_coord = "Updated coordinate"
with warnings.catch_warnings(record=True) as warning_list:
warnings.simplefilter("always")
result = amend_metadata(cube, 'new_cube_name', np.dtype,
updated_coords, updated_attributes,
warnings_on=True)
self.assertTrue(any(item.category == UserWarning
for item in warning_list))
self.assertTrue(any(warning_msg_attr in str(item)
for item in warning_list))
self.assertTrue(any(warning_msg_coord in str(item)
for item in warning_list))
self.assertEqual(result.attributes['new_attribute'],
'new_value')
def process(self, cube, timestep):
"""
Extrapolates input cube data and updates validity time. The input
cube should have precisely two non-scalar dimension coordinates
(spatial x/y), and is expected to be in a projection such that grid
spacing is the same (or very close) at all points within the spatial
domain. The input cube should also have a "time" coordinate.
Args:
cube (iris.cube.Cube):
The 2D cube containing data to be advected
timestep (datetime.timedelta):
Advection time step
Returns:
advected_cube (iris.cube.Cube):
New cube with updated time and extrapolated data. New data
are filled with np.nan and masked where source data were
out of bounds (ie where data could not be advected from outside
the cube domain).
"""
# check that the input cube has precisely two non-scalar dimension
# coordinates (spatial x/y) and a scalar time coordinate
check_input_coords(cube, require_time=True)
# check spatial coordinates match those of plugin velocities
if (cube.coord(axis="x") != self.x_coord
or cube.coord(axis="y") != self.y_coord):
raise InvalidCubeError("Input data grid does not match advection "
"velocities")
# derive velocities in "grid squares per second"
def grid_spacing(coord):
"""Calculate grid spacing along a given spatial axis"""
new_coord = coord.copy()
new_coord.convert_units('m')
return np.float32(np.diff((new_coord).points)[0])
grid_vel_x = self.vel_x.data / grid_spacing(cube.coord(axis="x"))
grid_vel_y = self.vel_y.data / grid_spacing(cube.coord(axis="y"))
# raise a warning if data contains unmasked NaNs
nan_count = np.count_nonzero(~np.isfinite(cube.data))
if nan_count > 0:
warnings.warn("input data contains unmasked NaNs")
# perform advection and create output cube
advected_data = self._advect_field(cube.data, grid_vel_x, grid_vel_y,
timestep.total_seconds())
advected_cube = cube.copy(data=advected_data)
# increment output cube time and add a "forecast_period" coordinate
original_datetime, = \
(cube.coord("time").units).num2date(cube.coord("time").points)
new_datetime = original_datetime + timestep
new_time = (cube.coord("time").units).date2num(new_datetime)
advected_cube.coord("time").points = new_time
advected_cube.coord("time").convert_units(
"seconds since 1970-01-01 00:00:00")
advected_cube.coord("time").points = (np.around(
advected_cube.coord("time").points).astype(np.int64))
try:
advected_cube.coord("forecast_reference_time").convert_units(
"seconds since 1970-01-01 00:00:00")
except CoordinateNotFoundError:
frt_coord = cube.coord("time").copy()
frt_coord.rename("forecast_reference_time")
advected_cube.add_aux_coord(frt_coord)
advected_cube.coord("forecast_reference_time").convert_units(
"seconds since 1970-01-01 00:00:00")
frt_points = np.around(
advected_cube.coord("forecast_reference_time").points).astype(
np.int64)
advected_cube.coord("forecast_reference_time").points = frt_points
forecast_period_seconds = np.int32(timestep.total_seconds())
forecast_period_coord = AuxCoord(forecast_period_seconds,
standard_name="forecast_period",
units="s")
try:
advected_cube.remove_coord("forecast_period")
except CoordinateNotFoundError:
pass
advected_cube.add_aux_coord(forecast_period_coord)
# Modify the source attribute to describe the advected field as a
# Nowcast
if "institution" in advected_cube.attributes.keys():
advected_cube.attributes["source"] = ("{} Nowcast".format(
advected_cube.attributes["institution"]))
else:
advected_cube.attributes["source"] = "Nowcast"
add_history_attribute(advected_cube, "Nowcast")
advected_cube = amend_metadata(advected_cube, **self.metadata_dict)
return advected_cube
def main(argv=None):
"""
Standardise a source cube. Available options are regridding (bilinear or
nearest-neighbour, optionally with land-mask awareness), updating meta-data
and converting float64 data to float32. A check for float64 data compliance
can be made by only specify a source NetCDF file with no other arguments.
"""
parser = ArgParser(
description='Standardise a source data cube. Three main options are '
'available; fixing float64 data, regridding and updating '
'metadata. If regridding then additional options are '
'available to use bilinear or nearest-neighbour '
'(optionally with land-mask awareness) modes. If only a '
'source file is specified with no other arguments, then '
'an exception will be raised if float64 data are found on '
'the source.')
parser.add_argument('source_data_filepath',
metavar='SOURCE_DATA',
help='A cube of data that is to be standardised and '
'optionally fixed for float64 data, regridded '
'and meta data changed')
parser.add_argument("--output_filepath",
metavar="OUTPUT_FILE",
default=None,
help="The output path for the processed NetCDF. "
"If only a source file is specified and no "
"output file, then the source will be checked"
"for float64 data.")
regrid_group = parser.add_argument_group("Regridding options")
regrid_group.add_argument(
"--target_grid_filepath",
metavar="TARGET_GRID",
help=('If specified then regridding of the source '
'against the target grid is enabled. If also using '
'landmask-aware regridding, then this must be land_binary_mask '
'data.'))
regrid_group.add_argument(
"--regrid_mode",
default='bilinear',
choices=['bilinear', 'nearest', 'nearest-with-mask'],
help=('Selects which regridding technique to use. Default uses '
'iris.analysis.Linear(); "nearest" uses Nearest() (Use for less '
'continuous fields, e.g. precipitation.); "nearest-with-mask" '
'ensures that target data are sourced from points with the same '
'mask value (Use for coast-line-dependent variables like '
'temperature).'))
regrid_group.add_argument(
"--extrapolation_mode",
default='nanmask',
help='Mode to use for extrapolating data into regions '
'beyond the limits of the source_data domain. '
'Refer to online documentation for iris.analysis. '
'Modes are: '
'extrapolate - The extrapolation points will '
'take their value from the nearest source point. '
'nan - The extrapolation points will be be '
'set to NaN. '
'error - A ValueError exception will be raised, '
'notifying an attempt to extrapolate. '
'mask - The extrapolation points will always be '
'masked, even if the source data is not a '
'MaskedArray. '
'nanmask - If the source data is a MaskedArray '
'the extrapolation points will be masked. '
'Otherwise they will be set to NaN. '
'Defaults to nanmask.')
regrid_group.add_argument(
"--input_landmask_filepath",
metavar="INPUT_LANDMASK_FILE",
help=("A path to a NetCDF file describing the land_binary_mask on "
"the source-grid if coastline-aware regridding is required."))
regrid_group.add_argument(
"--landmask_vicinity",
metavar="LANDMASK_VICINITY",
default=25000.,
type=float,
help=("Radius of vicinity to search for a coastline, in metres. "
"Default value; 25000 m"))
parser.add_argument("--fix_float64",
action='store_true',
default=False,
help="Check and fix cube for float64 data. Without "
"this option an exception will be raised if "
"float64 data is found but no fix applied.")
parser.add_argument("--json_file",
metavar="JSON_FILE",
default=None,
help='Filename for the json file containing required '
'changes that will be applied '
'to the metadata. Defaults to None.')
args = parser.parse_args(args=argv)
if args.target_grid_filepath or args.json_file or args.fix_float64:
if not args.output_filepath:
msg = ("An argument has been specified that requires an output "
"filepath but none has been provided")
raise ValueError(msg)
if (args.input_landmask_filepath
and "nearest-with-mask" not in args.regrid_mode):
msg = ("Land-mask file supplied without appropriate regrid_mode. "
"Use --regrid_mode=nearest-with-mask.")
raise ValueError(msg)
if args.input_landmask_filepath and not args.target_grid_filepath:
msg = ("Cannot specify input_landmask_filepath without "
"target_grid_filepath")
raise ValueError(msg)
# source file data path is a mandatory argument
output_data = load_cube(args.source_data_filepath)
if args.fix_float64:
check_cube_not_float64(output_data, fix=True)
else:
check_cube_not_float64(output_data, fix=False)
# Re-grid with options:
# if a target grid file has been specified, then regrid optionally
# applying float64 data check, metadata change, Iris nearest and
# extrapolation mode as required.
if args.target_grid_filepath:
target_grid = load_cube(args.target_grid_filepath)
regridder = iris.analysis.Linear(
extrapolation_mode=args.extrapolation_mode)
if args.regrid_mode in ["nearest", "nearest-with-mask"]:
regridder = iris.analysis.Nearest(
extrapolation_mode=args.extrapolation_mode)
output_data = output_data.regrid(target_grid, regridder)
if args.regrid_mode in ["nearest-with-mask"]:
if not args.input_landmask_filepath:
msg = ("An argument has been specified that requires an input "
"landmask filepath but none has been provided")
raise ValueError(msg)
source_landsea = load_cube(args.input_landmask_filepath)
if "land_binary_mask" not in source_landsea.name():
msg = ("Expected land_binary_mask in input_landmask_filepath "
"but found {}".format(repr(source_landsea)))
warnings.warn(msg)
if "land_binary_mask" not in target_grid.name():
msg = ("Expected land_binary_mask in target_grid_filepath "
"but found {}".format(repr(target_grid)))
warnings.warn(msg)
output_data = RegridLandSea(
vicinity_radius=args.landmask_vicinity).process(
output_data, source_landsea, target_grid)
target_grid_attributes = ({
k: v
for (k, v) in target_grid.attributes.items()
if 'mosg__' in k or 'institution' in k
})
amend_metadata(output_data, attributes=target_grid_attributes)
# Change metadata only option:
# if output file path and json metadata file specified,
# change the metadata
if args.json_file:
with open(args.json_file, 'r') as input_file:
metadata_dict = json.load(input_file)
output_data = amend_metadata(output_data, **metadata_dict)
# Check and fix for float64 data only option:
if args.fix_float64:
check_cube_not_float64(output_data, fix=True)
if args.output_filepath:
save_netcdf(output_data, args.output_filepath)
def process(orography,
landmask,
site_list,
metadata_dict=None,
all_methods=False,
land_constraint=None,
minimum_dz=None,
search_radius=None,
node_limit=None,
site_coordinate_system=None,
site_x_coordinate=None,
site_y_coordinate=None):
"""Module to create neighbour cubes for extracting spot data.
Determine grid point coordinates within the provided cubes that neighbour
spot data sites defined within the provided JSON/Dictionary.
If no options are set the returned cube will contain the nearest neighbour
found for each site. Other constrained neighbour finding methods can be
set with options below.
1. Nearest neighbour.
2. Nearest land point neighbour.
3. Nearest neighbour with minimum height difference.
4. Nearest land point neighbour with minimum height difference.
Args:
orography (iris.cube.Cube):
Cube of model orography for the model grid on which neighbours are
being found.
landmask (iris.cube.Cube):
Cube of model land mask for the model grid on which neighbours are
being found.
site_list (dict):
Dictionary that contains the spot sites for which neighbouring grid
points are to be found.
metadata_dict (dict):
Dictionary that can be used to modify the metadata of the
returned cube.
Default is None.
all_methods (bool):
If True, this will return a cube containing the nearest grid point
neighbours to spot sites as defined by each possible combination
of constraints.
Default is False.
land_constraint (bool):
If True, this will return a cube containing the nearest grid point
neighbours to spot sites that are also land points. May be used
with the minimum_dz option.
Default is None.
minimum_dz (bool):
If True, this will return a cube containing the nearest grid point
neighbour to each spot site that is found, within a given search
radius, to minimise the height difference between the two. May be
used with the land_constraint option.
Default is None.
search_radius (float):
The radius in metres about a spot site within which to search for
a grid point neighbour that is land or which has a smaller height
difference than the nearest.
Default is None.
node_limit (int):
When searching within the defined search_radius for suitable
neighbours, a KDTree is constructed. This node_limit prevents the
tree from becoming too large for large search radii. A default of
36 will be set, which is to say the nearest 36 grid points will be
considered. If the search radius is likely to contain more than
36 points, this value should be increased to ensure all point
are considered.
Default is None.
site_coordinate_system (cartopy coordinate system):
The coordinate system in which the site coordinates are provided
within the site list. This must be provided as the name of a
cartopy coordinate system. The Default will become PlateCarree.
This can be a complete definition, including parameters required
to modify a default system. e.g
Miller(central_longitude=90)
If a globe is required this can be specified as
Globe(semimajor_axis=100, semiminor_axis=100)
Default is None.
site_x_coordinate (str):
The key that identifies site x coordinates in the provided site
dictionary. Defaults to longitude.
Default is None.
site_y_coordinate (str):
The key that identifies site y coordinates in the provided site
dictionary. Defaults to latitude.
Default is None.
Returns:
result (iris.cube.Cube):
The processed Cube.
Raises:
ValueError:
If all_methods is used with land_constraint or minimum_dz.
"""
# Check valid options have been selected.
if all_methods is True and (land_constraint or minimum_dz):
raise ValueError(
'Cannot use all_methods option with other constraints.')
# Filter kwargs for those expected by plugin and which are set.
# This preserves the plugin defaults for unset options.
args = {
'land_constraint': land_constraint,
'minimum_dz': minimum_dz,
'search_radius': search_radius,
'site_coordinate_system': site_coordinate_system,
'site_x_coordinate': site_x_coordinate,
'node_limit': node_limit,
'site_y_coordinate': site_y_coordinate
}
fargs = (site_list, orography, landmask)
kwargs = {k: v for (k, v) in args.items() if v is not None}
# Deal with coordinate systems for sites other than PlateCarree.
if 'site_coordinate_system' in kwargs.keys():
scrs = kwargs['site_coordinate_system']
kwargs['site_coordinate_system'] = safe_eval(scrs, ccrs,
PROJECTION_LIST)
# Call plugin to generate neighbour cubes
if all_methods:
methods = [{
**kwargs, 'land_constraint': False,
'minimum_dz': False
}, {
**kwargs, 'land_constraint': True,
'minimum_dz': False
}, {
**kwargs, 'land_constraint': False,
'minimum_dz': True
}, {
**kwargs, 'land_constraint': True,
'minimum_dz': True
}]
all_methods = iris.cube.CubeList([])
for method in methods:
all_methods.append(NeighbourSelection(**method).process(*fargs))
squeezed_cubes = iris.cube.CubeList([])
for index, cube in enumerate(all_methods):
cube.coord('neighbour_selection_method').points = np.int32(index)
squeezed_cubes.append(iris.util.squeeze(cube))
result = merge_cubes(squeezed_cubes)
else:
result = NeighbourSelection(**kwargs).process(*fargs)
result = enforce_coordinate_ordering(
result,
['spot_index', 'neighbour_selection_method', 'grid_attributes'])
# Modify final metadata as described by provided JSON file.
if metadata_dict:
result = amend_metadata(result, **metadata_dict)
return result
def process(neighbour_cube,
diagnostic_cube,
lapse_rate_cube=None,
apply_lapse_rate_correction=False,
land_constraint=False,
minimum_dz=False,
extract_percentiles=None,
ecc_bounds_warning=False,
metadata_dict=None,
suppress_warnings=False):
"""Module to run spot data extraction.
Extract diagnostic data from gridded fields for spot data sites. It is
possible to apply a temperature lapse rate adjustment to temperature data
that helps to account for differences between the spot site's real altitude
and that of the grid point from which the temperature data is extracted.
Args:
neighbour_cube (iris.cube.Cube):
Cube of spot-data neighbours and the spot site information.
diagnostic_cube (iris.cube.Cube):
Cube containing the diagnostic data to be extracted.
lapse_rate_cube (iris.cube.Cube):
Cube containing temperature lapse rates. If this cube is provided
and a screen temperature cube is being processed, the lapse rates
will be used to adjust the temperature to better represent each
spot's site-altitude.
apply_lapse_rate_correction (bool):
If True, and a lapse rate cube has been provided, extracted
screen temperature will be adjusted to better match the altitude
of the spot site for which they have been extracted.
Default is False.
land_constraint (bool):
If True, the neighbour cube will be interrogated for grid point
neighbours that were identified using a land constraint. This means
that the grid points should be land points except for sites where
none were found within the search radius when the neighbour cube
was created. May be used with minimum_dz.
Default is False.
minimum_dz (bool):
If True, the neighbour cube will be interrogated for grid point
neighbours that were identified using the minimum height
difference constraint. These are grid points that were found to be
the closest in altitude to the spot site within the search radius
defined when the neighbour cube was created. May be used with
land_constraint.
Default is False.
extract_percentiles (list or int):
If set to a percentile value or a list of percentile values,
data corresponding to those percentiles will be returned. For
example [25, 50, 75] will result in the 25th, 50th and 75th
percentiles being returned from a cube of probabilities,
percentiles or realizations.
Note that for percentiles inputs, the desired percentile(s) must
exist in the input cube.
Default is None.
ecc_bounds_warning (bool):
If True, where calculated percentiles are outside the ECC bounds
range, raises a warning rather than an exception.
Default is False.
metadata_dict (dict):
If provided, this dictionary can be used to modify the metadata
of the returned cube.
Default is None.
suppress_warnings (bool):
Suppress warning output. This option should only be used if it
is known that warnings will be generated but they are not required.
Default is None.
Returns:
result (iris.cube.Cube):
The processed cube.
Raises:
ValueError:
If the percentile diagnostic cube does not contain the requested
percentile value.
ValueError:
If the lapse rate cube was provided but the diagnostic being
processed is not air temperature.
ValueError:
If the lapse rate cube provided does not have the name
"air_temperature_lapse_rate"
ValueError:
If the lapse rate cube does not contain a single valued height
coordinate.
Warns:
warning:
If diagnostic cube is not a known probabilistic type.
warning:
If a lapse rate cube was provided, but the height of the
temperature does not match that of the data used.
warning:
If a lapse rate cube was not provided, but the option to apply
the lapse rate correction was enabled.
"""
neighbour_selection_method = NeighbourSelection(
land_constraint=land_constraint,
minimum_dz=minimum_dz).neighbour_finding_method_name()
plugin = SpotExtraction(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(neighbour_cube, diagnostic_cube)
# If a probability or percentile diagnostic cube is provided, extract
# the given percentile if available. This is done after the spot-extraction
# to minimise processing time; usually there are far fewer spot sites than
# grid points.
if extract_percentiles is not None:
try:
perc_coordinate = find_percentile_coordinate(result)
except CoordinateNotFoundError:
if 'probability_of_' in result.name():
result = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=ecc_bounds_warning).process(
result, percentiles=extract_percentiles)
result = iris.util.squeeze(result)
elif result.coords('realization', dim_coords=True):
fast_percentile_method = (False if np.ma.isMaskedArray(
result.data) else True)
result = PercentileConverter(
'realization',
percentiles=extract_percentiles,
fast_percentile_method=fast_percentile_method).process(
result)
else:
msg = ('Diagnostic cube is not a known probabilistic type. '
'The {} percentile could not be extracted. Extracting '
'data from the cube including any leading '
'dimensions.'.format(extract_percentiles))
if not suppress_warnings:
warnings.warn(msg)
else:
constraint = [
'{}={}'.format(perc_coordinate.name(), extract_percentiles)
]
perc_result = extract_subcube(result, constraint)
if perc_result is not None:
result = perc_result
else:
msg = ('The percentile diagnostic cube does not contain the '
'requested percentile value. Requested {}, available '
'{}'.format(extract_percentiles,
perc_coordinate.points))
raise ValueError(msg)
# Check whether a lapse rate cube has been provided and we are dealing with
# temperature data and the lapse-rate option is enabled.
if apply_lapse_rate_correction and lapse_rate_cube:
if not result.name() == "air_temperature":
msg = ("A lapse rate cube was provided, but the diagnostic being "
"processed is not air temperature and cannot be adjusted.")
raise ValueError(msg)
if not lapse_rate_cube.name() == 'air_temperature_lapse_rate':
msg = ("A cube has been provided as a lapse rate cube but does "
"not have the expected name air_temperature_lapse_rate: "
"{}".format(lapse_rate_cube.name()))
raise ValueError(msg)
try:
lapse_rate_height_coord = lapse_rate_cube.coord("height")
except (ValueError, CoordinateNotFoundError):
msg = ("Lapse rate cube does not contain a single valued height "
"coordinate. This is required to ensure it is applied to "
"equivalent temperature data.")
raise ValueError(msg)
# Check the height of the temperature data matches that used to
# calculate the lapse rates. If so, adjust temperatures using the lapse
# rate values.
if diagnostic_cube.coord("height") == lapse_rate_height_coord:
plugin = SpotLapseRateAdjust(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(result, neighbour_cube, lapse_rate_cube)
elif not suppress_warnings:
warnings.warn(
"A lapse rate cube was provided, but the height of the "
"temperature data does not match that of the data used "
"to calculate the lapse rates. As such the temperatures "
"were not adjusted with the lapse rates.")
elif apply_lapse_rate_correction and not lapse_rate_cube:
if not suppress_warnings:
warnings.warn(
"A lapse rate cube was not provided, but the option to "
"apply the lapse rate correction was enabled. No lapse rate "
"correction could be applied.")
# Modify final metadata as described by provided JSON file.
if metadata_dict:
result = amend_metadata(result, **metadata_dict)
# Remove the internal model_grid_hash attribute if present.
result.attributes.pop('model_grid_hash', None)
return result
def main(argv=None):
"""Calculate temperature lapse rates."""
parser = ArgParser(
description='Calculate temperature lapse rates in units of K m-1 '
'over a given orography grid. ')
parser.add_argument('temperature_filepath',
metavar='INPUT_TEMPERATURE_FILE',
help='A path to an input NetCDF temperature file to'
'be processed. ')
parser.add_argument('--orography_filepath',
metavar='INPUT_OROGRAPHY_FILE',
help='A path to an input NetCDF orography file. ')
parser.add_argument('--land_sea_mask_filepath',
metavar='LAND_SEA_MASK_FILE',
help='A path to an input NetCDF land/sea mask file. ')
parser.add_argument('output_filepath',
metavar='OUTPUT_FILE',
help='The output path for the processed temperature '
'lapse rates NetCDF. ')
parser.add_argument('--max_height_diff',
metavar='MAX_HEIGHT_DIFF',
type=float,
default=35,
help='Maximum allowable height difference between the '
'central point and points in the neighbourhood '
'over which the lapse rate will be calculated '
'(metres).')
parser.add_argument('--nbhood_radius',
metavar='NBHOOD_RADIUS',
type=int,
default=7,
help='Radius of neighbourhood around each point. '
'The neighbourhood will be a square array with '
'side length 2*nbhood_radius + 1.')
parser.add_argument('--max_lapse_rate',
metavar='MAX_LAPSE_RATE',
type=float,
default=-3 * DALR,
help='Maximum lapse rate allowed which must be '
'provided in units of K m-1. Default is -3*DALR')
parser.add_argument('--min_lapse_rate',
metavar='MIN_LAPSE_RATE',
type=float,
default=DALR,
help='Minimum lapse rate allowed which must be '
'provided in units of K m-1. Default is the DALR')
parser.add_argument('--return_dalr',
action='store_true',
default=False,
help='Flag to return a cube containing the dry '
'adiabatic lapse rate rather than calculating '
'the true lapse rate.')
args = parser.parse_args(args=argv)
if args.min_lapse_rate > args.max_lapse_rate:
msg = 'Minimum lapse rate specified is greater than the maximum.'
raise ValueError(msg)
if args.max_height_diff < 0:
msg = 'Maximum height difference specified is less than zero.'
raise ValueError(msg)
if args.nbhood_radius < 0:
msg = 'Neighbourhood radius specified is less than zero.'
raise ValueError(msg)
temperature_cube = load_cube(args.temperature_filepath)
if args.return_dalr:
result = temperature_cube.copy(
data=np.full_like(temperature_cube.data, U_DALR.points[0]))
result.rename('air_temperature_lapse_rate')
result.units = U_DALR.units
else:
orography_cube = load_cube(args.orography_filepath)
land_sea_mask_cube = load_cube(args.land_sea_mask_filepath)
result = LapseRate(max_height_diff=args.max_height_diff,
nbhood_radius=args.nbhood_radius,
max_lapse_rate=args.max_lapse_rate,
min_lapse_rate=args.min_lapse_rate).process(
temperature_cube, orography_cube,
land_sea_mask_cube)
attributes = {
"title": "delete",
"source": "delete",
"history": "delete",
"um_version": "delete"
}
result = amend_metadata(result, attributes=attributes)
save_netcdf(result, args.output_filepath)
def process(self,
cube_list,
new_diagnostic_name,
revised_coords=None,
revised_attributes=None,
expanded_coord=None):
"""
Create a combined cube.
Args:
cube_list (iris.cube.CubeList):
Cube List contain the cubes to combine.
new_diagnostic_name (str):
New name for the combined diagnostic.
Keyword Args:
revised_coords (dict or None):
Revised coordinates for combined cube.
revised_attributes (dict or None):
Revised attributes for combined cube.
expanded_coord (dict or None):
Coordinates to be expanded as a key, with the value
indicating whether the upper or mid point of the coordinate
should be used as the point value, e.g.
{'time': 'upper'}.
Returns:
result (iris.cube.Cube):
Cube containing the combined data.
Raises:
TypeError: If cube_list is not an iris.cube.CubeList.
ValueError: If the cubelist contains only one cube.
"""
if not isinstance(cube_list, iris.cube.CubeList):
msg = ('Expecting data to be an instance of iris.cube.CubeList '
'but is {}.'.format(type(cube_list)))
raise TypeError(msg)
if len(cube_list) < 2:
msg = 'Expecting 2 or more cubes in cube_list'
raise ValueError(msg)
# resulting cube will be based on the first cube.
data_type = cube_list[0].dtype
result = cube_list[0].copy()
for ind in range(1, len(cube_list)):
cube1, cube2 = (resolve_metadata_diff(
result.copy(),
cube_list[ind].copy(),
warnings_on=self.warnings_on))
result = self.combine(cube1, cube2)
if self.operation == 'mean':
result.data = result.data / len(cube_list)
# If cube has coord bounds that we want to expand
if expanded_coord:
result = expand_bounds(result, cube_list, expanded_coord)
result = amend_metadata(result,
new_diagnostic_name,
data_type,
revised_coords,
revised_attributes,
warnings_on=self.warnings_on)
return result
def process(self, cube1, cube2, boxsize=30):
"""
Extracts data from input cubes, performs dimensionless advection
displacement calculation, and creates new cubes with advection
velocities in metres per second. Each input cube should have precisely
two non-scalar dimension coordinates (spatial x/y), and are expected to
be in a projection such that grid spacing is the same (or very close)
at all points within the spatial domain. Each input cube must also
have a scalar "time" coordinate.
Args:
cube1 (iris.cube.Cube):
2D cube from (earlier) time 1
cube2 (iris.cube.Cube):
2D cube from (later) time 2
Kwargs:
boxsize (int):
The side length of the square box over which to solve the
optical flow constraint. This should be greater than the
data smoothing radius.
Returns:
(tuple) : tuple containing:
**ucube** (iris.cube.Cube):
2D cube of advection velocities in the x-direction
**vcube** (iris.cube.Cube):
2D cube of advection velocities in the y-direction
"""
# clear existing parameters
self.data_smoothing_radius = None
self.boxsize = None
# check the nature of the input cubes, and raise a warning if they are
# not both precipitation
if cube1.name() != cube2.name():
msg = 'Input cubes contain different data types {} and {}'
raise ValueError(msg.format(cube1.name(), cube2.name()))
data_name = cube1.name().lower()
if "rain" not in data_name and "precipitation" not in data_name:
msg = ('Input data are of non-precipitation type {}. Plugin '
'parameters have not been tested and may not be appropriate'
' for this variable.')
warnings.warn(msg.format(cube1.name()))
# check cubes have exactly two spatial dimension coordinates and a
# scalar time coordinate
check_input_coords(cube1, require_time=True)
check_input_coords(cube2, require_time=True)
# check cube dimensions match
if (cube1.coord(axis="x") != cube2.coord(axis="x") or
cube1.coord(axis="y") != cube2.coord(axis="y")):
raise InvalidCubeError("Input cubes on unmatched grids")
# check grids are equal area
check_if_grid_is_equal_area(cube1)
check_if_grid_is_equal_area(cube2)
# convert units to mm/hr as these avoid the need to manipulate tiny
# decimals
try:
cube1 = cube1.copy()
cube2 = cube2.copy()
cube1.convert_units('mm/hr')
cube2.convert_units('mm/hr')
except ValueError as err:
msg = ('Input data are in units that cannot be converted to mm/hr '
'which are the required units for use with optical flow.')
raise ValueError(msg) from err
# check time difference is positive
time1 = (cube1.coord("time").units).num2date(
cube1.coord("time").points[0])
time2 = (cube2.coord("time").units).num2date(
cube2.coord("time").points[0])
cube_time_diff = time2 - time1
if cube_time_diff.total_seconds() <= 0:
msg = "Expected positive time difference cube2 - cube1: got {} s"
raise InvalidCubeError(msg.format(cube_time_diff.total_seconds()))
# if time difference is greater 15 minutes, increase data smoothing
# radius so that larger advection displacements can be resolved
if cube_time_diff.total_seconds() > 900:
data_smoothing_radius_km = self.data_smoothing_radius_km * (
cube_time_diff.total_seconds()/900.)
else:
data_smoothing_radius_km = self.data_smoothing_radius_km
# calculate smoothing radius in grid square units
new_coord = cube1.coord(axis='x').copy()
new_coord.convert_units('km')
grid_length_km = np.float32(np.diff((new_coord).points)[0])
data_smoothing_radius = \
int(data_smoothing_radius_km / grid_length_km)
# Fail verbosely if data smoothing radius is too small and will
# trigger silent failures downstream
if data_smoothing_radius < 3:
msg = ("Input data smoothing radius {} too small (minimum 3 "
"grid squares)")
raise ValueError(msg.format(data_smoothing_radius))
# Fail if self.boxsize is less than data smoothing radius
self.boxsize = boxsize
if self.boxsize < data_smoothing_radius:
msg = ("Box size {} too small (should not be less than data "
"smoothing radius {})")
raise ValueError(
msg.format(self.boxsize, data_smoothing_radius))
# extract 2-dimensional data arrays
data1 = next(cube1.slices([cube1.coord(axis='y'),
cube1.coord(axis='x')])).data
data2 = next(cube2.slices([cube2.coord(axis='y'),
cube2.coord(axis='x')])).data
# fill any mask with 0 values so fill_values are not spread into the
# domain when smoothing the fields.
if np.ma.is_masked(data1):
data1 = data1.filled(0)
if np.ma.is_masked(data2):
data2 = data2.filled(0)
# if input arrays have no non-zero values, set velocities to zero here
# and raise a warning
if (np.allclose(data1, np.zeros(data1.shape)) or
np.allclose(data2, np.zeros(data2.shape))):
msg = ("No non-zero data in input fields: setting optical flow "
"velocities to zero")
warnings.warn(msg)
ucomp = np.zeros(data1.shape, dtype=np.float32)
vcomp = np.zeros(data2.shape, dtype=np.float32)
else:
# calculate dimensionless displacement between the two input fields
ucomp, vcomp = self.process_dimensionless(data1, data2, 1, 0,
data_smoothing_radius)
# convert displacements to velocities in metres per second
for vel in [ucomp, vcomp]:
vel *= np.float32(1000.*grid_length_km)
vel /= cube_time_diff.total_seconds()
# create velocity output cubes based on metadata from later input cube
x_coord = cube2.coord(axis="x")
y_coord = cube2.coord(axis="y")
t_coord = cube2.coord("time")
ucube = iris.cube.Cube(
ucomp, long_name="precipitation_advection_x_velocity",
units="m s-1", dim_coords_and_dims=[(y_coord, 0), (x_coord, 1)])
ucube.add_aux_coord(t_coord)
ucube = amend_metadata(ucube, **self.metadata_dict)
vcube = iris.cube.Cube(
vcomp, long_name="precipitation_advection_y_velocity",
units="m s-1", dim_coords_and_dims=[(y_coord, 0), (x_coord, 1)])
vcube.add_aux_coord(t_coord)
vcube = amend_metadata(vcube, **self.metadata_dict)
return ucube, vcube
def process(output_data, target_grid=None, source_landsea=None,
metadata_dict=None, regrid_mode='bilinear',
extrapolation_mode='nanmask', landmask_vicinity=25000,
fix_float64=False):
"""Standardises a cube by one or more of regridding, updating meta-data etc
Standardise a source cube. Available options are regridding
(bi-linear or nearest-neighbour, optionally with land-mask
awareness), updating meta-data and converting float64 data to
float32. A check for float64 data compliance can be made by only
specifying a source cube with no other arguments.
Args:
output_data (iris.cube.Cube):
Output cube. If the only argument, then it is checked bor float64
data.
target_grid (iris.cube.Cube):
If specified, then regridding of the source against the target
grid is enabled. If also using landmask-aware regridding then this
must be land_binary_mask data.
Default is None.
source_landsea (iris.cube.Cube):
A cube describing the land_binary_mask on the source-grid if
coastline-aware regridding is required.
Default is None.
metadata_dict (dict):
Dictionary containing required changes that will be applied to
the metadata.
Default is None.
regrid_mode (str):
Selects which regridding techniques to use. Default uses
iris.analysis.Linear(); "nearest" uses Nearest() (Use for less
continuous fields, e.g precipitation.); "nearest-with-mask"
ensures that target data are sources from points with the same
mask value (Use for coast-line-dependant variables
like temperature).
extrapolation_mode (str):
Mode to use for extrapolating data into regions beyond the limits
of the source_data domain. Refer to online documentation for
iris.analysis.
Modes are -
extrapolate -The extrapolation points will take their values
from the nearest source point.
nan - The extrapolation points will be set to NaN.
error - A ValueError exception will be raised notifying an attempt
to extrapolate.
mask - The extrapolation points will always be masked, even if
the source data is not a MaskedArray.
nanmask - If the source data is a MaskedArray the extrapolation
points will be masked. Otherwise they will be set to NaN.
Defaults is 'nanmask'.
landmask_vicinity (float):
Radius of vicinity to search for a coastline, in metres.
Defaults is 25000 m
fix_float64 (bool):
If True, checks and fixes cube for float64 data. Without this
option an exception will be raised if float64 data is found but no
fix applied.
Default is False.
Returns:
output_data (iris.cube.Cube):
Processed cube.
Raises:
ValueError:
If source landsea is supplied but regrid mode not
nearest-with-mask.
ValueError:
If source landsea is supplied but not target grid.
ValueError:
If regrid_mode is "nearest-with-mask" but no landmask cube has
been provided.
Warns:
warning:
If the 'source_landsea' did not have a cube named land_binary_mask.
warning:
If the 'target_grid' did not have a cube named land_binary_mask.
"""
if (source_landsea and
"nearest-with-mask" not in regrid_mode):
msg = ("Land-mask file supplied without appropriate regrid_mode. "
"Use --regrid_mode=nearest-with-mask.")
raise ValueError(msg)
if source_landsea and not target_grid:
msg = ("Cannot specify input_landmask_filepath without "
"target_grid_filepath")
raise ValueError(msg)
# Process
# Re-grid with options:
check_cube_not_float64(output_data, fix=fix_float64)
# if a target grid file has been specified, then regrid optionally
# applying float64 data check, metadata change, Iris nearest and
# extrapolation mode as required.
if target_grid:
regridder = iris.analysis.Linear(
extrapolation_mode=extrapolation_mode)
if regrid_mode in ["nearest", "nearest-with-mask"]:
regridder = iris.analysis.Nearest(
extrapolation_mode=extrapolation_mode)
output_data = output_data.regrid(target_grid, regridder)
if regrid_mode in ["nearest-with-mask"]:
if not source_landsea:
msg = ("An argument has been specified that requires an input "
"landmask cube but none has been provided")
raise ValueError(msg)
if "land_binary_mask" not in source_landsea.name():
msg = ("Expected land_binary_mask in input_landmask cube "
"but found {}".format(repr(source_landsea)))
warnings.warn(msg)
if "land_binary_mask" not in target_grid.name():
msg = ("Expected land_binary_mask in target_grid cube "
"but found {}".format(repr(target_grid)))
warnings.warn(msg)
output_data = RegridLandSea(
vicinity_radius=landmask_vicinity).process(
output_data, source_landsea, target_grid)
target_grid_attributes = (
{k: v for (k, v) in target_grid.attributes.items()
if 'mosg__' in k or 'institution' in k})
amend_metadata(output_data, attributes=target_grid_attributes)
# Change metadata only option:
# if output file path and json metadata file specified,
# change the metadata
if metadata_dict:
output_data = amend_metadata(output_data, **metadata_dict)
check_cube_not_float64(output_data, fix=fix_float64)
return output_data
def main(argv=None):
"""Load in arguments and start spotdata extraction process."""
parser = ArgParser(
description="Extract diagnostic data from gridded fields for spot data"
" sites. It is possible to apply a temperature lapse rate adjustment"
" to temperature data that helps to account for differences between"
" the spot sites real altitude and that of the grid point from which"
" the temperature data is extracted.")
# Input and output files required.
parser.add_argument("neighbour_filepath", metavar="NEIGHBOUR_FILEPATH",
help="Path to a NetCDF file of spot-data neighbours. "
"This file also contains the spot site information.")
parser.add_argument("diagnostic_filepath", metavar="DIAGNOSTIC_FILEPATH",
help="Path to a NetCDF file containing the diagnostic "
"data to be extracted.")
parser.add_argument("temperature_lapse_rate_filepath",
metavar="LAPSE_RATE_FILEPATH", nargs='?',
help="(Optional) Filepath to a NetCDF file containing"
" temperature lapse rates. If this cube is provided,"
" and a screen temperature cube is being processed,"
" the lapse rates will be used to adjust the"
" temperatures to better represent each spot's"
" site-altitude.")
parser.add_argument("output_filepath", metavar="OUTPUT_FILEPATH",
help="The output path for the resulting NetCDF")
parser.add_argument(
"--apply_lapse_rate_correction",
default=False, action="store_true",
help="If the option is set and a lapse rate cube has been "
"provided, extracted screen temperatures will be adjusted to "
"better match the altitude of the spot site for which they have "
"been extracted.")
method_group = parser.add_argument_group(
title="Neighbour finding method",
description="If none of these options are set, the nearest grid point "
"to a spot site will be used without any other constraints.")
method_group.add_argument(
"--land_constraint", default=False, action='store_true',
help="If set the neighbour cube will be interrogated for grid point"
" neighbours that were identified using a land constraint. This means"
" that the grid points should be land points except for sites where"
" none were found within the search radius when the neighbour cube was"
" created. May be used with minimum_dz.")
method_group.add_argument(
"--minimum_dz", default=False, action='store_true',
help="If set the neighbour cube will be interrogated for grid point"
" neighbours that were identified using a minimum height difference"
" constraint. These are grid points that were found to be the closest"
" in altitude to the spot site within the search radius defined when"
" the neighbour cube was created. May be used with land_constraint.")
percentile_group = parser.add_argument_group(
title="Extract percentiles",
description="Extract particular percentiles from probabilistic, "
"percentile, or realization inputs. If deterministic input is "
"provided a warning is raised and all leading dimensions are included "
"in the returned spot-data cube.")
percentile_group.add_argument(
"--extract_percentiles", default=None, nargs='+', type=int,
help="If set to a percentile value or a list of percentile values, "
"data corresponding to those percentiles will be returned. For "
"example setting '--extract_percentiles 25 50 75' will result in the "
"25th, 50th, and 75th percentiles being returned from a cube of "
"probabilities, percentiles, or realizations. Note that for "
"percentile inputs, the desired percentile(s) must exist in the input "
"cube.")
parser.add_argument(
"--ecc_bounds_warning", default=False, action="store_true",
help="If True, where calculated percentiles are outside the ECC "
"bounds range, raise a warning rather than an exception.")
meta_group = parser.add_argument_group("Metadata")
meta_group.add_argument(
"--metadata_json", metavar="METADATA_JSON", default=None,
help="If provided, this JSON file can be used to modify the metadata "
"of the returned netCDF file. Defaults to None.")
output_group = parser.add_argument_group("Suppress Verbose output")
# This CLI may be used to prepare data for verification without knowing the
# form of the input, be it deterministic, realizations or probabilistic.
# A warning is normally raised when attempting to extract a percentile from
# deterministic data as this is not possible; the spot-extraction of the
# entire cube is returned. When preparing data for verification we know
# that we will produce a large number of these warnings when passing in
# deterministic data. This option to suppress warnings is provided to
# reduce the amount of unneeded logging information that is written out.
output_group.add_argument(
"--suppress_warnings", default=False, action="store_true",
help="Suppress warning output. This option should only be used if "
"it is known that warnings will be generated but they are not "
"required.")
args = parser.parse_args(args=argv)
neighbour_cube = load_cube(args.neighbour_filepath)
diagnostic_cube = load_cube(args.diagnostic_filepath)
neighbour_selection_method = NeighbourSelection(
land_constraint=args.land_constraint,
minimum_dz=args.minimum_dz).neighbour_finding_method_name()
plugin = SpotExtraction(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(neighbour_cube, diagnostic_cube)
# If a probability or percentile diagnostic cube is provided, extract
# the given percentile if available. This is done after the spot-extraction
# to minimise processing time; usually there are far fewer spot sites than
# grid points.
if args.extract_percentiles:
try:
perc_coordinate = find_percentile_coordinate(result)
except CoordinateNotFoundError:
if 'probability_of_' in result.name():
result = GeneratePercentilesFromProbabilities(
ecc_bounds_warning=args.ecc_bounds_warning).process(
result, percentiles=args.extract_percentiles)
result = iris.util.squeeze(result)
elif result.coords('realization', dim_coords=True):
fast_percentile_method = (
False if np.ma.isMaskedArray(result.data) else True)
result = PercentileConverter(
'realization', percentiles=args.extract_percentiles,
fast_percentile_method=fast_percentile_method).process(
result)
else:
msg = ('Diagnostic cube is not a known probabilistic type. '
'The {} percentile could not be extracted. Extracting '
'data from the cube including any leading '
'dimensions.'.format(
args.extract_percentiles))
if not args.suppress_warnings:
warnings.warn(msg)
else:
constraint = ['{}={}'.format(perc_coordinate.name(),
args.extract_percentiles)]
perc_result = extract_subcube(result, constraint)
if perc_result is not None:
result = perc_result
else:
msg = ('The percentile diagnostic cube does not contain the '
'requested percentile value. Requested {}, available '
'{}'.format(args.extract_percentiles,
perc_coordinate.points))
raise ValueError(msg)
# Check whether a lapse rate cube has been provided and we are dealing with
# temperature data and the lapse-rate option is enabled.
if (args.temperature_lapse_rate_filepath and
args.apply_lapse_rate_correction):
if not result.name() == "air_temperature":
msg = ("A lapse rate cube was provided, but the diagnostic being "
"processed is not air temperature and cannot be adjusted.")
raise ValueError(msg)
lapse_rate_cube = load_cube(args.temperature_lapse_rate_filepath)
if not lapse_rate_cube.name() == 'air_temperature_lapse_rate':
msg = ("A cube has been provided as a lapse rate cube but does "
"not have the expected name air_temperature_lapse_rate: "
"{}".format(lapse_rate_cube.name()))
raise ValueError(msg)
try:
lapse_rate_height_coord = lapse_rate_cube.coord("height")
except (ValueError, CoordinateNotFoundError):
msg = ("Lapse rate cube does not contain a single valued height "
"coordinate. This is required to ensure it is applied to "
"equivalent temperature data.")
raise ValueError(msg)
# Check the height of the temperature data matches that used to
# calculate the lapse rates. If so, adjust temperatures using the lapse
# rate values.
if diagnostic_cube.coord("height") == lapse_rate_height_coord:
plugin = SpotLapseRateAdjust(
neighbour_selection_method=neighbour_selection_method)
result = plugin.process(result, neighbour_cube, lapse_rate_cube)
else:
msg = ("A lapse rate cube was provided, but the height of "
"the temperature data does not match that of the data used "
"to calculate the lapse rates. As such the temperatures "
"were not adjusted with the lapse rates.")
if not args.suppress_warnings:
warnings.warn(msg)
elif (args.apply_lapse_rate_correction and
not args.temperature_lapse_rate_filepath):
msg = ("A lapse rate cube was not provided, but the option to "
"apply the lapse rate correction was enabled. No lapse rate "
"correction could be applied.")
if not args.suppress_warnings:
warnings.warn(msg)
# Modify final metadata as described by provided JSON file.
if args.metadata_json:
with open(args.metadata_json, 'r') as input_file:
metadata_dict = json.load(input_file)
result = amend_metadata(result, **metadata_dict)
# Remove the internal model_grid_hash attribute if present.
result.attributes.pop('model_grid_hash', None)
# Save the spot data cube.
save_netcdf(result, args.output_filepath)
def process(temperature_cube,
orography_cube,
land_sea_mask_cube,
max_height_diff=35,
nbhood_radius=7,
max_lapse_rate=3 * DALR,
min_lapse_rate=DALR,
return_dalr=False):
"""Calculate temperature lapse rates in units of K m-1 over orography grid.
Args:
temperature_cube (iris.cube.Cube):
A cube of air temperature to be processed (K).
orography_cube (iris.cube.Cube):
A Cube containing orography data (metres).
land_sea_mask_cube (iris.cube.Cube):
A cube containing a binary land-sea mask.
True for land-points.
False for sea.
max_height_diff (float):
Maximum allowable height difference between the central point and
points in the neighbourhood over which the lapse rate will be
calculated.
Default is 35.
nbhood_radius (int):
Radius of neighbourhood around each point. The neighbourhood
will be a square array with side length 2*nbhood_radius + 1.
The default value of 7 is from the reference paper.
max_lapse_rate (float):
Maximum lapse rate allowed.
Default is 3*improver.constants.DALR.
min_lapse_rate (float):
Minimum lapse rate allowed.
Default is improver.constants.DALR.
return_dalr (bool):
If True, returns a cube containing the dry adiabatic lapse rate
rather than calculating the true lapse rate.
Returns:
result (iris.cube.Cube):
Cube containing lapse rate (K m-1)
Raises:
ValueError:
If minimum lapse rate is greater than maximum.
ValueError:
If Maximum height difference is less than zero.
ValueError:
If neighbourhood radius is less than zero.
"""
if min_lapse_rate > max_lapse_rate:
msg = 'Minimum lapse rate specified is greater than the maximum.'
raise ValueError(msg)
if max_height_diff < 0:
msg = 'Maximum height difference specified is less than zero.'
raise ValueError(msg)
if nbhood_radius < 0:
msg = 'Neighbourhood radius specified is less than zero.'
raise ValueError(msg)
if return_dalr:
result = temperature_cube.copy(
data=np.full_like(temperature_cube.data, U_DALR.points[0]))
result.rename('air_temperature_lapse_rate')
result.units = U_DALR.units
else:
result = LapseRate(max_height_diff=max_height_diff,
nbhood_radius=nbhood_radius,
max_lapse_rate=max_lapse_rate,
min_lapse_rate=min_lapse_rate).process(
temperature_cube, orography_cube,
land_sea_mask_cube)
attributes = {
"title": "delete",
"source": "delete",
"history": "delete",
"um_version": "delete"
}
result = amend_metadata(result, attributes=attributes)
return result
def main(argv=None):
"""Load in arguments and get going."""
description = (
"Determine grid point coordinates within the provided cubes that "
"neighbour spot data sites defined within the provided JSON "
"file. If no options are set the returned netCDF file will contain the"
" nearest neighbour found for each site. Other constrained neighbour "
"finding methods can be set with options below.")
options = ("\n\nThese methods are:\n\n 1. nearest neighbour\n"
" 2. nearest land point neighbour\n"
" 3. nearest neighbour with minimum height difference\n"
" 4. nearest land point neighbour with minimum height "
"difference")
parser = ArgParser(description=('\n'.join(wrap(description, width=79)) +
options),
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument("site_list_filepath",
metavar="SITE_LIST_FILEPATH",
help="Path to a JSON file that contains the spot sites"
" for which neighbouring grid points are to be found.")
parser.add_argument("orography_filepath",
metavar="OROGRAPHY_FILEPATH",
help="Path to a NetCDF file of model orography for the"
" model grid on which neighbours are being found.")
parser.add_argument("landmask_filepath",
metavar="LANDMASK_FILEPATH",
help="Path to a NetCDF file of model land mask for the"
" model grid on which neighbours are being found.")
parser.add_argument("output_filepath",
metavar="OUTPUT_FILEPATH",
help="The output path for the resulting NetCDF")
parser.add_argument(
"--all_methods",
default=False,
action='store_true',
help="If set this will return a cube containing the nearest grid point"
" neighbours to spot sites as defined by each possible combination of"
" constraints.")
group = parser.add_argument_group('Apply constraints to neighbour choice')
group.add_argument(
"--land_constraint",
default=False,
action='store_true',
help="If set this will return a cube containing the nearest grid point"
" neighbours to spot sites that are also land points. May be used with"
" the minimum_dz option.")
group.add_argument(
"--minimum_dz",
default=False,
action='store_true',
help="If set this will return a cube containing the nearest grid point"
" neighbour to each spot site that is found, within a given search"
" radius, to minimise the height difference between the two. May be"
" used with the land_constraint option.")
group.add_argument(
"--search_radius",
metavar="SEARCH_RADIUS",
type=float,
help="The radius in metres about a spot site within which to search"
" for a grid point neighbour that is land or which has a smaller "
" height difference than the nearest. The default value is 10000m "
"(10km).")
group.add_argument(
"--node_limit",
metavar="NODE_LIMIT",
type=int,
help="When searching within the defined search_radius for suitable "
"neighbours, a KDTree is constructed. This node_limit prevents the "
"tree from becoming too large for large search radii. A default of 36"
" is set, which is to say the nearest 36 grid points will be "
"considered. If the search_radius is likely to contain more than 36 "
"points, this value should be increased to ensure all points are "
"considered.")
s_group = parser.add_argument_group('Site list options')
s_group.add_argument(
"--site_coordinate_system",
metavar="SITE_COORDINATE_SYSTEM",
help="The coordinate system in which the site coordinates are provided"
" within the site list. This must be provided as the name of a cartopy"
" coordinate system. The default is a PlateCarree system, with site"
" coordinates given by latitude/longitude pairs. This can be a"
" complete definition, including parameters required to modify a"
" default system, e.g. Miller(central_longitude=90). If a globe is"
" required this can be specified as e.g."
" Globe(semimajor_axis=100, semiminor_axis=100).")
s_group.add_argument(
"--site_x_coordinate",
metavar="SITE_X_COORDINATE",
help="The x coordinate key within the JSON file. The plugin default is"
" 'longitude', but can be changed using this option if required.")
s_group.add_argument(
"--site_y_coordinate",
metavar="SITE_Y_COORDINATE",
help="The y coordinate key within the JSON file. The plugin default is"
" 'latitude', but can be changed using this option if required.")
meta_group = parser.add_argument_group("Metadata")
meta_group.add_argument(
"--metadata_json",
metavar="METADATA_JSON",
default=None,
help="If provided, this JSON file can be used to modify the metadata "
"of the returned netCDF file. Defaults to None.")
args = parser.parse_args(args=argv)
# Open input files
with open(args.site_list_filepath, 'r') as site_file:
sitelist = json.load(site_file)
orography = load_cube(args.orography_filepath)
landmask = load_cube(args.landmask_filepath)
fargs = (sitelist, orography, landmask)
# Filter kwargs for those expected by plugin and which are set.
# This preserves the plugin defaults for unset options.
kwarg_list = [
'land_constraint', 'minimum_dz', 'search_radius',
'site_coordinate_system', 'site_x_coordinate', 'node_limit',
'site_y_coordinate'
]
kwargs = {
k: v
for (k, v) in vars(args).items() if k in kwarg_list and v is not None
}
# Deal with coordinate systems for sites other than PlateCarree.
if 'site_coordinate_system' in kwargs.keys():
scrs = kwargs['site_coordinate_system']
kwargs['site_coordinate_system'] = safe_eval(scrs, ccrs,
PROJECTION_LIST)
# Check valid options have been selected.
if args.all_methods is True and (kwargs['land_constraint'] is True
or kwargs['minimum_dz'] is True):
raise ValueError(
'Cannot use all_methods option with other constraints.')
# Call plugin to generate neighbour cubes
if args.all_methods:
methods = []
methods.append({
**kwargs, 'land_constraint': False,
'minimum_dz': False
})
methods.append({
**kwargs, 'land_constraint': True,
'minimum_dz': False
})
methods.append({
**kwargs, 'land_constraint': False,
'minimum_dz': True
})
methods.append({**kwargs, 'land_constraint': True, 'minimum_dz': True})
all_methods = iris.cube.CubeList([])
for method in methods:
all_methods.append(NeighbourSelection(**method).process(*fargs))
squeezed_cubes = iris.cube.CubeList([])
for index, cube in enumerate(all_methods):
cube.coord('neighbour_selection_method').points = index
squeezed_cubes.append(iris.util.squeeze(cube))
result = merge_cubes(squeezed_cubes)
else:
result = NeighbourSelection(**kwargs).process(*fargs)
result = enforce_coordinate_ordering(
result,
['spot_index', 'neighbour_selection_method', 'grid_attributes'])
# Modify final metadata as described by provided JSON file.
if args.metadata_json:
with open(args.metadata_json, 'r') as input_file:
metadata_dict = json.load(input_file)
result = amend_metadata(result, **metadata_dict)
# Save the neighbour cube
save_netcdf(result, args.output_filepath)
