def missing_additional_data(self, method, ancillary_data, additional_data):
"""Test that the plugin copes with missing additional data."""
plugin = Plugin(method)
with self.assertRaises(KeyError):
plugin.process(self.cube, self.sites, self.neighbour_list,
ancillary_data, additional_data)
def test_extracted_value_deep_valley(self):
"""Test that the plugin returns the correct value.
Site set to be 100m or 70m below the land surface (90m or 60m below sea
level). The enforcement of a maximum extrapolation down into valleys
should result in the two site altitudes returning the same temperature.
This is an extrapolation scenario, an 'unresolved valley'."""
# Temperatures set up to mimic a cold night with an inversion where
# valley temperatures may be expected to fall considerably due to
# katabatic drainage.
t_level0 = np.ones((1, 20, 20)) * 0.
t_level1 = np.ones((1, 20, 20)) * 1.
t_level2 = np.ones((1, 20, 20)) * 2.
t_data = np.vstack((t_level0, t_level1, t_level2))
t_data.resize((3, 20, 20))
self.ad['temperature_on_height_levels'].data = t_data
cube = self.cube.extract(self.time_extract)
cube.data = cube.data * 0.0
self.sites['100']['altitude'] = -90.
self.neighbour_list['dz'] = -100.
plugin = Plugin(self.method)
result_dz = plugin.process(cube, self.sites, self.neighbour_list,
self.ancillary_data, self.ad, **self.kwargs)
self.sites['100']['altitude'] = -60.
self.neighbour_list['dz'] = -70.
result_70 = plugin.process(cube, self.sites, self.neighbour_list,
self.ancillary_data, self.ad, **self.kwargs)
self.assertEqual(result_dz.data, result_70.data)
def different_projection(self, method, ancillary_data, additional_data,
expected, **kwargs):
"""Test that the plugin copes with non-lat/lon grids."""
src_crs = ccrs.PlateCarree()
trg_crs = ccrs.TransverseMercator(central_latitude=0,
central_longitude=0)
trg_crs_iris = coord_systems.TransverseMercator(0, 0, 0, 0, 1.0)
lons = [-50, 50]
lats = [-25, 25]
x, y = [], []
for lon, lat in zip(lons, lats):
x_trg, y_trg = trg_crs.transform_point(lon, lat, src_crs)
x.append(x_trg)
y.append(y_trg)
new_x = DimCoord(np.linspace(x[0], x[1], 20),
standard_name='projection_x_coordinate',
units='m',
coord_system=trg_crs_iris)
new_y = DimCoord(np.linspace(y[0], y[1], 20),
standard_name='projection_y_coordinate',
units='m',
coord_system=trg_crs_iris)
new_cube = Cube(np.zeros(400).reshape(20, 20),
long_name="air_temperature",
dim_coords_and_dims=[(new_y, 0), (new_x, 1)],
units="K")
cube = self.cube.copy()
cube = cube.regrid(new_cube, iris.analysis.Nearest())
if ancillary_data is not None:
ancillary_data['orography'] = ancillary_data['orography'].regrid(
new_cube, iris.analysis.Nearest())
if additional_data is not None:
for ad in additional_data.keys():
additional_data[ad] = additional_data[ad].regrid(
new_cube, iris.analysis.Nearest())
# Define neighbours on this new projection
self.neighbour_list['i'] = 11
self.neighbour_list['j'] = 11
plugin = Plugin(method)
with iris.FUTURE.context(cell_datetime_objects=True):
cube = cube.extract(self.time_extract)
result = plugin.process(cube, self.sites, self.neighbour_list,
ancillary_data, additional_data, **kwargs)
self.assertEqual(cube.coord_system(), trg_crs_iris)
self.assertAlmostEqual(result.data, expected)
self.assertEqual(result.coord(axis='y').name(), 'latitude')
self.assertEqual(result.coord(axis='x').name(), 'longitude')
self.assertAlmostEqual(result.coord(axis='y').points, 4.74)
self.assertAlmostEqual(result.coord(axis='x').points, 9.47)
def return_type(self, method, ancillary_data, additional_data, **kwargs):
"""Test that the plugin returns an iris.cube.Cube."""
plugin = Plugin(method)
cube = self.cube.extract(self.time_extract)
result = plugin.process(cube, self.sites, self.neighbour_list,
ancillary_data, additional_data, **kwargs)
self.assertIsInstance(result, Cube)
def extracted_value(self, method, ancillary_data, additional_data,
expected, **kwargs):
"""Test that the plugin returns the correct value."""
plugin = Plugin(method)
cube = self.cube.extract(self.time_extract)
result = plugin.process(cube, self.sites, self.neighbour_list,
ancillary_data, additional_data, **kwargs)
self.assertArrayAlmostEqual(result.data, expected)
def extracted_value(self, method, ancillary_data, additional_data,
expected, **kwargs):
"""Test that the plugin returns the correct value."""
plugin = Plugin(method)
with iris.FUTURE.context(cell_datetime_objects=True):
cube = self.cube.extract(self.time_extract)
result = plugin.process(cube, self.sites, self.neighbour_list,
ancillary_data, additional_data, **kwargs)
self.assertAlmostEqual(result.data, expected)
def test_invalid_method(self):
"""Test that the plugin can handle an invalid method being passed
in."""
plugin = Plugin('quantum_interpolation')
msg = 'Unknown method'
cube = self.cube.extract(self.time_extract)
with self.assertRaisesRegex(AttributeError, msg):
plugin.process(cube, self.sites, self.neighbour_list, {}, None,
**self.kwargs)
def different_projection(self, method, ancillary_data, additional_data,
expected, **kwargs):
"""Test that the plugin copes with non-lat/lon grids."""
trg_crs = None
src_crs = ccrs.PlateCarree()
trg_crs = ccrs.LambertConformal(central_longitude=50,
central_latitude=10)
trg_crs_iris = coord_systems.LambertConformal(central_lon=50,
central_lat=10)
lons = self.cube.coord('longitude').points
lats = self.cube.coord('latitude').points
x, y = [], []
for lon, lat in zip(lons, lats):
x_trg, y_trg = trg_crs.transform_point(lon, lat, src_crs)
x.append(x_trg)
y.append(y_trg)
new_x = AuxCoord(x,
standard_name='projection_x_coordinate',
units='m',
coord_system=trg_crs_iris)
new_y = AuxCoord(y,
standard_name='projection_y_coordinate',
units='m',
coord_system=trg_crs_iris)
cube = Cube(self.cube.data,
long_name="air_temperature",
dim_coords_and_dims=[(self.cube.coord('time'), 0)],
aux_coords_and_dims=[(new_y, 1), (new_x, 2)],
units="K")
plugin = Plugin(method)
with iris.FUTURE.context(cell_datetime_objects=True):
cube = cube.extract(self.time_extract)
result = plugin.process(cube, self.sites, self.neighbour_list,
ancillary_data, additional_data, **kwargs)
self.assertEqual(cube.coord_system(), trg_crs_iris)
self.assertAlmostEqual(result.data, expected)
self.assertEqual(result.coord(axis='y').name(), 'latitude')
self.assertEqual(result.coord(axis='x').name(), 'longitude')
self.assertAlmostEqual(result.coord(axis='y').points, 4.74)
self.assertAlmostEqual(result.coord(axis='x').points, 9.47)
def process(self, input_cube):
"""Convert each point to a truth value based on provided threshold
values. The truth value may or may not be fuzzy depending upon if
fuzzy_bounds are supplied.
Args:
input_cube (iris.cube.Cube):
Cube to threshold. The code is dimension-agnostic.
Returns:
cube (iris.cube.Cube):
Cube after a threshold has been applied. The data within this
cube will contain values between 0 and 1 to indicate whether
a given threshold has been exceeded or not.
The cube meta-data will contain:
* input_cube name prepended with `probability_of_`
* threshold dimension coordinate with same units as input_cube
* threshold attribute (above or below threshold)
* cube units set to (1).
Raises:
ValueError: if a np.nan value is detected within the input cube.
"""
thresholded_cubes = iris.cube.CubeList()
if np.isnan(input_cube.data).any():
raise ValueError("Error: NaN detected in input cube data")
for threshold, bounds in zip(self.thresholds, self.fuzzy_bounds):
cube = input_cube.copy()
if bounds[0] == bounds[1]:
truth_value = cube.data > threshold
else:
truth_value = np.where(
cube.data < threshold,
rescale(cube.data,
data_range=(bounds[0], threshold),
scale_range=(0., 0.5),
clip=True),
rescale(cube.data,
data_range=(threshold, bounds[1]),
scale_range=(0.5, 1.),
clip=True),
)
truth_value = truth_value.astype(np.float64)
if self.below_thresh_ok:
truth_value = 1. - truth_value
cube.data = truth_value
coord = iris.coords.DimCoord(threshold,
long_name="threshold",
units=cube.units)
cube.add_aux_coord(coord)
cube = iris.util.new_axis(cube, 'threshold')
thresholded_cubes.append(cube)
cube, = thresholded_cubes.concatenate()
# TODO: Correct when formal cf-standards exists
# Force the metadata to temporary conventions
if self.below_thresh_ok:
cube.attributes.update({'relative_to_threshold': 'below'})
else:
cube.attributes.update({'relative_to_threshold': 'above'})
cube.rename("probability_of_{}".format(cube.name()))
cube.units = Unit(1)
cube = ExtractData.make_stat_coordinate_first(cube)
return cube
def process_diagnostic(diagnostic,
neighbours,
sites,
forecast_times,
data_path,
ancillary_data,
output_path=None):
"""
Extract data and write output for a given diagnostic.
Args:
-----
diagnostic : string
String naming the diagnostic to be processed.
neighbours : numpy.array
Array of neigbouring grid points that are associated with sites
in the SortedDictionary of sites.
sites : dict
A dictionary containing the properties of spotdata sites.
forecast_times : list[datetime.datetime objects]
A list of datetimes representing forecast times for which data is
required.
data_path : string
Path to diagnostic data files.
ancillary_data : dict
A dictionary containing additional model data that is needed.
e.g. {'orography': <cube of orography>}
output_path : str
Path to which output file containing processed diagnostic should be
written.
Returns:
--------
None
Raises:
-------
IOError : If no relevant data cubes are found at given path.
Exception : No spotdata returned.
"""
# Search directory structure for all files relevant to current diagnostic.
files_to_read = [
os.path.join(dirpath, filename)
for dirpath, _, files in os.walk(data_path) for filename in files
if diagnostic['filepath'] in filename
]
if not files_to_read:
raise IOError('No relevant data files found in {}.'.format(data_path))
# Load cubes into an iris.cube.CubeList.
cubes = Load('multi_file').process(files_to_read,
diagnostic['diagnostic_name'])
# Grab the relevant set of grid point neighbours for the neighbour finding
# method being used by this diagnostic.
neighbour_hash = construct_neighbour_hash(diagnostic['neighbour_finding'])
neighbour_list = neighbours[neighbour_hash]
# Check if additional diagnostics are needed (e.g. multi-level data).
# If required, load into the additional_diagnostics dictionary.
additional_diagnostics = get_method_prerequisites(
diagnostic['interpolation_method'], data_path)
# Create empty iris.cube.CubeList to hold extracted data cubes.
resulting_cubes = CubeList()
# Get optional kwargs that may be set to override defaults.
optionals = [
'upper_level', 'lower_level', 'no_neighbours', 'dz_tolerance',
'dthetadz_threshold', 'dz_max_adjustment'
]
kwargs = {}
if ancillary_data.get('config_constants') is not None:
for optional in optionals:
constant = ancillary_data.get('config_constants').get(optional)
if constant is not None:
kwargs[optional] = constant
# Loop over forecast times.
for a_time in forecast_times:
# Extract Cube from CubeList at current time.
time_extract = datetime_constraint(a_time)
cube = extract_cube_at_time(cubes, a_time, time_extract)
if cube is None:
# If no cube is available at given time, try the next time.
continue
ad = {}
if additional_diagnostics is not None:
# Extract additional diagnostcs at current time.
ad = extract_ad_at_time(additional_diagnostics, a_time,
time_extract)
args = (cube, sites, neighbour_list, ancillary_data, ad)
# Extract diagnostic data using defined method.
resulting_cubes.append(
ExtractData(diagnostic['interpolation_method']).process(
*args, **kwargs))
# Concatenate CubeList into Cube, creating a time DimCoord, and write out.
if resulting_cubes:
cube_out, = resulting_cubes.concatenate()
WriteOutput('as_netcdf', dir_path=output_path).process(cube_out)
else:
raise Exception('No data available at given forecast times.')
# If set in the configuration, extract the diagnostic maxima and minima
# values.
if diagnostic['extrema']:
extrema_cubes = ExtractExtrema(24, start_hour=9).process(cube_out)
extrema_cubes = extrema_cubes.merge()
for extrema_cube in extrema_cubes:
WriteOutput('as_netcdf',
dir_path=output_path).process(extrema_cube)
def process_diagnostic(diagnostics, neighbours, sites, ancillary_data,
diagnostic_name):
"""
Extract data and write output for a given diagnostic.
Args:
diagnostics (dict):
Dictionary containing information regarding how the diagnostics
are to be processed.
For example::
{
"temperature": {
"diagnostic_name": "air_temperature",
"extrema": true,
"filepath": "temperature_at_screen_level",
"interpolation_method":
"model_level_temperature_lapse_rate",
"neighbour_finding": {
"land_constraint": false,
"method": "fast_nearest_neighbour",
"vertical_bias": null
}
}
}
neighbours (numpy.array):
Array of neigbouring grid points that are associated with sites
in the SortedDictionary of sites.
sites (dict):
A dictionary containing the properties of spotdata sites.
ancillary_data (dict):
A dictionary containing additional model data that is needed.
e.g. {'orography': <cube of orography>}
diagnostic_name (string):
A string matching the keys in the diagnostics dictionary that
will be used to access information regarding how the diagnostic
is to be processed.
Returns:
(tuple): tuple containing:
**resulting_cube** (iris.cube.Cube or None):
Cube after extracting the diagnostic requested using the
desired extraction method.
None is returned if the "resulting_cubes" is an empty CubeList
after processing.
**extrema_cubes** (iris.cube.CubeList or None):
CubeList containing extrema values, if the 'extrema' diagnostic
is requested.
None is returned if the value for diagnostic_dict["extrema"]
is False, so that the extrema calculation is not required.
"""
diagnostic_dict = diagnostics[diagnostic_name]
# Grab the relevant set of grid point neighbours for the neighbour finding
# method being used by this diagnostic.
neighbour_hash = (construct_neighbour_hash(
diagnostic_dict['neighbour_finding']))
neighbour_list = neighbours[neighbour_hash]
# Get optional kwargs that may be set to override defaults.
optionals = [
'upper_level', 'lower_level', 'no_neighbours', 'dz_tolerance',
'dthetadz_threshold', 'dz_max_adjustment'
]
kwargs = {}
if ancillary_data.get('config_constants') is not None:
for optional in optionals:
constant = ancillary_data.get('config_constants').get(optional)
if constant is not None:
kwargs[optional] = constant
# Create a list of datetimes to loop through.
forecast_times = []
for cube in diagnostic_dict["data"]:
time = cube.coord("time")
forecast_times.extend(time.units.num2date(time.points))
# Create empty iris.cube.CubeList to hold extracted data cubes.
resulting_cubes = CubeList()
# Loop over forecast times.
for a_time in forecast_times:
# Extract Cube from CubeList at current time.
time_extract = datetime_constraint(a_time)
cube = extract_cube_at_time(diagnostic_dict["data"], a_time,
time_extract)
if cube is None:
# If no cube is available at given time, try the next time.
continue
ad = {}
if diagnostic_dict["additional_data"] is not None:
# Extract additional diagnostics at current time.
ad = extract_ad_at_time(diagnostic_dict["additional_data"], a_time,
time_extract)
args = (cube, sites, neighbour_list, ancillary_data, ad)
# Extract diagnostic data using defined method.
resulting_cubes.append(
ExtractData(diagnostic_dict['interpolation_method']).process(
*args, **kwargs))
if resulting_cubes:
# Concatenate CubeList into Cube for cubes with different
# forecast times.
resulting_cube = resulting_cubes.concatenate_cube()
else:
resulting_cube = None
if diagnostic_dict['extrema']:
extrema_cubes = (ExtractExtrema(24, start_hour=9).process(
resulting_cube.copy()))
extrema_cubes = extrema_cubes.merge()
else:
extrema_cubes = None
return resulting_cube, extrema_cubes
