def _add_levels(cube, levels=13):
clist = CubeList()
for level in range(levels):
mln = DimCoord(level, standard_name='model_level_number')
other = cube.copy()
other.add_aux_coord(mln)
clist.append(other)
return clist.merge_cube()
def _create_cube(self, filenames, variable):
import numpy as np
from cis.data_io.hdf import _read_hdf4
from cis.data_io import hdf_vd
from iris.cube import Cube, CubeList
from iris.coords import DimCoord, AuxCoord
from cis.time_util import calculate_mid_time, cis_standard_time_unit
from cis.data_io.hdf_sd import get_metadata
from cf_units import Unit
variables = ['XDim:GlobalGrid', 'YDim:GlobalGrid', variable]
logging.info("Listing coordinates: " + str(variables))
cube_list = CubeList()
# Read each file individually, let Iris do the merging at the end.
for f in filenames:
sdata, vdata = _read_hdf4(f, variables)
lat_points = np.linspace(-90., 90., hdf_vd.get_data(vdata['YDim:GlobalGrid']))
lon_points = np.linspace(-180., 180., hdf_vd.get_data(vdata['XDim:GlobalGrid']))
lat_coord = DimCoord(lat_points, standard_name='latitude', units='degrees')
lon_coord = DimCoord(lon_points, standard_name='longitude', units='degrees')
# create time coordinate using the midpoint of the time delta between the start date and the end date
start_datetime = self._get_start_date(f)
end_datetime = self._get_end_date(f)
mid_datetime = calculate_mid_time(start_datetime, end_datetime)
logging.debug("Using {} as datetime for file {}".format(mid_datetime, f))
time_coord = AuxCoord(mid_datetime, standard_name='time', units=cis_standard_time_unit,
bounds=[start_datetime, end_datetime])
var = sdata[variable]
metadata = get_metadata(var)
try:
units = Unit(metadata.units)
except ValueError:
logging.warning("Unable to parse units '{}' in {} for {}.".format(metadata.units, f, variable))
units = None
cube = Cube(_get_MODIS_SDS_data(sdata[variable]),
dim_coords_and_dims=[(lon_coord, 1), (lat_coord, 0)],
aux_coords_and_dims=[(time_coord, None)],
var_name=metadata._name, long_name=metadata.long_name, units=units)
cube_list.append(cube)
# Merge the cube list across the scalar time coordinates before returning a single cube.
return cube_list.merge_cube()
def test_statsmodels_members(self):
"""
Test that the plugin raises the desired warning if the statsmodels
module is not found for when the predictor is the ensemble members.
"""
warnings.simplefilter("always")
import imp
try:
statsmodels_found = imp.find_module('statsmodels')
statsmodels_found = True
except ImportError:
statsmodels_found = False
cube = self.cube
historic_forecasts = CubeList([])
for index in [1.0, 2.0, 3.0, 4.0, 5.0]:
temp_cube = cube.copy()
temp_cube.coord("time").points = (temp_cube.coord("time").points -
index)
historic_forecasts.append(temp_cube)
historic_forecasts.concatenate_cube()
current_forecast_predictor = cube
truth = cube.collapsed("realization", iris.analysis.MAX)
distribution = "gaussian"
desired_units = "degreesC"
predictor_of_mean_flag = "members"
no_of_members = 3
estimate_coefficients_from_linear_model_flag = True
if not statsmodels_found:
with warnings.catch_warnings(record=True) as warning_list:
plugin = Plugin(distribution,
desired_units,
predictor_of_mean_flag=predictor_of_mean_flag)
self.assertTrue(len(warning_list) == 1)
self.assertTrue(
any(item.category == UserWarning for item in warning_list))
self.assertTrue("The statsmodels can not be imported" in str(
warning_list[0]))
def _create_historic_forecasts(cube, number_of_days=5):
"""
Function to create a set of pseudo historic forecast cubes, based on the
input cube, and assuming that there will be one forecast per day at the
same hour of the day.
"""
historic_forecasts = CubeList([])
no_of_hours_in_day = 24
time_range = np.linspace(no_of_hours_in_day,
no_of_hours_in_day * number_of_days,
num=number_of_days,
endpoint=True)
for index in time_range:
temp_cube = cube.copy()
temp_cube.coord("forecast_reference_time").points = (
temp_cube.coord("forecast_reference_time").points - index)
temp_cube.coord("time").points = temp_cube.coord("time").points - index
temp_cube.data -= 2
historic_forecasts.append(temp_cube)
historic_forecast = concatenate_cubes(historic_forecasts)
return historic_forecast
def _create_truth(cube):
"""
Function to create truth cubes, based on the input cube, and assuming that
there will be one forecast per day at the same hour of the day.
"""
truth = CubeList([])
time_range = [24.0, 48.0, 72.0, 96.0, 120.0]
for index in time_range:
temp_cube = cube.copy()
index -= temp_cube.coord("forecast_period").points[0]
temp_cube.coord("forecast_reference_time").points = (
temp_cube.coord("forecast_reference_time").points - index)
temp_cube.coord("time").points = temp_cube.coord("time").points - index
temp_cube.coord("forecast_period").points = 0
temp_cube.data -= 3
temp_cube = temp_cube.collapsed("realization", iris.analysis.MAX)
temp_cube.remove_coord("realization")
temp_cube.cell_methods = {}
truth.append(temp_cube)
truth = concatenate_cubes(truth)
return truth
def test_ice_large_with_fc(self):
"""Test that large VII probs do increase zero lightning risk when
forecast lead time is non-zero (three forecast_period points)"""
self.ice_cube.data[:, 1, 1] = 1.
self.fg_cube.data[1, 1] = 0.
frt_point = self.fg_cube.coord('forecast_reference_time').points[0]
fg_cube_input = CubeList([])
for fc_time in np.array([1, 2.5, 3]) * 3600: # seconds
fg_cube_next = self.fg_cube.copy()
fg_cube_next.coord('time').points = [frt_point + fc_time]
fg_cube_next.coord('forecast_period').points = [fc_time]
fg_cube_input.append(squeeze(fg_cube_next))
fg_cube_input = fg_cube_input.merge_cube()
expected = fg_cube_input.copy()
# expected.data contains all ones except:
expected.data[0, 1, 1] = 0.54
expected.data[1, 1, 1] = 0.0
expected.data[2, 1, 1] = 0.0
result = self.plugin.apply_ice(fg_cube_input,
self.ice_cube)
self.assertArrayAlmostEqual(result.data, expected.data)
class Test_extract(tests.IrisTest):
def setUp(self):
self.scalar_cubes = CubeList()
for i in range(5):
for letter in 'abcd':
self.scalar_cubes.append(Cube(i, long_name=letter))
def test_scalar_cube_name_constraint(self):
# Test the name based extraction of a CubeList containing scalar cubes.
res = self.scalar_cubes.extract('a')
expected = CubeList([Cube(i, long_name='a') for i in range(5)])
self.assertEqual(res, expected)
def test_scalar_cube_data_constraint(self):
# Test the extraction of a CubeList containing scalar cubes
# when using a cube_func.
val = 2
constraint = iris.Constraint(cube_func=lambda c: c.data == val)
res = self.scalar_cubes.extract(constraint)
expected = CubeList([Cube(val, long_name=letter) for letter in 'abcd'])
self.assertEqual(res, expected)
def test_statsmodels_realizations(self, warning_list=None):
"""
Test that the plugin raises the desired warning if the statsmodels
module is not found for when the predictor is the ensemble
realizations.
"""
import imp
try:
statsmodels_found = imp.find_module('statsmodels')
statsmodels_found = True
except ImportError:
statsmodels_found = False
cube = self.cube
historic_forecasts = CubeList([])
for index in [1.0, 2.0, 3.0, 4.0, 5.0]:
temp_cube = cube.copy()
temp_cube.coord("time").points = (temp_cube.coord("time").points -
index)
historic_forecasts.append(temp_cube)
historic_forecasts.concatenate_cube()
current_forecast_predictor = cube
truth = cube.collapsed("realization", iris.analysis.MAX)
distribution = "gaussian"
desired_units = "degreesC"
predictor_of_mean_flag = "realizations"
no_of_realizations = 3
estimate_coefficients_from_linear_model_flag = True
if not statsmodels_found:
plugin = Plugin(distribution,
desired_units,
predictor_of_mean_flag=predictor_of_mean_flag)
warning_msg = "The statsmodels can not be imported"
self.assertTrue(
any(item.category == ImportWarning for item in warning_list))
self.assertTrue(
any(warning_msg in str(item) for item in warning_list))
def main():
# Parameters to compare between forecasts
path = datadir + 'deterministic/'
filename = 'rp_physics.nc'
name = 'Temperature [K]'
pressure = 500
lead_time = 7*24
cs = iris.Constraint(
name=name, pressure=pressure, forecast_period=lead_time)
# Load full precision reference forecast
cube = iris.load_cube(path + filename, cs)
# Calculate the errors with each different precision used as the `truth`
diffs = CubeList()
for pseudo_truth in cube.slices_over('precision'):
# Add the precision of the `truth` cube as another coordinate
p = pseudo_truth.coord('precision').points[0]
p = AuxCoord(p, long_name='reference_precision')
# Calculate the errors
diff = rms_diff(cube, pseudo_truth)
diff.add_aux_coord(p)
# Store the errors in the cubelist
diffs.append(diff)
# Combine all the errors into a single cube with dimensions of
# precision vs reference_precision
diffs = diffs.merge_cube()
# Plot the errors
qplt.pcolor(diffs, vmin=0, cmap='cubehelix_r')
precisions = cube.coord('precision').points
plt.xticks(precisions)
plt.yticks(precisions)
plt.show()
return
def realization_cubes_fixture() -> CubeList:
"""Set up a single realization cube in parameter space"""
realizations = [0, 1, 2, 3]
data = np.ones((len(realizations), 2, 2), dtype=np.float32)
times = [datetime(2017, 11, 10, hour) for hour in [4, 5, 6]]
cubes = CubeList()
for time in times:
cubes.append(
set_up_variable_cube(
data,
realizations=realizations,
spatial_grid="equalarea",
time=time,
frt=datetime(2017, 11, 10, 1),
))
cube = cubes.merge_cube()
sliced_cubes = CubeList(cube.slices_over("realization"))
[
s.attributes.update({"history": f"20171110T{i:02d}00Z"})
for i, s in enumerate(sliced_cubes)
]
return sliced_cubes
def interpolate_to_cubes_mean(cubelist_in,
grid_along,
grid_across,
height_level_borders=None):
from iris.cube import CubeList
cubelist_out_along = CubeList()
cubelist_out_across = CubeList()
for variable in cubelist_in:
if variable.coord('geopotential_height').ndim == 1:
cube_along = my_interpolate_3D2D_mean(
variable,
grid_along,
coordinates=[
'geopotential_height', 'projection_y_coordinate',
'projection_x_coordinate'
],
method='linear',
height_level_borders=height_level_borders)
cube_across = my_interpolate_3D2D_mean(
variable,
grid_across,
coordinates=[
'geopotential_height', 'projection_y_coordinate',
'projection_x_coordinate'
],
method='linear',
height_level_borders=height_level_borders)
# if variable.coord('geopotential_height').ndim>1:
# cube_along=my_interpolate_3D2D_altitude(variable,grid_along,coordinates_in=['model_level_number','projection_y_coordinate','projection_x_coordinate'],coordinates_out=['geopotential_height','x_dx'],method='linear')
# cube_across=my_interpolate_3D2D_altitude(variable,grid_across,coordinates_in=['model_level_number','projection_y_coordinate','projection_x_coordinate'],coordinates_out=['geopotential_height','x_dx'],method='linear')
cubelist_out_along.append(cube_along)
cubelist_out_across.append(cube_across)
del cube_along
del cube_across
return cubelist_out_along, cubelist_out_across
def sum_profile_mask(cubelist_in, height_levels_borders, mask_cell):
from iris.cube import CubeList
from iris.analysis import SUM
from dask.array.ma import masked_invalid
cubelist_out = CubeList()
for variable in cubelist_in:
#Sum up values for height slices and mask for all cubes in cubelist_in:
cube_cell_profile = collapse_profile_mask(
variable_cube=variable,
height_levels_borders=height_levels_borders,
mask_cell=mask_cell,
coordinate='geopotential_height',
method=SUM)
cube_cell_profile.rename(variable.name())
cube_cell_profile.data = masked_invalid(cube_cell_profile.core_data())
coord_names = [coord.name() for coord in cube_cell_profile.coords()]
coord_names.remove('time')
coord_names.remove('geopotential_height')
for coord in coord_names:
cube_cell_profile.remove_coord(coord)
cubelist_out.append(cube_cell_profile)
return cubelist_out
def get_1d_two_param_cube(params=None, n_samples=10):
"""
Create an ensemble of 1d cubes perturbed over two idealised parameter
spaces. One of params or n_samples must be provided
:param np.array params: A list of params to sample the ensemble over
:param int n_samples: The number of params to sample (between 0. and 1.)
:return:
"""
from iris.cube import CubeList
if params is None:
params = np.linspace(np.zeros((2, )), np.ones((2, )), n_samples)
cubes = CubeList([])
for j, p in enumerate(params):
c = make_dummy_1d_cube(j)
# Perturb base data to represent some change in a parameter
c.data *= simple_polynomial_fn_two_param(*p)
cubes.append(c)
ensemble = cubes.concatenate_cube()
return ensemble
def test_filter_realizations(realization_cubes, short_realizations):
"""Run filter_realizations with realization time series where 0 or more are short of the
final time step"""
if short_realizations == 0:
cubes = realization_cubes
expected_realization_points = [0, 1, 2, 3]
else:
cubes = CubeList(realization_cubes[:-short_realizations])
cubes.append(realization_cubes[-short_realizations][:-1])
expected_realization_points = [0, 1, 2, 3][:-short_realizations]
result = filter_realizations(cubes)
assert isinstance(result, Cube)
assert np.allclose(cubes[0].coord("time").points,
result.coord("time").points)
assert np.allclose(
result.coord("realization").points, expected_realization_points)
if short_realizations == 3:
# History attribute is retained if there are no differing values
assert result.attributes["history"] == cubes[0].attributes["history"]
else:
# History attribute is removed if differing values are supplied
assert "history" not in result.attributes.keys()
def process(self, cube: Cube) -> Cube:
"""
Ensure that the cube passed to the maximum_within_vicinity method is
2d and subsequently merged back together.
Args:
cube:
Thresholded cube.
Returns:
Cube containing the occurrences within a vicinity for each
xy 2d slice, which have been merged back together.
"""
max_cubes = CubeList([])
for cube_slice in cube.slices(
[cube.coord(axis="y"), cube.coord(axis="x")]):
max_cubes.append(self.maximum_within_vicinity(cube_slice))
result_cube = max_cubes.merge_cube()
# Put dimensions back if they were there before.
result_cube = check_cube_coordinates(cube, result_cube)
return result_cube
def truth_spot(truth_grid):
truth_data_spot = truth_grid[0, ...].data.reshape((2, 9))
truths_spot_list = CubeList()
for day in range(5, 7):
time_coords = construct_scalar_time_coords(
datetime(2017, 11, day, 4, 0), None, datetime(2017, 11, day, 4, 0),
)
time_coords = [t[0] for t in time_coords]
truths_spot_list.append(
build_spotdata_cube(
truth_data_spot,
name="probability_of_air_temperature_above_threshold",
units="1",
altitude=_dummy_point_locations,
latitude=_dummy_point_locations,
longitude=_dummy_point_locations,
wmo_id=_dummy_string_ids,
additional_dims=[_threshold_coord],
scalar_coords=time_coords,
)
)
truths_spot = truths_spot_list.merge_cube()
return truths_spot
def input_cubes():
"""Generate input cubes on a Global grid"""
data = np.zeros((3, 3), dtype=np.float32)
cubes = CubeList([])
cubes.append(
set_up_variable_cube(
data.copy(),
name="lwe_convective_precipitation_rate",
units="m s-1",
grid_spacing=10,
domain_corner=(-90, -180),
attributes=GLOBAL_ATTRIBUTES,
))
cubes.append(
set_up_variable_cube(
data.copy(),
name="lwe_stratiform_precipitation_rate",
units="m s-1",
grid_spacing=10,
domain_corner=(-90, -180),
attributes=GLOBAL_ATTRIBUTES,
))
return cubes
def correct_analyses(cubes):
newcubes = CubeList()
for cube in cubes:
# Squeeze cubes dimensions
newcube = squeeze(cube)
# Give time coordinate proper name
newcube.coord('t').rename('time')
# Correct dimensional coordinates
z, y, x, t = newcube.coords()
z.rename('level_height')
z.units = 'm'
z.attributes = {'positive': 'up'}
y.rename('latitude')
y.coord_system = lat.coord_system
y.units = lat.units
x.rename('longitude')
x.coord_system = lon.coord_system
x.units = lon.units
newcubes.append(newcube)
# Correct cube names
for before, after in name_pairs:
newcubes.extract(before)[0].rename(after)
# Correct units
for name, unit in units:
newcubes.extract(name)[0].units = unit
return newcubes
def create_data_object(self, filenames, variable):
from netCDF4 import Dataset
from biggus import OrthoArrayAdapter
from iris.cube import Cube, CubeList
from iris.coords import DimCoord
from iris.fileformats.netcdf import NetCDFDataProxy
from datetime import datetime
from os.path import basename
from cis.time_util import cis_standard_time_unit
from cis.data_io.gridded_data import make_from_cube
import numpy as np
cubes = CubeList()
for f in filenames:
# Open the file
ds = Dataset(f)
# E.g. 'NO2.COLUMN.VERTICAL.TROPOSPHERIC.CS30_BACKSCATTER.SOLAR'
v = ds.variables[variable]
# Get the coords
lat = ds.variables['LATITUDE']
lon = ds.variables['LONGITUDE']
# Create a biggus adaptor over the data
scale_factor = getattr(v, 'scale_factor', None)
add_offset = getattr(v, 'add_offset', None)
if scale_factor is None and add_offset is None:
v_dtype = v.datatype
elif scale_factor is not None:
v_dtype = scale_factor.dtype
else:
v_dtype = add_offset.dtype
# proxy = NetCDFDataProxy(v.shape, v_dtype, f, variable, float(v.VAR_FILL_VALUE))
# a = OrthoArrayAdapter(proxy)
# Mask out all invalid values (NaN, Inf, etc)
a = np.ma.masked_invalid(v[:])
# Set everything negative to NaN
a = np.ma.masked_less(a, 0.0)
# Just read the lat and lon in directly
lat_coord = DimCoord(lat[:], standard_name='latitude', units='degrees', long_name=lat.VAR_DESCRIPTION)
lon_coord = DimCoord(lon[:], standard_name='longitude', units='degrees', long_name=lon.VAR_DESCRIPTION)
# Pull the date out of the filename
fname = basename(f)
dt = datetime.strptime(fname[:10], "%Y_%m_%d")
t_coord = DimCoord(cis_standard_time_unit.date2num(dt), standard_name='time', units=cis_standard_time_unit)
c = Cube(a, long_name=getattr(v, "VAR_DESCRIPTION", None), units=getattr(v, "VAR_UNITS", None),
dim_coords_and_dims=[(lat_coord, 0), (lon_coord, 1)])
c.add_aux_coord(t_coord)
# Close the file
ds.close()
cubes.append(c)
# We have a scalar time coord and no conflicting metadata so this should just create one cube...
merged = cubes.merge_cube()
# Return as a CIS GriddedData object
return make_from_cube(merged)
class AtmosFlow:
"""
Atmospheric Flow
Used to calculate meteorological parameters from the given cubes.
Derived quantities are stored as cached properties to save computational
time.
Calculating quantites that involve on horizontal derivatives are only
true if used in cartesian coordinates, e.g. on the output from a LAM model
with constant grid spacing. Use `prepare_cube_on_model_levels` to prepare
cubes for `AtmosFlow` on a cartesian grid.
Attributes
----------
cubes: iris.cube.CubeList
list of cubes representing meteorological parameters
main_cubes: iris.cube.CubeList
list of non-scalar cubes
wind_cmpnt: iris.cube.CubeList
list of u,v,w-wind components
{x,y,z}coord: iris.coord.Coord
Coordinates in the respective dimensions
pres: iris.cube.Cube
Pressure created from a coordinate, if possible
lats: iris.cube.Cube
latitudes
fcor: iris.cube.Cube
Coriolis parameter (taken at 45N by default)
d{x,y}: iris.cube.Cube
Grid spacing (if cartesian=True)
"""
def __init__(self, cartesian=True, **kw_vars):
"""
Parameters
----------
cartesian: bool (default True)
Cartesian coord system flag
**kw_vars: dict of iris cubes
meteorological parameters
Examples
--------
Initialise an `AtmosFlow` object with 3 wind components
>>> AF = AtmosFlow(u=u_cart, v=v_cart, w=w_cart)
and calculate relative vorticity:
>>> rv = AF.rel_vort
"""
self.__dict__.update(kw_vars)
self.cartesian = cartesian
self.cubes = CubeList(filter(iscube, self.__dict__.values()))
self.main_cubes = CubeList(filter(iscube_and_not_scalar,
self.__dict__.values()))
self.wind_cmpnt = CubeList(filter(None,
[getattr(self, 'u', None),
getattr(self, 'v', None),
getattr(self, 'w', None)]))
thecube = self.main_cubes[0]
check_coords(self.main_cubes)
# Get the dim_coord, or None if none exist, for the xyz dimensions
self.xcoord = thecube.coord(axis='X', dim_coords=True)
self.ycoord = thecube.coord(axis='Y', dim_coords=True)
self.zcoord = thecube.coord(axis='Z')
if self.zcoord.units.is_convertible('Pa'):
# Check if the vertical coordinate is pressure
self.zmode = 'pcoord'
for cube in self.main_cubes:
if self.zcoord in cube.dim_coords:
clean_pressure_coord(cube)
self.pres = coords.pres_coord_to_cube(thecube)
self.cubes.append(self.pres)
if not hasattr(self, 'lats'):
try:
_, lats = grid.unrotate_lonlat_grids(thecube)
except (ValueError, AttributeError):
lats = np.array([45.])
self.lats = Cube(lats,
units='degrees',
standard_name='latitude')
self.fcor = mcalc.coriolis_parameter(self.lats)
self.fcor.convert_units('s-1')
if self.cartesian:
for ax, rot_name in zip(('x', 'y'),
('grid_longitude', 'grid_latitude')):
for cube in self.cubes:
if rot_name in [i.name() for i in cube.coords(axis=ax)]:
cube.remove_coord(rot_name)
try:
_dx = thecube.attributes['um_res'].to_flt('m')
except KeyError:
_dx = 1.
self.dx = Cube(_dx, units='m')
self.dy = Cube(_dx, units='m')
# Non-spherical coords?
# self.horiz_cs = thecube.coord(axis='x', dim_coords=True).coord_system
self.horiz_cs = thecube.coord_system()
self._spherical_coords = isinstance(self.horiz_cs,
(iris.coord_systems.GeogCS,
iris.coord_systems.RotatedGeogCS))
# todo: interface for spherical coordinates switch?
# assert not self._spherical_coords,\
# 'Only non-spherical coordinates are allowed ...'
if self.cartesian and self._spherical_coords:
warnings.warn('Cubes are in spherical coordinates!'
'\n Use `replace_lonlat_dimcoord_with_cart function`'
' to change coordinates!')
def __repr__(self):
msg = "arke `Atmospheric Flow` containing of:\n"
msg += "\n".join(tuple(i.name() for i in self.cubes))
return msg
def d_dx(self, name=None, alias=None):
r"""
Derivative of a cube along the x-axis
.. math::
\frac{\partial }{\partial x}
"""
if name is None and isinstance(alias, str):
v = getattr(self, alias)
else:
v = self.cubes.extract_strict(name)
return cube_deriv(v, self.xcoord)
def d_dy(self, name=None, alias=None):
r"""
Derivative of a cube along the y-axis
.. math::
\frac{\partial }{\partial y}
"""
if name is None and isinstance(alias, str):
v = getattr(self, alias)
else:
v = self.cubes.extract_strict(name)
return cube_deriv(v, self.ycoord)
def hgradmag(self, name=None, alias=None):
r"""
Magnitude of the horizontal gradient of a cube
.. math::
\sqrt[(\frac{\partial }{\partial y})^2
+(\frac{\partial }{\partial y})^2]
"""
dvdx2 = self.d_dx(name=name, alias=alias) ** 2
dvdy2 = self.d_dy(name=name, alias=alias) ** 2
return (dvdx2 + dvdy2) ** 0.5
@cached_property
def wspd(self):
r"""
Calculate wind speed (magnitude)
.. math::
\sqrt{u^2 + v^2 + w^2}
"""
res = 0
for cmpnt in self.wind_cmpnt:
res += cmpnt**2
res = res**0.5
res.rename('wind_speed')
return res
@cached_property
def tke(self):
r"""
Calculate total kinetic energy
.. math::
0.5(u^2 + v^2 + w^2)
"""
res = 0
for cmpnt in self.wind_cmpnt:
res += cmpnt**2
res = 0.5 * res # * self.density
res.convert_units('m2 s-2')
res.rename('total_kinetic_energy')
return res
@cached_property
def du_dx(self):
r"""
Derivative of u-wind along the x-axis
.. math::
u_x = \frac{\partial u}{\partial x}
"""
return cube_deriv(self.u, self.xcoord)
@cached_property
def du_dy(self):
r"""
Derivative of u-wind along the y-axis
.. math::
u_y = \frac{\partial u}{\partial y}
"""
return cube_deriv(self.u, self.ycoord)
@cached_property
def du_dz(self):
r"""
Derivative of u-wind along the z-axis
.. math::
u_z = \frac{\partial u}{\partial z}
"""
return cube_deriv(self.u, self.zcoord)
@cached_property
def dv_dx(self):
r"""
Derivative of v-wind along the x-axis
.. math::
v_x = \frac{\partial v}{\partial x}
"""
return cube_deriv(self.v, self.xcoord)
@cached_property
def dv_dy(self):
r"""
Derivative of v-wind along the y-axis
.. math::
v_y = \frac{\partial v}{\partial y}
"""
return cube_deriv(self.v, self.ycoord)
@cached_property
def dv_dz(self):
r"""
Derivative of v-wind along the z-axis
.. math::
v_z = \frac{\partial v}{\partial z}
"""
return cube_deriv(self.v, self.zcoord)
@cached_property
def dw_dx(self):
r"""
Derivative of w-wind along the x-axis
.. math::
w_x = \frac{\partial w}{\partial x}
"""
return cube_deriv(self.w, self.xcoord)
@cached_property
def dw_dy(self):
r"""
Derivative of w-wind along the y-axis
.. math::
w_y = \frac{\partial w}{\partial y}
"""
return cube_deriv(self.w, self.ycoord)
@cached_property
def dw_dz(self):
r"""
Derivative of w-wind along the z-axis
.. math::
w_z = \frac{\partial w}{\partial z}
"""
return cube_deriv(self.w, self.zcoord)
@cached_property
def rel_vort(self):
r"""
Calculate the vertical component of the vorticity vector
.. math::
\zeta = v_x - u_y
"""
res = self.dv_dx - self.du_dy
res.rename('atmosphere_relative_vorticity')
res.convert_units('s-1')
return res
@cached_property
def div_h(self):
r"""
Calculate the horizontal divergence
.. math::
D_h = u_x + v_y
"""
res = self.du_dx + self.dv_dy
res.rename('divergence_of_wind')
res.convert_units('s-1')
return res
@cached_property
def rel_vort_hadv(self):
r"""
Calculate the horizontal advection of relative vorticity
.. math::
\vec v\cdot \nabla \zeta
"""
res = (self.u*cube_deriv(self.rel_vort, self.xcoord) +
self.v*cube_deriv(self.rel_vort, self.ycoord))
res.rename('horizontal_advection_of_atmosphere_relative_vorticity')
res.convert_units('s-2')
return res
@cached_property
def rel_vort_vadv(self):
r"""
Calculate the vertical advection of relative vorticity
.. math::
w\frac{\partial \zeta}{\partial z}
"""
res = self.w*cube_deriv(self.rel_vort, self.zcoord)
res.rename('vertical_advection_of_atmosphere_relative_vorticity')
res.convert_units('s-2')
return res
@cached_property
def rel_vort_stretch(self):
r"""
Stretching term
.. math::
\nabla\cdot\vec v (\zeta+f)
"""
res = self.div_h * (self.rel_vort + self.fcor.data)
res.rename('stretching_term_of_atmosphere_relative_vorticity_budget')
res.convert_units('s-2')
return res
@cached_property
def rel_vort_tilt(self):
r"""
Tilting (twisting) term
.. math::
\vec k \cdot \nabla w\times\frac{\partial\vec v}{\partial z} =
\frac{\partial w}{\partial x}*\frac{\partial v}{\partial z} -
\frac{\partial w}{\partial y}*\frac{\partial u}{\partial z}
"""
res = self.dw_dx * self.dv_dz - self.dw_dy * self.du_dz
res.rename('tilting_term_of_atmosphere_relative_vorticity_budget')
res.convert_units('s-2')
return res
@cached_property
def dfm_stretch(self):
r"""
Stretching deformation
.. math::
Def = u_x - v_y
"""
res = self.du_dx - self.dv_dy
res.rename('stretching_deformation_2d')
res.convert_units('s-1')
return res
@cached_property
def dfm_shear(self):
r"""
Shearing deformation
.. math::
Def' = u_y + v_x
"""
res = self.du_dy + self.dv_dx
res.rename('shearing_deformation_2d')
res.convert_units('s-1')
return res
@cached_property
def kvn(self):
r"""
Kinematic vorticity number
.. math::
W_k=\frac{||\Omega||}{||S||}=
\frac{\sqrt{\zeta^2}}{\sqrt{D_h^2 + Def^2 + Def'^2}}
where
.. math::
\zeta=v_x - u_y
D_h = u_x + v_y
Def = u_x - v_y
Def' = u_y + v_x
Reference:
http://dx.doi.org/10.3402/tellusa.v68.29464
"""
numerator = self.rel_vort
denominator = (self.div_h**2 +
self.dfm_stretch**2 +
self.dfm_shear**2)**0.5
res = numerator/denominator
res.rename('kinematic_vorticity_number_2d')
return res
@cached_property
def density(self):
r"""
Air density
.. math::
\rho = \frac{p}{R_d T}
"""
p = self.cubes.extract_strict('air_pressure')
rd = AuxCoord(mconst.Rd.data, units=mconst.Rd.units)
try:
temp = self.cubes.extract_strict('air_temperature')
except iris.exceptions.ConstraintMismatchError:
temp = self.temp
res = p / (temp * rd)
res.rename('air_density')
res.convert_units('kg m-3')
self.cubes.append(res)
return res
@cached_property
def theta(self):
r"""
Air potential temperature
If temperature is given:
.. math::
\theta = T (p_0/p)^{R_d/c_p}}
"""
try:
th = self.cubes.extract_strict('air_potential_temperature')
except iris.exceptions.ConstraintMismatchError:
p = self.cubes.extract_strict('air_pressure')
temp = self.cubes.extract_strict('air_temperature')
th = mcalc.potential_temperature(p, temp)
th.rename('air_potential_temperature')
th.convert_units('K')
self.cubes.append(th)
return th
@cached_property
def temp(self):
r"""
Air temperature
If potential temperature is given:
.. math::
T = \theta (p/p_0)^{R_d/c_p}}
"""
try:
t = self.cubes.extract_strict('air_temperature')
except iris.exceptions.ConstraintMismatchError:
p0 = AuxCoord(mconst.P0.data, units=mconst.P0.units)
kappa = mconst.kappa.data
p = self.cubes.extract_strict('air_pressure')
t = self.theta * (p / p0) ** kappa
t.rename('air_temperature')
t.convert_units('K')
self.cubes.append(t)
return t
@cached_property
def mixr(self):
r"""
Water vapour mixing ratio
"""
try:
mixr = self.cubes.extract_strict('mixing_ratio')
except iris.exceptions.ConstraintMismatchError:
spechum = self.cubes.extract_strict('specific_humidity')
mixr = mcalc.specific_humidity_to_mixing_ratio(spechum)
self.cubes.append(mixr)
return mixr
@cached_property
def thetae(self):
r"""
Equivalent potential temperature
.. math::
\theta_e = \theta e^\frac{L_v r_s}{C_{pd} T}
"""
try:
th = self.cubes.extract_strict('equivalent_potential_temperature')
except iris.exceptions.ConstraintMismatchError:
p = self.cubes.extract_strict('air_pressure')
temp = self.cubes.extract_strict('air_temperature')
spechum = self.cubes.extract_strict('specific_humidity')
mixr = mcalc.specific_humidity_to_mixing_ratio(spechum)
e = mcalc.vapor_pressure(p, mixr)
dew = mcalc.dewpoint(e)
dew.convert_units('K')
th = mcalc.equivalent_potential_temperature(p, temp, dew)
th.rename('equivalent_potential_temperature')
th.convert_units('K')
self.cubes.append(th)
return th
@cached_property
def relh(self):
r""" Relative humdity """
try:
rh = self.cubes.extract_strict('relative_humidity')
except iris.exceptions.ConstraintMismatchError:
p = self.cubes.extract_strict('air_pressure')
temp = self.cubes.extract_strict('air_temperature')
spechum = self.cubes.extract_strict('specific_humidity')
rh = mcalc.specific_to_relative_humidity(p, temp, spechum)
rh.rename('relative_humidity')
rh.convert_units('1')
self.cubes.append(rh)
return rh
@cached_property
def specific_volume(self):
r"""
Air Specific Volume
.. math::
\alpha = \rho^{-1}
"""
res = self.density ** (-1)
res.rename('air_specific_volume')
self.main_cubes.append(res)
return res
@cached_property
def dp_dx(self):
r"""
Derivative of pressure along the x-axis
.. math::
p_x = \frac{\partial p}{\partial x}
"""
return cube_deriv(self.pres, self.xcoord)
@cached_property
def dp_dy(self):
r"""
Derivative of pressure along the y-axis
.. math::
p_y = \frac{\partial p}{\partial y}
"""
return cube_deriv(self.pres, self.ycoord)
@cached_property
def dsv_dx(self):
r"""
Derivative of specific volume along the x-axis
.. math::
\alpha_x = \frac{\partial\alpha}{\partial x}
"""
return cube_deriv(self.specific_volume, self.xcoord)
@cached_property
def dsv_dy(self):
r"""
Derivative of specific volume along the y-axis
.. math::
\alpha_y = \frac{\partial\alpha}{\partial y}
"""
return cube_deriv(self.specific_volume, self.ycoord)
@cached_property
def baroclinic_term(self):
r"""
Baroclinic term of relative vorticity budget
.. math::
\frac{\partial p}{\partial x}\frac{\partial\alpha}{\partial y}
- \frac{\partial p}{\partial y}\frac{\partial\alpha}{\partial x}
"""
res = (self.dp_dx * self.dsv_dy
- self.dp_dy * self.dsv_dx)
res.rename('baroclinic_term_of_atmosphere_relative_vorticity_budget')
res.convert_units('s-1')
return res
@cached_property
def brunt_vaisala_squared(self):
r"""
Brunt–Väisälä frequency squared
(pressure coordinates)
.. math::
N^2 = -\frac{g^2 \rho}{\theta}\frac{\partial \theta}{\partial p}
"""
dthdp = cube_deriv(self.theta, self.zcoord)
g2 = -1 * mconst.g**2
g2 = AuxCoord(g2.data, units=g2.units)
res = self.density / self.theta * dthdp * g2
res.rename('square_of_brunt_vaisala_frequency_in_air')
res.convert_units('s-2')
return res
@cached_property
def eady_growth_rate(self):
r"""
Eady growth rate
(pressure coordinates)
.. math::
EGR = 0.31 * \frac{f}{N}\frac{\partial V}{\partial z}
= 0.31 * \frac{-\rho g f}{N}\frac{\partial V}{\partial p}
"""
factor = -1 * mconst.egr_factor * self.fcor * mconst.g
dvdp = cube_deriv(self.wspd, self.zcoord)
dvdp.data = abs(dvdp.data)
res = (self.density * factor.data * AuxCoord(1, units=factor.units)
* dvdp / self.brunt_vaisala_squared**0.5)
res.rename('eady_growth_rate')
res.convert_units('s-1')
return res
@cached_property
def kinematic_frontogenesis(self):
r"""
2D Kinematic frontogenesis from MetPy package
.. math::
F=\frac{1}{2}\left|\nabla \theta\right|[D cos(2\beta)-\delta]
"""
res = mcalc.frontogenesis(self.theta, self.u, self.v,
self.dx, self.dy, dim_order='yx')
res.rename('kinematic_frontogenesis')
res.convert_units('K m-1 s-1')
return res
def interpolate_to_cubes(cubelist_in,
grid_along,
grid_across,
z_coord='model_level_number'):
from iris.cube import CubeList
cubelist_out_along = CubeList()
cubelist_out_across = CubeList()
for variable in cubelist_in:
if z_coord == 'model_level_number':
cube_along = my_interpolate_3D2D(variable,
grid_along,
coordinates=[
'model_level_number',
'projection_y_coordinate',
'projection_x_coordinate'
],
method='linear')
cube_across = my_interpolate_3D2D(variable,
grid_across,
coordinates=[
'model_level_number',
'projection_y_coordinate',
'projection_x_coordinate'
],
method='linear')
elif z_coord == 'geopotential_height':
if variable.coord('geopotential_height').ndim == 1:
cube_along = my_interpolate_3D2D(variable,
grid_along,
coordinates=[
'geopotential_height',
'projection_y_coordinate',
'projection_x_coordinate'
],
method='linear')
cube_across = my_interpolate_3D2D(
variable,
grid_across,
coordinates=[
'geopotential_height', 'projection_y_coordinate',
'projection_x_coordinate'
],
method='linear')
if variable.coord('geopotential_height').ndim > 1:
cube_along = my_interpolate_3D2D_altitude(
variable,
grid_along,
coordinates_in=[
'model_level_number', 'projection_y_coordinate',
'projection_x_coordinate'
],
coordinates_out=['geopotential_height', 'x_dx'],
method='linear')
cube_across = my_interpolate_3D2D_altitude(
variable,
grid_across,
coordinates_in=[
'model_level_number', 'projection_y_coordinate',
'projection_x_coordinate'
],
coordinates_out=['geopotential_height', 'x_dx'],
method='linear')
else:
raise ValueError(
'z_coord must be model_level_number or geopotential_height ')
cubelist_out_along.append(cube_along)
cubelist_out_across.append(cube_across)
del cube_along
del cube_across
return cubelist_out_along, cubelist_out_across
def extract_cell_cubes_subset(cubelist_in,
mask,
track,
cell,
z_coord='model_level_number',
height_levels=None):
from iris.analysis import SUM
from iris import Constraint
from iris.cube import CubeList
from iris.coords import AuxCoord
import numpy as np
from tobac import mask_cell, mask_cell_surface, get_bounding_box
from copy import deepcopy
track_i = track[track['cell'] == cell]
cubelist_cell_integrated_out = CubeList()
cubelist_cell_sum = CubeList()
for time_i in track_i['time'].values:
logging.debug('start extracting cubes for cell ' + str(cell) +
' and time ' + str(time_i))
constraint_time = Constraint(time=time_i)
mask_i = mask.extract(constraint_time)
mask_cell_i = mask_cell(mask_i, cell, track_i, masked=False)
mask_cell_surface_i = mask_cell_surface(mask_i,
cell,
track_i,
masked=False,
z_coord=z_coord)
x_dim = mask_cell_surface_i.coord_dims('projection_x_coordinate')[0]
y_dim = mask_cell_surface_i.coord_dims('projection_y_coordinate')[0]
x_coord = mask_cell_surface_i.coord('projection_x_coordinate')
y_coord = mask_cell_surface_i.coord('projection_y_coordinate')
if (mask_cell_surface_i.core_data() > 0).any():
box_mask_i = get_bounding_box(mask_cell_surface_i.core_data(),
buffer=1)
box_mask = [[
x_coord.points[box_mask_i[x_dim][0]],
x_coord.points[box_mask_i[x_dim][1]]
],
[
y_coord.points[box_mask_i[y_dim][0]],
y_coord.points[box_mask_i[y_dim][1]]
]]
else:
box_mask = [[np.nan, np.nan], [np.nan, np.nan]]
width = 20
dx = 500
x = track_i[track_i['time'].values ==
time_i]['projection_x_coordinate'].values[0]
y = track_i[track_i['time'].values ==
time_i]['projection_y_coordinate'].values[0]
n_add_width = 2
box_slice = [[
x - (width + n_add_width) * dx, x + (width + n_add_width) * dx
], [y - (width + n_add_width) * dx, y + (width + n_add_width) * dx]]
x_min = np.nanmin([box_mask[0][0], box_slice[0][0]])
x_max = np.nanmax([box_mask[0][1], box_slice[0][1]])
y_min = np.nanmin([box_mask[1][0], box_slice[1][0]])
y_max = np.nanmax([box_mask[1][1], box_slice[1][1]])
constraint_x = Constraint(projection_x_coordinate=lambda cell: int(
x_min) < cell < int(x_max))
constraint_y = Constraint(projection_y_coordinate=lambda cell: int(
y_min) < cell < int(y_max))
constraint = constraint_time & constraint_x & constraint_y
mask_cell_i = mask_cell_i.extract(constraint_x & constraint_y)
mask_cell_surface_i = mask_cell_surface_i.extract(constraint_x
& constraint_y)
cubelist_i = cubelist_in.extract(constraint)
cubelist_cell_sum.extend(
sum_profile_mask(cubelist_i, height_levels, mask_cell_i))
cubelist_cell_sum_out = cubelist_cell_sum.merge()
for cube in cubelist_cell_sum_out:
cell_time_coord = AuxCoord(
track_i['time_cell'].dt.total_seconds().values,
units='s',
long_name='time_cell')
cube.add_aux_coord(cell_time_coord, cube.coord_dims('time')[0])
for cube in cubelist_cell_sum_out:
cubelist_cell_integrated_out.append(
cube.collapsed(('geopotential_height'), SUM))
track_cell_integrated = deepcopy(track_i)
#
for cube in cubelist_cell_integrated_out:
track_cell_integrated[cube.name()] = cube.core_data()
return cubelist_cell_sum_out, cubelist_cell_integrated_out, track_cell_integrated
def run_spotdata(diagnostics,
ancillary_data,
sites,
config_constants,
use_multiprocessing=False):
"""
A routine that calls the components of the spotdata code. This includes
building site data into a suitable format, finding grid neighbours to
those sites with the chosen method, and then extracting data with the
chosen method. The final results are written out to new irregularly
gridded iris.cube.Cubes.
Args:
diagnostics (dict):
Dictionary containing the information regarding the methods that
will be applied for a specific diagnostic, as well as the data
following loading in a cube, and any additional data required to
be able to compute the methods requested.
For example::
{
"temperature": {
"diagnostic_name": "air_temperature",
"extrema": True,
"filepath": "temperature_at_screen_level",
"interpolation_method": "use_nearest",
"neighbour_finding": {
"land_constraint": False,
"method": "fast_nearest_neighbour",
"vertical_bias": None
"data": iris.cube.CubeList
"additional_data" : iris.cube.CubeList
}
}
}
ancillary_data (dict):
Dictionary containing named ancillary data; the key gives the name
and the item is the iris.cube.Cube of data.
sites (dict):
Contains:
latitudes (list of ints/floats or None):
A list of latitudes for running for a custom set of
sites. The order should correspond to the subsequent latitudes
and altitudes variables to construct each site.
longitudes (list of ints/floats or None):
A list of longitudes for running for a custom set of
sites.
altitudes (list of ints/floats or None):
A list of altitudes for running for a custom set of
sites.
site_ids (list of ints or None):
A list of site_ids to associate with the above
constructed sites. This must be ordered the same as the
latitudes/longitudes/altitudes lists.
config_constants (dict):
Dictionary defining constants to be used in methods that have
tolerances that may be set. e.g. maximum vertical extrapolation/
interpolation of temperatures using a temperature lapse rate
method.
use_multiprocessing (boolean):
A switch determining whether to use multiprocessing in the data
extraction step.
Returns:
(tuple): tuple containing:
**resulting_cube** (iris.cube.Cube or None):
Cube after extracting the diagnostic requested using the
desired extraction method.
None is returned if the "resulting_cubes" is an empty CubeList
after processing.
**extrema_cubes** (iris.cube.CubeList or None):
CubeList containing extrema values, if the 'extrema' diagnostic
is requested.
None is returned if the value for diagnostic_dict["extrema"]
is False, so that the extrema calculation is not required.
"""
# Read in constants to use; if not available, defaults will be used.
neighbour_kwargs = {}
if config_constants is not None:
no_neighbours = config_constants.get('no_neighbours')
if no_neighbours is not None:
neighbour_kwargs['no_neighbours'] = no_neighbours
# Add configuration constants to ancillaries (may be None if unset).
ancillary_data['config_constants'] = config_constants
# Set up site-grid point neighbour list using default method. Other IGPS
# methods will use this as a starting point so it must always be done.
# Assumes orography file is on the same grid as the diagnostic data.
neighbours = {}
default_neighbours = {
'method': 'fast_nearest_neighbour',
'vertical_bias': None,
'land_constraint': False
}
default_hash = construct_neighbour_hash(default_neighbours)
neighbours[default_hash] = PointSelection(**default_neighbours).process(
ancillary_data['orography'],
sites,
ancillary_data=ancillary_data,
**neighbour_kwargs)
# Set up site-grid point neighbour lists for all IGPS methods being used.
for key in diagnostics.keys():
neighbour_finding = diagnostics[key]['neighbour_finding']
neighbour_hash = construct_neighbour_hash(neighbour_finding)
# Check if defined neighbour method results already exist.
if neighbour_hash not in neighbours.keys():
# If not, find neighbours with new method.
neighbours[neighbour_hash] = (PointSelection(
**neighbour_finding).process(
ancillary_data['orography'],
sites,
ancillary_data=ancillary_data,
default_neighbours=neighbours[default_hash],
**neighbour_kwargs))
if use_multiprocessing:
# Process diagnostics on separate threads if multiprocessing is
# selected. Determine number of diagnostics to establish
# multiprocessing pool size.
n_diagnostic_threads = min(len(diagnostics.keys()), mp.cpu_count())
# Establish multiprocessing pool - each diagnostic processed on its
# own thread.
diagnostic_pool = mp.Pool(processes=n_diagnostic_threads)
diagnostic_keys = [
diagnostic_name for diagnostic_name in diagnostics.keys()
]
result = (diagnostic_pool.map_async(
partial(process_diagnostic, diagnostics, neighbours, sites,
ancillary_data), diagnostic_keys))
diagnostic_pool.close()
diagnostic_pool.join()
resulting_cubes = CubeList()
extrema_cubes = CubeList()
for result in result.get():
resulting_cubes.append(result[0])
extrema_cubes.append(result[1:])
else:
# Process diagnostics serially on one thread.
resulting_cubes = CubeList()
extrema_cubes = CubeList()
for key in diagnostics.keys():
resulting_cube, extrema_cubelist = (process_diagnostic(
diagnostics, neighbours, sites, ancillary_data, key))
resulting_cubes.append(resulting_cube)
extrema_cubes.append(extrema_cubelist)
return resulting_cubes, extrema_cubes
def wxcode_series_fixture(
data,
cube_type,
offset_reference_times: bool,
model_id_attr: bool,
record_run_attr: bool,
) -> Tuple[bool, CubeList]:
"""Generate a time series of weather code cubes for combination to create
a period representative code. When offset_reference_times is set, each
successive cube will have a reference time one hour older."""
time = TARGET_TIME
ntimes = len(data)
wxcubes = CubeList()
for i in range(ntimes):
wxtime = time - timedelta(hours=i)
wxbounds = [wxtime - timedelta(hours=1), wxtime]
if offset_reference_times:
wxfrt = time - timedelta(hours=18) - timedelta(hours=i)
else:
wxfrt = time - timedelta(hours=18)
wxdata = np.ones((2, 2), dtype=np.int8)
wxdata[0, 0] = data[i]
if cube_type == "gridded":
wxcubes.append(
set_up_wxcube(data=wxdata, time=wxtime, time_bounds=wxbounds, frt=wxfrt)
)
else:
time_coords = construct_scalar_time_coords(wxtime, wxbounds, wxfrt)
time_coords = [crd for crd, _ in time_coords]
latitudes = np.array([50, 52, 54, 56])
longitudes = np.array([-4, -2, 0, 2])
altitudes = wmo_ids = unique_site_id = np.arange(4)
unique_site_id_key = "met_office_site_id"
wxcubes.append(
build_spotdata_cube(
wxdata.flatten(),
"weather_code",
1,
altitudes,
latitudes,
longitudes,
wmo_ids,
unique_site_id=unique_site_id,
unique_site_id_key=unique_site_id_key,
scalar_coords=time_coords,
)
)
# Add a blendtime coordinate as UK weather symbols are constructed
# from model blended data.
blend_time = wxcubes[-1].coord("forecast_reference_time").copy()
blend_time.rename("blend_time")
wxcubes[-1].add_aux_coord(blend_time)
if model_id_attr:
if i == 0:
wxcubes[-1].attributes.update({MODEL_ID_ATTR: "uk_det uk_ens"})
else:
wxcubes[-1].attributes.update({MODEL_ID_ATTR: "uk_ens"})
if record_run_attr:
ukv_time = wxfrt - timedelta(hours=1)
enukx_time = wxfrt - timedelta(hours=3)
if i == 0:
wxcubes[-1].attributes.update(
{
RECORD_RUN_ATTR: f"uk_det:{ukv_time:{TIME_FORMAT}}:\nuk_ens:{enukx_time:{TIME_FORMAT}}:" # noqa: E501
}
)
else:
wxcubes[-1].attributes.update(
{RECORD_RUN_ATTR: f"uk_ens:{enukx_time:{TIME_FORMAT}}:"}
)
return model_id_attr, record_run_attr, offset_reference_times, wxcubes
def segmentation_2D(track,
field,
dxy,
threshold=0,
target='maximum',
method='watershed',
max_distance=None):
"""
Function using watershedding or random walker to determine cloud volumes associated with tracked updrafts
Parameters:
track: pandas.DataFrame
output from trackpy/maketrack
field_in: iris.cube.Cube
containing the 3D (time,x,y) field to perform the watershedding on
threshold: float
threshold for the watershedding field to be used for the mask
target: string
Switch to determine if algorithm looks strating from maxima or minima in input field (maximum: starting from maxima (default), minimum: starting from minima)
method: str ('method')
flag determining the algorithm to use (currently watershedding implemented)
Output:
segmentation_out: iris.cube.Cube
Cloud mask, 0 outside and integer numbers according to track inside the clouds
"""
import numpy as np
from skimage.morphology import watershed
# from skimage.segmentation import random_walker
import logging
from iris.cube import CubeList
from iris.util import new_axis
from scipy.ndimage import distance_transform_edt
logging.info('Start wateshedding 2D')
# CubeList to store individual segmentation masks
segmentation_out_list = CubeList()
track['ncells'] = 0
if max_distance is not None:
max_distance_pixel = np.ceil(max_distance / dxy)
field_time = field.slices_over('time')
for i, field_i in enumerate(field_time):
# Create cube of the same dimensions and coordinates as input data to store mask:
segmentation_out_i = 1 * field_i
segmentation_out_i.rename('segmentation_mask')
segmentation_out_i.units = 1
data_i = field_i.core_data()
time_i = field_i.coord('time').units.num2date(
field_i.coord('time').points[0])
tracks_i = track[track['time'] == time_i]
# mask data outside region above/below threshold and invert data if tracking maxima:
if target == 'maximum':
unmasked = data_i > threshold
data_i_segmentation = -1 * data_i
elif target == 'minimum':
unmasked = data_i < threshold
data_i_segmentation = data_i
else:
raise ValueError('unknown type of target')
markers = np.zeros_like(unmasked).astype(np.int32)
for index, row in tracks_i.iterrows():
markers[int(row['hdim_1']), int(row['hdim_2'])] = row['feature']
markers[~unmasked] = 0
if method == 'watershed':
segmentation_mask_i = watershed(data_i_segmentation,
markers.astype(np.int32),
mask=unmasked)
# elif method=='random_walker':
# #res1 = random_walker(Mask, markers,mode='cg')
# res1=random_walker(data_i_segmentation, markers.astype(np.int32),
# beta=130, mode='bf', tol=0.001, copy=True, multichannel=False, return_full_prob=False, spacing=None)
else:
raise ValueError('unknown method, must be watershed')
# remove everything from the individual masks that is more than max_distance_pixel away from the markers
if max_distance is not None:
for feature in tracks_i['feature']:
D = distance_transform_edt((markers != feature).astype(int))
segmentation_mask_i[np.bitwise_and(
segmentation_mask_i == feature,
D > max_distance_pixel)] = 0
segmentation_out_i.data = segmentation_mask_i
# using merge throws error, so cubes with time promoted to DimCoord and using concatenate:
# segmentation_out_list.append(segmentation_out_i)
segmentation_out_i_temp = new_axis(segmentation_out_i,
scalar_coord='time')
segmentation_out_list.append(segmentation_out_i_temp)
# count number of grid cells asoociated to each tracked cell and write that into DataFrame:
values, count = np.unique(segmentation_mask_i, return_counts=True)
counts = dict(zip(values, count))
for index, row in tracks_i.iterrows():
if row['feature'] in counts.keys():
track.loc[index, 'ncells'] = counts[row['feature']]
logging.debug('Finished segmentation 2D for ' +
time_i.strftime('%Y-%m-%d_%H:%M:%S'))
#merge individual masks in CubeList into one Cube:
# using merge throws error, so cubes with time promoted to DimCoord and using concatenate:
# segmentation_out=segmentation_out_list.merge_cube()
segmentation_out = segmentation_out_list.concatenate_cube()
logging.debug('Finished segmentation 2D')
return segmentation_out, track
def _get_cube(self,
file_list,
climatology=False,
overlay_probability_levels=False):
"""
Get an iris cube based on the given files using selection criteria
from the input_data.
@param file_list (list[str]): a list of file name to retrieve data from
@param climatology (boolean): if True extract the climatology data
@param overlay_probability_levels (boolean): if True only include the
10th, 50th and 90th percentile data
@return an iris cube, maybe 'None' if overlay_probability_levels=True
"""
if climatology is True:
LOG.info("_get_cube for climatology")
elif overlay_probability_levels is True:
LOG.info("_get_cube, overlay probability levels")
else:
LOG.info("_get_cube")
if LOG.getEffectiveLevel() == logging.DEBUG:
LOG.debug("_get_cube from %s files", len(file_list))
for fpath in file_list:
LOG.debug(" - FILE: %s", fpath)
# Load the cubes
cubes = CubeList()
try:
for file_path in file_list:
f_list = glob.glob(file_path)
cube_list = [iris.load_cube(f) for f in f_list]
cubes.extend(cube_list)
except IOError as ex:
if overlay_probability_levels is True:
# not all variables have corresponding probabilistic data
return None
for file_name in file_list:
file_name = file_name.split("*")[0]
if not path.exists(file_name):
LOG.error("File not found: %s", file_name)
raise UKCPDPDataNotFoundException from ex
if overlay_probability_levels is True:
collection = COLLECTION_PROB
else:
collection = self.input_data.get_value(InputType.COLLECTION)
# Remove time_bnds cubes
if collection == COLLECTION_PROB:
unfiltered_cubes = cubes
cubes = CubeList()
for cube in unfiltered_cubes:
if cube.name() != "time_bnds":
cubes.append(cube)
# Different creation dates will stop cubes concatenating, so lets
# remove them
for cube in cubes:
coords = cube.coords(var_name="creation_date")
for coord in coords:
cube.remove_coord(coord)
if len(cubes) == 0:
LOG.warning("No data was retrieved from the following files:%s",
file_list)
raise UKCPDPDataNotFoundException(
"No data found for given selection options")
LOG.debug("First cube:\n%s", cubes[0])
LOG.debug("Concatenate cubes:\n%s", cubes)
iris.experimental.equalise_cubes.equalise_attributes(cubes)
unify_time_units(cubes)
try:
cube = cubes.concatenate_cube()
except iris.exceptions.ConcatenateError as ex:
LOG.error("Failed to concatenate cubes:\n%s\n%s", ex, cubes)
error_cubes = CubeList()
for error_cube in cubes:
error_cubes.append(error_cube)
try:
LOG.info("Appending %s",
error_cube.coord("ensemble_member_id").points[0])
except iris.exceptions.CoordinateNotFoundError:
pass
try:
error_cubes.concatenate_cube()
except iris.exceptions.ConcatenateError as ex:
message = ""
try:
message = " {}".format(
error_cube.coord("ensemble_member_id").points[0])
except iris.exceptions.CoordinateNotFoundError:
pass
LOG.error(
"Error when concatenating cube%s:\n%s\n%s",
message,
ex,
error_cube,
)
break
# pylint: disable=W0707
raise UKCPDPDataNotFoundException(
"No data found for given selection options")
LOG.debug("Concatenated cube:\n%s", cube)
if climatology is True:
# generate a time slice constraint based on the baseline
time_slice_constraint = self._time_slice_selector(True)
else:
# generate a time slice constraint
time_slice_constraint = self._time_slice_selector(False)
if time_slice_constraint is not None:
cube = cube.extract(time_slice_constraint)
if cube is None:
if time_slice_constraint is not None:
LOG.warning(
"Time slice constraint resulted in no cubes being "
"returned: %s",
time_slice_constraint,
)
raise UKCPDPDataNotFoundException(
"Selection constraints resulted in no data being"
" selected")
# generate a temporal constraint
temporal_constraint = self._get_temporal_selector()
if temporal_constraint is not None:
cube = cube.extract(temporal_constraint)
if cube is None:
if temporal_constraint is not None:
LOG.warning(
"Temporal constraint resulted in no cubes being "
"returned: %s",
temporal_constraint,
)
raise UKCPDPDataNotFoundException(
"Selection constraints resulted in no data being"
" selected")
# extract 10, 50 and 90 percentiles
if overlay_probability_levels is True:
cube = get_probability_levels(cube, False)
# generate an area constraint
area_constraint = self._get_spatial_selector(cube, collection)
if area_constraint is not None:
cube = cube.extract(area_constraint)
if self.input_data.get_area_type() == AreaType.BBOX:
# Make sure we still have x, y dimension coordinated for
# bboxes
cube = self._promote_x_y_coords(cube)
if cube is None:
if area_constraint is not None:
LOG.warning(
"Area constraint resulted in no cubes being "
"returned: %s",
area_constraint,
)
raise UKCPDPDataNotFoundException(
"Selection constraints resulted in no data being"
" selected")
return cube
def subset_data(
cube: Cube,
grid_spec: Optional[Dict[str, Dict[str, int]]] = None,
site_list: Optional[List] = None,
) -> Cube:
"""Extract a spatial cutout or subset of sites from data
to generate suite reference outputs.
Args:
cube:
Input dataset
grid_spec:
Dictionary containing bounding grid points and an integer "thinning
factor" for each of UK and global grid, to create cutouts. Eg a
"thinning factor" of 10 would mean every 10th point being taken for
the cutout. The expected dictionary has keys that are spatial coordinate
names, with values that are dictionaries with "min", "max" and "thin" keys.
site_list:
List of WMO site IDs to extract. These IDs must match the type and format
of the "wmo_id" coordinate on the input spot cube.
Returns:
Subset of input cube as specified by input constraints
Raises:
ValueError:
If site_list is not provided for a spot data cube
ValueError:
If the spot data cube does not contain any of the required sites
ValueError:
If grid_spec is not provided for a gridded cube
ValueError:
If grid_spec does not contain entries for the spatial coordinates on
the input gridded data
ValueError:
If the grid_spec provided does not overlap with the cube domain
"""
if cube.coords("spot_index"):
if site_list is None:
raise ValueError("site_list required to extract from spot data")
constraint = Constraint(
coord_values={"wmo_id": lambda x: x in site_list})
result = cube.extract(constraint)
if result is None:
raise ValueError(
f"Cube does not contain any of the required sites: {site_list}"
)
else:
if grid_spec is None:
raise ValueError("grid_spec required to extract from gridded data")
x_coord = cube.coord(axis="x").name()
y_coord = cube.coord(axis="y").name()
for coord in [y_coord, x_coord]:
if coord not in grid_spec:
raise ValueError(
f"Cube coordinates {y_coord}, {x_coord} are not present within "
f"{grid_spec.keys()}")
def _create_cutout(cube, grid_spec):
"""Given a gridded data cube and boundary limits for cutout dimensions,
create cutout. Expects cube on either lat-lon or equal area grid.
"""
x_coord = cube.coord(axis="x").name()
y_coord = cube.coord(axis="y").name()
xmin = grid_spec[x_coord]["min"]
xmax = grid_spec[x_coord]["max"]
ymin = grid_spec[y_coord]["min"]
ymax = grid_spec[y_coord]["max"]
# need to use cube intersection for circular coordinates (longitude)
if x_coord == "longitude":
lat_constraint = Constraint(
latitude=lambda y: ymin <= y.point <= ymax)
cutout = cube.extract(lat_constraint)
if cutout is None:
return cutout
cutout = cutout.intersection(longitude=(xmin, xmax),
ignore_bounds=True)
# intersection creates a new coordinate with default datatype - we
# therefore need to re-cast to meet the IMPROVER standard
cutout.coord("longitude").points = cutout.coord(
"longitude").points.astype(FLOAT_DTYPE)
if cutout.coord("longitude").bounds is not None:
cutout.coord("longitude").bounds = cutout.coord(
"longitude").bounds.astype(FLOAT_DTYPE)
else:
x_constraint = Constraint(
projection_x_coordinate=lambda x: xmin <= x.point <= xmax)
y_constraint = Constraint(
projection_y_coordinate=lambda y: ymin <= y.point <= ymax)
cutout = cube.extract(x_constraint & y_constraint)
return cutout
cutout = _create_cutout(cube, grid_spec)
if cutout is None:
raise ValueError(
"Cube domain does not overlap with cutout specified:\n"
f"{x_coord}: {grid_spec[x_coord]}, {y_coord}: {grid_spec[y_coord]}"
)
original_coords = get_dim_coord_names(cutout)
thin_x = grid_spec[x_coord]["thin"]
thin_y = grid_spec[y_coord]["thin"]
result_list = CubeList()
try:
for subcube in cutout.slices([y_coord, x_coord]):
result_list.append(subcube[::thin_y, ::thin_x])
except ValueError as cause:
# error is raised if X or Y coordinate are single-valued (non-dimensional)
if "iterator" in str(cause) and "dimension" in str(cause):
raise ValueError(
"Function does not support single point extraction")
else:
raise
result = result_list.merge_cube()
enforce_coordinate_ordering(result, original_coords)
return result
def forecast_dataframe_to_cube(
df: DataFrame,
training_dates: DatetimeIndex,
forecast_period: int,
) -> Cube:
"""Convert a forecast DataFrame into an iris Cube. The percentiles
within the forecast DataFrame are rebadged as realizations.
Args:
df:
DataFrame expected to contain the following columns: forecast,
blend_time, forecast_period, forecast_reference_time, time,
wmo_id, percentile, diagnostic, latitude, longitude, period,
height, cf_name, units. Any other columns are ignored.
training_dates:
Datetimes spanning the training period.
forecast_period:
Forecast period in seconds as an integer.
Returns:
Cube containing the forecasts from the training period.
"""
fp_point = pd.Timedelta(int(forecast_period), unit="seconds")
cubelist = CubeList()
for adate in training_dates:
time_df = df.loc[(df["time"] == adate)
& (df["forecast_period"] == fp_point)]
time_df = _preprocess_temporal_columns(time_df)
if time_df.empty:
continue
# The following columns are expected to contain one unique value
# per column.
for col in ["period", "height", "cf_name", "units", "diagnostic"]:
_unique_check(time_df, col)
if time_df["period"].isna().all():
time_bounds = None
fp_bounds = None
else:
period = time_df["period"].values[0]
time_bounds = [adate - period, adate]
fp_bounds = [fp_point - period, fp_point]
time_coord = _define_time_coord(adate, time_bounds)
height_coord = _define_height_coord(time_df["height"].values[0])
fp_coord = AuxCoord(
np.array(fp_point.total_seconds(),
dtype=TIME_COORDS["forecast_period"].dtype),
"forecast_period",
bounds=fp_bounds if fp_bounds is None else [
np.array(f.total_seconds(),
dtype=TIME_COORDS["forecast_period"].dtype)
for f in fp_bounds
],
units=TIME_COORDS["forecast_period"].units,
)
frt_coord = AuxCoord(
np.array(
time_df["forecast_reference_time"].values[0].timestamp(),
dtype=TIME_COORDS["forecast_reference_time"].dtype,
),
"forecast_reference_time",
units=TIME_COORDS["forecast_reference_time"].units,
)
for percentile in sorted(df["percentile"].unique()):
perc_coord = DimCoord(np.float32(percentile),
long_name="percentile",
units="%")
perc_df = time_df.loc[time_df["percentile"] == percentile]
cube = build_spotdata_cube(
perc_df["forecast"].astype(np.float32),
perc_df["cf_name"].values[0],
perc_df["units"].values[0],
perc_df["altitude"].astype(np.float32),
perc_df["latitude"].astype(np.float32),
perc_df["longitude"].astype(np.float32),
perc_df["wmo_id"].values.astype("U5"),
scalar_coords=[
time_coord,
frt_coord,
fp_coord,
perc_coord,
height_coord,
],
)
cubelist.append(cube)
if not cubelist:
return
cube = cubelist.merge_cube()
return RebadgePercentilesAsRealizations()(cube)
def process_diagnostic(diagnostic,
neighbours,
sites,
forecast_times,
data_path,
ancillary_data,
output_path=None):
"""
Extract data and write output for a given diagnostic.
Args:
-----
diagnostic : string
String naming the diagnostic to be processed.
neighbours : numpy.array
Array of neigbouring grid points that are associated with sites
in the SortedDictionary of sites.
sites : dict
A dictionary containing the properties of spotdata sites.
forecast_times : list[datetime.datetime objects]
A list of datetimes representing forecast times for which data is
required.
data_path : string
Path to diagnostic data files.
ancillary_data : dict
A dictionary containing additional model data that is needed.
e.g. {'orography': <cube of orography>}
output_path : str
Path to which output file containing processed diagnostic should be
written.
Returns:
--------
None
Raises:
-------
IOError : If no relevant data cubes are found at given path.
Exception : No spotdata returned.
"""
# Search directory structure for all files relevant to current diagnostic.
files_to_read = [
os.path.join(dirpath, filename)
for dirpath, _, files in os.walk(data_path) for filename in files
if diagnostic['filepath'] in filename
]
if not files_to_read:
raise IOError('No relevant data files found in {}.'.format(data_path))
# Load cubes into an iris.cube.CubeList.
cubes = Load('multi_file').process(files_to_read,
diagnostic['diagnostic_name'])
# Grab the relevant set of grid point neighbours for the neighbour finding
# method being used by this diagnostic.
neighbour_hash = construct_neighbour_hash(diagnostic['neighbour_finding'])
neighbour_list = neighbours[neighbour_hash]
# Check if additional diagnostics are needed (e.g. multi-level data).
# If required, load into the additional_diagnostics dictionary.
additional_diagnostics = get_method_prerequisites(
diagnostic['interpolation_method'], data_path)
# Create empty iris.cube.CubeList to hold extracted data cubes.
resulting_cubes = CubeList()
# Get optional kwargs that may be set to override defaults.
optionals = [
'upper_level', 'lower_level', 'no_neighbours', 'dz_tolerance',
'dthetadz_threshold', 'dz_max_adjustment'
]
kwargs = {}
if ancillary_data.get('config_constants') is not None:
for optional in optionals:
constant = ancillary_data.get('config_constants').get(optional)
if constant is not None:
kwargs[optional] = constant
# Loop over forecast times.
for a_time in forecast_times:
# Extract Cube from CubeList at current time.
time_extract = datetime_constraint(a_time)
cube = extract_cube_at_time(cubes, a_time, time_extract)
if cube is None:
# If no cube is available at given time, try the next time.
continue
ad = {}
if additional_diagnostics is not None:
# Extract additional diagnostcs at current time.
ad = extract_ad_at_time(additional_diagnostics, a_time,
time_extract)
args = (cube, sites, neighbour_list, ancillary_data, ad)
# Extract diagnostic data using defined method.
resulting_cubes.append(
ExtractData(diagnostic['interpolation_method']).process(
*args, **kwargs))
# Concatenate CubeList into Cube, creating a time DimCoord, and write out.
if resulting_cubes:
cube_out, = resulting_cubes.concatenate()
WriteOutput('as_netcdf', dir_path=output_path).process(cube_out)
else:
raise Exception('No data available at given forecast times.')
# If set in the configuration, extract the diagnostic maxima and minima
# values.
if diagnostic['extrema']:
extrema_cubes = ExtractExtrema(24, start_hour=9).process(cube_out)
extrema_cubes = extrema_cubes.merge()
for extrema_cube in extrema_cubes:
WriteOutput('as_netcdf',
dir_path=output_path).process(extrema_cube)
def process_diagnostic(diagnostics, neighbours, sites, ancillary_data,
diagnostic_name):
"""
Extract data and write output for a given diagnostic.
Args:
diagnostics (dict):
Dictionary containing information regarding how the diagnostics
are to be processed.
For example::
{
"temperature": {
"diagnostic_name": "air_temperature",
"extrema": true,
"filepath": "temperature_at_screen_level",
"interpolation_method":
"model_level_temperature_lapse_rate",
"neighbour_finding": {
"land_constraint": false,
"method": "fast_nearest_neighbour",
"vertical_bias": null
}
}
}
neighbours (numpy.array):
Array of neigbouring grid points that are associated with sites
in the SortedDictionary of sites.
sites (dict):
A dictionary containing the properties of spotdata sites.
ancillary_data (dict):
A dictionary containing additional model data that is needed.
e.g. {'orography': <cube of orography>}
diagnostic_name (string):
A string matching the keys in the diagnostics dictionary that
will be used to access information regarding how the diagnostic
is to be processed.
Returns:
(tuple): tuple containing:
**resulting_cube** (iris.cube.Cube or None):
Cube after extracting the diagnostic requested using the
desired extraction method.
None is returned if the "resulting_cubes" is an empty CubeList
after processing.
**extrema_cubes** (iris.cube.CubeList or None):
CubeList containing extrema values, if the 'extrema' diagnostic
is requested.
None is returned if the value for diagnostic_dict["extrema"]
is False, so that the extrema calculation is not required.
"""
diagnostic_dict = diagnostics[diagnostic_name]
# Grab the relevant set of grid point neighbours for the neighbour finding
# method being used by this diagnostic.
neighbour_hash = (construct_neighbour_hash(
diagnostic_dict['neighbour_finding']))
neighbour_list = neighbours[neighbour_hash]
# Get optional kwargs that may be set to override defaults.
optionals = [
'upper_level', 'lower_level', 'no_neighbours', 'dz_tolerance',
'dthetadz_threshold', 'dz_max_adjustment'
]
kwargs = {}
if ancillary_data.get('config_constants') is not None:
for optional in optionals:
constant = ancillary_data.get('config_constants').get(optional)
if constant is not None:
kwargs[optional] = constant
# Create a list of datetimes to loop through.
forecast_times = []
for cube in diagnostic_dict["data"]:
time = cube.coord("time")
forecast_times.extend(time.units.num2date(time.points))
# Create empty iris.cube.CubeList to hold extracted data cubes.
resulting_cubes = CubeList()
# Loop over forecast times.
for a_time in forecast_times:
# Extract Cube from CubeList at current time.
time_extract = datetime_constraint(a_time)
cube = extract_cube_at_time(diagnostic_dict["data"], a_time,
time_extract)
if cube is None:
# If no cube is available at given time, try the next time.
continue
ad = {}
if diagnostic_dict["additional_data"] is not None:
# Extract additional diagnostics at current time.
ad = extract_ad_at_time(diagnostic_dict["additional_data"], a_time,
time_extract)
args = (cube, sites, neighbour_list, ancillary_data, ad)
# Extract diagnostic data using defined method.
resulting_cubes.append(
ExtractData(diagnostic_dict['interpolation_method']).process(
*args, **kwargs))
if resulting_cubes:
# Concatenate CubeList into Cube for cubes with different
# forecast times.
resulting_cube = resulting_cubes.concatenate_cube()
else:
resulting_cube = None
if diagnostic_dict['extrema']:
extrema_cubes = (ExtractExtrema(24, start_hour=9).process(
resulting_cube.copy()))
extrema_cubes = extrema_cubes.merge()
else:
extrema_cubes = None
return resulting_cube, extrema_cubes
def setUp(self):
"""
Create a cube containing a regular lat-lon grid.
Data is striped horizontally,
e.g.
1 1 1 1 1 1
1 1 1 1 1 1
2 2 2 2 2 2
2 2 2 2 2 2
3 3 3 3 3 3
3 3 3 3 3 3
"""
data = np.ones((12, 12))
data[0:4, :] = 1
data[4:8, :] = 2
data[8:, :] = 3
latitudes = np.linspace(-90, 90, 12)
longitudes = np.linspace(-180, 180, 12)
latitude = DimCoord(
latitudes,
standard_name="latitude",
units="degrees",
coord_system=GeogCS(6371229.0),
)
longitude = DimCoord(
longitudes,
standard_name="longitude",
units="degrees",
coord_system=GeogCS(6371229.0),
circular=True,
)
# Use time of 2017-02-17 06:00:00
time = DimCoord(
[1487311200],
standard_name="time",
units=cf_units.Unit("seconds since 1970-01-01 00:00:00",
calendar="gregorian"),
)
long_time_coord = DimCoord(
list(range(1487311200, 1487397600, 3600)),
standard_name="time",
units=cf_units.Unit("seconds since 1970-01-01 00:00:00",
calendar="gregorian"),
)
time_dt = dt(2017, 2, 17, 6, 0)
time_extract = Constraint(
time=lambda cell: cell.point == PartialDateTime(
time_dt.year, time_dt.month, time_dt.day, time_dt.hour))
cube = Cube(
data.reshape((1, 12, 12)),
long_name="air_temperature",
dim_coords_and_dims=[(time, 0), (latitude, 1), (longitude, 2)],
units="K",
)
long_cube = Cube(
np.arange(3456).reshape(24, 12, 12),
long_name="air_temperature",
dim_coords_and_dims=[(long_time_coord, 0), (latitude, 1),
(longitude, 2)],
units="K",
)
orography = Cube(
np.ones((12, 12)),
long_name="surface_altitude",
dim_coords_and_dims=[(latitude, 0), (longitude, 1)],
units="m",
)
# Western half of grid at altitude 0, eastern half at 10.
# Note that the pressure_on_height_levels data is left unchanged,
# so it is as if there is a sharp front running up the grid with
# differing pressures on either side at equivalent heights above
# the surface (e.g. east 1000hPa at 0m AMSL, west 1000hPa at 10m AMSL).
# So there is higher pressure in the west.
orography.data[0:10] = 0
orography.data[10:] = 10
ancillary_data = {}
ancillary_data["orography"] = orography
additional_data = {}
adlist = CubeList()
adlist.append(cube)
additional_data["air_temperature"] = adlist
data_indices = [list(data.nonzero()[0]), list(data.nonzero()[1])]
self.cube = cube
self.long_cube = long_cube
self.data = data
self.time_dt = time_dt
self.time_extract = time_extract
self.data_indices = data_indices
self.ancillary_data = ancillary_data
self.additional_data = additional_data
def plot_hydrometeors_composite_integrated(Hydrometeors_Composite,
maxvalue=None,
aggregate_min=None,
mp=None,
xlim=None,
ylim=None,
xlim_profile=None,
ylim_integrated=None,
title=None,
figsize=(20 / 2.54, 10 / 2.54),
height_ratios=[1.8, 1],
width_ratios=[4, 1]):
from mpdiag import hydrometeors_colors
from iris.analysis import MEAN, SUM
from iris.coord_categorisation import add_categorised_coord
from iris.cube import CubeList
from copy import deepcopy
dict_hydrometeors_colors, dict_hydrometeors_names = hydrometeors_colors(
microphysics_scheme=mp)
if xlim is None:
xlim = [
Hydrometeors_Composite[0].coord('time').points[0],
Hydrometeors_Composite[0].coord('time').points[-1]
]
if ylim is None:
ylim = [
Hydrometeors_Composite[0].coord('geopotential_height').points[0] /
1000,
Hydrometeors_Composite[0].coord('geopotential_height').points[-1] /
1000
]
fig, ax = plt.subplots(
nrows=2,
ncols=2,
#sharex='col', sharey='row',
gridspec_kw={
'height_ratios': height_ratios,
'width_ratios': width_ratios
},
figsize=figsize)
fig.subplots_adjust(left=0.1,
right=0.95,
bottom=0.1,
top=0.93,
wspace=0.1,
hspace=0.2)
Hydrometeors_Composite_copy = deepcopy(Hydrometeors_Composite)
if aggregate_min is not None:
def get_min(coord, value):
minutes = value
return np.floor(
minutes / aggregate_min) * aggregate_min + aggregate_min / 2
hydrometeors_aggregated = CubeList()
for cube in Hydrometeors_Composite_copy:
if aggregate_min == 5:
add_categorised_coord(cube, 'time_aggregated', 'time', get_min)
hydrometeors_aggregated.append(
cube.aggregated_by(['time_aggregated'], MEAN))
hydrometeors_piecharts = hydrometeors_aggregated
else:
hydrometeors_piecharts = hydrometeors_Composite
plot_hydrometeors_color_time(hydrometeors_piecharts,
Aux=None,
axes=ax[0, 0],
microphysics_scheme=mp,
scaling='linear',
minvalue=0,
maxvalue=maxvalue,
vscale=maxvalue,
piecharts_rasterized=False,
legend_piecharts=True,
fontsize_legend=6,
legend_piecharts_pos=(1.05, -0.25),
legend_overlay=False,
legend_overlay_pos=(1, -0.6),
overlay=False,
xlabel=False,
ylabel=False,
xlim=xlim,
ylim=ylim,
scale=True,
unit_scale='kg m$^{-1}$ s$^{-1}$',
fontsize_scale=6,
x_shift=0)
ax[0, 0].plot([0, 0], [0, 20000], color='grey', ls='-')
for cube in Hydrometeors_Composite:
color = dict_hydrometeors_colors[cube.name()]
ax[0, 1].plot(cube.collapsed(('time'), MEAN).data,
cube.coord('geopotential_height').points / 1000,
color=color)
ax[1, 0].plot(cube.coord('time').points,
cube.collapsed(('geopotential_height'), SUM).data,
color=color)
#ax1[1,0].set_ylim(0,1000)
# ax[1,0].plot([0,0],[0,2e10],color='grey',ls='-')
ax[1, 1].axis('off')
ax[0, 0].set_ylabel('altitude (km)')
ax[1, 0].set_xlabel('time (min)')
ax[1, 0].set_xlim(xlim)
ax[1, 0].set_ylim(ylim_integrated)
ax[0, 1].set_ylim(ylim)
ax[0, 1].set_xlim(xlim_profile)
ax[1, 0].set_ylabel('integrated (kg $^{-1}$)')
ax[1, 0].ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
ax[0, 1].ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
ax[0, 0].xaxis.set_tick_params(labelbottom=False)
ax[0, 1].yaxis.set_tick_params(labelleft=False)
ax[0, 1].set_xlabel('integrated (kg m$^{-1}$)', labelpad=10)
if title:
ax[0, 0].set_title(title, loc='left')
return fig
def create_data_object(self, filenames, variable, index_offset=1):
from cis.data_io.hdf_vd import get_data
from cis.data_io.hdf_vd import VDS
from pyhdf.error import HDF4Error
from cis.data_io import hdf_sd
from iris.coords import DimCoord, AuxCoord
from iris.cube import Cube, CubeList
from cis.data_io.gridded_data import GriddedData
from cis.time_util import cis_standard_time_unit
from datetime import datetime
from iris.util import new_axis
import numpy as np
logging.debug("Creating data object for variable " + variable)
variables = ["Pressure_Mean"]
logging.info("Listing coordinates: " + str(variables))
variables.append(variable)
# reading data from files
sdata = {}
for filename in filenames:
try:
sds_dict = hdf_sd.read(filename, variables)
except HDF4Error as e:
raise IOError(str(e))
for var in list(sds_dict.keys()):
utils.add_element_to_list_in_dict(sdata, var, sds_dict[var])
# work out size of data arrays
# the coordinate variables will be reshaped to match that.
# NOTE: This assumes that all Caliop_L1 files have the same altitudes.
# If this is not the case, then the following line will need to be changed
# to concatenate the data from all the files and not just arbitrarily pick
# the altitudes from the first file.
alt_data = self._get_calipso_data(hdf_sd.HDF_SDS(filenames[0], 'Altitude_Midpoint'))[0, :]
alt_coord = DimCoord(alt_data, standard_name='altitude', units='km')
alt_coord.convert_units('m')
lat_data = self._get_calipso_data(hdf_sd.HDF_SDS(filenames[0], 'Latitude_Midpoint'))[0, :]
lat_coord = DimCoord(lat_data, standard_name='latitude', units='degrees_north')
lon_data = self._get_calipso_data(hdf_sd.HDF_SDS(filenames[0], 'Longitude_Midpoint'))[0, :]
lon_coord = DimCoord(lon_data, standard_name='longitude', units='degrees_east')
cubes = CubeList()
for f in filenames:
t = get_data(VDS(f, "Nominal_Year_Month"), True)[0]
time_data = cis_standard_time_unit.date2num(datetime(int(t[0:4]), int(t[4:6]), 15))
time_coord = AuxCoord(time_data, long_name='Profile_Time', standard_name='time',
units=cis_standard_time_unit)
# retrieve data + its metadata
var = sdata[variable]
metadata = hdf.read_metadata(var, "SD")
data = self._get_calipso_data(hdf_sd.HDF_SDS(f, variable))
pres_data = self._get_calipso_data(hdf_sd.HDF_SDS(f, 'Pressure_Mean'))
pres_coord = AuxCoord(pres_data, standard_name='air_pressure', units='hPa')
if data.ndim == 2:
# pres_coord = new_axis()
cube = Cube(data, long_name=metadata.long_name or variable, units=self.clean_units(metadata.units),
dim_coords_and_dims=[(lat_coord, 0), (lon_coord, 1)],
aux_coords_and_dims=[(time_coord, ())])
# Promote the time scalar coord to a length one dimension
new_cube = new_axis(cube, 'time')
cubes.append(new_cube)
elif data.ndim == 3:
# pres_coord = new_axis()
cube = Cube(data, long_name=metadata.long_name or variable, units=self.clean_units(metadata.units),
dim_coords_and_dims=[(lat_coord, 0), (lon_coord, 1), (alt_coord, 2)],
aux_coords_and_dims=[(time_coord, ())])
# Promote the time scalar coord to a length one dimension
new_cube = new_axis(cube, 'time')
# Then add the (extended) pressure coord so that it is explicitly a function of time
new_cube.add_aux_coord(pres_coord[np.newaxis, ...], (0, 1, 2, 3))
cubes.append(new_cube)
else:
raise ValueError("Unexpected number of dimensions for CALIOP data: {}".format(data.ndim))
# Concatenate the cubes from each file into a single GriddedData object
gd = GriddedData.make_from_cube(cubes.concatenate_cube())
return gd
def process(self, cube):
"""
Calculate extrema values for diagnostic in cube over the period given
from the start_hour, both set at initialisation.
Args:
-----
cube : iris.cube.Cube
Cube of diagnostic data with a utc_offset coordinate.
Returns:
--------
period_cubes : iris.cube.CubeList
CubeList of diagnostic extrema cubes.
"""
# Change to 64 bit to avoid the 2038 problem with any time
# manipulations on units in seconds since the epoch.
cube.coord('time').points = cube.coord('time').points.astype(np.int64)
# Adjust times on cube to be local to each site.
local_tz_cube = make_local_time_cube(cube)
# Starts at start_hour on first available day, runs until start_hour on
# final_date.
start_time, end_time = get_datetime_limits(local_tz_cube.coord('time'),
self.start_hour)
num_periods = int(
np.ceil(
(end_time - start_time).total_seconds() / 3600 / self.period))
starts = [
start_time + datetime.timedelta(hours=i * self.period)
for i in range(num_periods)
]
ends = [
time + datetime.timedelta(hours=self.period) for time in starts
]
# Extract extrema values over desired time periods, producing a cube
# for each period.
period_cubes = CubeList()
for period_start, period_end in zip(starts, ends):
extrema_constraint = datetime_constraint(period_start, period_end)
with iris.FUTURE.context(cell_datetime_objects=True):
cube_over_period = local_tz_cube.extract(extrema_constraint)
if cube_over_period is not None:
# Ensure time dimension of resulting cube reflects period.
mid_time = dt_to_utc_hours(period_start +
(period_end - period_start) / 2)
bounds = [
dt_to_utc_hours(period_start),
dt_to_utc_hours(period_end)
]
extremas = [['max', iris.analysis.MAX],
['min', iris.analysis.MIN]]
for name, method in extremas:
cube_out = cube_over_period.collapsed('time', method)
cube_out.long_name = cube_out.name() + '_' + name
cube_out.standard_name = None
cube_out.coord('time').convert_units(
'hours since 1970-01-01 00:00:00')
cube_out.coord('time').points = mid_time
cube_out.coord('time').bounds = bounds
period_cubes.append(cube_out)
return period_cubes
def process(self, cube: Cube) -> Cube:
"""
Produces the vicinity processed data. The input data is sliced to
yield y-x slices to which the maximum_within_vicinity method is applied.
The different vicinity radii (if multiple) are looped over and a
coordinate recording the radius used is added to each resulting cube.
A single cube is returned with the leading coordinates of the input cube
preserved. If a single vicinity radius is provided, a new scalar
radius_of_vicinity coordinate will be found on the returned cube. If
multiple radii are provided, this coordinate will be a dimension
coordinate following any probabilistic / realization coordinates.
Args:
cube:
Thresholded cube.
Returns:
Cube containing the occurrences within a vicinity for each radius,
calculated for each yx slice, which have been merged to yield a
single cube.
Raises:
ValueError: Cube and land mask have differing spatial coordinates.
"""
if self.land_mask_cube and not spatial_coords_match(
[cube, self.land_mask_cube]):
raise ValueError(
"Supplied cube do not have the same spatial coordinates and land mask"
)
if not self.native_grid_point_radius:
grid_point_radii = [
distance_to_number_of_grid_cells(cube, radius)
for radius in self.radii
]
else:
grid_point_radii = self.radii
radii_cubes = CubeList()
# List of non-spatial dimensions to restore as leading on the output.
leading_dimensions = [
crd.name() for crd in cube.coords(dim_coords=True)
if not crd.coord_system
]
for radius, grid_point_radius in zip(self.radii, grid_point_radii):
max_cubes = CubeList([])
for cube_slice in cube.slices(
[cube.coord(axis="y"),
cube.coord(axis="x")]):
max_cubes.append(
self.maximum_within_vicinity(cube_slice,
grid_point_radius))
result_cube = max_cubes.merge_cube()
# Put dimensions back if they were there before.
result_cube = check_cube_coordinates(cube, result_cube)
# Add a coordinate recording the vicinity radius applied to the data.
self._add_vicinity_coordinate(result_cube, radius)
radii_cubes.append(result_cube)
# Merge cubes produced for each vicinity radius.
result_cube = radii_cubes.merge_cube()
# Enforce order of leading dimensions on the output to match the input.
enforce_coordinate_ordering(result_cube, leading_dimensions)
if is_probability(result_cube):
result_cube.rename(in_vicinity_name_format(result_cube.name()))
else:
result_cube.rename(f"{result_cube.name()}_in_vicinity")
return result_cube
