def test_set_lazy(self):
# Setting new lazy bounds realises them.
coord = DimCoord(self.pts_real, bounds=self.bds_lazy)
new_bounds = self.bds_lazy + 102.3
coord.bounds = new_bounds
result = coord.core_bounds()
self.assertEqualRealArraysAndDtypes(result, new_bounds.compute())
def test_real_points(self):
# Getting real points returns a copy
coord = DimCoord(self.pts_real)
result = coord.core_points()
self.assertArraysDoNotShareData(
result, self.pts_real,
'Points are the same array as the provided data.')
def test_set_lazy(self):
# Setting new lazy points realises them.
coord = DimCoord(self.pts_real)
new_pts = self.pts_lazy + 102.3
coord.points = new_pts
result = coord.core_points()
self.assertEqualRealArraysAndDtypes(result, new_pts.compute())
def test_real_points(self):
# Getting real points does not change or copy them.
coord = DimCoord(self.pts_real)
result = coord.core_points()
self.assertArraysShareData(
result, self.pts_real,
'Points are not the same array as the provided data.')
def test_real_points(self):
data = self.pts_real
coord = DimCoord(data)
result = coord.core_points()
self.assertArraysShareData(
result, self.pts_real,
'core_points() are not the same data as the internal array.')
def test_short_time_interval(self):
coord = DimCoord([5], standard_name='time',
units='days since 1970-01-01')
expected = ("DimCoord([1970-01-06 00:00:00], standard_name='time', "
"calendar='gregorian')")
result = coord.__str__()
self.assertEqual(expected, result)
def test_fail_not_monotonic(self):
# Setting real bounds requires that they are monotonic.
coord = DimCoord(self.pts_real, bounds=self.bds_real)
msg = 'strictly monotonic'
with self.assertRaisesRegexp(ValueError, msg):
coord.bounds = np.array([[3.0, 2.0], [1.0, 0.0], [2.0, 1.0]])
self.assertArrayEqual(coord.bounds, self.bds_real)
def test_fail_bad_shape(self):
# Setting real points requires matching shape.
coord = DimCoord(self.pts_real, bounds=self.bds_real)
msg = 'The shape of the bounds array should be'
with self.assertRaisesRegexp(ValueError, msg):
coord.bounds = np.array([1.0, 2.0, 3.0])
self.assertArrayEqual(coord.bounds, self.bds_real)
def _make_partially_collapsed_coord(self, coord, grid, guessed_axis):
"""
Make a new DimCoord which represents a partially collapsed (aggregated into bins) coordinate.
This dimcoord will have a grid
:type coord: data_io.Coord.Coord
:param coord: Coordinate to partially collapse
:type grid: aggregation.aggregation_grid.AggregationGrid
:param grid: grid on which this coordinate will aggregate
:type guessed_axis: str
:param guessed_axis: String identifier of the axis to which this coordinate belongs (e.g. 'T', 'X')
:return: DimCoord
"""
if grid.is_time or guessed_axis == 'T':
# Ensure that the limits are date/times.
dt = parse_datetime.convert_datetime_components_to_datetime(grid.start, True)
grid_start = Subset._convert_datetime_to_coord_unit(coord, dt)
dt = parse_datetime.convert_datetime_components_to_datetime(grid.end, False)
grid_end = Subset._convert_datetime_to_coord_unit(coord, dt)
grid_delta = grid.delta
else:
# Assume to be a non-time axis
(grid_start, grid_end) = Subset._fix_non_circular_limits(float(grid.start), float(grid.end))
grid_delta = float(grid.delta)
new_coordinate_grid = aggregation_grid_array(grid_start, grid_end, grid_delta, grid.is_time, coord)
new_coord = DimCoord(new_coordinate_grid, var_name=coord.name(), standard_name=coord.standard_name,
units=coord.units)
if len(new_coord.points) == 1:
new_coord.bounds = [[grid_start, grid_end]]
else:
new_coord.guess_bounds()
return new_coord
def test_long_time_interval__bounded(self):
coord = DimCoord([5, 6], standard_name='time',
units='years since 1970-01-01')
coord.guess_bounds()
expected = ("DimCoord([5 6], bounds=[[4.5 5.5]\n [5.5 6.5]], "
"standard_name='time', calendar='gregorian')")
result = coord.__str__()
self.assertEqual(expected, result)
def test_fail_bad_shape(self):
# Setting real points requires matching shape.
points = [1.0, 2.0]
coord = DimCoord(points)
msg = 'Require data with shape \(2,\), got \(3,\)'
with self.assertRaisesRegexp(ValueError, msg):
coord.points = np.array([1.0, 2.0, 3.0])
self.assertArrayEqual(coord.points, points)
def test_set_real(self):
# Setting points does not copy, but makes a readonly view.
coord = DimCoord(self.pts_real)
new_pts = self.pts_real + 102.3
coord.points = new_pts
result = coord.core_points()
self.assertArraysShareData(
result, new_pts,
'Points are not the same data as the assigned array.')
def test_1d_coord_no_bounds_warning(self):
coord = DimCoord([0, 1, 2], standard_name='latitude')
msg = "Coordinate 'latitude' is not bounded, guessing contiguous " \
"bounds."
with warnings.catch_warnings():
# Cause all warnings to raise Exceptions
warnings.simplefilter("error")
with self.assertRaisesRegexp(Warning, msg):
coord.contiguous_bounds()
def test_set_real(self):
# Setting points copies the data
coord = DimCoord(self.pts_real)
new_pts = self.pts_real + 102.3
coord.points = new_pts
result = coord.core_points()
self.assertArraysDoNotShareData(
result, new_pts,
'Points are the same data as the assigned array.')
def test_set_real(self):
# Setting bounds does not copy, but makes a readonly view.
coord = DimCoord(self.pts_real, bounds=self.bds_real)
new_bounds = self.bds_real + 102.3
coord.bounds = new_bounds
result = coord.core_bounds()
self.assertArraysShareData(
result, new_bounds,
'Bounds are not the same data as the assigned array.')
def test_short_time_interval__bounded(self):
coord = DimCoord([5, 6], standard_name='time',
units='days since 1970-01-01')
coord.guess_bounds()
expected = ("DimCoord([1970-01-06 00:00:00, 1970-01-07 00:00:00], "
"bounds=[[1970-01-05 12:00:00, 1970-01-06 12:00:00],\n"
" [1970-01-06 12:00:00, 1970-01-07 12:00:00]], "
"standard_name='time', calendar='gregorian')")
result = coord.__str__()
self.assertEqual(expected, result)
def setUp(self):
self.sigma = DimCoord(np.arange(5, dtype=np.float) * 10, long_name="sigma", units="1")
self.eta = AuxCoord(np.arange(4, dtype=np.float).reshape(2, 2), long_name="eta", units="m")
self.depth = AuxCoord(np.arange(4, dtype=np.float).reshape(2, 2) * 10, long_name="depth", units="m")
self.depth_c = AuxCoord([15], long_name="depth_c", units="m")
self.nsigma = AuxCoord([3], long_name="nsigma")
self.zlev = DimCoord(np.arange(5, dtype=np.float) * 10, long_name="zlev", units="m")
self.kwargs = dict(
sigma=self.sigma, eta=self.eta, depth=self.depth, depth_c=self.depth_c, nsigma=self.nsigma, zlev=self.zlev
)
def test__discontiguity_in_bounds_call(self):
# Check that :meth:`iris.coords.Coord._discontiguity_in_bounds` is
# called.
coord = DimCoord([1, 3], bounds=[[0, 2], [2, 4]])
with mock.patch('iris.coords.Coord._discontiguity_in_bounds'
) as discontiguity_check:
# Discontiguity returns two objects that are unpacked in
# `coord.is_contiguous`.
discontiguity_check.return_value = [None, None]
coord.is_contiguous(rtol=1e-1, atol=1e-3)
discontiguity_check.assert_called_with(rtol=1e-1, atol=1e-3)
def test_dim_1d(self):
# Numeric coords should not be serialised.
coord = DimCoord(points=np.array([2, 4, 6, 8]),
bounds=np.array([[1, 3], [3, 5], [5, 7], [7, 9]]))
for units in ['unknown', 'no_unit', 1, 'K']:
coord.units = units
collapsed_coord = coord.collapsed()
self.assertArrayEqual(collapsed_coord.points,
np.mean(coord.points))
self.assertArrayEqual(collapsed_coord.bounds,
[[coord.bounds.min(), coord.bounds.max()]])
def test_formula_terms_p0_bounded(self):
coord_a = DimCoord(range(5))
coord_p0 = DimCoord(1, bounds=[0, 2], var_name='p0')
self.provides['coordinates'].extend([(coord_a, 'a'), (coord_p0, 'p0')])
self.requires['formula_terms'] = dict(a='a', b='b', ps='ps', p0='p0')
with warnings.catch_warnings(record=True) as warn:
warnings.simplefilter('always')
_load_aux_factory(self.engine, self.cube)
self.assertEqual(len(warn), 1)
msg = 'Ignoring atmosphere hybrid sigma pressure scalar ' \
'coordinate {!r} bounds.'.format(coord_p0.name())
self.assertEqual(msg, warn[0].message.message)
def test_longitude_0_360_one_degree(self):
x = np.arange(0.5, 360.5, 1)
y = np.array([50.5, 51.5])
values = np.arange(len(y) * len(x)).reshape((len(y), len(x)))
latitude = DimCoord(y, standard_name='latitude', units='degrees')
longitude = DimCoord(x, standard_name='longitude', units='degrees')
data = make_from_cube(Cube(values, dim_coords_and_dims=[(latitude, 0), (longitude, 1)]))
out_values, out_x, out_y= Generic2DPlot._cube_manipulation(data, longitude.contiguous_bounds(),
latitude.contiguous_bounds())
x_bounds = np.arange(0, 361, 1)
y_bounds = np.array([50, 51, 52])
assert_arrays_equal(out_x, x_bounds)
assert_arrays_equal(out_y, y_bounds)
def setUp(self):
self.sigma = DimCoord(np.arange(5, dtype=np.float) * 10,
long_name='sigma', units='1')
self.eta = AuxCoord(np.arange(4, dtype=np.float).reshape(2, 2),
long_name='eta', units='m')
self.depth = AuxCoord(np.arange(4, dtype=np.float).reshape(2, 2) * 10,
long_name='depth', units='m')
self.depth_c = AuxCoord([15], long_name='depth_c', units='m')
self.nsigma = AuxCoord([3], long_name='nsigma')
self.zlev = DimCoord(np.arange(5, dtype=np.float) * 10,
long_name='zlev', units='m')
self.kwargs = dict(sigma=self.sigma, eta=self.eta, depth=self.depth,
depth_c=self.depth_c, nsigma=self.nsigma,
zlev=self.zlev)
class Test_nearest_neighbour_index__descending(tests.IrisTest):
def setUp(self):
points = [270., 180., 90., 0.]
self.coord = DimCoord(points, circular=False,
units='degrees')
def _test_nearest_neighbour_index(self, target, bounds=False,
circular=False):
_bounds = [[340, 260], [260, 100], [100, 10], [10, -20]]
ext_pnts = [-70, -10, 110, 275, 370]
if bounds:
self.coord.bounds = _bounds
self.coord.circular = circular
results = [self.coord.nearest_neighbour_index(ind) for ind in ext_pnts]
self.assertEqual(results, target)
def test_nobounds(self):
target = [3, 3, 2, 0, 0]
self._test_nearest_neighbour_index(target)
def test_nobounds_circular(self):
target = [0, 3, 2, 0, 3]
self._test_nearest_neighbour_index(target, circular=True)
def test_bounded(self):
target = [3, 3, 1, 0, 0]
self._test_nearest_neighbour_index(target, bounds=True)
def test_bounded_circular(self):
target = [0, 3, 1, 0, 3]
self._test_nearest_neighbour_index(target, bounds=True, circular=True)
def test_lat_lon_decreasing_no_bounds(self):
x = np.array([0.5, -0.5])
y = np.array([51.5, 50.5])
values = np.array([[1, 2], [3, 4]])
latitude = DimCoord(y, standard_name='latitude', units='degrees')
longitude = DimCoord(x, standard_name='longitude', units='degrees')
data = make_from_cube(Cube(values, dim_coords_and_dims=[(latitude, 0), (longitude, 1)]))
out_values, out_x, out_y= Generic2DPlot._cube_manipulation(data, longitude.contiguous_bounds(),
latitude.contiguous_bounds())
expected_x = np.array([1, 0, -1])
expected_y = np.array([52, 51, 50])
expected_v = np.array([[1, 2],
[3, 4]])
assert_arrays_equal(out_x, expected_x)
assert_arrays_equal(out_y, expected_y)
assert_arrays_equal(out_values, expected_v)
def test_writable_points(self):
coord1 = DimCoord(np.arange(5),
bounds=[[0, 1], [1, 2], [2, 3], [3, 4], [4, 5]])
coord2 = coord1.copy()
msg = 'destination is read-only'
with self.assertRaisesRegexp(ValueError, msg):
coord1.points[:] = 0
with self.assertRaisesRegexp(ValueError, msg):
coord2.points[:] = 0
with self.assertRaisesRegexp(ValueError, msg):
coord1.bounds[:] = 0
with self.assertRaisesRegexp(ValueError, msg):
coord2.bounds[:] = 0
def test_formula_terms_p0_bounded(self):
coord_a = DimCoord(np.arange(5))
coord_p0 = DimCoord(1, bounds=[0, 2], var_name="p0")
self.provides["coordinates"].extend([(coord_a, "a"), (coord_p0, "p0")])
self.requires["formula_terms"] = dict(a="a", b="b", ps="ps", p0="p0")
with warnings.catch_warnings(record=True) as warn:
warnings.simplefilter("always")
_load_aux_factory(self.engine, self.cube)
self.assertEqual(len(warn), 1)
msg = "Ignoring atmosphere hybrid sigma pressure scalar " "coordinate {!r} bounds.".format(coord_p0.name())
self.assertEqual(msg, warn[0].message.message)
def _add_iris_coord(cube, name, points, dim, calendar=None):
"""
Add a Coord to a Cube from a Pandas index or columns array.
If no calendar is specified for a time series, Gregorian is assumed.
"""
units = Unit("unknown")
if calendar is None:
calendar = iris.unit.CALENDAR_GREGORIAN
# Convert pandas datetime objects to python datetime obejcts.
if isinstance(points, pandas.tseries.index.DatetimeIndex):
points = np.array([i.to_datetime() for i in points])
# Convert datetime objects to Iris' current datetime representation.
if points.dtype == object:
dt_types = (datetime.datetime, netcdftime.datetime)
if all([isinstance(i, dt_types) for i in points]):
units = Unit("hours since epoch", calendar=calendar)
points = units.date2num(points)
points = np.array(points)
if (np.issubdtype(points.dtype, np.number) and
iris.util.monotonic(points, strict=True)):
coord = DimCoord(points, units=units)
coord.rename(name)
cube.add_dim_coord(coord, dim)
else:
coord = AuxCoord(points, units=units)
coord.rename(name)
cube.add_aux_coord(coord, dim)
def uv_cubes(x=None, y=None):
"""Return u, v cubes with a grid in a rotated pole CRS."""
cs = iris.coord_systems.RotatedGeogCS(grid_north_pole_latitude=37.5,
grid_north_pole_longitude=177.5)
if x is None:
x = np.linspace(311.9, 391.1, 6)
if y is None:
y = np.linspace(-23.6, 24.8, 5)
x2d, y2d = np.meshgrid(x, y)
u = 10 * (2 * np.cos(2 * np.deg2rad(x2d) + 3 * np.deg2rad(y2d + 30)) ** 2)
v = 20 * np.cos(6 * np.deg2rad(x2d))
lon = DimCoord(x, standard_name='grid_longitude', units='degrees',
coord_system=cs)
lat = DimCoord(y, standard_name='grid_latitude', units='degrees',
coord_system=cs)
u_cube = Cube(u, standard_name='x_wind', units='m/s')
v_cube = Cube(v, standard_name='y_wind', units='m/s')
for cube in (u_cube, v_cube):
cube.add_dim_coord(lat.copy(), 0)
cube.add_dim_coord(lon.copy(), 1)
return u_cube, v_cube
class Test_guess_bounds(tests.IrisTest):
def setUp(self):
self.coord = DimCoord(np.array([-160, -120, 0, 30, 150, 170]),
units='degree', standard_name='longitude',
circular=True)
def test_non_circular(self):
self.coord.circular = False
self.coord.guess_bounds()
target = np.array([[-180., -140.], [-140., -60.], [-60., 15.],
[15., 90.], [90., 160.], [160., 180.]])
self.assertArrayEqual(target, self.coord.bounds)
def test_circular_increasing(self):
self.coord.guess_bounds()
target = np.array([[-175., -140.], [-140., -60.], [-60., 15.],
[15., 90.], [90., 160.], [160., 185.]])
self.assertArrayEqual(target, self.coord.bounds)
def test_circular_decreasing(self):
self.coord.points = self.coord.points[::-1]
self.coord.guess_bounds()
target = np.array([[185., 160.], [160., 90.], [90., 15.],
[15., -60.], [-60., -140.], [-140., -175.]])
self.assertArrayEqual(target, self.coord.bounds)
def test_circular_increasing_alt_range(self):
self.coord.points = np.array([10, 30, 90, 150, 210, 220])
self.coord.guess_bounds()
target = np.array([[-65., 20.], [20., 60.], [60., 120.],
[120., 180.], [180., 215.], [215., 295.]])
self.assertArrayEqual(target, self.coord.bounds)
def test_circular_decreasing_alt_range(self):
self.coord.points = np.array([10, 30, 90, 150, 210, 220])[::-1]
self.coord.guess_bounds()
target = np.array([[295, 215], [215, 180], [180, 120], [120, 60],
[60, 20], [20, -65]])
self.assertArrayEqual(target, self.coord.bounds)
def uv_cubes_3d(ref_cube, n_realization=3):
"""
Return 3d u, v cubes with a grid in a rotated pole CRS taken from
the provided 2d cube, by adding a realization dimension
coordinate bound to teh zeroth dimension.
"""
lat = ref_cube.coord('grid_latitude')
lon = ref_cube.coord('grid_longitude')
x2d, y2d = np.meshgrid(lon.points, lat.points)
u = 10 * (2 * np.cos(2 * np.deg2rad(x2d) + 3 * np.deg2rad(y2d + 30)) ** 2)
v = 20 * np.cos(6 * np.deg2rad(x2d))
# Multiply slices by factor to give variation over 0th dim.
factor = np.arange(1, n_realization + 1).reshape(n_realization, 1, 1)
u = factor * u
v = factor * v
realization = DimCoord(np.arange(n_realization), 'realization')
u_cube = Cube(u, standard_name='x_wind', units='m/s')
v_cube = Cube(v, standard_name='y_wind', units='m/s')
for cube in (u_cube, v_cube):
cube.add_dim_coord(realization.copy(), 0)
cube.add_dim_coord(lat.copy(), 1)
cube.add_dim_coord(lon.copy(), 2)
return u_cube, v_cube
def test_real_bounds(self):
coord = DimCoord(self.pts_real, bounds=self.bds_real)
result = coord.core_bounds()
self.assertArraysDoNotShareData(
result, self.bds_real,
'core_bounds() are the same data as the internal array.')
def create_data_object(self, filenames, variable, index_offset=1):
from cis.data_io.hdf_vd import get_data
from cis.data_io.hdf_vd import VDS
from pyhdf.error import HDF4Error
from cis.data_io import hdf_sd
from iris.coords import DimCoord, AuxCoord
from iris.cube import Cube
from cis.data_io.gridded_data import GriddedData
from cis.time_util import cis_standard_time_unit
logging.debug("Creating data object for variable " + variable)
variables = ['Latitude', 'Longitude', "Profile_Time", "Pressure"]
logging.info("Listing coordinates: " + str(variables))
variables.append(variable)
# reading data from files
sdata = {}
for filename in filenames:
try:
sds_dict = hdf_sd.read(filename, variables)
except HDF4Error as e:
raise IOError(str(e))
for var in list(sds_dict.keys()):
utils.add_element_to_list_in_dict(sdata, var, sds_dict[var])
alt_name = "altitude"
logging.info("Additional coordinates: '" + alt_name + "'")
# work out size of data arrays
# the coordinate variables will be reshaped to match that.
# NOTE: This assumes that all Caliop_L1 files have the same altitudes.
# If this is not the case, then the following line will need to be changed
# to concatenate the data from all the files and not just arbitrarily pick
# the altitudes from the first file.
alt_data = get_data(VDS(filenames[0], "Lidar_Data_Altitudes"), True)
alt_coord = DimCoord(alt_data, standard_name='altitude', units='km')
alt_coord.convert_units('m')
lat_data = hdf.read_data(sdata['Latitude'],
self._get_calipso_data)[:, index_offset]
lat_coord = AuxCoord(lat_data, standard_name='latitude')
pres_data = hdf.read_data(sdata['Pressure'], self._get_calipso_data)
pres_coord = AuxCoord(pres_data,
standard_name='air_pressure',
units='hPa')
# longitude
lon = sdata['Longitude']
lon_data = hdf.read_data(lon, self._get_calipso_data)[:, index_offset]
lon_coord = AuxCoord(lon_data, standard_name='longitude')
# profile time, x
time = sdata['Profile_Time']
time_data = hdf.read_data(time, self._get_calipso_data)[:,
index_offset]
time_coord = DimCoord(time_data,
long_name='Profile_Time',
standard_name='time',
units="seconds since 1993-01-01 00:00:00")
time_coord.convert_units(cis_standard_time_unit)
# retrieve data + its metadata
var = sdata[variable]
metadata = hdf.read_metadata(var, "SD")
if variable in MIXED_RESOLUTION_VARIABLES:
logging.warning(
"Using Level 2 resolution profile for mixed resolution variable {}. See CALIPSO "
"documentation for more details".format(variable))
data = hdf.read_data(var, self._get_mixed_resolution_calipso_data)
else:
data = hdf.read_data(var, self._get_calipso_data)
cube = Cube(data,
long_name=metadata.long_name,
units=self.clean_units(metadata.units),
dim_coords_and_dims=[(alt_coord, 1), (time_coord, 0)],
aux_coords_and_dims=[(lat_coord, (0, )),
(lon_coord, (0, )),
(pres_coord, (0, 1))])
gd = GriddedData.make_from_cube(cube)
return gd
def test_valid(self):
coords_and_dims = _convert_scalar_realization_coords(lbrsvd4=21)
self.assertEqual(coords_and_dims,
[(DimCoord([21], standard_name='realization'), None)])
def setUp(self):
self.cube_a = Cube([[1, 2, 3], [4, 5, 6]], long_name='a')
self.x_coord = DimCoord([11, 12, 13], long_name='x')
self.y_coord = DimCoord([21, 22], long_name='y')
self.cube_a.add_dim_coord(self.x_coord, 1)
self.cube_a.add_dim_coord(self.y_coord, 0)
def test_dup(self):
c1 = self.ref_co_1d
c2 = DimCoord([1, 2])
self._coords_eq(c1, c2)
print experiment_id
sys.stdout.flush()
cube = iris.load_cube(fu, glob_tc)
#pdb.set_trace()
mean_list = []
for pressure_cube in (cube.slices(
['time', 'grid_latitude', 'grid_longitude'])):
time_coords = pressure_cube.coord('time')
add_hour_of_day(pressure_cube, time_coords)
pc_time_merge = pressure_cube.aggregated_by('hour', iris.analysis.MEAN)
pc_time_merge.add_dim_coord(DimCoord(points=pc_time_merge.coords('time')[0].bounds[:,0].flatten(),\
long_name='time', standard_name='time',units=pc_time_merge.coords('time')[0].units),0)
mean_list.extend(iris.cube.CubeList([pc_time_merge]))
#pdb.set_trace()
mean = iris.cube.CubeList(mean_list).merge_cube()
mean.coord('grid_latitude').guess_bounds()
mean.coord('grid_longitude').guess_bounds()
if dm == 'temp':
mean.rename('potential_temperature')
mean.units = iris.unit.Unit('K')
if dm == 'sp_hum':
mean.rename('specific_humidity')
def build_spotdata_cube(
data,
name,
units,
altitude,
latitude,
longitude,
wmo_id,
scalar_coords=None,
neighbour_methods=None,
grid_attributes=None,
additional_dims=None,
):
"""
Function to build a spotdata cube with expected dimension and auxiliary
coordinate structure.
It can be used to create spot data cubes. In this case the data is the
spot data values at each site, and the coordinates that describe each site.
It can also be used to create cubes which describe the grid points that are
used to extract each site from a gridded field, for different selection
method. The selection methods are specified by the neighbour_methods
coordinate. The grid_attribute coordinate encapsulates information required
to extract data, for example the x/y indices that identify the grid point
neighbour.
.. See the documentation for examples of these cubes.
.. include:: extended_documentation/spotdata/build_spotdata_cube/
build_spotdata_cube_examples.rst
Args:
data (float or numpy.ndarray):
Float spot data or array of data points from several sites.
The spot index should be the last dimension if the array is
multi-dimensional (see optional additional dimensions below).
name (str):
Cube name (eg 'air_temperature')
units (str):
Cube units (eg 'K')
altitude (float or numpy.ndarray):
Float or 1d array of site altitudes in metres
latitude (float or numpy.ndarray):
Float or 1d array of site latitudes in degrees
longitude (float or numpy.ndarray):
Float or 1d array of site longitudes in degrees
wmo_id (str or list of str):
String or list of site 5-digit WMO identifiers
scalar_coords (list of iris.coords.AuxCoord):
Optional list of iris.coords.AuxCoord instances
neighbour_methods (list of str):
Optional list of neighbour method names, e.g. 'nearest'
grid_attributes (list of str):
Optional list of grid attribute names, e.g. x-index, y-index
additional_dims (list of iris.coords.DimCoord):
Optional list of additional dimensions to preceed the spot data dimension.
Returns:
iris.cube.Cube:
A cube containing the extracted spot data with spot data being the final dimension.
"""
# construct auxiliary coordinates
alt_coord = AuxCoord(altitude, "altitude", units="m")
lat_coord = AuxCoord(latitude, "latitude", units="degrees")
lon_coord = AuxCoord(longitude, "longitude", units="degrees")
id_coord = AuxCoord(wmo_id, long_name="wmo_id", units="no_unit")
aux_coords_and_dims = []
# append scalar coordinates
if scalar_coords is not None:
for coord in scalar_coords:
aux_coords_and_dims.append((coord, None))
# construct dimension coordinates
if np.isscalar(data):
data = np.array([data])
spot_index = DimCoord(
np.arange(data.shape[-1], dtype=np.int32), long_name="spot_index", units="1"
)
dim_coords_and_dims = []
current_dim = 0
if neighbour_methods is not None:
neighbour_methods_coord = DimCoord(
np.arange(len(neighbour_methods), dtype=np.int32),
long_name="neighbour_selection_method",
units="1",
)
neighbour_methods_key = AuxCoord(
neighbour_methods,
long_name="neighbour_selection_method_name",
units="no_unit",
)
dim_coords_and_dims.append((neighbour_methods_coord, current_dim))
aux_coords_and_dims.append((neighbour_methods_key, current_dim))
current_dim += 1
if grid_attributes is not None:
grid_attributes_coord = DimCoord(
np.arange(len(grid_attributes), dtype=np.int32),
long_name="grid_attributes",
units="1",
)
grid_attributes_key = AuxCoord(
grid_attributes, long_name="grid_attributes_key", units="no_unit"
)
dim_coords_and_dims.append((grid_attributes_coord, current_dim))
aux_coords_and_dims.append((grid_attributes_key, current_dim))
current_dim += 1
if additional_dims is not None:
for coord in additional_dims:
dim_coords_and_dims.append((coord, current_dim))
current_dim += 1
dim_coords_and_dims.append((spot_index, current_dim))
for coord in [alt_coord, lat_coord, lon_coord, id_coord]:
aux_coords_and_dims.append((coord, current_dim))
# create output cube
spot_cube = iris.cube.Cube(
data,
long_name=name,
units=units,
dim_coords_and_dims=dim_coords_and_dims,
aux_coords_and_dims=aux_coords_and_dims,
)
# rename to force a standard name to be set if name is valid
spot_cube.rename(name)
return spot_cube
def add_coordinate(
incube,
coord_points,
coord_name,
coord_units=None,
dtype=np.float32,
order=None,
is_datetime=False,
attributes=None,
):
"""
Function to duplicate a sample cube with an additional coordinate to create
a cubelist. The cubelist is merged to create a single cube, which can be
reordered to place the new coordinate in the required position.
Args:
incube (iris.cube.Cube):
Cube to be duplicated.
coord_points (list or numpy.ndarray):
Values for the coordinate.
coord_name (str):
Long name of the coordinate to be added.
coord_units (str):
Coordinate unit required.
dtype (type):
Datatype for coordinate points.
order (list):
Optional list of integers to reorder the dimensions on the new
merged cube. For example, if the new coordinate is required to
be in position 1 on a 4D cube, use order=[1, 0, 2, 3] to swap the
new coordinate position with that of the original leading
coordinate.
is_datetime (bool):
If "true", the leading coordinate points have been given as a
list of datetime objects and need converting. In this case the
"coord_units" argument is overridden and the time points provided
in seconds. The "dtype" argument is overridden and set to int64.
attributes (dict):
Optional coordinate attributes.
Returns:
iris.cube.Cube:
Cube containing an additional dimension coordinate.
"""
# if the coordinate already exists as a scalar coordinate, remove it
cube = incube.copy()
try:
cube.remove_coord(coord_name)
except CoordinateNotFoundError:
pass
# if new coordinate points are provided as datetimes, convert to seconds
if is_datetime:
coord_units = TIME_COORDS["time"].units
dtype = TIME_COORDS["time"].dtype
new_coord_points = [_create_time_point(val) for val in coord_points]
coord_points = new_coord_points
cubes = iris.cube.CubeList([])
for val in coord_points:
temp_cube = cube.copy()
temp_cube.add_aux_coord(
DimCoord(
np.array([val], dtype=dtype),
long_name=coord_name,
units=coord_units,
attributes=attributes,
))
# recalculate forecast period if time or frt have been updated
if ("time" in coord_name and coord_units is not None
and Unit(coord_units).is_time_reference()):
forecast_period = forecast_period_coord(
temp_cube, force_lead_time_calculation=True)
try:
temp_cube.replace_coord(forecast_period)
except CoordinateNotFoundError:
temp_cube.add_aux_coord(forecast_period)
cubes.append(temp_cube)
new_cube = cubes.merge_cube()
if order is not None:
new_cube.transpose(order)
return new_cube
def set_up_variable_cube(
data,
name="air_temperature",
units="K",
spatial_grid="latlon",
time=datetime(2017, 11, 10, 4, 0),
time_bounds=None,
frt=datetime(2017, 11, 10, 0, 0),
realizations=None,
include_scalar_coords=None,
attributes=None,
standard_grid_metadata=None,
):
"""
Set up a cube containing a single variable field with:
- x/y spatial dimensions (equal area or lat / lon)
- optional leading "realization" dimension
- "time", "forecast_reference_time" and "forecast_period" scalar coords
- option to specify additional scalar coordinates
- configurable attributes
Args:
data (numpy.ndarray):
2D (y-x ordered) or 3D (realization-y-x ordered) array of data
to put into the cube.
name (str):
Variable name (standard / long)
units (str):
Variable units
spatial_grid (str):
What type of x/y coordinate values to use. Permitted values are
"latlon" or "equalarea".
time (datetime.datetime):
Single cube validity time
time_bounds (tuple or list of datetime.datetime instances):
Lower and upper bound on time point, if required
frt (datetime.datetime):
Single cube forecast reference time
realizations (list or numpy.ndarray):
List of forecast realizations. If not present, taken from the
leading dimension of the input data array (if 3D).
include_scalar_coords (list):
List of iris.coords.DimCoord or AuxCoord instances of length 1.
attributes (dict):
Optional cube attributes.
standard_grid_metadata (str):
Recognised mosg__model_configuration for which to set up Met
Office standard grid attributes. Should be 'uk_det', 'uk_ens',
'gl_det' or 'gl_ens'.
"""
# construct spatial dimension coordimates
ypoints = data.shape[-2]
xpoints = data.shape[-1]
y_coord, x_coord = construct_xy_coords(ypoints, xpoints, spatial_grid)
# construct realization dimension for 3D data, and dim_coords list
ndims = len(data.shape)
if ndims == 3:
if realizations is not None:
if len(realizations) != data.shape[0]:
raise ValueError(
"Cannot generate {} realizations from data of shape "
"{}".format(len(realizations), data.shape))
realizations = np.array(realizations)
if issubclass(realizations.dtype.type, np.integer):
# expect integer realizations
realizations = realizations.astype(np.int32)
else:
# option needed for percentile & probability cube setup
realizations = realizations.astype(np.float32)
else:
realizations = np.arange(data.shape[0]).astype(np.int32)
realization_coord = DimCoord(realizations, "realization", units="1")
dim_coords = [(realization_coord, 0), (y_coord, 1), (x_coord, 2)]
elif ndims == 2:
dim_coords = [(y_coord, 0), (x_coord, 1)]
else:
raise ValueError(
"Expected 2 or 3 dimensions on input data: got {}".format(ndims))
# construct list of aux_coords_and_dims
scalar_coords = construct_scalar_time_coords(time, time_bounds, frt)
if include_scalar_coords is not None:
for coord in include_scalar_coords:
scalar_coords.append((coord, None))
# set up attributes
cube_attrs = {}
if standard_grid_metadata is not None:
cube_attrs.update(MOSG_GRID_DEFINITION[standard_grid_metadata])
if attributes is not None:
cube_attrs.update(attributes)
# create data cube
cube = iris.cube.Cube(
data,
units=units,
attributes=cube_attrs,
dim_coords_and_dims=dim_coords,
aux_coords_and_dims=scalar_coords,
)
cube.rename(name)
# don't allow unit tests to set up invalid cubes
check_mandatory_standards(cube)
return cube
def ensemble_member(cube, field):
"""Add an 'ensemble member' coord to the cube, if present in the field."""
ensemble_member = getattr(field, "ensemble_member")
if ensemble_member != field.int_mdi:
cube.add_aux_coord(DimCoord(ensemble_member, "realization"))
def experiment(cube, field):
"""Add an 'experiment number' to the cube, if present in the field."""
if field.experiment_num != field.int_mdi:
cube.add_aux_coord(
DimCoord(field.experiment_num, long_name="experiment_number"))
def test_pseudo_level(self):
pseudo_level = 123
coord = DimCoord(pseudo_level, long_name='pseudo_level', units='1')
self.cube.add_aux_coord(coord)
lbuser5_produced = _pp_save_ppfield_values(self.cube).lbuser[4]
self.assertEqual(pseudo_level, lbuser5_produced)
def get_time_coord(time_point):
"""Time coordinate."""
return DimCoord([time_point],
var_name='time',
standard_name='time',
units='days since 6453-2-1')
def to_cube(self):
"""Return a new :class:`~iris.cube.Cube` from this ABFField."""
cube = iris.cube.Cube(self.data)
# Name.
if self.format.lower() == "abf":
cube.rename("leaf_area_index")
elif self.format.lower() == "abl":
cube.rename("FAPAR")
else:
msg = "Unknown ABF/ABL format: {}".format(self.format)
raise iris.exceptions.TranslationError(msg)
cube.units = "%"
# Grid.
step = 1.0 / 12.0
llcs = GeogCS(semi_major_axis=6378137.0, semi_minor_axis=6356752.31424)
x_coord = DimCoord(np.arange(X_SIZE) * step + (step / 2) - 180,
standard_name="longitude",
units="degrees",
coord_system=llcs)
y_coord = DimCoord(np.arange(Y_SIZE) * step + (step / 2) - 90,
standard_name="latitude",
units="degrees",
coord_system=llcs)
x_coord.guess_bounds()
y_coord.guess_bounds()
cube.add_dim_coord(x_coord, 1)
cube.add_dim_coord(y_coord, 0)
# Time.
if self.period == "a":
start = 1
end = 15
elif self.period == "b":
start = 16
end = calendar.monthrange(self.year, self.month)[1]
else:
raise iris.exceptions.TranslationError("Unknown period: "
"{}".format(self.period))
start = datetime.date(year=self.year, month=self.month, day=start)
end = datetime.date(year=self.year, month=self.month, day=end)
# Convert to "days since 0001-01-01".
# Iris will have proper datetime objects in the future.
# This step will not be necessary.
start = start.toordinal() - 1
end = end.toordinal() - 1
# TODO: Should we put the point in the middle of the period instead?
cube.add_aux_coord(
AuxCoord(start,
standard_name="time",
units="days since 0001-01-01",
bounds=[start, end]))
# TODO: Do they only come from Boston?
# Attributes.
cube.attributes["source"] = "Boston University"
return cube
class TestConcatenate(unittest.TestCase):
"""Tests for :func:`esmvalcore.preprocessor._io.concatenate`."""
def setUp(self):
"""Start tests."""
self._model_coord = DimCoord([1., 2.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
self.raw_cubes = []
self._add_cube([1., 2.], [1., 2.])
self._add_cube([3., 4.], [3., 4.])
self._add_cube([5., 6.], [5., 6.])
def _add_cube(self, data, coord):
self.raw_cubes.append(
Cube(data,
var_name='sample',
dim_coords_and_dims=((self._model_coord.copy(coord), 0), )))
def test_concatenate(self):
"""Test concatenation of two cubes."""
concatenated = _io.concatenate(self.raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1, 2, 3, 4, 5, 6]))
def test_concatenate_noop(self):
"""Test concatenation of a single cube."""
concatenated = _io.concatenate([self.raw_cubes[0]])
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1, 2]))
def test_concatenate_with_overlap(self):
"""Test concatenation of time overalapping cubes."""
self._add_cube([6.5, 7.5], [6., 7.])
concatenated = _io.concatenate(self.raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points,
np.array([1., 2., 3., 4., 5., 6., 7.]))
np.testing.assert_array_equal(concatenated.data,
np.array([1., 2., 3., 4., 5., 6.5, 7.5]))
def test_concatenate_with_overlap_2(self):
"""Test a more generic case."""
self._add_cube([65., 75., 100.], [9., 10., 11.])
self._add_cube([65., 75., 100.], [7., 8., 9.])
concatenated = _io.concatenate(self.raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points,
np.array([1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11.]))
def test_concatenate_with_overlap_3(self):
"""Test a more generic case."""
self._add_cube([65., 75., 100.], [9., 10., 11.])
self._add_cube([65., 75., 100., 100., 100., 112.],
[7., 8., 9., 10., 11., 12.])
concatenated = _io.concatenate(self.raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points,
np.array([1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12.]))
def test_concatenate_with_overlap_same_start(self):
"""Test a more generic case."""
cube1 = self.raw_cubes[0]
raw_cubes = [
cube1,
]
time_coord = DimCoord([1., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
raw_cubes.append(
Cube([33., 55.],
var_name='sample',
dim_coords_and_dims=((time_coord, 0), )))
concatenated = _io.concatenate(raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1., 7.]))
raw_cubes.reverse()
concatenated = _io.concatenate(raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1., 7.]))
def test_concatenate_with_iris_exception(self):
"""Test a more generic case."""
time_coord_1 = DimCoord([1.5, 5., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube1 = Cube([33., 55., 77.],
var_name='sample',
dim_coords_and_dims=((time_coord_1, 0), ))
time_coord_2 = DimCoord([1., 5., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube2 = Cube([33., 55., 77.],
var_name='sample',
dim_coords_and_dims=((time_coord_2, 0), ))
cubes_single_ovlp = [cube2, cube1]
with self.assertRaises(ConcatenateError):
_io.concatenate(cubes_single_ovlp)
def test_concatenate_no_time_coords(self):
"""Test a more generic case."""
time_coord_1 = DimCoord([1.5, 5., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube1 = Cube([33., 55., 77.],
var_name='sample',
dim_coords_and_dims=((time_coord_1, 0), ))
ap_coord_2 = DimCoord([1., 5., 7.],
var_name='air_pressure',
standard_name='air_pressure',
units='m',
attributes={'positive': 'down'})
cube2 = Cube([33., 55., 77.],
var_name='sample',
dim_coords_and_dims=((ap_coord_2, 0), ))
with self.assertRaises(ValueError):
_io.concatenate([cube1, cube2])
def test_concatenate_with_order(self):
"""Test a more generic case."""
time_coord_1 = DimCoord([1.5, 2., 5., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube1 = Cube([33., 44., 55., 77.],
var_name='sample',
dim_coords_and_dims=((time_coord_1, 0), ))
time_coord_2 = DimCoord([1., 2., 5., 7., 100.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube2 = Cube([33., 44., 55., 77., 1000.],
var_name='sample',
dim_coords_and_dims=((time_coord_2, 0), ))
cubes_ordered = [cube2, cube1]
concatenated = _io.concatenate(cubes_ordered)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1., 2., 5., 7.,
100.]))
cubes_reverse = [cube1, cube2]
concatenated = _io.concatenate(cubes_reverse)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1., 2., 5., 7.,
100.]))
def test_fail_on_calendar_concatenate_with_overlap(self):
"""Test fail of concatenation with overlap."""
time_coord = DimCoord([3., 7000.],
var_name='time',
standard_name='time',
units=Unit('days since 1950-01-01',
calendar='360_day'))
self.raw_cubes.append(
Cube([33., 55.],
var_name='sample',
dim_coords_and_dims=((time_coord, 0), )))
with self.assertRaises(TypeError):
_io.concatenate(self.raw_cubes)
def test_fail_on_units_concatenate_with_overlap(self):
"""Test fail of concatenation with overlap."""
time_coord_1 = DimCoord([3., 7000.],
var_name='time',
standard_name='time',
units=Unit('days since 1950-01-01',
calendar='360_day'))
time_coord_2 = DimCoord([3., 9000.],
var_name='time',
standard_name='time',
units=Unit('days since 1950-01-01',
calendar='360_day'))
time_coord_3 = DimCoord([3., 9000.],
var_name='time',
standard_name='time',
units=Unit('days since 1850-01-01',
calendar='360_day'))
raw_cubes = []
raw_cubes.append(
Cube([33., 55.],
var_name='sample',
dim_coords_and_dims=((time_coord_1, 0), )))
raw_cubes.append(
Cube([33., 55.],
var_name='sample',
dim_coords_and_dims=((time_coord_2, 0), )))
raw_cubes.append(
Cube([33., 55.],
var_name='sample',
dim_coords_and_dims=((time_coord_3, 0), )))
with self.assertRaises(ValueError):
_io.concatenate(raw_cubes)
def test_fail_metadata_differs(self):
"""Test exception raised if two cubes have different metadata."""
self.raw_cubes[0].units = 'm'
self.raw_cubes[1].units = 'K'
with self.assertRaises(ValueError):
_io.concatenate(self.raw_cubes)
def test_fix_attributes(self):
"""Test fixing attributes for concatenation."""
identical_attrs = {
'int': 42,
'float': 3.1415,
'bool': True,
'str': 'Hello, world',
'list': [1, 1, 2, 3, 5, 8, 13],
'tuple': (1, 2, 3, 4, 5),
'dict': {
1: 'one',
2: 'two',
3: 'three'
},
'nparray': np.arange(42),
}
differing_attrs = [
{
'new_int': 0,
'new_str': 'hello',
'new_nparray': np.arange(3),
'mix': np.arange(2),
},
{
'new_int': 1,
'new_str': 'world',
'new_list': [1, 1, 2],
'new_tuple': (0, 1),
'new_dict': {
0: 'zero',
},
'mix': {
1: 'one',
},
},
{
'new_str': '!',
'new_list': [1, 1, 2, 3],
'new_tuple': (1, 2, 3),
'new_dict': {
0: 'zeroo',
1: 'one',
},
'new_nparray': np.arange(2),
'mix': False,
},
]
resulting_attrs = {
'new_int': '0;1',
'new_str': 'hello;world;!',
'new_nparray': '[0 1 2];[0 1]',
'new_list': '[1, 1, 2];[1, 1, 2, 3]',
'new_tuple': '(0, 1);(1, 2, 3)',
'new_dict': "{0: 'zero'};{0: 'zeroo', 1: 'one'}",
'mix': "[0 1];{1: 'one'};False",
}
resulting_attrs.update(identical_attrs)
for idx in range(3):
self.raw_cubes[idx].attributes = identical_attrs
self.raw_cubes[idx].attributes.update(differing_attrs[idx])
_io._fix_cube_attributes(self.raw_cubes) # noqa
for cube in self.raw_cubes:
self.assertEqual(cube.attributes, resulting_attrs)
def convert(year, month, ndays, remove=False):
"""
Now need to:
- convert to q, RH , e, tw, DPD
- aggregate to daily averages
- regrid to 1by1 gridboxes
"""
MDI = -999.
# Set up null_cube with desired gridding format to use as a template
# Does this have to have the same time dimensions?
# ndays = np.int(p_cube.data[:,0,0] / 24)
time = DimCoord(np.arange(ndays * 24), standard_name='time', units='hours')
latitude = DimCoord(
np.linspace(89.5, -89.5, 180), # DIFF FROM OTHER ERA5 DOWNLOAD
# latitude = DimCoord(np.linspace(90, -90, 181),
standard_name='latitude',
long_name='gridbox centre latitude',
units='degrees_north')
longitude = DimCoord(
np.linspace(-179.5, 179.5, 360), # DIFF FROM OTHER ERA5 DOWNLOAD
# longitude = DimCoord(np.linspace(0, 359, 360),
standard_name='longitude',
long_name='gridbox centre longitude',
units='degrees_east')
null_cube = Cube(np.zeros((ndays * 24, 180, 360), np.float32),
dim_coords_and_dims=[(time, 0), (latitude, 1),
(longitude, 2)])
print('Check null_cube for new grid')
# pdb.set_trace()
# START OF LSM************************************************
# read in land_sea_mask
variable = "land_sea_mask"
lsm_cube = iris.load(
os.path.join(DataLoc,
"{}{:02d}_hourly_{}.nc".format(year, month, variable)))
#pdb.set_trace()
# convert from list to cube
lsm_cube = lsm_cube[0] #
# REgrid to 1by1 degree - cash the source, template, gridding type for later use - faster
regridder = iris.analysis.Linear().regridder(lsm_cube, null_cube)
lsm_cube_1by1 = regridder(lsm_cube)
print('Check lsm_cube_1by1 for new grid')
# pdb.set_trace()#
# remove old cube
lsm_cube = 0
lsm_cube_1by1 = lsm_cube_1by1[0, :, :]
# lsm_cube_1by1_field = lsm_cube_1by1.extract(iris.Constraint(time=0))
lsm_cube_1by1.units = "1"
print(lsm_cube_1by1)
print('Check lsm_cube_1by1 for land_sea_mask')
#pdb.set_trace()
# output
iris.save(lsm_cube_1by1,
os.path.join(
DataLoc,
"{}{:02d}_hourly_{}_ERA5.nc".format(year, month, variable)),
zlib=True)
print('Check lsm_cube_1by1 output')
# pdb.set_trace()
# END OF LSM************************************************************
# remove input files
if remove:
for variable in [
"2m_temperature", "2m_dewpoint_temperature", "surface_pressure"
]:
# for variable in ["2m_temperature", "2m_dewpoint_temperature", "surface_pressure", "land_sea_mask"]:
os.remove(
os.path.join(
DataLoc,
"{}{:02d}_hourly_{}.nc".format(year, month, variable)))
return # combine
def test_dtype_change(self):
c1 = self.ref_co_1d
c2 = DimCoord(np.array([1.0, 2.0], dtype=np.float64))
self._coords_eq(c1, c2)
def test_formula_terms_no_p0_term(self):
coord_a = DimCoord(np.arange(5), units='Pa')
self.provides['coordinates'].append((coord_a, 'a'))
self.requires['formula_terms'] = dict(a='a', b='b', ps='ps')
_load_aux_factory(self.engine, self.cube)
self._check_no_delta()
def setUp(self):
self.ref_co_1d = DimCoord([1, 2])
self.co_a = DimCoord([1, 2], long_name='a')
self.msg_metadata = "Coords 'a' have different metadata"
def sample_mesh_cube(nomesh_faces=None,
n_z=2,
with_parts=False,
**meshcoord_kwargs):
"""
Create a 2d test cube with 1 'normal' and 1 unstructured dimension (with a Mesh).
Result contains : dimcoords for both dims; an auxcoord on the unstructured dim; 2 mesh-coords.
By default, the mesh is provided by :func:`sample_mesh`, so coordinates and connectivity are not realistic.
Kwargs:
* nomesh_faces (int or None):
If set, don't add MeshCoords, so dim 1 is just a plain anonymous dim.
Set its length to the given value.
* n_z (int):
Length of the 'normal' dim. If 0, it is *omitted*.
* with_parts (bool):
If set, return all the constituent component coords
* meshcoord_kwargs (dict):
Extra controls passed to :func:`sample_meshcoord` for MeshCoord creation, to allow user-specified
location/mesh. The 'axis' key is not available, as we always add both an 'x' and 'y' MeshCOord.
Returns:
* cube : if with_parts not set
* (cube, parts) : if with_parts is set
'parts' is (mesh, dim0-dimcoord, dim1-dimcoord, dim1-auxcoord, x-meshcoord [or None], y-meshcoord [or None]).
"""
nomesh = nomesh_faces is not None
if nomesh:
mesh = None
n_faces = nomesh_faces
else:
mesh = meshcoord_kwargs.pop("mesh", None)
if mesh is None:
mesh = sample_mesh()
meshx, meshy = (sample_meshcoord(axis=axis,
mesh=mesh,
**meshcoord_kwargs)
for axis in ("x", "y"))
n_faces = meshx.shape[0]
mesh_dimco = DimCoord(np.arange(n_faces),
long_name="i_mesh_face",
units="1")
auxco_x = AuxCoord(np.zeros(n_faces), long_name="mesh_face_aux", units="1")
zco = DimCoord(np.arange(n_z), long_name="level", units=1)
cube = Cube(np.zeros((n_z, n_faces)), long_name="mesh_phenom")
cube.add_dim_coord(zco, 0)
if nomesh:
mesh_coords = []
else:
mesh_coords = [meshx, meshy]
cube.add_dim_coord(mesh_dimco, 1)
for co in mesh_coords + [auxco_x]:
cube.add_aux_coord(co, 1)
if not with_parts:
result = cube
else:
if nomesh:
meshx, meshy = None, None
parts = (mesh, zco, mesh_dimco, auxco_x, meshx, meshy)
result = (cube, parts)
return result
def setUp(self):
"""
Create a cube containing a regular lat-lon grid.
Data is striped horizontally,
e.g.
1 1 1 1 1 1
1 1 1 1 1 1
2 2 2 2 2 2
2 2 2 2 2 2
3 3 3 3 3 3
3 3 3 3 3 3
"""
data = np.ones((12, 12))
data[0:4, :] = 1
data[4:8, :] = 2
data[8:, :] = 3
latitudes = np.linspace(-90, 90, 12)
longitudes = np.linspace(-180, 180, 12)
latitude = DimCoord(latitudes,
standard_name='latitude',
units='degrees',
coord_system=GeogCS(6371229.0))
longitude = DimCoord(longitudes,
standard_name='longitude',
units='degrees',
coord_system=GeogCS(6371229.0),
circular=True)
# Use time of 2017-02-17 06:00:00
time = DimCoord([1487311200],
standard_name='time',
units=cf_units.Unit(
'seconds since 1970-01-01 00:00:00',
calendar='gregorian'))
long_time_coord = DimCoord(list(range(1487311200, 1487397600, 3600)),
standard_name='time',
units=cf_units.Unit(
'seconds since 1970-01-01 00:00:00',
calendar='gregorian'))
time_dt = dt(2017, 2, 17, 6, 0)
time_extract = Constraint(
time=lambda cell: cell.point == PartialDateTime(
time_dt.year, time_dt.month, time_dt.day, time_dt.hour))
cube = Cube(data.reshape((1, 12, 12)),
long_name="air_temperature",
dim_coords_and_dims=[(time, 0), (latitude, 1),
(longitude, 2)],
units="K")
long_cube = Cube(np.arange(3456).reshape(24, 12, 12),
long_name="air_temperature",
dim_coords_and_dims=[(long_time_coord, 0),
(latitude, 1), (longitude, 2)],
units="K")
orography = Cube(np.ones((12, 12)),
long_name="surface_altitude",
dim_coords_and_dims=[(latitude, 0), (longitude, 1)],
units="m")
# Western half of grid at altitude 0, eastern half at 10.
# Note that the pressure_on_height_levels data is left unchanged,
# so it is as if there is a sharp front running up the grid with
# differing pressures on either side at equivalent heights above
# the surface (e.g. east 1000hPa at 0m AMSL, west 1000hPa at 10m AMSL).
# So there is higher pressure in the west.
orography.data[0:10] = 0
orography.data[10:] = 10
ancillary_data = {}
ancillary_data['orography'] = orography
additional_data = {}
adlist = CubeList()
adlist.append(cube)
additional_data['air_temperature'] = adlist
data_indices = [list(data.nonzero()[0]), list(data.nonzero()[1])]
self.cube = cube
self.long_cube = long_cube
self.data = data
self.time_dt = time_dt
self.time_extract = time_extract
self.data_indices = data_indices
self.ancillary_data = ancillary_data
self.additional_data = additional_data
def test_1d_contiguous(self):
# Test that a 1D coordinate which is contiguous does not fail.
coord = DimCoord([1, 3, 5], bounds=[[0, 2], [2, 4], [4, 6]])
data = np.array([278, 300, 282])
_check_bounds_contiguity_and_mask(coord, data, atol=1e-3)
def test_1d_discontigous_masked(self):
# Test a 1D coordinate which is discontiguous but masked at
# discontiguities.
coord = DimCoord([1, 3, 5], bounds=[[0, 2], [2, 4], [5, 6]])
data = ma.array(np.array([278, 300, 282]), mask=[0, 1, 0])
_check_bounds_contiguity_and_mask(coord, data, atol=1e-3)
def test_no_bounds(self):
coord = DimCoord(self.pts_real)
result = coord.core_bounds()
self.assertIsNone(result)
def test_formula_terms_no_a_term(self):
coord_p0 = DimCoord(10, units='1')
self.provides['coordinates'].append((coord_p0, 'p0'))
self.requires['formula_terms'] = dict(a='p0', b='b', ps='ps')
_load_aux_factory(self.engine, self.cube)
self._check_no_delta()
def create_data_object(self, filenames, variable, index_offset=1):
from cis.data_io.hdf_vd import get_data
from cis.data_io.hdf_vd import VDS
from pyhdf.error import HDF4Error
from cis.data_io import hdf_sd
from iris.coords import DimCoord, AuxCoord
from iris.cube import Cube, CubeList
from cis.data_io.gridded_data import GriddedData
from cis.time_util import cis_standard_time_unit
from datetime import datetime
from iris.util import new_axis
import numpy as np
logging.debug("Creating data object for variable " + variable)
variables = ["Pressure_Mean"]
logging.info("Listing coordinates: " + str(variables))
variables.append(variable)
# reading data from files
sdata = {}
for filename in filenames:
try:
sds_dict = hdf_sd.read(filename, variables)
except HDF4Error as e:
raise IOError(str(e))
for var in list(sds_dict.keys()):
utils.add_element_to_list_in_dict(sdata, var, sds_dict[var])
# work out size of data arrays
# the coordinate variables will be reshaped to match that.
# NOTE: This assumes that all Caliop_L1 files have the same altitudes.
# If this is not the case, then the following line will need to be changed
# to concatenate the data from all the files and not just arbitrarily pick
# the altitudes from the first file.
alt_data = self._get_calipso_data(
hdf_sd.HDF_SDS(filenames[0], 'Altitude_Midpoint'))[0, :]
alt_coord = DimCoord(alt_data, standard_name='altitude', units='km')
alt_coord.convert_units('m')
lat_data = self._get_calipso_data(
hdf_sd.HDF_SDS(filenames[0], 'Latitude_Midpoint'))[0, :]
lat_coord = DimCoord(lat_data,
standard_name='latitude',
units='degrees_north')
lon_data = self._get_calipso_data(
hdf_sd.HDF_SDS(filenames[0], 'Longitude_Midpoint'))[0, :]
lon_coord = DimCoord(lon_data,
standard_name='longitude',
units='degrees_east')
cubes = CubeList()
for f in filenames:
t = get_data(VDS(f, "Nominal_Year_Month"), True)[0]
time_data = cis_standard_time_unit.date2num(
datetime(int(t[0:4]), int(t[4:6]), 15))
time_coord = AuxCoord(time_data,
long_name='Profile_Time',
standard_name='time',
units=cis_standard_time_unit)
# retrieve data + its metadata
var = sdata[variable]
metadata = hdf.read_metadata(var, "SD")
data = self._get_calipso_data(hdf_sd.HDF_SDS(f, variable))
pres_data = self._get_calipso_data(
hdf_sd.HDF_SDS(f, 'Pressure_Mean'))
pres_coord = AuxCoord(pres_data,
standard_name='air_pressure',
units='hPa')
if data.ndim == 2:
# pres_coord = new_axis()
cube = Cube(data,
long_name=metadata.long_name or variable,
units=self.clean_units(metadata.units),
dim_coords_and_dims=[(lat_coord, 0),
(lon_coord, 1)],
aux_coords_and_dims=[(time_coord, ())])
# Promote the time scalar coord to a length one dimension
new_cube = new_axis(cube, 'time')
cubes.append(new_cube)
elif data.ndim == 3:
# pres_coord = new_axis()
cube = Cube(data,
long_name=metadata.long_name or variable,
units=self.clean_units(metadata.units),
dim_coords_and_dims=[(lat_coord, 0),
(lon_coord, 1),
(alt_coord, 2)],
aux_coords_and_dims=[(time_coord, ())])
# Promote the time scalar coord to a length one dimension
new_cube = new_axis(cube, 'time')
# Then add the (extended) pressure coord so that it is explicitly a function of time
new_cube.add_aux_coord(pres_coord[np.newaxis, ...],
(0, 1, 2, 3))
cubes.append(new_cube)
else:
raise ValueError(
"Unexpected number of dimensions for CALIOP data: {}".
format(data.ndim))
# Concatenate the cubes from each file into a single GriddedData object
gd = GriddedData.make_from_cube(cubes.concatenate_cube())
return gd
def test_formula_terms_p0_non_scalar(self):
coord_p0 = DimCoord(np.arange(5))
self.provides['coordinates'].append((coord_p0, 'p0'))
self.requires['formula_terms'] = dict(p0='p0')
with self.assertRaises(ValueError):
_load_aux_factory(self.engine, self.cube)
def test_lazy_points(self):
lazy_data = self.pts_lazy
coord = DimCoord(lazy_data)
result = coord.core_points()
self.assertEqualRealArraysAndDtypes(result, self.pts_real)
def _make_cube(data, aux_coord=True, dim_coord=True, dtype=None):
"""
Create a 3d synthetic test cube.
"""
if dtype is None:
dtype = np.dtype('int8')
if not isinstance(data, np.ndarray):
data = np.empty(data, dtype=dtype)
z, y, x = data.shape
# Create the cube.
cm = CellMethod(method='mean',
coords='time',
intervals='20 minutes',
comments=None)
kwargs = dict(standard_name='air_temperature',
long_name='Air Temperature',
var_name='ta',
units='K',
attributes=dict(cube='attribute'),
cell_methods=(cm, ))
cube = iris.cube.Cube(data, **kwargs)
# Create a synthetic test vertical coordinate.
if dim_coord:
cube.add_dim_coord(_make_vcoord(z, dtype=dtype), 0)
# Create a synthetic test latitude coordinate.
data = np.arange(y, dtype=dtype) + 1
cs = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS)
kwargs = dict(standard_name='latitude',
long_name='Latitude',
var_name='lat',
units='degrees_north',
attributes=dict(latitude='attribute'),
coord_system=cs)
ycoord = DimCoord(data, **kwargs)
if data.size > 1:
ycoord.guess_bounds()
cube.add_dim_coord(ycoord, 1)
# Create a synthetic test longitude coordinate.
data = np.arange(x, dtype=dtype) + 1
kwargs = dict(standard_name='longitude',
long_name='Longitude',
var_name='lon',
units='degrees_east',
attributes=dict(longitude='attribute'),
coord_system=cs)
xcoord = DimCoord(data, **kwargs)
if data.size > 1:
xcoord.guess_bounds()
cube.add_dim_coord(xcoord, 2)
# Create a synthetic test 2d auxiliary coordinate
# that spans the vertical dimension.
if aux_coord:
data = np.arange(np.prod((z, y)), dtype=dtype).reshape(z, y)
kwargs = dict(standard_name=None,
long_name='Pressure Slice',
var_name='aplev',
units='hPa',
attributes=dict(positive='down'),
coord_system=None)
zycoord = AuxCoord(data, **kwargs)
cube.add_aux_coord(zycoord, (0, 1))
return cube
def test_1d_not_checked(self):
# Test a 1D coordinate, which is not checked as atol is not set.
coord = DimCoord([1, 3, 5], bounds=[[0, 2], [2, 4], [5, 6]])
data = np.array([278, 300, 282])
# Make sure contiguity checking doesn't throw an error
_check_bounds_contiguity_and_mask(coord, data)
