def test_concatenate_with_order(self):
"""Test a more generic case."""
time_coord_1 = DimCoord([1.5, 2., 5., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube1 = Cube([33., 44., 55., 77.],
var_name='sample',
dim_coords_and_dims=((time_coord_1, 0), ))
time_coord_2 = DimCoord([1., 2., 5., 7., 100.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube2 = Cube([33., 44., 55., 77., 1000.],
var_name='sample',
dim_coords_and_dims=((time_coord_2, 0), ))
cubes_ordered = [cube2, cube1]
concatenated = _io.concatenate(cubes_ordered)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1., 2., 5., 7.,
100.]))
cubes_reverse = [cube1, cube2]
concatenated = _io.concatenate(cubes_reverse)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1., 2., 5., 7.,
100.]))
def test_fail_on_units_concatenate_with_overlap(self):
"""Test fail of concatenation with overlap."""
time_coord_1 = DimCoord([3., 7000.],
var_name='time',
standard_name='time',
units=Unit('days since 1950-01-01',
calendar='360_day'))
time_coord_2 = DimCoord([3., 9000.],
var_name='time',
standard_name='time',
units=Unit('days since 1950-01-01',
calendar='360_day'))
time_coord_3 = DimCoord([3., 9000.],
var_name='time',
standard_name='time',
units=Unit('days since 1850-01-01',
calendar='360_day'))
raw_cubes = []
raw_cubes.append(
Cube([33., 55.],
var_name='sample',
dim_coords_and_dims=((time_coord_1, 0), )))
raw_cubes.append(
Cube([33., 55.],
var_name='sample',
dim_coords_and_dims=((time_coord_2, 0), )))
raw_cubes.append(
Cube([33., 55.],
var_name='sample',
dim_coords_and_dims=((time_coord_3, 0), )))
with self.assertRaises(ValueError):
_io.concatenate(raw_cubes)
def cmorization(in_dir, out_dir, cfg, _):
"""Cmorization func call."""
cmor_table = cfg['cmor_table']
glob_attrs = cfg['attributes']
# run the cmorization
for var, vals in cfg['variables'].items():
var_info = cmor_table.get_variable(vals['mip'], var)
glob_attrs['mip'] = vals['mip']
raw_info = {'name': vals['raw'], 'file': vals['file']}
inpfile = os.path.join(in_dir, cfg['filename'])
logger.info("CMORizing var %s from file type %s", var, inpfile)
years = range(1982, 2020)
months = ["0" + str(mo) for mo in range(1, 10)] + ["10", "11", "12"]
for year in years:
monthly_cubes = []
for month in months:
raw_info['file'] = inpfile.format(year=year, month=month)
logger.info("CMORizing var %s from file type %s", var,
raw_info['file'])
cube = extract_variable(var_info, raw_info, glob_attrs, year)
monthly_cubes.append(cube)
yearly_cube = concatenate(monthly_cubes)
save_variable(yearly_cube,
var,
out_dir,
glob_attrs,
unlimited_dimensions=['time'])
def test_concatenate_with_overlap_2(self):
"""Test a more generic case."""
self._add_cube([65., 75., 100.], [9., 10., 11.])
self._add_cube([65., 75., 100.], [7., 8., 9.])
concatenated = _io.concatenate(self.raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points,
np.array([1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11.]))
def test_concatenate_with_overlap(self):
"""Test concatenation of time overalapping cubes."""
self._add_cube([6.5, 7.5], [6., 7.])
concatenated = _io.concatenate(self.raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points,
np.array([1., 2., 3., 4., 5., 6., 7.]))
np.testing.assert_array_equal(concatenated.data,
np.array([1., 2., 3., 4., 5., 6.5, 7.5]))
def test_concatenate_no_time_coords(self):
"""Test a more generic case."""
time_coord_1 = DimCoord([1.5, 5., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube1 = Cube([33., 55., 77.],
var_name='sample',
dim_coords_and_dims=((time_coord_1, 0), ))
ap_coord_2 = DimCoord([1., 5., 7.],
var_name='air_pressure',
standard_name='air_pressure',
units='m',
attributes={'positive': 'down'})
cube2 = Cube([33., 55., 77.],
var_name='sample',
dim_coords_and_dims=((ap_coord_2, 0), ))
with self.assertRaises(ValueError):
_io.concatenate([cube1, cube2])
def test_concatenate_with_iris_exception(self):
"""Test a more generic case."""
time_coord_1 = DimCoord([1.5, 5., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube1 = Cube([33., 55., 77.],
var_name='sample',
dim_coords_and_dims=((time_coord_1, 0), ))
time_coord_2 = DimCoord([1., 5., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
cube2 = Cube([33., 55., 77.],
var_name='sample',
dim_coords_and_dims=((time_coord_2, 0), ))
cubes_single_ovlp = [cube2, cube1]
with self.assertRaises(ConcatenateError):
_io.concatenate(cubes_single_ovlp)
def test_concatenation_with_aux_factory(real_hybrid_pressure_cube_list):
"""Test actual concatenation of a cube with a derived coordinate."""
concatenated = _io.concatenate(real_hybrid_pressure_cube_list)
air_pressure_coord = concatenated.coord('air_pressure')
expected_coord = AuxCoord(
[[[[1.0]]], [[[1.0]]]],
bounds=[[[[[-50000.0, 150002.0]]]], [[[[-50000.0, 150002.0]]]]],
standard_name='air_pressure',
units='Pa',
)
assert air_pressure_coord == expected_coord
def test_concatenate_with_overlap_same_start(self):
"""Test a more generic case."""
cube1 = self.raw_cubes[0]
raw_cubes = [
cube1,
]
time_coord = DimCoord([1., 7.],
var_name='time',
standard_name='time',
units='days since 1950-01-01')
raw_cubes.append(
Cube([33., 55.],
var_name='sample',
dim_coords_and_dims=((time_coord, 0), )))
concatenated = _io.concatenate(raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1., 7.]))
raw_cubes.reverse()
concatenated = _io.concatenate(raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1., 7.]))
def _load_fx(fx_info, check_level):
"""Load and CMOR-check fx variables."""
fx_cubes = iris.cube.CubeList()
for fx_file in fx_info['filename']:
loaded_cube = load(fx_file, callback=concatenate_callback)
short_name = fx_info['short_name']
project = fx_info['project']
dataset = fx_info['dataset']
mip = fx_info['mip']
freq = fx_info['frequency']
loaded_cube = fix_metadata(loaded_cube,
short_name=short_name,
project=project,
dataset=dataset,
mip=mip,
frequency=freq,
check_level=check_level)
fx_cubes.append(loaded_cube[0])
fx_cube = concatenate(fx_cubes)
fx_cube = cmor_check_metadata(fx_cube,
cmor_table=project,
mip=mip,
short_name=short_name,
frequency=freq,
check_level=check_level)
fx_cube = fix_data(fx_cube,
short_name=short_name,
project=project,
dataset=dataset,
mip=mip,
frequency=freq,
check_level=check_level)
fx_cube = cmor_check_data(fx_cube,
cmor_table=project,
mip=mip,
short_name=fx_cube.var_name,
frequency=freq,
check_level=check_level)
return fx_cube
def test_fail_metadata_differs(self):
"""Test exception raised if two cubes have different metadata."""
self.raw_cubes[0].units = 'm'
self.raw_cubes[1].units = 'K'
with self.assertRaises(ValueError):
_io.concatenate(self.raw_cubes)
def test_concatenate_noop(self):
"""Test concatenation of a single cube."""
concatenated = _io.concatenate([self.raw_cubes[0]])
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1, 2]))
def test_concatenate_empty_cubes(self):
"""Test concatenation with empty :class:`iris.cube.CubeList`."""
empty_cubes = CubeList([])
result = _io.concatenate(empty_cubes)
assert result is empty_cubes
def test_concatenate(self):
"""Test concatenation of two cubes."""
concatenated = _io.concatenate(self.raw_cubes)
np.testing.assert_array_equal(
concatenated.coord('time').points, np.array([1, 2, 3, 4, 5, 6]))
def test_fail_with_duplicates(self):
"""Test exception raised if two cubes are overlapping."""
self.raw_cubes.append(self.raw_cubes[0].copy())
with self.assertRaises(ConcatenateError):
_io.concatenate(self.raw_cubes)
def test_concatenate(self):
"""Test concatenation of two cubes."""
concatenated = _io.concatenate(self.raw_cubes)
self.assertTrue(
(concatenated.coord('coord').points == np.array([1, 2, 3, 4, 5,
6])).all())
