def test_example_roundtrips(self):
# Check that save-and-loadback leaves Iris data unchanged,
# for data derived from each UGRID example CDL.
for ex_name, filepath in self.example_names_paths.items():
target_ncfile_path = str(self.temp_dir / f"{ex_name}.nc")
# Create a netcdf file from the test CDL.
check_call(f"ncgen {filepath} -k4 -o {target_ncfile_path}",
shell=True)
# Fill in blank data-variables.
_add_standard_data(target_ncfile_path)
# Load the original as Iris data
with PARSE_UGRID_ON_LOAD.context():
orig_cubes = iris.load(target_ncfile_path)
if "ex4" in ex_name:
# Discard the extra formula terms component cubes
# Saving these does not do what you expect
orig_cubes = orig_cubes.extract("datavar")
# Save-and-load-back to compare the Iris saved result.
resave_ncfile_path = str(self.temp_dir / f"{ex_name}_resaved.nc")
iris.save(orig_cubes, resave_ncfile_path)
with PARSE_UGRID_ON_LOAD.context():
savedloaded_cubes = iris.load(resave_ncfile_path)
# This should match the original exactly
# ..EXCEPT for our inability to compare meshes.
for orig, reloaded in zip(orig_cubes, savedloaded_cubes):
for cube in (orig, reloaded):
# Remove conventions attributes, which may differ.
cube.attributes.pop("Conventions", None)
# Remove var-names, which may differ.
cube.var_name = None
# Compare the mesh contents (as we can't compare actual meshes)
self.assertEqual(orig.location, reloaded.location)
orig_mesh = orig.mesh
reloaded_mesh = reloaded.mesh
self.assertEqual(orig_mesh.all_coords,
reloaded_mesh.all_coords)
self.assertEqual(
orig_mesh.all_connectivities,
reloaded_mesh.all_connectivities,
)
# Index the cubes to replace meshes with meshcoord-derived aux coords.
# This needs [:0] on the mesh dim, so do that on all dims.
keys = tuple([slice(0, None)] * orig.ndim)
orig = orig[keys]
reloaded = reloaded[keys]
# Resulting cubes, with collapsed mesh, should be IDENTICAL.
self.assertEqual(orig, reloaded)
def test_non_nc(self):
log_regex = r"Ignoring non-NetCDF file:.*"
with self.assertLogs(logger, level="INFO", msg_regex=log_regex):
with PARSE_UGRID_ON_LOAD.context():
meshes = load_meshes(
tests.get_data_path(["PP", "simple_pp", "global.pp"]))
self.assertDictEqual({}, meshes)
def test_no_mesh(self):
with PARSE_UGRID_ON_LOAD.context():
cube_list = load(
tests.get_data_path([
"NetCDF", "unstructured_grid", "theta_nodal_not_ugrid.nc"
]))
self.assertTrue(all([cube.mesh is None for cube in cube_list]))
def test_no_mesh(self):
with PARSE_UGRID_ON_LOAD.context():
meshes = load_meshes(
tests.get_data_path([
"NetCDF", "unstructured_grid", "theta_nodal_not_ugrid.nc"
]))
self.assertDictEqual({}, meshes)
def common_test(self, file_name, mesh_var_name):
with PARSE_UGRID_ON_LOAD.context():
mesh = load_mesh(
tests.get_data_path(["NetCDF", "unstructured_grid",
file_name]))
# NOTE: cannot use CML tests as this isn't supported for non-Cubes.
self.assertIsInstance(mesh, Mesh)
self.assertEqual(mesh.var_name, mesh_var_name)
def test_with_data(self):
nc_path = cdl_to_nc(self.ref_cdl)
with PARSE_UGRID_ON_LOAD.context():
meshes = load_meshes(nc_path)
files = list(meshes.keys())
self.assertEqual(1, len(files))
file_meshes = meshes[files[0]]
self.assertEqual(1, len(file_meshes))
mesh = file_meshes[0]
self.assertEqual("mesh", mesh.var_name)
def test_missing_mesh(self):
ref_cdl = self.ref_cdl.replace('face_data:mesh = "mesh"',
'face_data:mesh = "mesh2"')
nc_path = cdl_to_nc(ref_cdl)
# No error when mesh handling not activated.
_ = list(load_cubes(nc_path))
with PARSE_UGRID_ON_LOAD.context():
log_regex = r"File does not contain mesh.*"
with self.assertLogs(logger, level="DEBUG", msg_regex=log_regex):
_ = list(load_cubes(nc_path))
def test_var_name(self):
second_cdl, second_name = self.add_second_mesh()
cdls = [self.ref_cdl, second_cdl]
nc_paths = [cdl_to_nc(cdl) for cdl in cdls]
with PARSE_UGRID_ON_LOAD.context():
meshes = load_meshes(nc_paths, second_name)
files = list(meshes.keys())
self.assertEqual(1, len(files))
file_meshes = meshes[files[0]]
self.assertEqual(1, len(file_meshes))
self.assertEqual(second_name, file_meshes[0].var_name)
def test_multi_meshes(self):
ref_cdl, second_name = self.add_second_mesh()
nc_path = cdl_to_nc(ref_cdl)
with PARSE_UGRID_ON_LOAD.context():
meshes = load_meshes(nc_path)
files = list(meshes.keys())
self.assertEqual(1, len(files))
file_meshes = meshes[files[0]]
self.assertEqual(2, len(file_meshes))
mesh_names = [mesh.var_name for mesh in file_meshes]
self.assertIn("mesh", mesh_names)
self.assertIn(second_name, mesh_names)
def test_mixed_sources(self):
url = "http://foo"
file = TMP_DIR / f"{uuid4()}.nc"
file.touch()
glob = f"{TMP_DIR}/*.nc"
with PARSE_UGRID_ON_LOAD.context():
_ = load_meshes([url, glob])
file_uris = [
call[0][0] for call in self.format_agent_mock.call_args_list
]
for source in (url, Path(file).name):
self.assertIn(source, file_uris)
def setUpClass(cls):
cls.ref_cdl = """
netcdf mesh_test {
dimensions:
node = 3 ;
face = 1 ;
vertex = 3 ;
levels = 2 ;
variables:
int mesh ;
mesh:cf_role = "mesh_topology" ;
mesh:topology_dimension = 2 ;
mesh:node_coordinates = "node_x node_y" ;
mesh:face_coordinates = "face_x face_y" ;
mesh:face_node_connectivity = "face_nodes" ;
float node_x(node) ;
node_x:standard_name = "longitude" ;
float node_y(node) ;
node_y:standard_name = "latitude" ;
float face_x(face) ;
face_x:standard_name = "longitude" ;
float face_y(face) ;
face_y:standard_name = "latitude" ;
int face_nodes(face, vertex) ;
face_nodes:cf_role = "face_node_connectivity" ;
face_nodes:start_index = 0 ;
int levels(levels) ;
float node_data(levels, node) ;
node_data:coordinates = "node_x node_y" ;
node_data:location = "node" ;
node_data:mesh = "mesh" ;
float face_data(levels, face) ;
face_data:coordinates = "face_x face_y" ;
face_data:location = "face" ;
face_data:mesh = "mesh" ;
data:
mesh = 0;
node_x = 0., 2., 1.;
node_y = 0., 0., 1.;
face_x = 0.5;
face_y = 0.5;
face_nodes = 0, 1, 2;
levels = 1, 2;
node_data = 0., 0., 0.;
face_data = 0.;
}
"""
cls.nc_path = cdl_to_nc(cls.ref_cdl)
with PARSE_UGRID_ON_LOAD.context():
cls.mesh_cubes = list(load_cubes(cls.nc_path))
def test_no_mesh(self):
cdl_lines = self.ref_cdl.split("\n")
cdl_lines = filter(
lambda line: all(
[s not in line for s in (':mesh = "mesh"', "mesh_topology")]),
cdl_lines,
)
ref_cdl = "\n".join(cdl_lines)
nc_path = cdl_to_nc(ref_cdl)
with PARSE_UGRID_ON_LOAD.context():
meshes = load_meshes(nc_path)
self.assertDictEqual({}, meshes)
def test_no_data(self):
cdl_lines = self.ref_cdl.split("\n")
cdl_lines = filter(lambda line: ':mesh = "mesh"' not in line,
cdl_lines)
ref_cdl = "\n".join(cdl_lines)
nc_path = cdl_to_nc(ref_cdl)
with PARSE_UGRID_ON_LOAD.context():
meshes = load_meshes(nc_path)
files = list(meshes.keys())
self.assertEqual(1, len(files))
file_meshes = meshes[files[0]]
self.assertEqual(1, len(file_meshes))
mesh = file_meshes[0]
self.assertEqual("mesh", mesh.var_name)
def ugrid_load(uris, constraints=None, callback=None):
# TODO: remove constraint once files no longer have orphan connectivities.
orphan_connectivities = (
"Mesh2d_half_levels_edge_face_links",
"Mesh2d_half_levels_face_links",
"Mesh2d_half_levels_face_edges",
"Mesh2d_full_levels_edge_face_links",
"Mesh2d_full_levels_face_links",
"Mesh2d_full_levels_face_edges",
)
filter_orphan_connectivities = Constraint(
cube_func=lambda cube: cube.var_name not in orphan_connectivities)
if constraints is None:
constraints = filter_orphan_connectivities
else:
if not isinstance(constraints, Iterable):
constraints = [constraints]
constraints.append(filter_orphan_connectivities)
with PARSE_UGRID_ON_LOAD.context():
return load(uris, constraints, callback)
def test_example_result_cdls(self):
# Snapshot the result of saving the example cases.
for ex_name, filepath in self.example_names_paths.items():
target_ncfile_path = str(self.temp_dir / f"{ex_name}.nc")
# Create a netcdf file from the test CDL.
check_call(f"ncgen {filepath} -k4 -o {target_ncfile_path}",
shell=True)
# Fill in blank data-variables.
_add_standard_data(target_ncfile_path)
# Load as Iris data
with PARSE_UGRID_ON_LOAD.context():
cubes = iris.load(target_ncfile_path)
# Re-save, to check the save behaviour.
resave_ncfile_path = str(self.temp_dir / f"{ex_name}_resaved.nc")
iris.save(cubes, resave_ncfile_path)
# Check the output against a CDL snapshot.
refdir_relpath = (
"integration/experimental/ugrid_save/TestBasicSave/")
reffile_name = str(Path(filepath).name).replace(".nc", ".cdl")
reffile_path = refdir_relpath + reffile_name
self.assertCDL(resave_ncfile_path, reference_filename=reffile_path)
def common(ret_val):
self.load_meshes_mock.return_value = ret_val
with self.assertRaisesRegex(ValueError, "Expecting 1 mesh.*"):
with PARSE_UGRID_ON_LOAD.context():
_ = load_mesh([])
def create_synthetic_test_cube(self, **create_kwargs):
file_path = self.create_synthetic_file(**create_kwargs)
with PARSE_UGRID_ON_LOAD.context():
cube = load_cube(file_path)
return cube
def test_http(self):
url = "http://foo"
with PARSE_UGRID_ON_LOAD.context():
_ = load_meshes(url)
self.format_agent_mock.assert_called_with(url, None)
def test_returns_mesh(self):
with PARSE_UGRID_ON_LOAD.context():
mesh = load_mesh([])
self.assertEqual(mesh, "mesh")
def test_multi_files(self):
files_count = 3
nc_paths = [cdl_to_nc(self.ref_cdl) for _ in range(files_count)]
with PARSE_UGRID_ON_LOAD.context():
meshes = load_meshes(nc_paths)
self.assertEqual(files_count, len(meshes))
def test_invalid_scheme(self):
with self.assertRaisesRegex(ValueError,
"Iris cannot handle the URI scheme:.*"):
with PARSE_UGRID_ON_LOAD.context():
_ = load_meshes("foo://bar")
def test_calls_load_meshes(self):
args = [("file_1", "file_2"), "my_var_name"]
with PARSE_UGRID_ON_LOAD.context():
_ = load_mesh(args)
self.assertTrue(self.load_meshes_mock.called_with(args))