def test_read_mds_no_meta(all_mds_datadirs):
from xmitgcm.utils import read_mds
dirname, expected = all_mds_datadirs
shape = expected['shape']
ny, nx = shape[-2:]
if len(shape) == 4:
# we have an llc
nz, nface = shape[:2]
ny = nx * nface
else:
nz = shape[0]
dtype = expected['dtype']
shape_2d = (ny, nx)
shape_3d = shape_2d if nz == 1 else (nz, ) + shape_2d
prefixes = {'XC': shape_2d, 'hFacC': shape_3d}
for prefix, shape in prefixes.items():
basename = os.path.join(dirname, prefix)
with hide_file(dirname, prefix + '.meta'):
# can't read without specifying shape and dtype
with pytest.raises(IOError) as ioe:
res = read_mds(basename)
res = read_mds(basename, shape=shape, dtype=dtype)
assert isinstance(res, dict)
assert prefix in res
# should be dask by default
assert isinstance(res[prefix], dask.array.core.Array)
assert res[prefix].shape == shape
def test_read_mds(all_mds_datadirs):
"""Check that we can read mds data from .meta / .data pairs"""
dirname, expected = all_mds_datadirs
from xmitgcm.utils import read_mds
prefix = 'XC'
basename = os.path.join(dirname, prefix)
res = read_mds(basename)
assert isinstance(res, dict)
assert prefix in res
# should be dask by default
assert isinstance(res[prefix], dask.array.core.Array)
# try some options
res = read_mds(basename, force_dict=False, dask_delayed=False)
assert isinstance(res, np.memmap)
res = read_mds(basename, force_dict=False, use_mmap=False,
dask_delayed=False)
assert isinstance(res, np.ndarray)
# make sure endianness works
testval = res.newbyteorder('<')[0,0]
res_endian = read_mds(basename, force_dict=False, use_mmap=False,
endian='<', dask_delayed=False)
val_endian = res_endian[0,0]
np.testing.assert_allclose(testval, val_endian)
# try reading with iteration number
prefix = 'T'
basename = os.path.join(dirname, prefix)
iternum = expected['test_iternum']
res = read_mds(basename, iternum=iternum)
assert prefix in res
def test_read_mds_tokens(mds_datadirs_with_diagnostics):
from xmitgcm.utils import read_mds
dirname, expected = mds_datadirs_with_diagnostics
diagnostics = expected['diagnostics']
prefix = diagnostics[0]
basename = os.path.join(dirname, prefix)
iternum = expected['test_iternum']
data = read_mds(basename, iternum=iternum)
dask_keys = set()
for varname, da in data.items():
keys = list(da.dask.keys())
for k in keys:
token = k[0]
if 'mds' in token:
dask_keys.add(token)
assert len(dask_keys) == len(data)
def test_read_mds(all_mds_datadirs):
"""Check that we can read mds data from .meta / .data pairs"""
dirname, expected = all_mds_datadirs
from xmitgcm.utils import read_mds
prefix = 'XC'
basename = os.path.join(dirname, prefix)
# should be dask by default
res = read_mds(basename)
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], dask.array.core.Array)
# try some options
res = read_mds(basename, use_dask=False)
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], np.memmap)
res = read_mds(basename, use_dask=False, use_mmap=False)
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], np.ndarray)
res = read_mds(basename, chunks="2D")
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], dask.array.core.Array)
res = read_mds(basename, chunks="2D", use_dask=False)
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], np.memmap)
res = read_mds(basename, chunks="2D", use_dask=False, use_mmap=False)
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], np.ndarray)
# test the extra_metadata
if expected['geometry'] == 'llc':
emeta = {
'has_faces':
True,
'ny':
13 * 90,
'nx':
90,
'ny_facets': [3 * 90, 3 * 90, 90, 3 * 90, 3 * 90],
'face_facets': [0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 4],
'facet_orders': ['C', 'C', 'C', 'F', 'F'],
'face_offsets': [0, 1, 2, 0, 1, 2, 0, 0, 1, 2, 0, 1, 2],
'transpose_face': [
False, False, False, False, False, False, False, True, True,
True, True, True, True
]
}
else:
emeta = None
res = read_mds(basename,
chunks="2D",
use_dask=False,
use_mmap=False,
extra_metadata=emeta)
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], np.ndarray)
# make sure endianness works
res = read_mds(basename, use_dask=False, use_mmap=False)
testval = res[prefix].newbyteorder('<')[0, 0]
res_endian = read_mds(basename, use_mmap=False, endian='<', use_dask=False)
val_endian = res_endian[prefix][0, 0]
np.testing.assert_allclose(testval, val_endian)
# try reading with iteration number
prefix = 'T'
basename = os.path.join(dirname, prefix)
iternum = expected['test_iternum']
res = read_mds(basename, iternum=iternum)
assert prefix in res
assert isinstance(res[prefix], dask.array.core.Array)
# try some options
res = read_mds(basename, iternum=iternum, use_dask=False)
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], np.memmap)
res = read_mds(basename, iternum=iternum, use_dask=False, use_mmap=False)
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], np.ndarray)
res = read_mds(basename, iternum=iternum, chunks="2D")
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], dask.array.core.Array)
res = read_mds(basename, iternum=iternum, chunks="2D", use_dask=False)
assert isinstance(res, dict)
assert prefix in res
print(type(res[prefix]))
assert isinstance(res[prefix], np.ndarray) # should be memmap
res = read_mds(basename,
iternum=iternum,
chunks="2D",
use_dask=False,
use_mmap=False)
assert isinstance(res, dict)
assert prefix in res
assert isinstance(res[prefix], np.ndarray)
# check fails with bad nx, ny
if expected['geometry'] == 'llc':
emeta.update({'ny': 13 * 270, 'nx': 270})
with pytest.raises(AssertionError):
res = read_mds(basename,
iternum=iternum,
chunks="2D",
use_dask=False,
use_mmap=False,
extra_metadata=emeta)
