def test_io_xarray(tmpdir_factory, new_xarray, backend, complevel):
tmp_dir = tmpdir_factory.mktemp('tmp')
# save the object to disk
file_name = "test_file.h5"
data_inst = new_xarray
data_name = new_xarray.name
file_inst = PhiDataFile(str(tmp_dir / file_name), "w")
assert file_name in os.listdir(str(tmp_dir))
file_inst.write_xarray(data_inst, location='/data/',
complevel=complevel, backend=backend)
# read the object from disk
file_inst_2 = PhiDataFile(str(tmp_dir / file_name), "r")
# make sure the compression settings were correctly used
with file_inst_2.open('r', backend='pytables') as fh:
h5_filters = getattr(fh.root.data, data_name).filters
assert h5_filters.fletcher32
assert h5_filters.complevel == complevel
data_inst_2 = file_inst_2.read_xarray('/data/' + data_name)
# check that list_xarray() returns data_name
assert "/data/"+data_name in file_inst.list_xarray()
# check that we retrieve the original object
xr.testing.assert_identical(data_inst, data_inst_2)
shutil.rmtree(str(tmp_dir))
def test_read_xarray_mmap(tmpdir_factory, example_dataset, backend):
fh = PhiDataFile(example_dataset)
args = {'location': '/data/test_data', 'backend': backend}
X = fh.read_xarray(**args)
assert isinstance(X.values, np.ndarray)
with pytest.raises(ValueError, match='mmap=True is not compatible with'):
fh.read_xarray(chunks=(4, 2), mmap=True, **args)
with pytest.raises(ValueError, match='mmap=True is not compatible with'):
fh.read_xarray(index=(4, 2), mmap=True, **args)
X_m = fh.read_xarray(mmap=True, **args)
for attr in ['coords', 'dims', 'name', 'values']:
assert hasattr(X_m, attr)
assert not isinstance(X_m, np.ndarray)
assert_array_equal(X_m.values[:], X.values)
assert X_m.name == X.name
assert X_m.dims == X.dims
for key in X.coords:
assert_array_equal(X_m.coords[key], X.coords[key])
fh._fh.close()
def test_io_xarray_preserves_attrs(tmpdir_factory, new_xarray, backend):
tmp_dir = tmpdir_factory.mktemp('tmp')
file_name = "test_file.h5"
X = new_xarray
X.attrs['a'] = 'test'
file_inst = PhiDataFile(str(tmp_dir / file_name), "w")
file_inst.write_xarray(X, backend=backend)
X_2 = file_inst.read_xarray('/data/' + X.name)
# check attributes, order doesn't matter
assert dict(X.attrs) == dict(X_2.attrs)
shutil.rmtree(str(tmp_dir))
def example_dataset(tmpdir_factory, new_xarray):
tmp_dir = tmpdir_factory.mktemp('tmp')
data_inst = new_xarray
fname = str(tmp_dir / 'test.h5')
file_inst = PhiDataFile(fname, "w")
file_inst.write_xarray(data_inst, location='/data/')
yield fname
# clean up the temporary folder
try:
shutil.rmtree(str(tmp_dir))
except PermissionError:
warnings.warn('PermissionError: Failed to remove temporary file '
'%s on Windows' % str(tmp_dir))
def test_io_dask_support(tmpdir_factory, example_dataset, backend):
da = pytest.importorskip('dask.array')
fh = PhiDataFile(example_dataset)
with pytest.raises(ValueError, match='cannot be used together'):
fh.read_xarray('/data/test_data',
index=(2, 2),
chunks=(4, 2),
backend=backend)
X1 = fh.read_xarray('/data/test_data', backend=backend)
assert X1.chunks is None
assert not isinstance(X1.data, da.Array)
chunks = (50, 20, 30)
X2 = fh.read_xarray('/data/test_data', chunks=chunks, backend=backend)
assert X2.chunks is not None
assert isinstance(X2.data, da.Array)
# check that we can perform computations
assert X1.values.sum() == pytest.approx(X2.sum().compute().values)
assert fh._fh is not None
# reading an xarray in dask mode (when chunks are provided) does not
# close the file handler. Her we do it manually.
fh._fh.close()
def test_io_dask_support(tmpdir_factory, example_dataset):
da = pytest.importorskip('dask.array')
fh = PhiDataFile(example_dataset)
with pytest.raises(ValueError, match='cannot be used together'):
fh.read_xarray('/data/test_data', index=(2, 2), chunks=(4, 2))
X1 = fh.read_xarray('/data/test_data')
assert X1.chunks is None
assert not isinstance(X1.data, da.Array)
chunks = (50, 20, 30)
X2 = fh.read_xarray('/data/test_data', chunks=chunks)
assert X2.chunks is not None
assert isinstance(X2.data, da.Array)
# check that we can perform computations
assert X1.values.sum() == pytest.approx(X2.sum().compute().values)
Nx, Ny, Nw = 5, 5, 3
X = xr.DataArray(rng.rand(Nx, Ny, Nw),
dims=['x', 'y', 'w'],
coords={'x': np.arange(Nx), 'y': np.arange(Ny),
'w': np.linspace(2.3, 2.5, Nw)},
attrs={'scale_units': {'x': 'px', 'y': 'px', 'w': 'PHz.rad'}},
name='X')
#############################################################################
#
# The ``xarray.DataArray`` that we will use as an example, is as follows,
print(X)
fh = PhiDataFile('test_file.h5', 'w', force=True)
fh.write_xarray(X)
#############################################################################
#
# Now we will load this data back,
fh = PhiDataFile('test_file.h5', 'r')
X_out = fh.read_xarray('/data/X')
print(X)