def test_dask_distributed_read_netcdf_integration_test(loop, engine, autoclose,
nc_format):
if engine == 'h5netcdf' and autoclose:
pytest.skip('h5netcdf does not support autoclose')
if nc_format not in NC_FORMATS[engine]:
pytest.skip('invalid format for engine')
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6}
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename:
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
original = create_test_data()
original.to_netcdf(filename, engine=engine, format=nc_format)
with xr.open_dataset(filename,
chunks=chunks,
engine=engine,
autoclose=autoclose) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop, consolidated, compute):
if consolidated:
pytest.importorskip("zarr", minversion="2.2.1.dev2")
write_kwargs = {"consolidated": True}
read_kwargs = {"backend_kwargs": {"consolidated": True}}
else:
write_kwargs = read_kwargs = {}
chunks = {"dim1": 4, "dim2": 3, "dim3": 5}
with cluster() as (s, [a, b]):
with Client(s["address"], loop=loop):
original = create_test_data().chunk(chunks)
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS,
suffix=".zarrc") as filename:
maybe_futures = original.to_zarr(filename,
compute=compute,
**write_kwargs)
if not compute:
maybe_futures.compute()
with xr.open_dataset(filename,
chunks="auto",
engine="zarr",
**read_kwargs) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_read_netcdf_integration_test(loop, engine, autoclose,
nc_format):
if engine == 'h5netcdf' and autoclose:
pytest.skip('h5netcdf does not support autoclose')
if nc_format not in NC_FORMATS[engine]:
pytest.skip('invalid format for engine')
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6}
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename:
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
original = create_test_data()
original.to_netcdf(filename, engine=engine, format=nc_format)
with xr.open_dataset(filename,
chunks=chunks,
engine=engine,
autoclose=autoclose) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_integration_test(loop, engine):
with cluster() as (s, _):
with distributed.Client(s['address'], loop=loop):
original = create_test_data()
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename:
original.to_netcdf(filename, engine=engine)
with xr.open_dataset(filename, chunks=3, engine=engine) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop):
with cluster() as (s, _):
with distributed.Client(s['address'], loop=loop):
original = create_test_data()
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename:
original.to_zarr(filename)
with xr.open_zarr(filename) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_integration_test(loop, engine):
with cluster() as (s, _):
with distributed.Client(('127.0.0.1', s['port']), loop=loop):
original = create_test_data()
with create_tmp_file() as filename:
original.to_netcdf(filename, engine=engine)
restored = xr.open_dataset(filename, chunks=3, engine=engine)
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_integration_test(loop, engine):
with cluster() as (s, _):
with distributed.Client(('127.0.0.1', s['port']), loop=loop):
original = create_test_data()
with create_tmp_file() as filename:
original.to_netcdf(filename, engine=engine)
restored = xr.open_dataset(filename, chunks=3, engine=engine)
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_netcdf_integration_test_not_implemented(loop, engine):
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6}
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename:
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
original = create_test_data().chunk(chunks)
with raises_regex(NotImplementedError, 'distributed'):
original.to_netcdf(filename, engine=engine)
def test_dask_distributed_netcdf_integration_test_not_implemented(loop, engine):
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6}
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename:
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
original = create_test_data().chunk(chunks)
with raises_regex(NotImplementedError, 'distributed'):
original.to_netcdf(filename, engine=engine)
def test_dask_distributed_zarr_integration_test(loop):
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 5}
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
original = create_test_data().chunk(chunks)
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS,
suffix='.zarr') as filename:
original.to_zarr(filename)
with xr.open_zarr(filename) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop):
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 5}
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
original = create_test_data().chunk(chunks)
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS,
suffix='.zarr') as filename:
original.to_zarr(filename)
with xr.open_zarr(filename) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop, consolidated, compute):
if consolidated:
pytest.importorskip('zarr', minversion="2.2.1.dev2")
write_kwargs = dict(consolidated=True)
read_kwargs = dict(consolidated=True)
else:
write_kwargs = read_kwargs = {}
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 5}
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop):
original = create_test_data().chunk(chunks)
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS,
suffix='.zarrc') as filename:
maybe_futures = original.to_zarr(filename, compute=compute,
**write_kwargs)
if not compute:
maybe_futures.compute()
with xr.open_zarr(filename, **read_kwargs) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_netcdf_roundtrip(monkeypatch, loop,
engine, autoclose, nc_format):
monkeypatch.setenv('HDF5_USE_FILE_LOCKING', 'FALSE')
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6}
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename:
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
original = create_test_data().chunk(chunks)
original.to_netcdf(filename, engine=engine, format=nc_format)
with xr.open_dataset(filename,
chunks=chunks,
engine=engine,
autoclose=autoclose) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop, consolidated, compute):
if consolidated:
zarr = pytest.importorskip('zarr', minversion="2.2.1.dev2")
write_kwargs = dict(consolidated=True)
read_kwargs = dict(consolidated=True)
else:
write_kwargs = read_kwargs = {}
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 5}
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
original = create_test_data().chunk(chunks)
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS,
suffix='.zarrc') as filename:
maybe_futures = original.to_zarr(filename, compute=compute,
**write_kwargs)
if not compute:
maybe_futures.compute()
with xr.open_zarr(filename, **read_kwargs) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
def test_dask_distributed_netcdf_roundtrip(monkeypatch, loop,
engine, autoclose, nc_format):
monkeypatch.setenv('HDF5_USE_FILE_LOCKING', 'FALSE')
chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6}
with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename:
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as c:
original = create_test_data().chunk(chunks)
original.to_netcdf(filename, engine=engine, format=nc_format)
with xr.open_dataset(filename,
chunks=chunks,
engine=engine,
autoclose=autoclose) as restored:
assert isinstance(restored.var1.data, da.Array)
computed = restored.compute()
assert_allclose(original, computed)
