def test_dask_distributed_read_netcdf_integration_test(loop, engine, autoclose, nc_format): if engine == 'h5netcdf' and autoclose: pytest.skip('h5netcdf does not support autoclose') if nc_format not in NC_FORMATS[engine]: pytest.skip('invalid format for engine') chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: original = create_test_data() original.to_netcdf(filename, engine=engine, format=nc_format) with xr.open_dataset(filename, chunks=chunks, engine=engine, autoclose=autoclose) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop, consolidated, compute): if consolidated: pytest.importorskip("zarr", minversion="2.2.1.dev2") write_kwargs = {"consolidated": True} read_kwargs = {"backend_kwargs": {"consolidated": True}} else: write_kwargs = read_kwargs = {} chunks = {"dim1": 4, "dim2": 3, "dim3": 5} with cluster() as (s, [a, b]): with Client(s["address"], loop=loop): original = create_test_data().chunk(chunks) with create_tmp_file(allow_cleanup_failure=ON_WINDOWS, suffix=".zarrc") as filename: maybe_futures = original.to_zarr(filename, compute=compute, **write_kwargs) if not compute: maybe_futures.compute() with xr.open_dataset(filename, chunks="auto", engine="zarr", **read_kwargs) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_read_netcdf_integration_test(loop, engine, autoclose, nc_format): if engine == 'h5netcdf' and autoclose: pytest.skip('h5netcdf does not support autoclose') if nc_format not in NC_FORMATS[engine]: pytest.skip('invalid format for engine') chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: original = create_test_data() original.to_netcdf(filename, engine=engine, format=nc_format) with xr.open_dataset(filename, chunks=chunks, engine=engine, autoclose=autoclose) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_integration_test(loop, engine): with cluster() as (s, _): with distributed.Client(s['address'], loop=loop): original = create_test_data() with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: original.to_netcdf(filename, engine=engine) with xr.open_dataset(filename, chunks=3, engine=engine) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop): with cluster() as (s, _): with distributed.Client(s['address'], loop=loop): original = create_test_data() with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: original.to_zarr(filename) with xr.open_zarr(filename) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_integration_test(loop, engine): with cluster() as (s, _): with distributed.Client(('127.0.0.1', s['port']), loop=loop): original = create_test_data() with create_tmp_file() as filename: original.to_netcdf(filename, engine=engine) restored = xr.open_dataset(filename, chunks=3, engine=engine) assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_integration_test(loop, engine): with cluster() as (s, _): with distributed.Client(('127.0.0.1', s['port']), loop=loop): original = create_test_data() with create_tmp_file() as filename: original.to_netcdf(filename, engine=engine) restored = xr.open_dataset(filename, chunks=3, engine=engine) assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_netcdf_integration_test_not_implemented(loop, engine): chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: original = create_test_data().chunk(chunks) with raises_regex(NotImplementedError, 'distributed'): original.to_netcdf(filename, engine=engine)
def test_dask_distributed_netcdf_integration_test_not_implemented(loop, engine): chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: original = create_test_data().chunk(chunks) with raises_regex(NotImplementedError, 'distributed'): original.to_netcdf(filename, engine=engine)
def test_dask_distributed_zarr_integration_test(loop): chunks = {'dim1': 4, 'dim2': 3, 'dim3': 5} with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: original = create_test_data().chunk(chunks) with create_tmp_file(allow_cleanup_failure=ON_WINDOWS, suffix='.zarr') as filename: original.to_zarr(filename) with xr.open_zarr(filename) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop): chunks = {'dim1': 4, 'dim2': 3, 'dim3': 5} with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: original = create_test_data().chunk(chunks) with create_tmp_file(allow_cleanup_failure=ON_WINDOWS, suffix='.zarr') as filename: original.to_zarr(filename) with xr.open_zarr(filename) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop, consolidated, compute): if consolidated: pytest.importorskip('zarr', minversion="2.2.1.dev2") write_kwargs = dict(consolidated=True) read_kwargs = dict(consolidated=True) else: write_kwargs = read_kwargs = {} chunks = {'dim1': 4, 'dim2': 3, 'dim3': 5} with cluster() as (s, [a, b]): with Client(s['address'], loop=loop): original = create_test_data().chunk(chunks) with create_tmp_file(allow_cleanup_failure=ON_WINDOWS, suffix='.zarrc') as filename: maybe_futures = original.to_zarr(filename, compute=compute, **write_kwargs) if not compute: maybe_futures.compute() with xr.open_zarr(filename, **read_kwargs) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_netcdf_roundtrip(monkeypatch, loop, engine, autoclose, nc_format): monkeypatch.setenv('HDF5_USE_FILE_LOCKING', 'FALSE') chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: original = create_test_data().chunk(chunks) original.to_netcdf(filename, engine=engine, format=nc_format) with xr.open_dataset(filename, chunks=chunks, engine=engine, autoclose=autoclose) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_zarr_integration_test(loop, consolidated, compute): if consolidated: zarr = pytest.importorskip('zarr', minversion="2.2.1.dev2") write_kwargs = dict(consolidated=True) read_kwargs = dict(consolidated=True) else: write_kwargs = read_kwargs = {} chunks = {'dim1': 4, 'dim2': 3, 'dim3': 5} with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: original = create_test_data().chunk(chunks) with create_tmp_file(allow_cleanup_failure=ON_WINDOWS, suffix='.zarrc') as filename: maybe_futures = original.to_zarr(filename, compute=compute, **write_kwargs) if not compute: maybe_futures.compute() with xr.open_zarr(filename, **read_kwargs) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)
def test_dask_distributed_netcdf_roundtrip(monkeypatch, loop, engine, autoclose, nc_format): monkeypatch.setenv('HDF5_USE_FILE_LOCKING', 'FALSE') chunks = {'dim1': 4, 'dim2': 3, 'dim3': 6} with create_tmp_file(allow_cleanup_failure=ON_WINDOWS) as filename: with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as c: original = create_test_data().chunk(chunks) original.to_netcdf(filename, engine=engine, format=nc_format) with xr.open_dataset(filename, chunks=chunks, engine=engine, autoclose=autoclose) as restored: assert isinstance(restored.var1.data, da.Array) computed = restored.compute() assert_allclose(original, computed)