def test_fs_driver_backends():
from dask.bytes.hdfs3 import HDFS3HadoopFileSystem
from dask.bytes.pyarrow import PyArrowHadoopFileSystem
fs1, token1 = get_fs('hdfs')
assert isinstance(fs1, PyArrowHadoopFileSystem)
with dask.config.set(hdfs_driver='hdfs3'):
fs2, token2 = get_fs('hdfs')
assert isinstance(fs2, HDFS3HadoopFileSystem)
assert token1 != token2
with pytest.raises(ValueError):
with dask.config.set(hdfs_driver='not-a-valid-driver'):
get_fs('hdfs')
def test_fs_driver_backends():
from dask.bytes.hdfs3 import HDFS3HadoopFileSystem
from dask.bytes.pyarrow import PyArrowHadoopFileSystem
fs1, token1 = get_fs('hdfs')
assert isinstance(fs1, PyArrowHadoopFileSystem)
with dask.config.set(hdfs_driver='hdfs3'):
fs2, token2 = get_fs('hdfs')
assert isinstance(fs2, HDFS3HadoopFileSystem)
assert token1 != token2
with pytest.raises(ValueError):
with dask.config.set(hdfs_driver='not-a-valid-driver'):
get_fs('hdfs')
def __init__(self, path=None):
# from fsspec.registry import filesystem
from dask.bytes.core import get_fs
self.pdir = make_path_posix(path or conf.get('persist_path'))
path = posixpath.join(self.pdir, 'cat.yaml')
protocol = (self.pdir.split('://', 1)[0]
if "://" in self.pdir else 'file')
self.fs = get_fs(protocol)[0]
_maybe_add_rm(self.fs)
super(PersistStore, self).__init__(path)
def _open_dataset(self):
urlpath, protocol, options = infer_options(self.urlpath)
update_storage_options(options, self.storage_options)
self._fs, _ = get_fs(protocol, options)
if protocol != 'file':
self._mapper = get_mapper(protocol, self._fs, urlpath)
self._ds = xr.open_zarr(self._mapper, **self.kwargs)
else:
self._ds = xr.open_zarr(self.urlpath, **self.kwargs)
def _open_dataset(self):
import xarray as xr
from dask.bytes.core import get_fs, infer_options, \
update_storage_options
urlpath, protocol, options = infer_options(self.urlpath)
update_storage_options(options, self.storage_options)
self._fs, _ = get_fs(protocol, options)
if protocol != 'file':
self._mapper = get_mapper(protocol, self._fs, urlpath)
self._ds = xr.open_zarr(self._mapper, **self.kwargs)
else:
self._ds = xr.open_zarr(self.urlpath, **self.kwargs)
def open_store(url, data_vars, storage_options={}, **kwargs):
"""open zarr store."""
urlpath, protocol, options = infer_options(url)
update_storage_options(options, storage_options)
fs, _ = get_fs(protocol, options)
if protocol != 'file':
mapper = get_mapper(protocol, fs, urlpath)
ds = xr.open_zarr(mapper, **kwargs)
else:
ds = xr.open_zarr(urlpath, **kwargs)
if data_vars:
return set_coords(ds, data_vars)
else:
return ds
