def __new__(cls, path, *args, **kwargs):
if cls is DRPath:
if get_fs(path, rtype="class").is_remote:
cls = RemotePath
else:
cls = LocalPath
obj = cls(path, *args, **kwargs)
return obj
def glob(path, opts=None):
"""Filesystem-agnostic glob."""
return get_fs(path, opts=opts).glob(path)
def rmdir(path, opts=None):
"""Filesystem-agnostic rmdir."""
return get_fs(path, opts=opts).rmdir(path)
def makedirs(path, *args, opts=None, **kwargs):
"""Filesystem-agnostic makedirs."""
return get_fs(path, opts=opts).makedirs(path, *args, **kwargs)
def mv(src, dst, opts=None):
"""Filesystem-agnostic mv."""
return get_fs(src, opts=opts).mv(src, dst)
def open(path, mode, opts=None):
"""Filesystem-agnostic open."""
return get_fs(path, opts=opts).open(path, mode)
def exists(path, opts=None):
"""Filesystem-agnostic exists."""
return get_fs(path, opts=opts).exists(path)
def test_get_fs(path, fs):
assert isinstance(get_fs(path), fs)
def test_glob_files(s3_data_dir):
fs = get_fs("s3://s3-test-bucket/", rtype="instance")
res = fs.glob("s3://s3-test-bucket/dump/*.csv")
assert all([str(p).startswith("s3://s3-") for p in res])
assert len(res) == 10
def _get_path_class(path):
if get_fs(path).is_remote:
return RemotePath
else:
return Path
def test_get_fs_no_side_effect():
expected = config["fs_opts"]["s3"].get(dict).copy()
get_fs("s3://bucket", opts=dict(config_kwargs={'read_timeout': 600}))
assert config["fs_opts"]["s3"].get(dict) == expected
def test_config_get_fs(s3_opts_config):
res = {}
fs = get_fs("s3://bucket", res)
assert fs.fs.key == "test_config"
def fs(self):
return get_fs(self.path, opts=self.storage_options, rtype="instance")