def test_register_str(clear_registry):
with pytest.raises(ValueError):
get_filesystem_class("test")
register_implementation("test", "fsspec.AbstractFileSystem")
assert "test" not in registry
cls = get_filesystem_class("test")
assert cls is AbstractFileSystem
assert "test" in registry
def test_registry_readonly():
get_filesystem_class("file")
assert "file" in registry
assert "file" in list(registry)
with pytest.raises(ReadOnlyError):
del registry["file"]
with pytest.raises(ReadOnlyError):
registry["file"] = None
with pytest.raises(ReadOnlyError):
registry.clear()
def test_minversion_s3fs(protocol, module, minversion, oldversion, monkeypatch):
_registry.clear()
mod = pytest.importorskip(module, minversion)
assert get_filesystem_class("s3") is not None
_registry.clear()
monkeypatch.setattr(mod, "__version__", oldversion)
with pytest.raises(RuntimeError, match=minversion):
get_filesystem_class(protocol)
def test_entry_points_registered_on_import(clear_registry, clean_imports):
mock_ep = create_autospec(EntryPoint,
module="fsspec.spec.AbstractFileSystem")
mock_ep.name = "test" # this can't be set in the constructor...
if sys.version_info < (3, 8):
import_location = "importlib_metadata.entry_points"
else:
import_location = "importlib.metadata.entry_points"
with patch(import_location, return_value={"fsspec.specs": [mock_ep]}):
assert "test" not in registry
import fsspec # noqa
get_filesystem_class("test")
assert "test" in registry
def __init__(self, parsed_url, *args, **kwargs):
self._url = parsed_url
cls = get_filesystem_class(self._url.scheme)
url_kwargs = cls._get_kwargs_from_urls(
urllib.parse.urlunparse(self._url))
url_kwargs.update(kwargs)
self._fs = cls(**url_kwargs)
def test_register_fail(clear_registry):
register_implementation("test", "doesntexist.AbstractFileSystem")
with pytest.raises(ImportError):
get_filesystem_class("test")
with pytest.raises(ValueError):
register_implementation("test", "doesntexist.AbstractFileSystem")
register_implementation(
"test", "doesntexist.AbstractFileSystem", errtxt="hiho", clobber=True
)
with pytest.raises(ImportError) as e:
get_filesystem_class("test")
assert "hiho" in str(e.value)
register_implementation("test", AbstractFileSystem)
with pytest.raises(ValueError):
register_implementation("test", AbstractFileSystem)
register_implementation("test", AbstractFileSystem, clobber=True)
def _op3(dry_run=False):
logger.info("Moving {} to {}...".format(
asset_href, new_asset_href))
if not dry_run:
fs_source = get_filesystem_class(source_protocol)()
fs_dest.makedirs(os.path.dirname(new_asset_href),
exist_ok=True)
with fsspec.open(asset_href, 'rb') as f_src:
with fsspec.open(new_asset_href,
'wb') as f_dst:
f_dst.write(f_src.read())
fs_source.delete(asset_href)
def move_asset_file_to_item(item,
asset_href,
asset_subdirectory=None,
copy=False,
ignore_conflicts=False):
"""Moves an asset file to be alongside that item.
Args:
item (Item): The PySTAC Item
to perform the asset transformation on.
asset_href (str): The absolute HREF to the asset file.
asset_subdirectory (str or None): A subdirectory that will be used
to store the assets. If not supplied, the assets will be moved
or copied to the same directory as their item.
copy (bool): If False this function will move the asset file; if True,
the asset file will be copied.
ignore_conflicts (bool): If the asset destination file already exists,
this function will throw an error unless ignore_conflicts is True.
Returns:
str: The new absolute href for the asset file
"""
item_href = item.get_self_href()
if item_href is None:
raise ValueError(
'Self HREF is not available for item {}. This operation '
'requires that the Item HREFs are available.')
if not is_absolute_href(asset_href):
raise ValueError('asset_href msut be absolute.')
item_dir = os.path.dirname(item_href)
fname = os.path.basename(asset_href)
if asset_subdirectory is None:
target_dir = item_dir
else:
target_dir = os.path.join(item_dir, asset_subdirectory)
new_asset_href = os.path.join(target_dir, fname)
if asset_href != new_asset_href:
dest_protocol = split_protocol(new_asset_href)[0]
fs_dest = get_filesystem_class(dest_protocol)()
op = None
if fs_dest.exists(new_asset_href):
if not ignore_conflicts:
raise FileExistsError(
'{} already exists'.format(new_asset_href))
else:
if copy:
def _op1(dry_run=False):
logger.info("Copying {} to {}...".format(
asset_href, new_asset_href))
if not dry_run:
fs_dest.makedirs(os.path.dirname(new_asset_href),
exist_ok=True)
with fsspec.open(asset_href, 'rb') as f_src:
with fsspec.open(new_asset_href, 'wb') as f_dst:
f_dst.write(f_src.read())
op = _op1
else:
source_protocol = split_protocol(asset_href)[0]
if source_protocol == dest_protocol:
def _op2(dry_run=False):
logger.info("Moving {} to {}...".format(
asset_href, new_asset_href))
if not dry_run:
fs_dest.makedirs(os.path.dirname(new_asset_href),
exist_ok=True)
fs_dest.move(asset_href, new_asset_href)
op = _op2
else:
def _op3(dry_run=False):
logger.info("Moving {} to {}...".format(
asset_href, new_asset_href))
if not dry_run:
fs_source = get_filesystem_class(source_protocol)()
fs_dest.makedirs(os.path.dirname(new_asset_href),
exist_ok=True)
with fsspec.open(asset_href, 'rb') as f_src:
with fsspec.open(new_asset_href,
'wb') as f_dst:
f_dst.write(f_src.read())
fs_source.delete(asset_href)
op = _op3
if op is not None:
op(dry_run=False)
return new_asset_href
def test_register_cls(clear_registry):
with pytest.raises(ValueError):
get_filesystem_class("test")
register_implementation("test", AbstractFileSystem)
cls = get_filesystem_class("test")
assert cls is AbstractFileSystem
