def test_register_str(clear_registry): with pytest.raises(ValueError): get_filesystem_class("test") register_implementation("test", "fsspec.AbstractFileSystem") assert "test" not in registry cls = get_filesystem_class("test") assert cls is AbstractFileSystem assert "test" in registry
def test_registry_readonly(): get_filesystem_class("file") assert "file" in registry assert "file" in list(registry) with pytest.raises(ReadOnlyError): del registry["file"] with pytest.raises(ReadOnlyError): registry["file"] = None with pytest.raises(ReadOnlyError): registry.clear()
def test_minversion_s3fs(protocol, module, minversion, oldversion, monkeypatch): _registry.clear() mod = pytest.importorskip(module, minversion) assert get_filesystem_class("s3") is not None _registry.clear() monkeypatch.setattr(mod, "__version__", oldversion) with pytest.raises(RuntimeError, match=minversion): get_filesystem_class(protocol)
def test_entry_points_registered_on_import(clear_registry, clean_imports): mock_ep = create_autospec(EntryPoint, module="fsspec.spec.AbstractFileSystem") mock_ep.name = "test" # this can't be set in the constructor... if sys.version_info < (3, 8): import_location = "importlib_metadata.entry_points" else: import_location = "importlib.metadata.entry_points" with patch(import_location, return_value={"fsspec.specs": [mock_ep]}): assert "test" not in registry import fsspec # noqa get_filesystem_class("test") assert "test" in registry
def __init__(self, parsed_url, *args, **kwargs): self._url = parsed_url cls = get_filesystem_class(self._url.scheme) url_kwargs = cls._get_kwargs_from_urls( urllib.parse.urlunparse(self._url)) url_kwargs.update(kwargs) self._fs = cls(**url_kwargs)
def test_register_fail(clear_registry): register_implementation("test", "doesntexist.AbstractFileSystem") with pytest.raises(ImportError): get_filesystem_class("test") with pytest.raises(ValueError): register_implementation("test", "doesntexist.AbstractFileSystem") register_implementation( "test", "doesntexist.AbstractFileSystem", errtxt="hiho", clobber=True ) with pytest.raises(ImportError) as e: get_filesystem_class("test") assert "hiho" in str(e.value) register_implementation("test", AbstractFileSystem) with pytest.raises(ValueError): register_implementation("test", AbstractFileSystem) register_implementation("test", AbstractFileSystem, clobber=True)
def _op3(dry_run=False): logger.info("Moving {} to {}...".format( asset_href, new_asset_href)) if not dry_run: fs_source = get_filesystem_class(source_protocol)() fs_dest.makedirs(os.path.dirname(new_asset_href), exist_ok=True) with fsspec.open(asset_href, 'rb') as f_src: with fsspec.open(new_asset_href, 'wb') as f_dst: f_dst.write(f_src.read()) fs_source.delete(asset_href)
def move_asset_file_to_item(item, asset_href, asset_subdirectory=None, copy=False, ignore_conflicts=False): """Moves an asset file to be alongside that item. Args: item (Item): The PySTAC Item to perform the asset transformation on. asset_href (str): The absolute HREF to the asset file. asset_subdirectory (str or None): A subdirectory that will be used to store the assets. If not supplied, the assets will be moved or copied to the same directory as their item. copy (bool): If False this function will move the asset file; if True, the asset file will be copied. ignore_conflicts (bool): If the asset destination file already exists, this function will throw an error unless ignore_conflicts is True. Returns: str: The new absolute href for the asset file """ item_href = item.get_self_href() if item_href is None: raise ValueError( 'Self HREF is not available for item {}. This operation ' 'requires that the Item HREFs are available.') if not is_absolute_href(asset_href): raise ValueError('asset_href msut be absolute.') item_dir = os.path.dirname(item_href) fname = os.path.basename(asset_href) if asset_subdirectory is None: target_dir = item_dir else: target_dir = os.path.join(item_dir, asset_subdirectory) new_asset_href = os.path.join(target_dir, fname) if asset_href != new_asset_href: dest_protocol = split_protocol(new_asset_href)[0] fs_dest = get_filesystem_class(dest_protocol)() op = None if fs_dest.exists(new_asset_href): if not ignore_conflicts: raise FileExistsError( '{} already exists'.format(new_asset_href)) else: if copy: def _op1(dry_run=False): logger.info("Copying {} to {}...".format( asset_href, new_asset_href)) if not dry_run: fs_dest.makedirs(os.path.dirname(new_asset_href), exist_ok=True) with fsspec.open(asset_href, 'rb') as f_src: with fsspec.open(new_asset_href, 'wb') as f_dst: f_dst.write(f_src.read()) op = _op1 else: source_protocol = split_protocol(asset_href)[0] if source_protocol == dest_protocol: def _op2(dry_run=False): logger.info("Moving {} to {}...".format( asset_href, new_asset_href)) if not dry_run: fs_dest.makedirs(os.path.dirname(new_asset_href), exist_ok=True) fs_dest.move(asset_href, new_asset_href) op = _op2 else: def _op3(dry_run=False): logger.info("Moving {} to {}...".format( asset_href, new_asset_href)) if not dry_run: fs_source = get_filesystem_class(source_protocol)() fs_dest.makedirs(os.path.dirname(new_asset_href), exist_ok=True) with fsspec.open(asset_href, 'rb') as f_src: with fsspec.open(new_asset_href, 'wb') as f_dst: f_dst.write(f_src.read()) fs_source.delete(asset_href) op = _op3 if op is not None: op(dry_run=False) return new_asset_href
def test_register_cls(clear_registry): with pytest.raises(ValueError): get_filesystem_class("test") register_implementation("test", AbstractFileSystem) cls = get_filesystem_class("test") assert cls is AbstractFileSystem