def test_register_str(clear_registry):
with pytest.raises(ValueError):
get_filesystem_class("test")
register_implementation("test", "fsspec.AbstractFileSystem")
assert "test" not in registry
cls = get_filesystem_class("test")
assert cls is AbstractFileSystem
assert "test" in registry
def test_model_checkpoint_supports_fsspec_parameters(tmpdir):
import fsspec
from fsspec.registry import register_implementation
class FakeCloudStorage(fsspec.AbstractFileSystem):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._fs = fsspec.filesystem("file")
self.storage_options = storage_options
def ls(self, path, detail=True, **kwargs):
return self._fs.ls(join(tmpdir, path), detail, **kwargs)
def cp_file(self, path1, path2, **kwargs):
self._fs.cp_file(join(tmpdir, path1), join(tmpdir, path2),
**kwargs)
def _rm(self, path):
self._fs._rm(join(tmpdir, path))
def created(self, path):
return self._fs.created(join(tmpdir, path))
def modified(self, path):
return self._fs.modified(join(tmpdir, path))
def makedirs(self, path, exist_ok=False):
self._fs.makedirs(join(tmpdir, path), exist_ok=exist_ok)
register_implementation("fake", FakeCloudStorage)
storage_options = {"username": "******", "password": "******"}
mc = ModelCheckpoint(dirpath="fake://checkpoints",
save_top_k=-1,
monitor="epoch",
filename="test",
storage_options=storage_options)
trainer = Trainer(
max_epochs=2,
limit_train_batches=1,
limit_val_batches=1,
default_root_dir=tmpdir,
callbacks=[mc],
logger=False,
weights_summary=None,
progress_bar_refresh_rate=0,
)
trainer.fit(BoringModel())
assert mc.dirpath == "checkpoints"
assert mc._fs.storage_options == storage_options
def test_register_fail(clear_registry):
register_implementation("test", "doesntexist.AbstractFileSystem")
with pytest.raises(ImportError):
get_filesystem_class("test")
register_implementation("test", "doesntexist.AbstractFileSystem")
with pytest.raises(ValueError):
register_implementation("test",
"doesntexist.AbstractFileSystem",
clobber=False)
register_implementation("test",
"doesntexist.AbstractFileSystem",
errtxt="hiho",
clobber=True)
with pytest.raises(ImportError) as e:
get_filesystem_class("test")
assert "hiho" in str(e.value)
register_implementation("test", AbstractFileSystem)
with pytest.raises(ValueError):
register_implementation("test", AbstractFileSystem, clobber=False)
register_implementation("test", AbstractFileSystem, clobber=True)
def test_register_cls(clear_registry):
with pytest.raises(ValueError):
get_filesystem_class("test")
register_implementation("test", AbstractFileSystem)
cls = get_filesystem_class("test")
assert cls is AbstractFileSystem
def clear_registry():
register_implementation("mock", DummyTestFS)
try:
yield
finally:
_registry.clear()
headers = {"Authorization": f"Token {self.api_token}"}
else:
headers = None
r = requests.get(url, headers=headers)
if r.status_code == 404:
raise FileNotFoundError(path)
r.raise_for_status()
data = r.json()
if self.resource == "inputs" and self.field is None:
result = data
elif self.resource == "inputs" and self.field != "adjustment":
result = data[self.field]
elif self.resource == "inputs" and self.field == "adjustment":
result = data[self.field]
if self.section is not None:
result = result[self.section]
elif self.resource == "outputs":
result = data["outputs"]["downloadable"]
elif self.resource in ("title", "owner"):
result = {self.resource: data[self.resource]}
elif self.resource is None:
result = dict(data, outputs=base_url)
else:
raise FileNotFoundError()
return MemoryFile(None, None, json.dumps(result).encode("utf-8"))
register_implementation("cs", CSFileSystem)
