def test_clients_use_fs(with_fs: Path) -> None:
assert get_fs_client() is None
# Use the default path
use_fs()
client = get_fs_client()
assert isinstance(client, BucketClientFS)
assert client.root.exists()
# Use with False disables the client
use_fs(False)
client = get_fs_client()
assert client is None
# Can use a given path
use_fs(str(with_fs))
client = get_fs_client()
assert isinstance(client, BucketClientFS)
assert client.root == with_fs
# Can use a pathlib.Path
use_fs(with_fs / "sub_folder")
client = get_fs_client()
assert isinstance(client, BucketClientFS)
assert client.root == with_fs / "sub_folder"
use_fs(False)
def test_scandir_custom_class(bucket: str) -> None:
use_fs(True)
client = BucketClientFS()
root = Pathy(f"gs://{bucket}/")
scandir = MockScanDir(client=client, path=root)
blobs = [b for b in scandir]
assert len(blobs) == 1
def test_api_raises_with_no_known_bucket_clients_for_a_scheme(temp_folder):
accessor = BucketsAccessor()
path = Pathy("foo://foo")
with pytest.raises(ValueError):
accessor.client(path)
# Setting a fallback FS adapter fixes the problem
use_fs(str(temp_folder))
assert isinstance(accessor.client(path), BucketClientFS)
def pathy_fixture():
import tempfile
import shutil
from pathy import use_fs, Pathy
temp_folder = tempfile.mkdtemp(prefix="thinc-pathy")
use_fs(temp_folder)
root = Pathy("gs://test-bucket")
root.mkdir(exist_ok=True)
yield root
use_fs(False)
shutil.rmtree(temp_folder)
def test_api_export_spacy_model(temp_folder: Path) -> None:
"""spaCy model loading is one of the things we need to support"""
use_fs(temp_folder)
bucket = Pathy("gs://my-bucket/")
bucket.mkdir(exist_ok=True)
model = spacy.blank("en")
output_path = Pathy("gs://my-bucket/models/my_model")
model.to_disk(output_path)
sorted_entries = sorted([str(p) for p in output_path.glob("*")])
expected_entries = [
"gs://my-bucket/models/my_model/meta.json",
"gs://my-bucket/models/my_model/tokenizer",
"gs://my-bucket/models/my_model/vocab",
]
assert sorted_entries == expected_entries
def with_adapter(
adapter: str, bucket: str, other_bucket: str
) -> Generator[str, None, None]:
tmp_dir = None
scheme = "gs"
if adapter == "gcs":
# Use GCS
use_fs(False)
credentials = gcs_credentials_from_env()
if credentials is not None:
set_client_params("gs", credentials=credentials)
elif adapter == "fs":
# Use local file-system in a temp folder
tmp_dir = tempfile.mkdtemp()
use_fs(tmp_dir)
bucket_one = Pathy.from_bucket(bucket)
if not bucket_one.exists():
bucket_one.mkdir()
bucket_two = Pathy.from_bucket(other_bucket)
if not bucket_two.exists():
bucket_two.mkdir()
else:
raise ValueError("invalid adapter, nothing is configured")
# execute the test
yield scheme
if adapter == "fs" and tmp_dir is not None:
# Cleanup fs temp folder
shutil.rmtree(tmp_dir)
use_fs(False)
use_fs_cache(False)
def with_adapter(adapter: str, bucket: str, other_bucket: str):
tmp_dir = None
if adapter == "gcs":
# Use GCS (with system credentials)
use_fs(False)
elif adapter == "fs":
# Use local file-system in a temp folder
tmp_dir = tempfile.mkdtemp()
use_fs(tmp_dir)
bucket_one = Pathy.from_bucket(bucket)
if not bucket_one.exists():
bucket_one.mkdir()
bucket_two = Pathy.from_bucket(other_bucket)
if not bucket_two.exists():
bucket_two.mkdir()
else:
raise ValueError("invalid adapter, nothing is configured")
# execute the test
yield
if adapter == "fs" and tmp_dir is not None:
# Cleanup fs temp folder
shutil.rmtree(tmp_dir)
use_fs(False)
use_fs_cache(False)
def test_s3_import_error_missing_deps() -> None:
use_fs(False)
with pytest.raises(ImportError):
get_client("s3")
def with_fs(temp_folder: Path) -> Generator[Path, None, None]:
yield temp_folder
# Turn off FS adapter
use_fs(False)
def test_scandir_next(bucket: str) -> None:
use_fs(True)
client = BucketClientFS()
root = Pathy(f"gs://{bucket}/")
scandir = MockScanDir(client=client, path=root)
assert next(scandir.__next__()).name == "test-bucket" # type:ignore
def test_scandir_abstract_methods(bucket: str) -> None:
use_fs(True)
client = BucketClientFS()
root = Pathy(f"gs://{bucket}/")
with pytest.raises(NotImplementedError):
AbstractScanDir(client=client, path=root) # type:ignore
def with_fs(temp_folder):
yield temp_folder
# Turn off FS adapter
use_fs(False)
def test_clients_get_client_respects_use_fs_override() -> None:
use_fs(True)
assert isinstance(get_client("gs"), BucketClientFS)
use_fs(False)
