def test_clients_use_fs(with_fs: Path) -> None: assert get_fs_client() is None # Use the default path use_fs() client = get_fs_client() assert isinstance(client, BucketClientFS) assert client.root.exists() # Use with False disables the client use_fs(False) client = get_fs_client() assert client is None # Can use a given path use_fs(str(with_fs)) client = get_fs_client() assert isinstance(client, BucketClientFS) assert client.root == with_fs # Can use a pathlib.Path use_fs(with_fs / "sub_folder") client = get_fs_client() assert isinstance(client, BucketClientFS) assert client.root == with_fs / "sub_folder" use_fs(False)
def test_scandir_custom_class(bucket: str) -> None: use_fs(True) client = BucketClientFS() root = Pathy(f"gs://{bucket}/") scandir = MockScanDir(client=client, path=root) blobs = [b for b in scandir] assert len(blobs) == 1
def test_api_raises_with_no_known_bucket_clients_for_a_scheme(temp_folder): accessor = BucketsAccessor() path = Pathy("foo://foo") with pytest.raises(ValueError): accessor.client(path) # Setting a fallback FS adapter fixes the problem use_fs(str(temp_folder)) assert isinstance(accessor.client(path), BucketClientFS)
def pathy_fixture(): import tempfile import shutil from pathy import use_fs, Pathy temp_folder = tempfile.mkdtemp(prefix="thinc-pathy") use_fs(temp_folder) root = Pathy("gs://test-bucket") root.mkdir(exist_ok=True) yield root use_fs(False) shutil.rmtree(temp_folder)
def test_api_export_spacy_model(temp_folder: Path) -> None: """spaCy model loading is one of the things we need to support""" use_fs(temp_folder) bucket = Pathy("gs://my-bucket/") bucket.mkdir(exist_ok=True) model = spacy.blank("en") output_path = Pathy("gs://my-bucket/models/my_model") model.to_disk(output_path) sorted_entries = sorted([str(p) for p in output_path.glob("*")]) expected_entries = [ "gs://my-bucket/models/my_model/meta.json", "gs://my-bucket/models/my_model/tokenizer", "gs://my-bucket/models/my_model/vocab", ] assert sorted_entries == expected_entries
def with_adapter( adapter: str, bucket: str, other_bucket: str ) -> Generator[str, None, None]: tmp_dir = None scheme = "gs" if adapter == "gcs": # Use GCS use_fs(False) credentials = gcs_credentials_from_env() if credentials is not None: set_client_params("gs", credentials=credentials) elif adapter == "fs": # Use local file-system in a temp folder tmp_dir = tempfile.mkdtemp() use_fs(tmp_dir) bucket_one = Pathy.from_bucket(bucket) if not bucket_one.exists(): bucket_one.mkdir() bucket_two = Pathy.from_bucket(other_bucket) if not bucket_two.exists(): bucket_two.mkdir() else: raise ValueError("invalid adapter, nothing is configured") # execute the test yield scheme if adapter == "fs" and tmp_dir is not None: # Cleanup fs temp folder shutil.rmtree(tmp_dir) use_fs(False) use_fs_cache(False)
def with_adapter(adapter: str, bucket: str, other_bucket: str): tmp_dir = None if adapter == "gcs": # Use GCS (with system credentials) use_fs(False) elif adapter == "fs": # Use local file-system in a temp folder tmp_dir = tempfile.mkdtemp() use_fs(tmp_dir) bucket_one = Pathy.from_bucket(bucket) if not bucket_one.exists(): bucket_one.mkdir() bucket_two = Pathy.from_bucket(other_bucket) if not bucket_two.exists(): bucket_two.mkdir() else: raise ValueError("invalid adapter, nothing is configured") # execute the test yield if adapter == "fs" and tmp_dir is not None: # Cleanup fs temp folder shutil.rmtree(tmp_dir) use_fs(False) use_fs_cache(False)
def test_s3_import_error_missing_deps() -> None: use_fs(False) with pytest.raises(ImportError): get_client("s3")
def with_fs(temp_folder: Path) -> Generator[Path, None, None]: yield temp_folder # Turn off FS adapter use_fs(False)
def test_scandir_next(bucket: str) -> None: use_fs(True) client = BucketClientFS() root = Pathy(f"gs://{bucket}/") scandir = MockScanDir(client=client, path=root) assert next(scandir.__next__()).name == "test-bucket" # type:ignore
def test_scandir_abstract_methods(bucket: str) -> None: use_fs(True) client = BucketClientFS() root = Pathy(f"gs://{bucket}/") with pytest.raises(NotImplementedError): AbstractScanDir(client=client, path=root) # type:ignore
def with_fs(temp_folder): yield temp_folder # Turn off FS adapter use_fs(False)
def test_clients_get_client_respects_use_fs_override() -> None: use_fs(True) assert isinstance(get_client("gs"), BucketClientFS) use_fs(False)