def with_adapter(adapter: str, bucket: str, other_bucket: str):
tmp_dir = None
if adapter == "gcs":
# Use GCS (with system credentials)
use_fs(False)
elif adapter == "fs":
# Use local file-system in a temp folder
tmp_dir = tempfile.mkdtemp()
use_fs(tmp_dir)
bucket_one = Pathy.from_bucket(bucket)
if not bucket_one.exists():
bucket_one.mkdir()
bucket_two = Pathy.from_bucket(other_bucket)
if not bucket_two.exists():
bucket_two.mkdir()
else:
raise ValueError("invalid adapter, nothing is configured")
# execute the test
yield
if adapter == "fs" and tmp_dir is not None:
# Cleanup fs temp folder
shutil.rmtree(tmp_dir)
use_fs(False)
use_fs_cache(False)
def with_adapter(
adapter: str, bucket: str, other_bucket: str
) -> Generator[str, None, None]:
tmp_dir = None
scheme = "gs"
if adapter == "gcs":
# Use GCS
use_fs(False)
credentials = gcs_credentials_from_env()
if credentials is not None:
set_client_params("gs", credentials=credentials)
elif adapter == "fs":
# Use local file-system in a temp folder
tmp_dir = tempfile.mkdtemp()
use_fs(tmp_dir)
bucket_one = Pathy.from_bucket(bucket)
if not bucket_one.exists():
bucket_one.mkdir()
bucket_two = Pathy.from_bucket(other_bucket)
if not bucket_two.exists():
bucket_two.mkdir()
else:
raise ValueError("invalid adapter, nothing is configured")
# execute the test
yield scheme
if adapter == "fs" and tmp_dir is not None:
# Cleanup fs temp folder
shutil.rmtree(tmp_dir)
use_fs(False)
use_fs_cache(False)
def test_api_use_fs_cache(with_adapter: str, with_fs: str,
bucket: str) -> None:
path = Pathy(f"{with_adapter}://{bucket}/{ENV_ID}/directory/foo.txt")
path.write_text("---")
assert isinstance(path, Pathy)
with pytest.raises(ValueError):
Pathy.to_local(path)
use_fs_cache(Path(with_fs) / "sub_folder")
source_file: Path = Pathy.to_local(path)
foo_timestamp = Path(f"{source_file}.time")
assert foo_timestamp.exists()
orig_cache_time = foo_timestamp.read_text()
# fetch from the local cache
cached_file: Path = Pathy.to_local(f"{path}")
assert cached_file == source_file
cached_cache_time = foo_timestamp.read_text()
assert orig_cache_time == cached_cache_time, "cached blob timestamps should match"
# Update the blob
time.sleep(1.0)
path.write_text('{ "cool" : true }')
# Fetch the updated blob
Pathy.to_local(path)
updated_cache_time = foo_timestamp.read_text()
assert updated_cache_time != orig_cache_time, "cached timestamp did not change"
def test_api_path_to_local(with_adapter: str, bucket: str) -> None:
root: Pathy = Pathy.from_bucket(bucket) / "to_local"
foo_blob: Pathy = root / "foo"
foo_blob.write_text("---")
assert isinstance(foo_blob, Pathy)
use_fs_cache()
# Cache a blob
cached: Path = Pathy.to_local(foo_blob)
second_cached: Path = Pathy.to_local(foo_blob)
assert isinstance(cached, Path)
assert cached.exists() and cached.is_file(), "local file should exist"
assert second_cached == cached, "must be the same path"
assert second_cached.stat() == cached.stat(), "must have the same stat"
# Cache a folder hierarchy with blobs
complex_folder = root / "complex"
for i in range(3):
folder = f"folder_{i}"
for j in range(2):
gcs_blob: Pathy = complex_folder / folder / f"file_{j}.txt"
gcs_blob.write_text("---")
cached_folder: Path = Pathy.to_local(complex_folder)
assert isinstance(cached_folder, Path)
assert cached_folder.exists() and cached_folder.is_dir()
# Verify all the files exist in the file-system cache folder
for i in range(3):
folder = f"folder_{i}"
for j in range(2):
iter_blob: Path = cached_folder / folder / f"file_{j}.txt"
assert iter_blob.exists()
assert iter_blob.read_text() == "---"
clear_fs_cache()
assert not cached.exists(), "cache clear should delete file"