def with_adapter(adapter: str, bucket: str, other_bucket: str): tmp_dir = None if adapter == "gcs": # Use GCS (with system credentials) use_fs(False) elif adapter == "fs": # Use local file-system in a temp folder tmp_dir = tempfile.mkdtemp() use_fs(tmp_dir) bucket_one = Pathy.from_bucket(bucket) if not bucket_one.exists(): bucket_one.mkdir() bucket_two = Pathy.from_bucket(other_bucket) if not bucket_two.exists(): bucket_two.mkdir() else: raise ValueError("invalid adapter, nothing is configured") # execute the test yield if adapter == "fs" and tmp_dir is not None: # Cleanup fs temp folder shutil.rmtree(tmp_dir) use_fs(False) use_fs_cache(False)
def with_adapter( adapter: str, bucket: str, other_bucket: str ) -> Generator[str, None, None]: tmp_dir = None scheme = "gs" if adapter == "gcs": # Use GCS use_fs(False) credentials = gcs_credentials_from_env() if credentials is not None: set_client_params("gs", credentials=credentials) elif adapter == "fs": # Use local file-system in a temp folder tmp_dir = tempfile.mkdtemp() use_fs(tmp_dir) bucket_one = Pathy.from_bucket(bucket) if not bucket_one.exists(): bucket_one.mkdir() bucket_two = Pathy.from_bucket(other_bucket) if not bucket_two.exists(): bucket_two.mkdir() else: raise ValueError("invalid adapter, nothing is configured") # execute the test yield scheme if adapter == "fs" and tmp_dir is not None: # Cleanup fs temp folder shutil.rmtree(tmp_dir) use_fs(False) use_fs_cache(False)
def test_api_use_fs_cache(with_adapter: str, with_fs: str, bucket: str) -> None: path = Pathy(f"{with_adapter}://{bucket}/{ENV_ID}/directory/foo.txt") path.write_text("---") assert isinstance(path, Pathy) with pytest.raises(ValueError): Pathy.to_local(path) use_fs_cache(Path(with_fs) / "sub_folder") source_file: Path = Pathy.to_local(path) foo_timestamp = Path(f"{source_file}.time") assert foo_timestamp.exists() orig_cache_time = foo_timestamp.read_text() # fetch from the local cache cached_file: Path = Pathy.to_local(f"{path}") assert cached_file == source_file cached_cache_time = foo_timestamp.read_text() assert orig_cache_time == cached_cache_time, "cached blob timestamps should match" # Update the blob time.sleep(1.0) path.write_text('{ "cool" : true }') # Fetch the updated blob Pathy.to_local(path) updated_cache_time = foo_timestamp.read_text() assert updated_cache_time != orig_cache_time, "cached timestamp did not change"
def test_api_path_to_local(with_adapter: str, bucket: str) -> None: root: Pathy = Pathy.from_bucket(bucket) / "to_local" foo_blob: Pathy = root / "foo" foo_blob.write_text("---") assert isinstance(foo_blob, Pathy) use_fs_cache() # Cache a blob cached: Path = Pathy.to_local(foo_blob) second_cached: Path = Pathy.to_local(foo_blob) assert isinstance(cached, Path) assert cached.exists() and cached.is_file(), "local file should exist" assert second_cached == cached, "must be the same path" assert second_cached.stat() == cached.stat(), "must have the same stat" # Cache a folder hierarchy with blobs complex_folder = root / "complex" for i in range(3): folder = f"folder_{i}" for j in range(2): gcs_blob: Pathy = complex_folder / folder / f"file_{j}.txt" gcs_blob.write_text("---") cached_folder: Path = Pathy.to_local(complex_folder) assert isinstance(cached_folder, Path) assert cached_folder.exists() and cached_folder.is_dir() # Verify all the files exist in the file-system cache folder for i in range(3): folder = f"folder_{i}" for j in range(2): iter_blob: Path = cached_folder / folder / f"file_{j}.txt" assert iter_blob.exists() assert iter_blob.read_text() == "---" clear_fs_cache() assert not cached.exists(), "cache clear should delete file"