def test_api_use_fs_cache(with_adapter, with_fs: str, bucket: str):
path = Pathy(f"gs://{bucket}/directory/foo.txt")
path.write_text("---")
assert isinstance(path, Pathy)
with pytest.raises(ValueError):
Pathy.to_local(path)
use_fs_cache(with_fs)
source_file: Path = Pathy.to_local(path)
foo_timestamp = Path(f"{source_file}.time")
assert foo_timestamp.exists()
orig_cache_time = foo_timestamp.read_text()
# fetch from the local cache
cached_file: Path = Pathy.to_local(path)
assert cached_file == source_file
cached_cache_time = foo_timestamp.read_text()
assert orig_cache_time == cached_cache_time, "cached blob timestamps should match"
# Update the blob
time.sleep(0.1)
path.write_text('{ "cool" : true }')
# Fetch the updated blob
Pathy.to_local(path)
updated_cache_time = foo_timestamp.read_text()
assert updated_cache_time != orig_cache_time, "cached timestamp did not change"
def test_api_glob(with_adapter: str, bucket: str) -> None:
for i in range(3):
path = Pathy(f"gs://{bucket}/glob/{i}.file")
path.write_text("---")
for i in range(2):
path = Pathy(f"gs://{bucket}/glob/{i}/dir/file.txt")
path.write_text("---")
assert list(Pathy(f"gs://{bucket}/glob/").glob("*.test")) == []
assert sorted(list(Pathy(f"gs://{bucket}/glob/").glob("*.file"))) == [
Pathy(f"gs://{bucket}/glob/0.file"),
Pathy(f"gs://{bucket}/glob/1.file"),
Pathy(f"gs://{bucket}/glob/2.file"),
]
assert list(Pathy(f"gs://{bucket}/glob/0/").glob("*/*.txt")) == [
Pathy(f"gs://{bucket}/glob/0/dir/file.txt"),
]
assert sorted(Pathy(f"gs://{bucket}").glob("*lob/")) == [
Pathy(f"gs://{bucket}/glob"),
]
# Recursive matches
assert sorted(list(Pathy(f"gs://{bucket}/glob/").glob("**/*.txt"))) == [
Pathy(f"gs://{bucket}/glob/0/dir/file.txt"),
Pathy(f"gs://{bucket}/glob/1/dir/file.txt"),
]
# rglob adds the **/ for you
assert sorted(list(Pathy(f"gs://{bucket}/glob/").rglob("*.txt"))) == [
Pathy(f"gs://{bucket}/glob/0/dir/file.txt"),
Pathy(f"gs://{bucket}/glob/1/dir/file.txt"),
]
def test_cli_rm_file(with_adapter, bucket: str):
source = f"gs://{bucket}/cli_rm_file/file.txt"
path = Pathy(source)
path.write_text("---")
assert path.exists()
assert runner.invoke(app, ["rm", source]).exit_code == 0
assert not path.exists()
def test_api_is_file(with_adapter: str, bucket: str) -> None:
path = Pathy(f"gs://{bucket}/is_file/subfolder/another/my.file")
path.write_text("---")
# The full file is a file
assert path.is_file() is True
# Each parent node in the path is only a directory
for parent in path.parents:
assert parent.is_file() is False
def test_api_unlink_path(with_adapter: str, bucket: str) -> None:
path = Pathy(f"gs://{bucket}/unlink/404.txt")
with pytest.raises(FileNotFoundError):
path.unlink()
path = Pathy(f"gs://{bucket}/unlink/foo.txt")
path.write_text("---")
assert path.exists()
path.unlink()
assert not path.exists()
def test_api_stat(with_adapter: str, bucket: str) -> None:
path = Pathy("fake-bucket-1234-0987/fake-key")
with pytest.raises(ValueError):
path.stat()
path = Pathy(f"gs://{bucket}/foo.txt")
path.write_text("a-a-a-a-a-a-a")
stat = path.stat()
assert isinstance(stat, BlobStat)
assert stat.size is not None and stat.size > 0
assert stat.last_modified is not None and stat.last_modified > 0
with pytest.raises(ValueError):
assert Pathy(f"gs://{bucket}").stat()
with pytest.raises(FileNotFoundError):
assert Pathy(f"gs://{bucket}/nonexistant_file.txt").stat()
def test_api_owner(with_adapter: str, bucket: str) -> None:
# Raises for invalid file
with pytest.raises(FileNotFoundError):
Pathy(f"gs://{bucket}/write_text/not_a_valid_blob").owner()
path = Pathy(f"gs://{bucket}/write_text/file.txt")
path.write_text("---")
# TODO: How to set file owner to non-None in GCS? Then assert here.
#
# NOTE: The owner is always set when using the filesystem adapter, so
# we can't assert the same behavior here until we fix the above
# todo comment.
path.owner()
# dumb assert means we didn't raise
assert True
def test_api_iterdir(with_adapter: str, bucket: str) -> None:
# (n) files in a folder
for i in range(2):
path = Pathy(f"gs://{bucket}/iterdir/{i}.file")
path.write_text("---")
# 1 file in a subfolder
path = Pathy(f"gs://{bucket}/iterdir/sub/file.txt")
path.write_text("---")
path = Pathy(f"gs://{bucket}/iterdir/")
check = sorted(path.iterdir())
assert check == [
Pathy(f"gs://{bucket}/iterdir/0.file"),
Pathy(f"gs://{bucket}/iterdir/1.file"),
Pathy(f"gs://{bucket}/iterdir/sub"),
]
def test_api_exists(with_adapter: str, bucket: str) -> None:
path = Pathy("./fake-key")
with pytest.raises(ValueError):
path.exists()
# GCS buckets are globally unique, "test-bucket" exists so this
# raises an access error.
assert Pathy("gs://test-bucket/fake-key").exists() is False
# invalid bucket name
assert Pathy("gs://unknown-bucket-name-123987519875419").exists() is False
# valid bucket with invalid object
assert Pathy(f"gs://{bucket}/not_found_lol_nice.txt").exists() is False
path = Pathy(f"gs://{bucket}/directory/foo.txt")
path.write_text("---")
assert path.exists()
for parent in path.parents:
assert parent.exists()
def test_api_readwrite_text(with_adapter: str, bucket: str) -> None:
path = Pathy(f"gs://{bucket}/write_text/file.txt")
path.write_text("---")
with path.open() as file_obj:
assert file_obj.read() == "---"
assert path.read_text() == "---"
def test_api_is_dir(with_adapter: str, bucket: str) -> None:
path = Pathy(f"gs://{bucket}/is_dir/subfolder/another/my.file")
path.write_text("---")
assert path.is_dir() is False
for parent in path.parents:
assert parent.is_dir() is True
def test_api_open_for_read(with_adapter, bucket: str):
path = Pathy(f"gs://{bucket}/read/file.txt")
path.write_text("---")
with path.open() as file_obj:
assert file_obj.read() == "---"
assert path.read_text() == "---"
