def test_composite_objects():
with _get_temp_gcs_path() as remote_path:
with _get_temp_local_path() as local_path:
contents = b"0" * 2 * 2**20
with open(local_path, "wb") as f:
f.write(contents)
sp.run(
[
"gsutil",
"-o",
"GSUtil:parallel_composite_upload_threshold=1M",
"cp",
local_path,
remote_path,
],
check=True,
)
assert hashlib.md5(contents).hexdigest() == bf.md5(remote_path)
assert hashlib.md5(contents).hexdigest() == bf.md5(remote_path)
with tempfile.TemporaryDirectory() as tmpdir:
with bf.BlobFile(remote_path,
"rb",
cache_dir=tmpdir,
streaming=False) as f:
assert f.read() == contents
def test_md5(ctx):
contents = b"meow!"
meow_hash = hashlib.md5(contents).hexdigest()
with ctx() as path:
_write_contents(path, contents)
assert bf.md5(path) == meow_hash
with bf.BlobFile(path, "wb") as f:
f.write(contents)
assert bf.md5(path) == meow_hash
with bf.BlobFile(path, "wb") as f:
f.write(contents)
assert bf.md5(path) == meow_hash
def test_invalid_paths(base_path):
for suffix in ["", "/", "//", "/invalid.file", "/invalid/dir/"]:
path = base_path + suffix
print(path)
if path.endswith("/"):
expected_error = IsADirectoryError
else:
expected_error = FileNotFoundError
list(bf.glob(path))
if suffix == "":
for pattern in ["*", "**"]:
try:
list(bf.glob(path + pattern))
except bf.Error as e:
assert "Wildcards cannot be used" in e.message
else:
for pattern in ["*", "**"]:
list(bf.glob(path + pattern))
with pytest.raises(FileNotFoundError):
list(bf.listdir(path))
assert not bf.exists(path)
assert not bf.isdir(path)
with pytest.raises(expected_error):
bf.remove(path)
if suffix in ("", "/"):
try:
bf.rmdir(path)
except bf.Error as e:
assert "Cannot delete bucket" in e.message
else:
bf.rmdir(path)
with pytest.raises(NotADirectoryError):
bf.rmtree(path)
with pytest.raises(FileNotFoundError):
bf.stat(path)
if base_path == AZURE_INVALID_CONTAINER_NO_ACCOUNT:
with pytest.raises(bf.Error):
bf.get_url(path)
else:
bf.get_url(path)
with pytest.raises(FileNotFoundError):
bf.md5(path)
with pytest.raises(bf.Error):
bf.makedirs(path)
list(bf.walk(path))
with tempfile.TemporaryDirectory() as tmpdir:
local_path = os.path.join(tmpdir, "test.txt")
with pytest.raises(expected_error):
bf.copy(path, local_path)
with open(local_path, "w") as f:
f.write("meow")
with pytest.raises(expected_error):
bf.copy(local_path, path)
for streaming in [False, True]:
with pytest.raises(expected_error):
with bf.BlobFile(path, "rb", streaming=streaming) as f:
f.read()
with pytest.raises(expected_error):
with bf.BlobFile(path, "wb", streaming=streaming) as f:
f.write(b"meow")
