def test_listdir(ctx):
contents = b"meow!"
with ctx() as path:
dirpath = bf.dirname(path)
bf.makedirs(dirpath)
a_path = bf.join(dirpath, "a")
with bf.BlobFile(a_path, "wb") as w:
w.write(contents)
b_path = bf.join(dirpath, "b")
with bf.BlobFile(b_path, "wb") as w:
w.write(contents)
bf.makedirs(bf.join(dirpath, "c"))
expected = ["a", "b", "c"]
assert sorted(list(bf.listdir(dirpath))) == expected
dirpath = _convert_https_to_az(dirpath)
assert sorted(list(bf.listdir(dirpath))) == expected
def _list_image_files_recursively(data_dir):
results = []
for entry in sorted(bf.listdir(data_dir)):
full_path = bf.join(data_dir, entry)
ext = entry.split(".")[-1]
if "." in entry and ext.lower() in ["jpg", "jpeg", "png", "gif"]:
results.append(full_path)
elif bf.isdir(full_path):
results.extend(_list_image_files_recursively(full_path))
return results
def main():
have_credentials = setup_google_credentials()
os.environ.update({
"CIBW_BUILD":
"cp36-macosx_x86_64 cp37-macosx_x86_64 cp38-macosx_x86_64 cp36-manylinux_x86_64 cp37-manylinux_x86_64 cp38-manylinux_x86_64 cp36-win_amd64 cp37-win_amd64 cp38-win_amd64",
"CIBW_BEFORE_BUILD":
"pip install -e procgen-build && python -u -m procgen_build.build_qt --output-dir /tmp/qt5",
"CIBW_TEST_EXTRAS":
"test",
# the --pyargs option causes pytest to use the installed procgen wheel
"CIBW_TEST_COMMAND":
"pytest --verbose --benchmark-disable --durations=16 --pyargs procgen",
# this is where build-qt.py will put the files
"CIBW_ENVIRONMENT":
"PROCGEN_CMAKE_PREFIX_PATH=/tmp/qt5/qt/build/qtbase/lib/cmake/Qt5",
# this is a bit too verbose normally
# "CIBW_BUILD_VERBOSITY": "3",
})
if platform.system() == "Darwin":
# cibuildwheel's python copy on mac os x sometimes fails with this error:
# [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1076)
urlretrieve(
"https://curl.haxx.se/ca/cacert.pem",
os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem",
)
os.environ[
"SSL_CERT_FILE"] = os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem"
elif platform.system() == "Linux":
# since we're inside a docker container, adjust the credentials path to point at the mounted location
if have_credentials:
os.environ["CIBW_ENVIRONMENT"] = (
os.environ["CIBW_ENVIRONMENT"] +
" GOOGLE_APPLICATION_CREDENTIALS=/host" +
os.environ["GOOGLE_APPLICATION_CREDENTIALS"])
if "TRAVIS_TAG" in os.environ:
# pass TRAVIS_TAG to the container so that it can build wheels with the correct version number
os.environ["CIBW_ENVIRONMENT"] = (os.environ["CIBW_ENVIRONMENT"] +
" TRAVIS_TAG=" +
os.environ["TRAVIS_TAG"])
elif platform.system() == "Windows":
init_vsvars()
run("pip install cibuildwheel==1.4.1")
run("cibuildwheel --output-dir wheelhouse")
if have_credentials:
print("upload wheels", platform.system())
input_dir = "wheelhouse"
output_dir = f"gs://{GCS_BUCKET}/builds/"
for filename in bf.listdir(input_dir):
src = bf.join(input_dir, filename)
dst = bf.join(output_dir, filename)
print(src, "=>", dst)
bf.copy(src, dst, overwrite=True)
def test_listdir_sharded(ctx):
contents = b"meow!"
with ctx() as path:
dirpath = bf.dirname(path)
with bf.BlobFile(bf.join(dirpath, "a"), "wb") as w:
w.write(contents)
with bf.BlobFile(bf.join(dirpath, "aa"), "wb") as w:
w.write(contents)
with bf.BlobFile(bf.join(dirpath, "b"), "wb") as w:
w.write(contents)
with bf.BlobFile(bf.join(dirpath, "ca"), "wb") as w:
w.write(contents)
bf.makedirs(bf.join(dirpath, "c"))
with bf.BlobFile(bf.join(dirpath, "c/a"), "wb") as w:
w.write(contents)
# this should also test shard_prefix_length=2 but that takes too long
assert sorted(list(bf.listdir(dirpath, shard_prefix_length=1))) == [
"a",
"aa",
"b",
"c",
"ca",
]
def test_invalid_paths(base_path):
for suffix in ["", "/", "//", "/invalid.file", "/invalid/dir/"]:
path = base_path + suffix
print(path)
if path.endswith("/"):
expected_error = IsADirectoryError
else:
expected_error = FileNotFoundError
list(bf.glob(path))
if suffix == "":
for pattern in ["*", "**"]:
try:
list(bf.glob(path + pattern))
except bf.Error as e:
assert "Wildcards cannot be used" in e.message
else:
for pattern in ["*", "**"]:
list(bf.glob(path + pattern))
with pytest.raises(FileNotFoundError):
list(bf.listdir(path))
assert not bf.exists(path)
assert not bf.isdir(path)
with pytest.raises(expected_error):
bf.remove(path)
if suffix in ("", "/"):
try:
bf.rmdir(path)
except bf.Error as e:
assert "Cannot delete bucket" in e.message
else:
bf.rmdir(path)
with pytest.raises(NotADirectoryError):
bf.rmtree(path)
with pytest.raises(FileNotFoundError):
bf.stat(path)
if base_path == AZURE_INVALID_CONTAINER_NO_ACCOUNT:
with pytest.raises(bf.Error):
bf.get_url(path)
else:
bf.get_url(path)
with pytest.raises(FileNotFoundError):
bf.md5(path)
with pytest.raises(bf.Error):
bf.makedirs(path)
list(bf.walk(path))
with tempfile.TemporaryDirectory() as tmpdir:
local_path = os.path.join(tmpdir, "test.txt")
with pytest.raises(expected_error):
bf.copy(path, local_path)
with open(local_path, "w") as f:
f.write("meow")
with pytest.raises(expected_error):
bf.copy(local_path, path)
for streaming in [False, True]:
with pytest.raises(expected_error):
with bf.BlobFile(path, "rb", streaming=streaming) as f:
f.read()
with pytest.raises(expected_error):
with bf.BlobFile(path, "wb", streaming=streaming) as f:
f.write(b"meow")