def test_listdir(ctx): contents = b"meow!" with ctx() as path: dirpath = bf.dirname(path) bf.makedirs(dirpath) a_path = bf.join(dirpath, "a") with bf.BlobFile(a_path, "wb") as w: w.write(contents) b_path = bf.join(dirpath, "b") with bf.BlobFile(b_path, "wb") as w: w.write(contents) bf.makedirs(bf.join(dirpath, "c")) expected = ["a", "b", "c"] assert sorted(list(bf.listdir(dirpath))) == expected dirpath = _convert_https_to_az(dirpath) assert sorted(list(bf.listdir(dirpath))) == expected
def _list_image_files_recursively(data_dir): results = [] for entry in sorted(bf.listdir(data_dir)): full_path = bf.join(data_dir, entry) ext = entry.split(".")[-1] if "." in entry and ext.lower() in ["jpg", "jpeg", "png", "gif"]: results.append(full_path) elif bf.isdir(full_path): results.extend(_list_image_files_recursively(full_path)) return results
def main(): have_credentials = setup_google_credentials() os.environ.update({ "CIBW_BUILD": "cp36-macosx_x86_64 cp37-macosx_x86_64 cp38-macosx_x86_64 cp36-manylinux_x86_64 cp37-manylinux_x86_64 cp38-manylinux_x86_64 cp36-win_amd64 cp37-win_amd64 cp38-win_amd64", "CIBW_BEFORE_BUILD": "pip install -e procgen-build && python -u -m procgen_build.build_qt --output-dir /tmp/qt5", "CIBW_TEST_EXTRAS": "test", # the --pyargs option causes pytest to use the installed procgen wheel "CIBW_TEST_COMMAND": "pytest --verbose --benchmark-disable --durations=16 --pyargs procgen", # this is where build-qt.py will put the files "CIBW_ENVIRONMENT": "PROCGEN_CMAKE_PREFIX_PATH=/tmp/qt5/qt/build/qtbase/lib/cmake/Qt5", # this is a bit too verbose normally # "CIBW_BUILD_VERBOSITY": "3", }) if platform.system() == "Darwin": # cibuildwheel's python copy on mac os x sometimes fails with this error: # [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1076) urlretrieve( "https://curl.haxx.se/ca/cacert.pem", os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem", ) os.environ[ "SSL_CERT_FILE"] = os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem" elif platform.system() == "Linux": # since we're inside a docker container, adjust the credentials path to point at the mounted location if have_credentials: os.environ["CIBW_ENVIRONMENT"] = ( os.environ["CIBW_ENVIRONMENT"] + " GOOGLE_APPLICATION_CREDENTIALS=/host" + os.environ["GOOGLE_APPLICATION_CREDENTIALS"]) if "TRAVIS_TAG" in os.environ: # pass TRAVIS_TAG to the container so that it can build wheels with the correct version number os.environ["CIBW_ENVIRONMENT"] = (os.environ["CIBW_ENVIRONMENT"] + " TRAVIS_TAG=" + os.environ["TRAVIS_TAG"]) elif platform.system() == "Windows": init_vsvars() run("pip install cibuildwheel==1.4.1") run("cibuildwheel --output-dir wheelhouse") if have_credentials: print("upload wheels", platform.system()) input_dir = "wheelhouse" output_dir = f"gs://{GCS_BUCKET}/builds/" for filename in bf.listdir(input_dir): src = bf.join(input_dir, filename) dst = bf.join(output_dir, filename) print(src, "=>", dst) bf.copy(src, dst, overwrite=True)
def test_listdir_sharded(ctx): contents = b"meow!" with ctx() as path: dirpath = bf.dirname(path) with bf.BlobFile(bf.join(dirpath, "a"), "wb") as w: w.write(contents) with bf.BlobFile(bf.join(dirpath, "aa"), "wb") as w: w.write(contents) with bf.BlobFile(bf.join(dirpath, "b"), "wb") as w: w.write(contents) with bf.BlobFile(bf.join(dirpath, "ca"), "wb") as w: w.write(contents) bf.makedirs(bf.join(dirpath, "c")) with bf.BlobFile(bf.join(dirpath, "c/a"), "wb") as w: w.write(contents) # this should also test shard_prefix_length=2 but that takes too long assert sorted(list(bf.listdir(dirpath, shard_prefix_length=1))) == [ "a", "aa", "b", "c", "ca", ]
def test_invalid_paths(base_path): for suffix in ["", "/", "//", "/invalid.file", "/invalid/dir/"]: path = base_path + suffix print(path) if path.endswith("/"): expected_error = IsADirectoryError else: expected_error = FileNotFoundError list(bf.glob(path)) if suffix == "": for pattern in ["*", "**"]: try: list(bf.glob(path + pattern)) except bf.Error as e: assert "Wildcards cannot be used" in e.message else: for pattern in ["*", "**"]: list(bf.glob(path + pattern)) with pytest.raises(FileNotFoundError): list(bf.listdir(path)) assert not bf.exists(path) assert not bf.isdir(path) with pytest.raises(expected_error): bf.remove(path) if suffix in ("", "/"): try: bf.rmdir(path) except bf.Error as e: assert "Cannot delete bucket" in e.message else: bf.rmdir(path) with pytest.raises(NotADirectoryError): bf.rmtree(path) with pytest.raises(FileNotFoundError): bf.stat(path) if base_path == AZURE_INVALID_CONTAINER_NO_ACCOUNT: with pytest.raises(bf.Error): bf.get_url(path) else: bf.get_url(path) with pytest.raises(FileNotFoundError): bf.md5(path) with pytest.raises(bf.Error): bf.makedirs(path) list(bf.walk(path)) with tempfile.TemporaryDirectory() as tmpdir: local_path = os.path.join(tmpdir, "test.txt") with pytest.raises(expected_error): bf.copy(path, local_path) with open(local_path, "w") as f: f.write("meow") with pytest.raises(expected_error): bf.copy(local_path, path) for streaming in [False, True]: with pytest.raises(expected_error): with bf.BlobFile(path, "rb", streaming=streaming) as f: f.read() with pytest.raises(expected_error): with bf.BlobFile(path, "wb", streaming=streaming) as f: f.write(b"meow")