def test_copy(parallel): contents = b"meow!" with _get_temp_local_path() as local_path1, _get_temp_local_path( ) as local_path2, _get_temp_local_path( ) as local_path3, _get_temp_gcs_path() as gcs_path1, _get_temp_gcs_path( ) as gcs_path2, _get_temp_as_path() as as_path1, _get_temp_as_path( ) as as_path2, _get_temp_as_path(account=AS_TEST_ACCOUNT2, container=AS_TEST_CONTAINER2) as as_path3: with pytest.raises(FileNotFoundError): bf.copy(gcs_path1, gcs_path2, parallel=parallel) with pytest.raises(FileNotFoundError): bf.copy(as_path1, as_path2, parallel=parallel) _write_contents(local_path1, contents) testcases = [ (local_path1, local_path2), (local_path1, gcs_path1), (gcs_path1, gcs_path2), (gcs_path2, as_path1), (as_path1, as_path2), (as_path2, as_path3), (as_path3, local_path3), ] for src, dst in testcases: h = bf.copy(src, dst, return_md5=True, parallel=parallel) assert h == hashlib.md5(contents).hexdigest() assert _read_contents(dst) == contents with pytest.raises(FileExistsError): bf.copy(src, dst, parallel=parallel) bf.copy(src, dst, overwrite=True, parallel=parallel) assert _read_contents(dst) == contents
def main(): have_credentials = setup_google_credentials() os.environ.update({ "CIBW_BUILD": "cp36-macosx_x86_64 cp37-macosx_x86_64 cp38-macosx_x86_64 cp36-manylinux_x86_64 cp37-manylinux_x86_64 cp38-manylinux_x86_64 cp36-win_amd64 cp37-win_amd64 cp38-win_amd64", "CIBW_BEFORE_BUILD": "pip install -e procgen-build && python -u -m procgen_build.build_qt --output-dir /tmp/qt5", "CIBW_TEST_EXTRAS": "test", # the --pyargs option causes pytest to use the installed procgen wheel "CIBW_TEST_COMMAND": "pytest --verbose --benchmark-disable --durations=16 --pyargs procgen", # this is where build-qt.py will put the files "CIBW_ENVIRONMENT": "PROCGEN_CMAKE_PREFIX_PATH=/tmp/qt5/qt/build/qtbase/lib/cmake/Qt5", # this is a bit too verbose normally # "CIBW_BUILD_VERBOSITY": "3", }) if platform.system() == "Darwin": # cibuildwheel's python copy on mac os x sometimes fails with this error: # [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1076) urlretrieve( "https://curl.haxx.se/ca/cacert.pem", os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem", ) os.environ[ "SSL_CERT_FILE"] = os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem" elif platform.system() == "Linux": # since we're inside a docker container, adjust the credentials path to point at the mounted location if have_credentials: os.environ["CIBW_ENVIRONMENT"] = ( os.environ["CIBW_ENVIRONMENT"] + " GOOGLE_APPLICATION_CREDENTIALS=/host" + os.environ["GOOGLE_APPLICATION_CREDENTIALS"]) if "TRAVIS_TAG" in os.environ: # pass TRAVIS_TAG to the container so that it can build wheels with the correct version number os.environ["CIBW_ENVIRONMENT"] = (os.environ["CIBW_ENVIRONMENT"] + " TRAVIS_TAG=" + os.environ["TRAVIS_TAG"]) elif platform.system() == "Windows": init_vsvars() run("pip install cibuildwheel==1.4.1") run("cibuildwheel --output-dir wheelhouse") if have_credentials: print("upload wheels", platform.system()) input_dir = "wheelhouse" output_dir = f"gs://{GCS_BUCKET}/builds/" for filename in bf.listdir(input_dir): src = bf.join(input_dir, filename) dst = bf.join(output_dir, filename) print(src, "=>", dst) bf.copy(src, dst, overwrite=True)
def test_copy(): contents = b"meow!" with _get_temp_local_path() as local_path1, _get_temp_local_path() as local_path2, _get_temp_local_path() as local_path3, _get_temp_gcs_path() as gcs_path1, _get_temp_gcs_path() as gcs_path2, _get_temp_as_path() as as_path1, _get_temp_as_path() as as_path2: with pytest.raises(FileNotFoundError): bf.copy(gcs_path1, gcs_path2) with pytest.raises(FileNotFoundError): bf.copy(as_path1, as_path2) _write_contents(local_path1, contents) testcases = [ (local_path1, local_path2), (local_path1, gcs_path1), (gcs_path1, gcs_path2), (gcs_path2, as_path1), (as_path1, as_path2), (as_path2, local_path3), ] for src, dst in testcases: h = bf.copy(src, dst, return_md5=True) assert h == hashlib.md5(contents).hexdigest() assert _read_contents(dst) == contents with pytest.raises(FileExistsError): bf.copy(src, dst) bf.copy(src, dst, overwrite=True) assert _read_contents(dst) == contents
def main(): parser = argparse.ArgumentParser() parser.add_argument("--remote-dir", required=True) parser.add_argument("--local-dir", required=True) parser.add_argument("--size", default=100_000_000, type=int) parser.add_argument("--loops", default=10, type=int) parser.add_argument("--verify", action="store_true") args = parser.parse_args() tests = [ ( "local_to_remote", bf.join(args.local_dir, f"file-{args.size}.bin"), bf.join(args.remote_dir, "file.bin"), ), ( "remote_to_local", bf.join(args.remote_dir, f"file-{args.size}.bin"), bf.join(args.local_dir, "file.bin"), ), ] for name, src, dst in tests: data = (b"meow" * 249 + b"mew\n") * (args.size // 1000) assert len(data) == args.size if not bf.exists(src) or bf.stat(src).size != args.size: with bf.BlobFile(src, "wb") as f: f.write(data) m = hashlib.md5() m.update(data) data_hash = m.hexdigest() with timer(f"{name}_serial", args.size * args.loops): for i in range(args.loops): dst_path = dst + str(i) bf.copy(src, dst_path) if args.verify: verify_hash(data_hash, dst_path) bf.remove(dst_path) with timer(f"{name}_parallel", args.size * args.loops): for i in range(args.loops): dst_path = dst + str(i) bf.copy(src, dst_path, parallel=True) if args.verify: verify_hash(data_hash, dst_path) bf.remove(dst_path)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--version", required=True) parser.add_argument("--for-real", action="store_true") args = parser.parse_args() with tempfile.TemporaryDirectory() as tmpdir: for filepath in bf.glob( f"gs://openai-procgen/builds/procgen-{args.version}-*.whl"): print(filepath) bf.copy(filepath, bf.join(tmpdir, bf.basename(filepath))) if args.for_real: options = [] else: options = ["--repository-url", "https://test.pypi.org/legacy/"] subprocess.run([ "python", "-m", "twine", "upload", *options, *bf.glob(bf.join(tmpdir, "*.whl")) ], check=True)
def test_invalid_paths(base_path): for suffix in ["", "/", "//", "/invalid.file", "/invalid/dir/"]: path = base_path + suffix print(path) if path.endswith("/"): expected_error = IsADirectoryError else: expected_error = FileNotFoundError list(bf.glob(path)) if suffix == "": for pattern in ["*", "**"]: try: list(bf.glob(path + pattern)) except bf.Error as e: assert "Wildcards cannot be used" in e.message else: for pattern in ["*", "**"]: list(bf.glob(path + pattern)) with pytest.raises(FileNotFoundError): list(bf.listdir(path)) assert not bf.exists(path) assert not bf.isdir(path) with pytest.raises(expected_error): bf.remove(path) if suffix in ("", "/"): try: bf.rmdir(path) except bf.Error as e: assert "Cannot delete bucket" in e.message else: bf.rmdir(path) with pytest.raises(NotADirectoryError): bf.rmtree(path) with pytest.raises(FileNotFoundError): bf.stat(path) if base_path == AZURE_INVALID_CONTAINER_NO_ACCOUNT: with pytest.raises(bf.Error): bf.get_url(path) else: bf.get_url(path) with pytest.raises(FileNotFoundError): bf.md5(path) with pytest.raises(bf.Error): bf.makedirs(path) list(bf.walk(path)) with tempfile.TemporaryDirectory() as tmpdir: local_path = os.path.join(tmpdir, "test.txt") with pytest.raises(expected_error): bf.copy(path, local_path) with open(local_path, "w") as f: f.write("meow") with pytest.raises(expected_error): bf.copy(local_path, path) for streaming in [False, True]: with pytest.raises(expected_error): with bf.BlobFile(path, "rb", streaming=streaming) as f: f.read() with pytest.raises(expected_error): with bf.BlobFile(path, "wb", streaming=streaming) as f: f.write(b"meow")
def test_copy_azure_public(): with _get_temp_as_path() as dst: bf.copy(AZURE_PUBLIC_URL, dst) assert _read_contents(dst)[:4] == AZURE_PUBLIC_URL_HEADER