def test_copy(parallel):
contents = b"meow!"
with _get_temp_local_path() as local_path1, _get_temp_local_path(
) as local_path2, _get_temp_local_path(
) as local_path3, _get_temp_gcs_path() as gcs_path1, _get_temp_gcs_path(
) as gcs_path2, _get_temp_as_path() as as_path1, _get_temp_as_path(
) as as_path2, _get_temp_as_path(account=AS_TEST_ACCOUNT2,
container=AS_TEST_CONTAINER2) as as_path3:
with pytest.raises(FileNotFoundError):
bf.copy(gcs_path1, gcs_path2, parallel=parallel)
with pytest.raises(FileNotFoundError):
bf.copy(as_path1, as_path2, parallel=parallel)
_write_contents(local_path1, contents)
testcases = [
(local_path1, local_path2),
(local_path1, gcs_path1),
(gcs_path1, gcs_path2),
(gcs_path2, as_path1),
(as_path1, as_path2),
(as_path2, as_path3),
(as_path3, local_path3),
]
for src, dst in testcases:
h = bf.copy(src, dst, return_md5=True, parallel=parallel)
assert h == hashlib.md5(contents).hexdigest()
assert _read_contents(dst) == contents
with pytest.raises(FileExistsError):
bf.copy(src, dst, parallel=parallel)
bf.copy(src, dst, overwrite=True, parallel=parallel)
assert _read_contents(dst) == contents
def main():
have_credentials = setup_google_credentials()
os.environ.update({
"CIBW_BUILD":
"cp36-macosx_x86_64 cp37-macosx_x86_64 cp38-macosx_x86_64 cp36-manylinux_x86_64 cp37-manylinux_x86_64 cp38-manylinux_x86_64 cp36-win_amd64 cp37-win_amd64 cp38-win_amd64",
"CIBW_BEFORE_BUILD":
"pip install -e procgen-build && python -u -m procgen_build.build_qt --output-dir /tmp/qt5",
"CIBW_TEST_EXTRAS":
"test",
# the --pyargs option causes pytest to use the installed procgen wheel
"CIBW_TEST_COMMAND":
"pytest --verbose --benchmark-disable --durations=16 --pyargs procgen",
# this is where build-qt.py will put the files
"CIBW_ENVIRONMENT":
"PROCGEN_CMAKE_PREFIX_PATH=/tmp/qt5/qt/build/qtbase/lib/cmake/Qt5",
# this is a bit too verbose normally
# "CIBW_BUILD_VERBOSITY": "3",
})
if platform.system() == "Darwin":
# cibuildwheel's python copy on mac os x sometimes fails with this error:
# [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1076)
urlretrieve(
"https://curl.haxx.se/ca/cacert.pem",
os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem",
)
os.environ[
"SSL_CERT_FILE"] = os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem"
elif platform.system() == "Linux":
# since we're inside a docker container, adjust the credentials path to point at the mounted location
if have_credentials:
os.environ["CIBW_ENVIRONMENT"] = (
os.environ["CIBW_ENVIRONMENT"] +
" GOOGLE_APPLICATION_CREDENTIALS=/host" +
os.environ["GOOGLE_APPLICATION_CREDENTIALS"])
if "TRAVIS_TAG" in os.environ:
# pass TRAVIS_TAG to the container so that it can build wheels with the correct version number
os.environ["CIBW_ENVIRONMENT"] = (os.environ["CIBW_ENVIRONMENT"] +
" TRAVIS_TAG=" +
os.environ["TRAVIS_TAG"])
elif platform.system() == "Windows":
init_vsvars()
run("pip install cibuildwheel==1.4.1")
run("cibuildwheel --output-dir wheelhouse")
if have_credentials:
print("upload wheels", platform.system())
input_dir = "wheelhouse"
output_dir = f"gs://{GCS_BUCKET}/builds/"
for filename in bf.listdir(input_dir):
src = bf.join(input_dir, filename)
dst = bf.join(output_dir, filename)
print(src, "=>", dst)
bf.copy(src, dst, overwrite=True)
def test_copy():
contents = b"meow!"
with _get_temp_local_path() as local_path1, _get_temp_local_path() as local_path2, _get_temp_local_path() as local_path3, _get_temp_gcs_path() as gcs_path1, _get_temp_gcs_path() as gcs_path2, _get_temp_as_path() as as_path1, _get_temp_as_path() as as_path2:
with pytest.raises(FileNotFoundError):
bf.copy(gcs_path1, gcs_path2)
with pytest.raises(FileNotFoundError):
bf.copy(as_path1, as_path2)
_write_contents(local_path1, contents)
testcases = [
(local_path1, local_path2),
(local_path1, gcs_path1),
(gcs_path1, gcs_path2),
(gcs_path2, as_path1),
(as_path1, as_path2),
(as_path2, local_path3),
]
for src, dst in testcases:
h = bf.copy(src, dst, return_md5=True)
assert h == hashlib.md5(contents).hexdigest()
assert _read_contents(dst) == contents
with pytest.raises(FileExistsError):
bf.copy(src, dst)
bf.copy(src, dst, overwrite=True)
assert _read_contents(dst) == contents
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--remote-dir", required=True)
parser.add_argument("--local-dir", required=True)
parser.add_argument("--size", default=100_000_000, type=int)
parser.add_argument("--loops", default=10, type=int)
parser.add_argument("--verify", action="store_true")
args = parser.parse_args()
tests = [
(
"local_to_remote",
bf.join(args.local_dir, f"file-{args.size}.bin"),
bf.join(args.remote_dir, "file.bin"),
),
(
"remote_to_local",
bf.join(args.remote_dir, f"file-{args.size}.bin"),
bf.join(args.local_dir, "file.bin"),
),
]
for name, src, dst in tests:
data = (b"meow" * 249 + b"mew\n") * (args.size // 1000)
assert len(data) == args.size
if not bf.exists(src) or bf.stat(src).size != args.size:
with bf.BlobFile(src, "wb") as f:
f.write(data)
m = hashlib.md5()
m.update(data)
data_hash = m.hexdigest()
with timer(f"{name}_serial", args.size * args.loops):
for i in range(args.loops):
dst_path = dst + str(i)
bf.copy(src, dst_path)
if args.verify:
verify_hash(data_hash, dst_path)
bf.remove(dst_path)
with timer(f"{name}_parallel", args.size * args.loops):
for i in range(args.loops):
dst_path = dst + str(i)
bf.copy(src, dst_path, parallel=True)
if args.verify:
verify_hash(data_hash, dst_path)
bf.remove(dst_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--version", required=True)
parser.add_argument("--for-real", action="store_true")
args = parser.parse_args()
with tempfile.TemporaryDirectory() as tmpdir:
for filepath in bf.glob(
f"gs://openai-procgen/builds/procgen-{args.version}-*.whl"):
print(filepath)
bf.copy(filepath, bf.join(tmpdir, bf.basename(filepath)))
if args.for_real:
options = []
else:
options = ["--repository-url", "https://test.pypi.org/legacy/"]
subprocess.run([
"python", "-m", "twine", "upload", *options,
*bf.glob(bf.join(tmpdir, "*.whl"))
],
check=True)
def test_invalid_paths(base_path):
for suffix in ["", "/", "//", "/invalid.file", "/invalid/dir/"]:
path = base_path + suffix
print(path)
if path.endswith("/"):
expected_error = IsADirectoryError
else:
expected_error = FileNotFoundError
list(bf.glob(path))
if suffix == "":
for pattern in ["*", "**"]:
try:
list(bf.glob(path + pattern))
except bf.Error as e:
assert "Wildcards cannot be used" in e.message
else:
for pattern in ["*", "**"]:
list(bf.glob(path + pattern))
with pytest.raises(FileNotFoundError):
list(bf.listdir(path))
assert not bf.exists(path)
assert not bf.isdir(path)
with pytest.raises(expected_error):
bf.remove(path)
if suffix in ("", "/"):
try:
bf.rmdir(path)
except bf.Error as e:
assert "Cannot delete bucket" in e.message
else:
bf.rmdir(path)
with pytest.raises(NotADirectoryError):
bf.rmtree(path)
with pytest.raises(FileNotFoundError):
bf.stat(path)
if base_path == AZURE_INVALID_CONTAINER_NO_ACCOUNT:
with pytest.raises(bf.Error):
bf.get_url(path)
else:
bf.get_url(path)
with pytest.raises(FileNotFoundError):
bf.md5(path)
with pytest.raises(bf.Error):
bf.makedirs(path)
list(bf.walk(path))
with tempfile.TemporaryDirectory() as tmpdir:
local_path = os.path.join(tmpdir, "test.txt")
with pytest.raises(expected_error):
bf.copy(path, local_path)
with open(local_path, "w") as f:
f.write("meow")
with pytest.raises(expected_error):
bf.copy(local_path, path)
for streaming in [False, True]:
with pytest.raises(expected_error):
with bf.BlobFile(path, "rb", streaming=streaming) as f:
f.read()
with pytest.raises(expected_error):
with bf.BlobFile(path, "wb", streaming=streaming) as f:
f.write(b"meow")
def test_copy_azure_public():
with _get_temp_as_path() as dst:
bf.copy(AZURE_PUBLIC_URL, dst)
assert _read_contents(dst)[:4] == AZURE_PUBLIC_URL_HEADER
