def test_hit_cache_size_limit(self, start_cluster, URI_cache_10_MB):
"""Test eviction happens when we exceed a nonzero (10MB) cache size."""
NUM_NODES = 3
cluster, address = start_cluster
for i in range(NUM_NODES - 1): # Head node already added.
cluster.add_node(
num_cpus=1, runtime_env_dir_name=f"node_{i}_runtime_resources")
with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir):
with open("test_file_1", "wb") as f:
f.write(os.urandom(8 * 1024 * 1024)) # 8 MiB
ray.init(address, runtime_env={"working_dir": tmp_dir})
@ray.remote
def f():
pass
ray.get(f.remote())
ray.shutdown()
with open("test_file_2", "wb") as f:
f.write(os.urandom(4 * 1024 * 1024))
os.remove("test_file_1")
ray.init(address, runtime_env={"working_dir": tmp_dir})
# Without the cache size limit, we would expect the local dir to be
# 12 MB. Since we do have a size limit, the first package must be
# GC'ed, leaving us with 4 MB. Sleep to give time for deletion.
time.sleep(5)
for node in cluster.list_all_nodes():
local_dir = os.path.join(node.get_runtime_env_dir_path(),
"working_dir_files")
assert 3 < get_directory_size_bytes(local_dir) / (1024**2) < 5
def test_hit_cache_size_limit(
self, start_cluster, URI_cache_10_MB, disable_temporary_uri_pinning
):
"""Test eviction happens when we exceed a nonzero (10MB) cache size."""
NUM_NODES = 3
cluster, address = start_cluster
for i in range(NUM_NODES - 1): # Head node already added.
cluster.add_node(
num_cpus=1, runtime_env_dir_name=f"node_{i}_runtime_resources"
)
print(f'Added node with runtime_env_dir_name "node_{i}_runtime_resources".')
print(f"Added all {NUM_NODES} nodes.")
with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir):
print("Entered tempfile context manager.")
with open("test_file_1", "wb") as f:
f.write(os.urandom(8 * 1024 * 1024)) # 8 MiB
print('Wrote random bytes to "test_file_1" file.')
ray.init(address, runtime_env={"working_dir": tmp_dir})
print(f'Initialized Ray at "{address}" with working_dir.')
@ray.remote
def f():
pass
ray.get(f.remote())
print('Created and received response from task "f".')
ray.shutdown()
print("Ray has been shut down.")
with open("test_file_2", "wb") as f:
f.write(os.urandom(4 * 1024 * 1024))
print('Wrote random bytes to "test_file_2".')
os.remove("test_file_1")
print('Removed "test_file_1".')
ray.init(address, runtime_env={"working_dir": tmp_dir})
print(f'Reinitialized Ray at address "{address}" with working_dir.')
# Without the cache size limit, we would expect the local dir to be
# 12 MB. Since we do have a size limit, the first package must be
# GC'ed, leaving us with 4 MB.
for idx, node in enumerate(cluster.list_all_nodes()):
local_dir = os.path.join(
node.get_runtime_env_dir_path(), "working_dir_files"
)
print("Created local_dir path.")
def local_dir_size_near_4mb():
return 3 < get_directory_size_bytes(local_dir) / (1024 ** 2) < 5
wait_for_condition(local_dir_size_near_4mb)
print(f"get_directory_size_bytes assertion {idx} passed.")
def test_runtime_env_override(call_ray_start):
# https://github.com/ray-project/ray/issues/16481
with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
ray.init(address="auto", namespace="test")
@ray.remote
class Child:
def getcwd(self):
import os
return os.getcwd()
def read(self, path):
return open(path).read()
def ready(self):
pass
@ray.remote
class Parent:
def spawn_child(self, name, runtime_env):
child = Child.options(
lifetime="detached", name=name, runtime_env=runtime_env
).remote()
ray.get(child.ready.remote())
Parent.options(lifetime="detached", name="parent").remote()
ray.shutdown()
with open("hello", "w") as f:
f.write("world")
job_config = ray.job_config.JobConfig(runtime_env={"working_dir": "."})
ray.init(address="auto", namespace="test", job_config=job_config)
os.remove("hello")
parent = ray.get_actor("parent")
env = ray.get_runtime_context().runtime_env
print("Spawning with env:", env)
ray.get(parent.spawn_child.remote("child", env))
child = ray.get_actor("child")
child_cwd = ray.get(child.getcwd.remote())
# Child should be in tmp runtime resource dir.
assert child_cwd != os.getcwd(), (child_cwd, os.getcwd())
assert ray.get(child.read.remote("hello")) == "world"
ray.shutdown()
def test_large_file_error(shutdown_only, option: str):
with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir):
# Write to two separate files, each of which is below the threshold to
# make sure the error is for the full package size.
size = GCS_STORAGE_MAX_SIZE // 2 + 1
with open("test_file_1", "wb") as f:
f.write(os.urandom(size))
with open("test_file_2", "wb") as f:
f.write(os.urandom(size))
with pytest.raises(ValueError):
if option == "working_dir":
ray.init(runtime_env={"working_dir": "."})
else:
ray.init(runtime_env={"py_modules": ["."]})
def test_inheritance(start_cluster, option: str):
"""Tests that child tasks/actors inherit URIs properly."""
cluster, address = start_cluster
with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
with open("hello", "w") as f:
f.write("world")
if option == "working_dir":
ray.init(address, runtime_env={"working_dir": "."})
elif option == "py_modules":
ray.init(address, runtime_env={"py_modules": ["."]})
@ray.remote
def get_env():
return ray.get_runtime_context().runtime_env
@ray.remote
class EnvGetter:
def get(self):
return ray.get_runtime_context().runtime_env
job_env = ray.get_runtime_context().runtime_env
assert ray.get(get_env.remote()) == job_env
eg = EnvGetter.remote()
assert ray.get(eg.get.remote()) == job_env
# Passing a new URI should work.
if option == "working_dir":
env = {"working_dir": S3_PACKAGE_URI}
elif option == "py_modules":
env = {"py_modules": [S3_PACKAGE_URI]}
new_env = ray.get(get_env.options(runtime_env=env).remote())
assert new_env != job_env
eg = EnvGetter.options(runtime_env=env).remote()
assert ray.get(eg.get.remote()) != job_env
# Passing a local directory should not work.
if option == "working_dir":
env = {"working_dir": "."}
elif option == "py_modules":
env = {"py_modules": ["."]}
with pytest.raises(ValueError):
get_env.options(runtime_env=env).remote()
with pytest.raises(ValueError):
EnvGetter.options(runtime_env=env).remote()
def test_requirements_files(start_cluster, field):
"""Test the use of requirements.txt and environment.yaml.
Tests that requirements files are parsed on the driver, not the cluster.
This is the desired behavior because the file paths only make sense on the
driver machine. The files do not exist on the remote cluster.
Also tests the common use case of specifying the option --extra-index-url
in a pip requirements.txt file.
"""
cluster, address = start_cluster
# cd into a temporary directory and pass in pip/conda requirements file
# using a relative path. Since the cluster has already been started,
# the cwd of the processes on the cluster nodes will not be this
# temporary directory. So if the nodes try to read the requirements file,
# this test should fail because the relative path won't make sense.
with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
pip_list = [
"--extra-index-url https://pypi.org/simple",
"pip-install-test==0.5",
]
if field == "conda":
conda_dict = {"dependencies": ["pip", {"pip": pip_list}]}
relative_filepath = "environment.yml"
conda_file = Path(relative_filepath)
conda_file.write_text(yaml.dump(conda_dict))
runtime_env = {"conda": relative_filepath}
elif field == "pip":
relative_filepath = "requirements.txt"
pip_file = Path(relative_filepath)
pip_file.write_text("\n".join(pip_list))
runtime_env = {"pip": relative_filepath}
ray.init(address, runtime_env=runtime_env)
@ray.remote
def f():
import pip_install_test # noqa: F401
return True
# Ensure that the runtime env has been installed.
assert ray.get(f.remote())
def test_large_file_boundary(shutdown_only, option: str):
"""Check that packages just under the max size work as expected."""
with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir):
size = GCS_STORAGE_MAX_SIZE - 1024 * 1024
with open("test_file", "wb") as f:
f.write(os.urandom(size))
if option == "working_dir":
ray.init(runtime_env={"working_dir": "."})
else:
ray.init(runtime_env={"py_modules": ["."]})
@ray.remote
class Test:
def get_size(self):
with open("test_file", "rb") as f:
return len(f.read())
t = Test.remote()
assert ray.get(t.get_size.remote()) == size
def runtime_env_option(request):
import_in_task_script = """
import ray
ray.init(address="auto")
@ray.remote
def f():
import pip_install_test
ray.get(f.remote())
"""
if request.param == "no_working_dir":
yield {
"runtime_env": {},
"entrypoint": "echo hello",
"expected_logs": "hello\n",
}
elif request.param in {
"local_working_dir",
"local_working_dir_zip",
"local_py_modules",
"working_dir_and_local_py_modules_whl",
}:
with tempfile.TemporaryDirectory() as tmp_dir:
path = Path(tmp_dir)
hello_file = path / "test.py"
with hello_file.open(mode="w") as f:
f.write("from test_module import run_test\n")
f.write("print(run_test())")
module_path = path / "test_module"
module_path.mkdir(parents=True)
test_file = module_path / "test.py"
with test_file.open(mode="w") as f:
f.write("def run_test():\n")
f.write(" return 'Hello from test_module!'\n") # noqa: Q000
init_file = module_path / "__init__.py"
with init_file.open(mode="w") as f:
f.write("from test_module.test import run_test\n")
if request.param == "local_working_dir":
yield {
"runtime_env": {
"working_dir": tmp_dir
},
"entrypoint": "python test.py",
"expected_logs": "Hello from test_module!\n",
}
elif request.param == "local_working_dir_zip":
local_zipped_dir = shutil.make_archive(
os.path.join(tmp_dir, "test"), "zip", tmp_dir)
yield {
"runtime_env": {
"working_dir": local_zipped_dir
},
"entrypoint": "python test.py",
"expected_logs": "Hello from test_module!\n",
}
elif request.param == "local_py_modules":
yield {
"runtime_env": {
"py_modules": [str(Path(tmp_dir) / "test_module")]
},
"entrypoint": ("python -c 'import test_module;"
"print(test_module.run_test())'"),
"expected_logs":
"Hello from test_module!\n",
}
elif request.param == "working_dir_and_local_py_modules_whl":
yield {
"runtime_env": {
"working_dir":
"s3://runtime-env-test/script_runtime_env.zip",
"py_modules": [
Path(os.path.dirname(__file__)) /
"pip_install_test-0.5-py3-none-any.whl"
],
},
"entrypoint":
("python script.py && python -c 'import pip_install_test'"
),
"expected_logs":
("Executing main() from script.py !!\n"
"Good job! You installed a pip module."),
}
else:
raise ValueError(
f"Unexpected pytest fixture option {request.param}")
elif request.param == "s3_working_dir":
yield {
"runtime_env": {
"working_dir": "s3://runtime-env-test/script_runtime_env.zip",
},
"entrypoint": "python script.py",
"expected_logs": "Executing main() from script.py !!\n",
}
elif request.param == "pip_txt":
with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
pip_list = ["pip-install-test==0.5"]
relative_filepath = "requirements.txt"
pip_file = Path(relative_filepath)
pip_file.write_text("\n".join(pip_list))
runtime_env = {
"pip": {
"packages": relative_filepath,
"pip_check": False
}
}
yield {
"runtime_env":
runtime_env,
"entrypoint": (f"python -c 'import pip_install_test' && "
f"python -c '{import_in_task_script}'"),
"expected_logs":
"Good job! You installed a pip module.",
}
elif request.param == "conda_yaml":
with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
conda_dict = {
"dependencies": ["pip", {
"pip": ["pip-install-test==0.5"]
}]
}
relative_filepath = "environment.yml"
conda_file = Path(relative_filepath)
conda_file.write_text(yaml.dump(conda_dict))
runtime_env = {"conda": relative_filepath}
yield {
"runtime_env": runtime_env,
"entrypoint": f"python -c '{import_in_task_script}'",
# TODO(architkulkarni): Uncomment after #22968 is fixed.
# "entrypoint": "python -c 'import pip_install_test'",
"expected_logs": "Good job! You installed a pip module.",
}
else:
assert False, f"Unrecognized option: {request.param}."
def test_runtime_env_interface():
# Test the interface related to working_dir
default_working_dir = "s3://bucket/key.zip"
modify_working_dir = "s3://bucket/key_A.zip"
runtime_env = RuntimeEnv(working_dir=default_working_dir)
runtime_env_dict = runtime_env.to_dict()
assert runtime_env.working_dir_uri() == default_working_dir
runtime_env["working_dir"] = modify_working_dir
runtime_env_dict["working_dir"] = modify_working_dir
assert runtime_env.working_dir_uri() == modify_working_dir
assert runtime_env.to_dict() == runtime_env_dict
# Test that the modification of working_dir also works on
# proto serialization
assert runtime_env_dict == RuntimeEnv.from_proto(
runtime_env.build_proto_runtime_env())
runtime_env.pop("working_dir")
assert runtime_env.to_dict() == {}
# Test the interface related to py_modules
init_py_modules = ["s3://bucket/key_1.zip", "s3://bucket/key_2.zip"]
addition_py_modules = ["s3://bucket/key_3.zip", "s3://bucket/key_4.zip"]
runtime_env = RuntimeEnv(py_modules=init_py_modules)
runtime_env_dict = runtime_env.to_dict()
assert set(runtime_env.py_modules_uris()) == set(init_py_modules)
runtime_env["py_modules"].extend(addition_py_modules)
runtime_env_dict["py_modules"].extend(addition_py_modules)
assert set(runtime_env.py_modules_uris()) == set(init_py_modules +
addition_py_modules)
assert runtime_env.to_dict() == runtime_env_dict
# Test that the modification of py_modules also works on
# proto serialization
assert runtime_env_dict == RuntimeEnv.from_proto(
runtime_env.build_proto_runtime_env())
runtime_env.pop("py_modules")
assert runtime_env.to_dict() == {}
# Test the interface related to env_vars
init_env_vars = {"A": "a", "B": "b"}
update_env_vars = {"C": "c"}
runtime_env = RuntimeEnv(env_vars=init_env_vars)
runtime_env_dict = runtime_env.to_dict()
runtime_env["env_vars"].update(update_env_vars)
runtime_env_dict["env_vars"].update(update_env_vars)
init_env_vars_copy = init_env_vars.copy()
init_env_vars_copy.update(update_env_vars)
assert runtime_env["env_vars"] == init_env_vars_copy
assert runtime_env_dict == runtime_env.to_dict()
# Test that the modification of env_vars also works on
# proto serialization
assert runtime_env_dict == RuntimeEnv.from_proto(
runtime_env.build_proto_runtime_env())
runtime_env.pop("env_vars")
assert runtime_env.to_dict() == {}
# Test the interface related to conda
conda_name = "conda"
modify_conda_name = "conda_A"
conda_config = {"dependencies": ["dep1", "dep2"]}
runtime_env = RuntimeEnv(conda=conda_name)
runtime_env_dict = runtime_env.to_dict()
assert runtime_env.has_conda()
assert runtime_env.conda_env_name() == conda_name
assert runtime_env.conda_config() is None
runtime_env["conda"] = modify_conda_name
runtime_env_dict["conda"] = modify_conda_name
assert runtime_env_dict == runtime_env.to_dict()
assert runtime_env.has_conda()
assert runtime_env.conda_env_name() == modify_conda_name
assert runtime_env.conda_config() is None
runtime_env["conda"] = conda_config
runtime_env_dict["conda"] = conda_config
assert runtime_env_dict == runtime_env.to_dict()
assert runtime_env.has_conda()
assert runtime_env.conda_env_name() is None
assert runtime_env.conda_config() == json.dumps(conda_config,
sort_keys=True)
# Test that the modification of conda also works on
# proto serialization
assert runtime_env_dict == RuntimeEnv.from_proto(
runtime_env.build_proto_runtime_env())
runtime_env.pop("conda")
assert runtime_env.to_dict() == {"_ray_commit": "{{RAY_COMMIT_SHA}}"}
# Test the interface related to pip
with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
requirement_file = os.path.join(tmpdir, "requirements.txt")
requirement_packages = ["dep5", "dep6"]
with open(requirement_file, "wt") as f:
for package in requirement_packages:
f.write(package)
f.write("\n")
pip_packages = ["dep1", "dep2"]
addition_pip_packages = ["dep3", "dep4"]
runtime_env = RuntimeEnv(pip=pip_packages)
runtime_env_dict = runtime_env.to_dict()
assert runtime_env.has_pip()
assert set(runtime_env.pip_config()["packages"]) == set(pip_packages)
assert runtime_env.virtualenv_name() is None
runtime_env["pip"]["packages"].extend(addition_pip_packages)
runtime_env_dict["pip"]["packages"].extend(addition_pip_packages)
# The default value of pip_check is False
runtime_env_dict["pip"]["pip_check"] = False
assert runtime_env_dict == runtime_env.to_dict()
assert runtime_env.has_pip()
assert set(
runtime_env.pip_config()["packages"]) == set(pip_packages +
addition_pip_packages)
assert runtime_env.virtualenv_name() is None
runtime_env["pip"] = requirement_file
runtime_env_dict["pip"] = requirement_packages
assert runtime_env.has_pip()
assert set(
runtime_env.pip_config()["packages"]) == set(requirement_packages)
assert runtime_env.virtualenv_name() is None
# The default value of pip_check is False
runtime_env_dict["pip"] = dict(packages=runtime_env_dict["pip"],
pip_check=False)
assert runtime_env_dict == runtime_env.to_dict()
# Test that the modification of pip also works on
# proto serialization
assert runtime_env_dict == RuntimeEnv.from_proto(
runtime_env.build_proto_runtime_env())
runtime_env.pop("pip")
assert runtime_env.to_dict() == {"_ray_commit": "{{RAY_COMMIT_SHA}}"}
# Test conflict
with pytest.raises(ValueError):
RuntimeEnv(pip=pip_packages, conda=conda_name)
runtime_env = RuntimeEnv(pip=pip_packages)
runtime_env["conda"] = conda_name
with pytest.raises(ValueError):
runtime_env.serialize()
# Test the interface related to container
container_init = {
"image": "anyscale/ray-ml:nightly-py38-cpu",
"worker_path": "/root/python/ray/workers/default_worker.py",
"run_options": ["--cap-drop SYS_ADMIN", "--log-level=debug"],
}
update_container = {"image": "test_modify"}
runtime_env = RuntimeEnv(container=container_init)
runtime_env_dict = runtime_env.to_dict()
assert runtime_env.has_py_container()
assert runtime_env.py_container_image() == container_init["image"]
assert runtime_env.py_container_worker_path(
) == container_init["worker_path"]
assert runtime_env.py_container_run_options(
) == container_init["run_options"]
runtime_env["container"].update(update_container)
runtime_env_dict["container"].update(update_container)
container_copy = container_init
container_copy.update(update_container)
assert runtime_env_dict == runtime_env.to_dict()
assert runtime_env.has_py_container()
assert runtime_env.py_container_image() == container_copy["image"]
assert runtime_env.py_container_worker_path(
) == container_copy["worker_path"]
assert runtime_env.py_container_run_options(
) == container_copy["run_options"]
# Test that the modification of container also works on
# proto serialization
assert runtime_env_dict == RuntimeEnv.from_proto(
runtime_env.build_proto_runtime_env())
runtime_env.pop("container")
assert runtime_env.to_dict() == {}
def test_pip_with_env_vars(start_cluster):
with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
TEST_ENV_NAME = "TEST_ENV_VARS"
TEST_ENV_VALUE = "TEST"
package_name = "test_package"
package_dir = os.path.join(tmpdir, package_name)
try_to_create_directory(package_dir)
setup_filename = os.path.join(package_dir, "setup.py")
setup_code = """import os
from setuptools import setup, find_packages
from setuptools.command.install import install
class InstallTestPackage(install):
# this function will be called when pip install this package
def run(self):
assert os.environ.get('{TEST_ENV_NAME}') == '{TEST_ENV_VALUE}'
setup(
name='test_package',
version='0.0.1',
packages=find_packages(),
cmdclass=dict(install=InstallTestPackage),
license="MIT",
zip_safe=False,
)
""".format(TEST_ENV_NAME=TEST_ENV_NAME, TEST_ENV_VALUE=TEST_ENV_VALUE)
with open(setup_filename, "wt") as f:
f.writelines(setup_code)
python_filename = os.path.join(package_dir, "test.py")
python_code = "import os; print(os.environ)"
with open(python_filename, "wt") as f:
f.writelines(python_code)
gz_filename = os.path.join(tmpdir, package_name + ".tar.gz")
subprocess.check_call(["tar", "-zcvf", gz_filename, package_name])
with pytest.raises(ray.exceptions.RuntimeEnvSetupError):
@ray.remote(runtime_env={
"env_vars": {
TEST_ENV_NAME: "failed"
},
"pip": [gz_filename],
})
def f1():
return True
ray.get(f1.remote())
@ray.remote(runtime_env={
"env_vars": {
TEST_ENV_NAME: TEST_ENV_VALUE
},
"pip": [gz_filename],
})
def f2():
return True
assert ray.get(f2.remote())
