def test_hit_cache_size_limit(self, start_cluster, URI_cache_10_MB): """Test eviction happens when we exceed a nonzero (10MB) cache size.""" NUM_NODES = 3 cluster, address = start_cluster for i in range(NUM_NODES - 1): # Head node already added. cluster.add_node( num_cpus=1, runtime_env_dir_name=f"node_{i}_runtime_resources") with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir): with open("test_file_1", "wb") as f: f.write(os.urandom(8 * 1024 * 1024)) # 8 MiB ray.init(address, runtime_env={"working_dir": tmp_dir}) @ray.remote def f(): pass ray.get(f.remote()) ray.shutdown() with open("test_file_2", "wb") as f: f.write(os.urandom(4 * 1024 * 1024)) os.remove("test_file_1") ray.init(address, runtime_env={"working_dir": tmp_dir}) # Without the cache size limit, we would expect the local dir to be # 12 MB. Since we do have a size limit, the first package must be # GC'ed, leaving us with 4 MB. Sleep to give time for deletion. time.sleep(5) for node in cluster.list_all_nodes(): local_dir = os.path.join(node.get_runtime_env_dir_path(), "working_dir_files") assert 3 < get_directory_size_bytes(local_dir) / (1024**2) < 5
def test_hit_cache_size_limit( self, start_cluster, URI_cache_10_MB, disable_temporary_uri_pinning ): """Test eviction happens when we exceed a nonzero (10MB) cache size.""" NUM_NODES = 3 cluster, address = start_cluster for i in range(NUM_NODES - 1): # Head node already added. cluster.add_node( num_cpus=1, runtime_env_dir_name=f"node_{i}_runtime_resources" ) print(f'Added node with runtime_env_dir_name "node_{i}_runtime_resources".') print(f"Added all {NUM_NODES} nodes.") with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir): print("Entered tempfile context manager.") with open("test_file_1", "wb") as f: f.write(os.urandom(8 * 1024 * 1024)) # 8 MiB print('Wrote random bytes to "test_file_1" file.') ray.init(address, runtime_env={"working_dir": tmp_dir}) print(f'Initialized Ray at "{address}" with working_dir.') @ray.remote def f(): pass ray.get(f.remote()) print('Created and received response from task "f".') ray.shutdown() print("Ray has been shut down.") with open("test_file_2", "wb") as f: f.write(os.urandom(4 * 1024 * 1024)) print('Wrote random bytes to "test_file_2".') os.remove("test_file_1") print('Removed "test_file_1".') ray.init(address, runtime_env={"working_dir": tmp_dir}) print(f'Reinitialized Ray at address "{address}" with working_dir.') # Without the cache size limit, we would expect the local dir to be # 12 MB. Since we do have a size limit, the first package must be # GC'ed, leaving us with 4 MB. for idx, node in enumerate(cluster.list_all_nodes()): local_dir = os.path.join( node.get_runtime_env_dir_path(), "working_dir_files" ) print("Created local_dir path.") def local_dir_size_near_4mb(): return 3 < get_directory_size_bytes(local_dir) / (1024 ** 2) < 5 wait_for_condition(local_dir_size_near_4mb) print(f"get_directory_size_bytes assertion {idx} passed.")
def test_runtime_env_override(call_ray_start): # https://github.com/ray-project/ray/issues/16481 with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir): ray.init(address="auto", namespace="test") @ray.remote class Child: def getcwd(self): import os return os.getcwd() def read(self, path): return open(path).read() def ready(self): pass @ray.remote class Parent: def spawn_child(self, name, runtime_env): child = Child.options( lifetime="detached", name=name, runtime_env=runtime_env ).remote() ray.get(child.ready.remote()) Parent.options(lifetime="detached", name="parent").remote() ray.shutdown() with open("hello", "w") as f: f.write("world") job_config = ray.job_config.JobConfig(runtime_env={"working_dir": "."}) ray.init(address="auto", namespace="test", job_config=job_config) os.remove("hello") parent = ray.get_actor("parent") env = ray.get_runtime_context().runtime_env print("Spawning with env:", env) ray.get(parent.spawn_child.remote("child", env)) child = ray.get_actor("child") child_cwd = ray.get(child.getcwd.remote()) # Child should be in tmp runtime resource dir. assert child_cwd != os.getcwd(), (child_cwd, os.getcwd()) assert ray.get(child.read.remote("hello")) == "world" ray.shutdown()
def test_large_file_error(shutdown_only, option: str): with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir): # Write to two separate files, each of which is below the threshold to # make sure the error is for the full package size. size = GCS_STORAGE_MAX_SIZE // 2 + 1 with open("test_file_1", "wb") as f: f.write(os.urandom(size)) with open("test_file_2", "wb") as f: f.write(os.urandom(size)) with pytest.raises(ValueError): if option == "working_dir": ray.init(runtime_env={"working_dir": "."}) else: ray.init(runtime_env={"py_modules": ["."]})
def test_inheritance(start_cluster, option: str): """Tests that child tasks/actors inherit URIs properly.""" cluster, address = start_cluster with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir): with open("hello", "w") as f: f.write("world") if option == "working_dir": ray.init(address, runtime_env={"working_dir": "."}) elif option == "py_modules": ray.init(address, runtime_env={"py_modules": ["."]}) @ray.remote def get_env(): return ray.get_runtime_context().runtime_env @ray.remote class EnvGetter: def get(self): return ray.get_runtime_context().runtime_env job_env = ray.get_runtime_context().runtime_env assert ray.get(get_env.remote()) == job_env eg = EnvGetter.remote() assert ray.get(eg.get.remote()) == job_env # Passing a new URI should work. if option == "working_dir": env = {"working_dir": S3_PACKAGE_URI} elif option == "py_modules": env = {"py_modules": [S3_PACKAGE_URI]} new_env = ray.get(get_env.options(runtime_env=env).remote()) assert new_env != job_env eg = EnvGetter.options(runtime_env=env).remote() assert ray.get(eg.get.remote()) != job_env # Passing a local directory should not work. if option == "working_dir": env = {"working_dir": "."} elif option == "py_modules": env = {"py_modules": ["."]} with pytest.raises(ValueError): get_env.options(runtime_env=env).remote() with pytest.raises(ValueError): EnvGetter.options(runtime_env=env).remote()
def test_requirements_files(start_cluster, field): """Test the use of requirements.txt and environment.yaml. Tests that requirements files are parsed on the driver, not the cluster. This is the desired behavior because the file paths only make sense on the driver machine. The files do not exist on the remote cluster. Also tests the common use case of specifying the option --extra-index-url in a pip requirements.txt file. """ cluster, address = start_cluster # cd into a temporary directory and pass in pip/conda requirements file # using a relative path. Since the cluster has already been started, # the cwd of the processes on the cluster nodes will not be this # temporary directory. So if the nodes try to read the requirements file, # this test should fail because the relative path won't make sense. with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir): pip_list = [ "--extra-index-url https://pypi.org/simple", "pip-install-test==0.5", ] if field == "conda": conda_dict = {"dependencies": ["pip", {"pip": pip_list}]} relative_filepath = "environment.yml" conda_file = Path(relative_filepath) conda_file.write_text(yaml.dump(conda_dict)) runtime_env = {"conda": relative_filepath} elif field == "pip": relative_filepath = "requirements.txt" pip_file = Path(relative_filepath) pip_file.write_text("\n".join(pip_list)) runtime_env = {"pip": relative_filepath} ray.init(address, runtime_env=runtime_env) @ray.remote def f(): import pip_install_test # noqa: F401 return True # Ensure that the runtime env has been installed. assert ray.get(f.remote())
def test_large_file_boundary(shutdown_only, option: str): """Check that packages just under the max size work as expected.""" with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir): size = GCS_STORAGE_MAX_SIZE - 1024 * 1024 with open("test_file", "wb") as f: f.write(os.urandom(size)) if option == "working_dir": ray.init(runtime_env={"working_dir": "."}) else: ray.init(runtime_env={"py_modules": ["."]}) @ray.remote class Test: def get_size(self): with open("test_file", "rb") as f: return len(f.read()) t = Test.remote() assert ray.get(t.get_size.remote()) == size
def runtime_env_option(request): import_in_task_script = """ import ray ray.init(address="auto") @ray.remote def f(): import pip_install_test ray.get(f.remote()) """ if request.param == "no_working_dir": yield { "runtime_env": {}, "entrypoint": "echo hello", "expected_logs": "hello\n", } elif request.param in { "local_working_dir", "local_working_dir_zip", "local_py_modules", "working_dir_and_local_py_modules_whl", }: with tempfile.TemporaryDirectory() as tmp_dir: path = Path(tmp_dir) hello_file = path / "test.py" with hello_file.open(mode="w") as f: f.write("from test_module import run_test\n") f.write("print(run_test())") module_path = path / "test_module" module_path.mkdir(parents=True) test_file = module_path / "test.py" with test_file.open(mode="w") as f: f.write("def run_test():\n") f.write(" return 'Hello from test_module!'\n") # noqa: Q000 init_file = module_path / "__init__.py" with init_file.open(mode="w") as f: f.write("from test_module.test import run_test\n") if request.param == "local_working_dir": yield { "runtime_env": { "working_dir": tmp_dir }, "entrypoint": "python test.py", "expected_logs": "Hello from test_module!\n", } elif request.param == "local_working_dir_zip": local_zipped_dir = shutil.make_archive( os.path.join(tmp_dir, "test"), "zip", tmp_dir) yield { "runtime_env": { "working_dir": local_zipped_dir }, "entrypoint": "python test.py", "expected_logs": "Hello from test_module!\n", } elif request.param == "local_py_modules": yield { "runtime_env": { "py_modules": [str(Path(tmp_dir) / "test_module")] }, "entrypoint": ("python -c 'import test_module;" "print(test_module.run_test())'"), "expected_logs": "Hello from test_module!\n", } elif request.param == "working_dir_and_local_py_modules_whl": yield { "runtime_env": { "working_dir": "s3://runtime-env-test/script_runtime_env.zip", "py_modules": [ Path(os.path.dirname(__file__)) / "pip_install_test-0.5-py3-none-any.whl" ], }, "entrypoint": ("python script.py && python -c 'import pip_install_test'" ), "expected_logs": ("Executing main() from script.py !!\n" "Good job! You installed a pip module."), } else: raise ValueError( f"Unexpected pytest fixture option {request.param}") elif request.param == "s3_working_dir": yield { "runtime_env": { "working_dir": "s3://runtime-env-test/script_runtime_env.zip", }, "entrypoint": "python script.py", "expected_logs": "Executing main() from script.py !!\n", } elif request.param == "pip_txt": with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir): pip_list = ["pip-install-test==0.5"] relative_filepath = "requirements.txt" pip_file = Path(relative_filepath) pip_file.write_text("\n".join(pip_list)) runtime_env = { "pip": { "packages": relative_filepath, "pip_check": False } } yield { "runtime_env": runtime_env, "entrypoint": (f"python -c 'import pip_install_test' && " f"python -c '{import_in_task_script}'"), "expected_logs": "Good job! You installed a pip module.", } elif request.param == "conda_yaml": with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir): conda_dict = { "dependencies": ["pip", { "pip": ["pip-install-test==0.5"] }] } relative_filepath = "environment.yml" conda_file = Path(relative_filepath) conda_file.write_text(yaml.dump(conda_dict)) runtime_env = {"conda": relative_filepath} yield { "runtime_env": runtime_env, "entrypoint": f"python -c '{import_in_task_script}'", # TODO(architkulkarni): Uncomment after #22968 is fixed. # "entrypoint": "python -c 'import pip_install_test'", "expected_logs": "Good job! You installed a pip module.", } else: assert False, f"Unrecognized option: {request.param}."
def test_runtime_env_interface(): # Test the interface related to working_dir default_working_dir = "s3://bucket/key.zip" modify_working_dir = "s3://bucket/key_A.zip" runtime_env = RuntimeEnv(working_dir=default_working_dir) runtime_env_dict = runtime_env.to_dict() assert runtime_env.working_dir_uri() == default_working_dir runtime_env["working_dir"] = modify_working_dir runtime_env_dict["working_dir"] = modify_working_dir assert runtime_env.working_dir_uri() == modify_working_dir assert runtime_env.to_dict() == runtime_env_dict # Test that the modification of working_dir also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("working_dir") assert runtime_env.to_dict() == {} # Test the interface related to py_modules init_py_modules = ["s3://bucket/key_1.zip", "s3://bucket/key_2.zip"] addition_py_modules = ["s3://bucket/key_3.zip", "s3://bucket/key_4.zip"] runtime_env = RuntimeEnv(py_modules=init_py_modules) runtime_env_dict = runtime_env.to_dict() assert set(runtime_env.py_modules_uris()) == set(init_py_modules) runtime_env["py_modules"].extend(addition_py_modules) runtime_env_dict["py_modules"].extend(addition_py_modules) assert set(runtime_env.py_modules_uris()) == set(init_py_modules + addition_py_modules) assert runtime_env.to_dict() == runtime_env_dict # Test that the modification of py_modules also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("py_modules") assert runtime_env.to_dict() == {} # Test the interface related to env_vars init_env_vars = {"A": "a", "B": "b"} update_env_vars = {"C": "c"} runtime_env = RuntimeEnv(env_vars=init_env_vars) runtime_env_dict = runtime_env.to_dict() runtime_env["env_vars"].update(update_env_vars) runtime_env_dict["env_vars"].update(update_env_vars) init_env_vars_copy = init_env_vars.copy() init_env_vars_copy.update(update_env_vars) assert runtime_env["env_vars"] == init_env_vars_copy assert runtime_env_dict == runtime_env.to_dict() # Test that the modification of env_vars also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("env_vars") assert runtime_env.to_dict() == {} # Test the interface related to conda conda_name = "conda" modify_conda_name = "conda_A" conda_config = {"dependencies": ["dep1", "dep2"]} runtime_env = RuntimeEnv(conda=conda_name) runtime_env_dict = runtime_env.to_dict() assert runtime_env.has_conda() assert runtime_env.conda_env_name() == conda_name assert runtime_env.conda_config() is None runtime_env["conda"] = modify_conda_name runtime_env_dict["conda"] = modify_conda_name assert runtime_env_dict == runtime_env.to_dict() assert runtime_env.has_conda() assert runtime_env.conda_env_name() == modify_conda_name assert runtime_env.conda_config() is None runtime_env["conda"] = conda_config runtime_env_dict["conda"] = conda_config assert runtime_env_dict == runtime_env.to_dict() assert runtime_env.has_conda() assert runtime_env.conda_env_name() is None assert runtime_env.conda_config() == json.dumps(conda_config, sort_keys=True) # Test that the modification of conda also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("conda") assert runtime_env.to_dict() == {"_ray_commit": "{{RAY_COMMIT_SHA}}"} # Test the interface related to pip with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir): requirement_file = os.path.join(tmpdir, "requirements.txt") requirement_packages = ["dep5", "dep6"] with open(requirement_file, "wt") as f: for package in requirement_packages: f.write(package) f.write("\n") pip_packages = ["dep1", "dep2"] addition_pip_packages = ["dep3", "dep4"] runtime_env = RuntimeEnv(pip=pip_packages) runtime_env_dict = runtime_env.to_dict() assert runtime_env.has_pip() assert set(runtime_env.pip_config()["packages"]) == set(pip_packages) assert runtime_env.virtualenv_name() is None runtime_env["pip"]["packages"].extend(addition_pip_packages) runtime_env_dict["pip"]["packages"].extend(addition_pip_packages) # The default value of pip_check is False runtime_env_dict["pip"]["pip_check"] = False assert runtime_env_dict == runtime_env.to_dict() assert runtime_env.has_pip() assert set( runtime_env.pip_config()["packages"]) == set(pip_packages + addition_pip_packages) assert runtime_env.virtualenv_name() is None runtime_env["pip"] = requirement_file runtime_env_dict["pip"] = requirement_packages assert runtime_env.has_pip() assert set( runtime_env.pip_config()["packages"]) == set(requirement_packages) assert runtime_env.virtualenv_name() is None # The default value of pip_check is False runtime_env_dict["pip"] = dict(packages=runtime_env_dict["pip"], pip_check=False) assert runtime_env_dict == runtime_env.to_dict() # Test that the modification of pip also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("pip") assert runtime_env.to_dict() == {"_ray_commit": "{{RAY_COMMIT_SHA}}"} # Test conflict with pytest.raises(ValueError): RuntimeEnv(pip=pip_packages, conda=conda_name) runtime_env = RuntimeEnv(pip=pip_packages) runtime_env["conda"] = conda_name with pytest.raises(ValueError): runtime_env.serialize() # Test the interface related to container container_init = { "image": "anyscale/ray-ml:nightly-py38-cpu", "worker_path": "/root/python/ray/workers/default_worker.py", "run_options": ["--cap-drop SYS_ADMIN", "--log-level=debug"], } update_container = {"image": "test_modify"} runtime_env = RuntimeEnv(container=container_init) runtime_env_dict = runtime_env.to_dict() assert runtime_env.has_py_container() assert runtime_env.py_container_image() == container_init["image"] assert runtime_env.py_container_worker_path( ) == container_init["worker_path"] assert runtime_env.py_container_run_options( ) == container_init["run_options"] runtime_env["container"].update(update_container) runtime_env_dict["container"].update(update_container) container_copy = container_init container_copy.update(update_container) assert runtime_env_dict == runtime_env.to_dict() assert runtime_env.has_py_container() assert runtime_env.py_container_image() == container_copy["image"] assert runtime_env.py_container_worker_path( ) == container_copy["worker_path"] assert runtime_env.py_container_run_options( ) == container_copy["run_options"] # Test that the modification of container also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("container") assert runtime_env.to_dict() == {}
def test_pip_with_env_vars(start_cluster): with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir): TEST_ENV_NAME = "TEST_ENV_VARS" TEST_ENV_VALUE = "TEST" package_name = "test_package" package_dir = os.path.join(tmpdir, package_name) try_to_create_directory(package_dir) setup_filename = os.path.join(package_dir, "setup.py") setup_code = """import os from setuptools import setup, find_packages from setuptools.command.install import install class InstallTestPackage(install): # this function will be called when pip install this package def run(self): assert os.environ.get('{TEST_ENV_NAME}') == '{TEST_ENV_VALUE}' setup( name='test_package', version='0.0.1', packages=find_packages(), cmdclass=dict(install=InstallTestPackage), license="MIT", zip_safe=False, ) """.format(TEST_ENV_NAME=TEST_ENV_NAME, TEST_ENV_VALUE=TEST_ENV_VALUE) with open(setup_filename, "wt") as f: f.writelines(setup_code) python_filename = os.path.join(package_dir, "test.py") python_code = "import os; print(os.environ)" with open(python_filename, "wt") as f: f.writelines(python_code) gz_filename = os.path.join(tmpdir, package_name + ".tar.gz") subprocess.check_call(["tar", "-zcvf", gz_filename, package_name]) with pytest.raises(ray.exceptions.RuntimeEnvSetupError): @ray.remote(runtime_env={ "env_vars": { TEST_ENV_NAME: "failed" }, "pip": [gz_filename], }) def f1(): return True ray.get(f1.remote()) @ray.remote(runtime_env={ "env_vars": { TEST_ENV_NAME: TEST_ENV_VALUE }, "pip": [gz_filename], }) def f2(): return True assert ray.get(f2.remote())