def test_serialize_deserialize(option): runtime_env = dict() if option == "pip_list": runtime_env["pip"] = ["pkg1", "pkg2"] elif option == "conda_name": runtime_env["conda"] = "env_name" elif option == "conda_dict": runtime_env["conda"] = {"dependencies": ["dep1", "dep2"]} elif option == "container": runtime_env["container"] = { "image": "anyscale/ray-ml:nightly-py38-cpu", "worker_path": "/root/python/ray/workers/default_worker.py", "run_options": ["--cap-drop SYS_ADMIN", "--log-level=debug"], } elif option == "plugins": runtime_env["plugins"] = { "class_path1": { "config1": "val1" }, "class_path2": "string_config", } else: raise ValueError("unexpected option " + str(option)) proto_runtime_env = RuntimeEnv(**runtime_env, _validate=False)._proto_runtime_env cls_runtime_env = RuntimeEnv.from_proto(proto_runtime_env) assert cls_runtime_env.to_dict() == runtime_env
def test_serialize_deserialize(option): runtime_env = dict() if option == "pip_list": runtime_env["pip"] = ["pkg1", "pkg2"] elif option == "pip_dict": runtime_env["pip"] = { "packages": ["pkg1", "pkg2"], "pip_check": False, "pip_version": "<22,>20", } elif option == "conda_name": runtime_env["conda"] = "env_name" elif option == "conda_dict": runtime_env["conda"] = {"dependencies": ["dep1", "dep2"]} elif option == "container": runtime_env["container"] = { "image": "anyscale/ray-ml:nightly-py38-cpu", "worker_path": "/root/python/ray/_private/workers/default_worker.py", "run_options": ["--cap-drop SYS_ADMIN", "--log-level=debug"], } elif option == "plugins": runtime_env["plugins"] = { "class_path1": { "config1": "val1" }, "class_path2": "string_config", } else: raise ValueError("unexpected option " + str(option)) proto_runtime_env = RuntimeEnv(**runtime_env, _validate=False).build_proto_runtime_env() cls_runtime_env = RuntimeEnv.from_proto(proto_runtime_env) cls_runtime_env_dict = cls_runtime_env.to_dict() if "pip" in runtime_env and isinstance(runtime_env["pip"], list): pip_config_in_cls_runtime_env = cls_runtime_env_dict.pop("pip") pip_config_in_runtime_env = runtime_env.pop("pip") assert { "packages": pip_config_in_runtime_env, "pip_check": False, } == pip_config_in_cls_runtime_env assert cls_runtime_env_dict == runtime_env
def test_runtime_env_interface(): # Test the interface related to working_dir default_working_dir = "s3://bucket/key.zip" modify_working_dir = "s3://bucket/key_A.zip" runtime_env = RuntimeEnv(working_dir=default_working_dir) runtime_env_dict = runtime_env.to_dict() assert runtime_env.working_dir_uri() == default_working_dir runtime_env["working_dir"] = modify_working_dir runtime_env_dict["working_dir"] = modify_working_dir assert runtime_env.working_dir_uri() == modify_working_dir assert runtime_env.to_dict() == runtime_env_dict # Test that the modification of working_dir also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("working_dir") assert runtime_env.to_dict() == {} # Test the interface related to py_modules init_py_modules = ["s3://bucket/key_1.zip", "s3://bucket/key_2.zip"] addition_py_modules = ["s3://bucket/key_3.zip", "s3://bucket/key_4.zip"] runtime_env = RuntimeEnv(py_modules=init_py_modules) runtime_env_dict = runtime_env.to_dict() assert set(runtime_env.py_modules_uris()) == set(init_py_modules) runtime_env["py_modules"].extend(addition_py_modules) runtime_env_dict["py_modules"].extend(addition_py_modules) assert set(runtime_env.py_modules_uris()) == set(init_py_modules + addition_py_modules) assert runtime_env.to_dict() == runtime_env_dict # Test that the modification of py_modules also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("py_modules") assert runtime_env.to_dict() == {} # Test the interface related to env_vars init_env_vars = {"A": "a", "B": "b"} update_env_vars = {"C": "c"} runtime_env = RuntimeEnv(env_vars=init_env_vars) runtime_env_dict = runtime_env.to_dict() runtime_env["env_vars"].update(update_env_vars) runtime_env_dict["env_vars"].update(update_env_vars) init_env_vars_copy = init_env_vars.copy() init_env_vars_copy.update(update_env_vars) assert runtime_env["env_vars"] == init_env_vars_copy assert runtime_env_dict == runtime_env.to_dict() # Test that the modification of env_vars also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("env_vars") assert runtime_env.to_dict() == {} # Test the interface related to conda conda_name = "conda" modify_conda_name = "conda_A" conda_config = {"dependencies": ["dep1", "dep2"]} runtime_env = RuntimeEnv(conda=conda_name) runtime_env_dict = runtime_env.to_dict() assert runtime_env.has_conda() assert runtime_env.conda_env_name() == conda_name assert runtime_env.conda_config() is None runtime_env["conda"] = modify_conda_name runtime_env_dict["conda"] = modify_conda_name assert runtime_env_dict == runtime_env.to_dict() assert runtime_env.has_conda() assert runtime_env.conda_env_name() == modify_conda_name assert runtime_env.conda_config() is None runtime_env["conda"] = conda_config runtime_env_dict["conda"] = conda_config assert runtime_env_dict == runtime_env.to_dict() assert runtime_env.has_conda() assert runtime_env.conda_env_name() is None assert runtime_env.conda_config() == json.dumps(conda_config, sort_keys=True) # Test that the modification of conda also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("conda") assert runtime_env.to_dict() == {"_ray_commit": "{{RAY_COMMIT_SHA}}"} # Test the interface related to pip with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir): requirement_file = os.path.join(tmpdir, "requirements.txt") requirement_packages = ["dep5", "dep6"] with open(requirement_file, "wt") as f: for package in requirement_packages: f.write(package) f.write("\n") pip_packages = ["dep1", "dep2"] addition_pip_packages = ["dep3", "dep4"] runtime_env = RuntimeEnv(pip=pip_packages) runtime_env_dict = runtime_env.to_dict() assert runtime_env.has_pip() assert set(runtime_env.pip_config()["packages"]) == set(pip_packages) assert runtime_env.virtualenv_name() is None runtime_env["pip"]["packages"].extend(addition_pip_packages) runtime_env_dict["pip"]["packages"].extend(addition_pip_packages) # The default value of pip_check is False runtime_env_dict["pip"]["pip_check"] = False assert runtime_env_dict == runtime_env.to_dict() assert runtime_env.has_pip() assert set( runtime_env.pip_config()["packages"]) == set(pip_packages + addition_pip_packages) assert runtime_env.virtualenv_name() is None runtime_env["pip"] = requirement_file runtime_env_dict["pip"] = requirement_packages assert runtime_env.has_pip() assert set( runtime_env.pip_config()["packages"]) == set(requirement_packages) assert runtime_env.virtualenv_name() is None # The default value of pip_check is False runtime_env_dict["pip"] = dict(packages=runtime_env_dict["pip"], pip_check=False) assert runtime_env_dict == runtime_env.to_dict() # Test that the modification of pip also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("pip") assert runtime_env.to_dict() == {"_ray_commit": "{{RAY_COMMIT_SHA}}"} # Test conflict with pytest.raises(ValueError): RuntimeEnv(pip=pip_packages, conda=conda_name) runtime_env = RuntimeEnv(pip=pip_packages) runtime_env["conda"] = conda_name with pytest.raises(ValueError): runtime_env.serialize() # Test the interface related to container container_init = { "image": "anyscale/ray-ml:nightly-py38-cpu", "worker_path": "/root/python/ray/workers/default_worker.py", "run_options": ["--cap-drop SYS_ADMIN", "--log-level=debug"], } update_container = {"image": "test_modify"} runtime_env = RuntimeEnv(container=container_init) runtime_env_dict = runtime_env.to_dict() assert runtime_env.has_py_container() assert runtime_env.py_container_image() == container_init["image"] assert runtime_env.py_container_worker_path( ) == container_init["worker_path"] assert runtime_env.py_container_run_options( ) == container_init["run_options"] runtime_env["container"].update(update_container) runtime_env_dict["container"].update(update_container) container_copy = container_init container_copy.update(update_container) assert runtime_env_dict == runtime_env.to_dict() assert runtime_env.has_py_container() assert runtime_env.py_container_image() == container_copy["image"] assert runtime_env.py_container_worker_path( ) == container_copy["worker_path"] assert runtime_env.py_container_run_options( ) == container_copy["run_options"] # Test that the modification of container also works on # proto serialization assert runtime_env_dict == RuntimeEnv.from_proto( runtime_env.build_proto_runtime_env()) runtime_env.pop("container") assert runtime_env.to_dict() == {}