def test_two_node_uri(two_node_cluster, working_dir): cluster, _ = two_node_cluster redis_address = cluster.address import ray._private.runtime_env as runtime_env import tempfile with tempfile.NamedTemporaryFile(suffix="zip") as tmp_file: pkg_name = runtime_env.get_project_package_name(working_dir, []) pkg_uri = runtime_env.Protocol.PIN_GCS.value + "://" + pkg_name runtime_env.create_project_package(working_dir, [], tmp_file.name) runtime_env.push_package(pkg_uri, tmp_file.name) runtime_env = f"""{{ "working_dir_uri": "{pkg_uri}" }}""" script = driver_script.format(redis_address=redis_address, working_dir=working_dir, runtime_env=runtime_env) out = run_string_as_driver(script) assert out.strip().split()[-1] == "1000"
def test_two_node_uri(two_node_cluster, working_dir, client_mode): cluster, _ = two_node_cluster (address, env, PKG_DIR) = start_client_server(cluster, client_mode) import ray._private.runtime_env as runtime_env import tempfile with tempfile.NamedTemporaryFile(suffix="zip") as tmp_file: pkg_name = runtime_env.get_project_package_name(working_dir, []) pkg_uri = runtime_env.Protocol.PIN_GCS.value + "://" + pkg_name runtime_env.create_project_package(working_dir, [], tmp_file.name) runtime_env.push_package(pkg_uri, tmp_file.name) runtime_env = f"""{{ "working_dir_uri": "{pkg_uri}" }}""" execute_statement = "print(sum(ray.get([run_test.remote()] * 1000)))" script = driver_script.format(**locals()) out = run_string_as_driver(script, env) assert out.strip().split()[-1] == "1000" assert len(list(Path(PKG_DIR).iterdir())) == 1
def load_package(config_path: str) -> "_RuntimePackage": """Load the code package given its config path. Args: config_path (str): The path to the configuration YAML that defines the package. For documentation on the packaging format, see the example YAML in ``example_pkg/ray_pkg.yaml``. Examples: >>> # Load from local. >>> my_pkg = load_package("~/path/to/my_pkg.yaml") >>> # Load from GitHub. >>> my_pkg = ray.util.load_package( ... "https://raw.githubusercontent.com/user/repo/refspec" ... "/path/to/package/my_pkg.yaml") >>> # Inspect the package runtime env. >>> print(my_pkg._runtime_env) ... {"conda": {...}, ... "docker": "anyscale-ml/ray-ml:nightly-py38-cpu", ... "working_dir": "https://github.com/demo/foo/blob/v3.0/project/"} >>> # Run remote functions from the package. >>> my_pkg.my_func.remote(1, 2) >>> # Create actors from the package. >>> actor = my_pkg.MyActor.remote(3, 4) >>> # Create new remote funcs in the same env as a package. >>> @ray.remote(runtime_env=my_pkg._runtime_env) >>> def f(): ... """ config_path = _download_from_github_if_needed(config_path) if not os.path.exists(config_path): raise ValueError("Config file does not exist: {}".format(config_path)) # TODO(ekl) validate schema? config = yaml.safe_load(open(config_path).read()) base_dir = os.path.abspath(os.path.dirname(config_path)) runtime_env = config["runtime_env"] # Autofill working directory by uploading to GCS storage. if "working_dir" not in runtime_env: pkg_name = runtime_support.get_project_package_name( working_dir=base_dir, py_modules=[]) pkg_uri = runtime_support.Protocol.GCS.value + "://" + pkg_name def do_register_package(): if not runtime_support.package_exists(pkg_uri): tmp_path = os.path.join(_pkg_tmp(), "_tmp{}".format(pkg_name)) runtime_support.create_project_package(working_dir=base_dir, py_modules=[], output_path=tmp_path) # TODO(ekl) does this get garbage collected correctly with the # current job id? runtime_support.push_package(pkg_uri, tmp_path) if not runtime_support.package_exists(pkg_uri): raise RuntimeError( "Failed to upload package {}".format(pkg_uri)) if ray.is_initialized(): do_register_package() else: ray.worker._post_init_hooks.append(do_register_package) runtime_env["working_dir"] = pkg_uri # Autofill conda config. conda_yaml = os.path.join(base_dir, "conda.yaml") if os.path.exists(conda_yaml): if "conda" in runtime_env: raise ValueError( "Both conda.yaml and conda: section found in package") runtime_env["conda"] = yaml.safe_load(open(conda_yaml).read()) if "stub_file" in config: # TODO(ekl) remove this path print("Warning: stub_file is deprecated, use interface_file instead") interface_file = config["stub_file"] else: interface_file = config["interface_file"] pkg = _RuntimePackage(name=config["name"], desc=config["description"], interface_file=os.path.join(base_dir, interface_file), runtime_env=runtime_env) return pkg