def test_two_node_uri(two_node_cluster, working_dir):
cluster, _ = two_node_cluster
redis_address = cluster.address
import ray._private.runtime_env as runtime_env
import tempfile
with tempfile.NamedTemporaryFile(suffix="zip") as tmp_file:
pkg_name = runtime_env.get_project_package_name(working_dir, [])
pkg_uri = runtime_env.Protocol.PIN_GCS.value + "://" + pkg_name
runtime_env.create_project_package(working_dir, [], tmp_file.name)
runtime_env.push_package(pkg_uri, tmp_file.name)
runtime_env = f"""{{ "working_dir_uri": "{pkg_uri}" }}"""
script = driver_script.format(redis_address=redis_address,
working_dir=working_dir,
runtime_env=runtime_env)
out = run_string_as_driver(script)
assert out.strip().split()[-1] == "1000"
def test_two_node_uri(two_node_cluster, working_dir, client_mode):
cluster, _ = two_node_cluster
(address, env, PKG_DIR) = start_client_server(cluster, client_mode)
import ray._private.runtime_env as runtime_env
import tempfile
with tempfile.NamedTemporaryFile(suffix="zip") as tmp_file:
pkg_name = runtime_env.get_project_package_name(working_dir, [])
pkg_uri = runtime_env.Protocol.PIN_GCS.value + "://" + pkg_name
runtime_env.create_project_package(working_dir, [], tmp_file.name)
runtime_env.push_package(pkg_uri, tmp_file.name)
runtime_env = f"""{{ "working_dir_uri": "{pkg_uri}" }}"""
execute_statement = "print(sum(ray.get([run_test.remote()] * 1000)))"
script = driver_script.format(**locals())
out = run_string_as_driver(script, env)
assert out.strip().split()[-1] == "1000"
assert len(list(Path(PKG_DIR).iterdir())) == 1
def load_package(config_path: str) -> "_RuntimePackage":
"""Load the code package given its config path.
Args:
config_path (str): The path to the configuration YAML that defines
the package. For documentation on the packaging format, see the
example YAML in ``example_pkg/ray_pkg.yaml``.
Examples:
>>> # Load from local.
>>> my_pkg = load_package("~/path/to/my_pkg.yaml")
>>> # Load from GitHub.
>>> my_pkg = ray.util.load_package(
... "https://raw.githubusercontent.com/user/repo/refspec"
... "/path/to/package/my_pkg.yaml")
>>> # Inspect the package runtime env.
>>> print(my_pkg._runtime_env)
... {"conda": {...},
... "docker": "anyscale-ml/ray-ml:nightly-py38-cpu",
... "working_dir": "https://github.com/demo/foo/blob/v3.0/project/"}
>>> # Run remote functions from the package.
>>> my_pkg.my_func.remote(1, 2)
>>> # Create actors from the package.
>>> actor = my_pkg.MyActor.remote(3, 4)
>>> # Create new remote funcs in the same env as a package.
>>> @ray.remote(runtime_env=my_pkg._runtime_env)
>>> def f(): ...
"""
config_path = _download_from_github_if_needed(config_path)
if not os.path.exists(config_path):
raise ValueError("Config file does not exist: {}".format(config_path))
# TODO(ekl) validate schema?
config = yaml.safe_load(open(config_path).read())
base_dir = os.path.abspath(os.path.dirname(config_path))
runtime_env = config["runtime_env"]
# Autofill working directory by uploading to GCS storage.
if "working_dir" not in runtime_env:
pkg_name = runtime_support.get_project_package_name(
working_dir=base_dir, py_modules=[])
pkg_uri = runtime_support.Protocol.GCS.value + "://" + pkg_name
def do_register_package():
if not runtime_support.package_exists(pkg_uri):
tmp_path = os.path.join(_pkg_tmp(), "_tmp{}".format(pkg_name))
runtime_support.create_project_package(working_dir=base_dir,
py_modules=[],
output_path=tmp_path)
# TODO(ekl) does this get garbage collected correctly with the
# current job id?
runtime_support.push_package(pkg_uri, tmp_path)
if not runtime_support.package_exists(pkg_uri):
raise RuntimeError(
"Failed to upload package {}".format(pkg_uri))
if ray.is_initialized():
do_register_package()
else:
ray.worker._post_init_hooks.append(do_register_package)
runtime_env["working_dir"] = pkg_uri
# Autofill conda config.
conda_yaml = os.path.join(base_dir, "conda.yaml")
if os.path.exists(conda_yaml):
if "conda" in runtime_env:
raise ValueError(
"Both conda.yaml and conda: section found in package")
runtime_env["conda"] = yaml.safe_load(open(conda_yaml).read())
if "stub_file" in config:
# TODO(ekl) remove this path
print("Warning: stub_file is deprecated, use interface_file instead")
interface_file = config["stub_file"]
else:
interface_file = config["interface_file"]
pkg = _RuntimePackage(name=config["name"],
desc=config["description"],
interface_file=os.path.join(base_dir,
interface_file),
runtime_env=runtime_env)
return pkg
