def current_ray_pip_specifier() -> Optional[str]:
"""The pip requirement specifier for the running version of Ray.
Returns:
A string which can be passed to `pip install` to install the
currently running Ray version, or None if running on a version
built from source locally (likely if you are developing Ray).
Examples:
Returns "ray[all]==1.4.0" if running the stable release
Returns "https://s3-us-west-2.amazonaws.com/ray-wheels/master/[..].whl"
if running the nightly or a specific commit
"""
if os.environ.get("RAY_CI_POST_WHEEL_TESTS"):
# Running in Buildkite CI after the wheel has been built.
# Wheels are at in the ray/.whl directory, and the present file is
# at ray/python/ray/workers. Use relative paths to allow for
# testing locally if needed.
return os.path.join(
Path(__file__).resolve().parents[3], ".whl", get_wheel_filename())
elif ray.__commit__ == "{{RAY_COMMIT_SHA}}":
# Running on a version built from source locally.
logger.warning(
"Current Ray version could not be detected, most likely "
"because you are using a version of Ray "
"built from source. If you wish to use runtime_env, "
"you can try building a wheel and including the wheel "
"explicitly as a pip dependency.")
return None
elif "dev" in ray.__version__:
# Running on a nightly wheel.
return get_master_wheel_url()
else:
return f"ray[all]=={ray.__version__}"
def test_pip_job_config(shutdown_only, pip_as_str):
"""Tests dynamic installation of pip packages in a task's runtime env."""
ray_wheel_path = os.path.join("/ray/.whl", get_wheel_filename())
if pip_as_str:
requirements_txt = f"""
{ray_wheel_path}
pip-install-test==0.5
opentelemetry-api==1.0.0rc1
opentelemetry-sdk==1.0.0rc1
"""
runtime_env = {"pip": requirements_txt}
else:
runtime_env = {
"pip": [
ray_wheel_path, "pip-install-test==0.5",
"opentelemetry-api==1.0.0rc1", "opentelemetry-sdk==1.0.0rc1"
]
}
ray.init(job_config=JobConfig(runtime_env=runtime_env))
@ray.remote
def f():
import pip_install_test # noqa
return True
with pytest.raises(ModuleNotFoundError):
# Ensure pip-install-test is not installed on the test machine
import pip_install_test # noqa
assert ray.get(f.remote())
def test_pip_task(shutdown_only, pip_as_str):
"""Tests pip installs in the runtime env specified in the job config."""
ray.init()
ray_wheel_path = os.path.join("/ray/.whl", get_wheel_filename())
if pip_as_str:
requirements_txt = f"""
{ray_wheel_path}
pip-install-test==0.5
opentelemetry-api==1.0.0rc1
opentelemetry-sdk==1.0.0rc1
"""
runtime_env = {"pip": requirements_txt}
else:
runtime_env = {
"pip": [
ray_wheel_path, "pip-install-test==0.5",
"opentelemetry-api==1.0.0rc1", "opentelemetry-sdk==1.0.0rc1"
]
}
@ray.remote
def f():
import pip_install_test # noqa
return True
with pytest.raises(ModuleNotFoundError):
# Ensure pip-install-test is not installed on the test machine
import pip_install_test # noqa
with pytest.raises(ray.exceptions.RayTaskError) as excinfo:
ray.get(f.remote())
assert "ModuleNotFoundError" in str(excinfo.value)
assert ray.get(f.options(runtime_env=runtime_env).remote())
def test_conda_create_job_config(shutdown_only):
"""Tests dynamic conda env creation in a runtime env in the JobConfig."""
ray_wheel_filename = get_wheel_filename()
# E.g. 3.6.13
python_micro_version_dots = ".".join(map(str, sys.version_info[:3]))
ray_wheel_path = os.path.join("/ray/.whl", ray_wheel_filename)
runtime_env = {
"conda": {
"dependencies": [
f"python={python_micro_version_dots}", "pip", {
"pip": [
ray_wheel_path, "pip-install-test==0.5",
"opentelemetry-api==1.0.0rc1",
"opentelemetry-sdk==1.0.0rc1"
]
}
]
}
}
ray.init(job_config=JobConfig(runtime_env=runtime_env))
@ray.remote
def f():
import pip_install_test # noqa
return True
with pytest.raises(ModuleNotFoundError):
# Ensure pip-install-test is not installed on the test machine
import pip_install_test # noqa
assert ray.get(f.remote())
def test_conda_create_task(shutdown_only):
"""Tests dynamic creation of a conda env in a task's runtime env."""
ray.init()
ray_wheel_filename = get_wheel_filename()
# E.g. 3.6.13
python_micro_version_dots = ".".join(map(str, sys.version_info[:3]))
ray_wheel_path = os.path.join("/ray/.whl", ray_wheel_filename)
runtime_env = {
"conda": {
"dependencies": [
f"python={python_micro_version_dots}", "pip", {
"pip": [
ray_wheel_path, "pip-install-test==0.5",
"opentelemetry-api==1.0.0rc1",
"opentelemetry-sdk==1.0.0rc1"
]
}
]
}
}
@ray.remote
def f():
import pip_install_test # noqa
return True
with pytest.raises(ModuleNotFoundError):
# Ensure pip-install-test is not installed on the test machine
import pip_install_test # noqa
with pytest.raises(ray.exceptions.RayTaskError) as excinfo:
ray.get(f.remote())
assert "ModuleNotFoundError" in str(excinfo.value)
assert ray.get(f.options(runtime_env=runtime_env).remote())
def test_get_wheel_filename():
ray_version = "2.0.0.dev0"
for sys_platform in ["darwin", "linux", "win32"]:
for py_version in ["36", "37", "38"]:
filename = get_wheel_filename(sys_platform, ray_version,
py_version)
prefix = "https://s3-us-west-2.amazonaws.com/ray-wheels/latest/"
url = f"{prefix}{filename}"
assert requests.head(url).status_code == 200