def current_ray_pip_specifier() -> Optional[str]:
"""The pip requirement specifier for the running version of Ray.
Returns:
A string which can be passed to `pip install` to install the
currently running Ray version, or None if running on a version
built from source locally (likely if you are developing Ray).
Examples:
Returns "ray[all]==1.4.0" if running the stable release
Returns "https://s3-us-west-2.amazonaws.com/ray-wheels/master/[..].whl"
if running the nightly or a specific commit
"""
logger = get_hook_logger()
if os.environ.get("RAY_CI_POST_WHEEL_TESTS"):
# Running in Buildkite CI after the wheel has been built.
# Wheels are at in the ray/.whl directory, and the present file is
# at ray/python/ray/workers. Use relative paths to allow for
# testing locally if needed.
return os.path.join(
Path(__file__).resolve().parents[3], ".whl", get_wheel_filename())
elif ray.__commit__ == "{{RAY_COMMIT_SHA}}":
# Running on a version built from source locally.
logger.warning(
"Current Ray version could not be detected, most likely "
"because you are using a version of Ray "
"built from source. If you wish to use runtime_env, "
"you can try building a wheel and including the wheel "
"explicitly as a pip dependency.")
return None
elif "dev" in ray.__version__:
# Running on a nightly wheel.
return get_master_wheel_url()
else:
return get_release_wheel_url()
def sleep_setup_runtime_env(runtime_env: dict, session_dir):
logger = get_hook_logger()
logger.info(f"Setting up runtime environment {runtime_env}")
logger.info("Simulating long runtime env setup. Sleeping for 15s...")
time.sleep(15)
logger.info("Finished sleeping for 15s")
return RuntimeEnvContext()
def current_ray_pip_specifier() -> Optional[str]:
"""The pip requirement specifier for the running version of Ray.
Returns:
A string which can be passed to `pip install` to install the
currently running Ray version, or None if running on a version
built from source locally (likely if you are developing Ray).
Examples:
Returns "https://s3-us-west-2.amazonaws.com/ray-wheels/[..].whl"
if running a stable release, a nightly or a specific commit
"""
logger = get_hook_logger()
if os.environ.get("RAY_CI_POST_WHEEL_TESTS"):
# Running in Buildkite CI after the wheel has been built.
# Wheels are at in the ray/.whl directory, and the present file is
# at ray/python/ray/workers. Use relative paths to allow for
# testing locally if needed.
return os.path.join(
Path(__file__).resolve().parents[3], ".whl", get_wheel_filename())
elif ray.__commit__ == "{{RAY_COMMIT_SHA}}":
# Running on a version built from source locally.
if os.environ.get("RAY_RUNTIME_ENV_LOCAL_DEV_MODE") != "1":
logger.warning(
"Current Ray version could not be detected, most likely "
"because you have manually built Ray from source. To use "
"runtime_env in this case, set the environment variable "
"RAY_RUNTIME_ENV_LOCAL_DEV_MODE=1.")
return None
elif "dev" in ray.__version__:
# Running on a nightly wheel.
return get_master_wheel_url()
else:
return get_release_wheel_url()
def get_or_create_conda_env(conda_env_path: str,
base_dir: Optional[str] = None) -> str:
"""
Given a conda YAML, creates a conda environment containing the required
dependencies if such a conda environment doesn't already exist. Returns the
name of the conda environment, which is based on a hash of the YAML.
Args:
conda_env_path: Path to a conda environment YAML file.
base_dir (str, optional): Directory to install the environment into via
the --prefix option to conda create. If not specified, will
install into the default conda directory (e.g. ~/anaconda3/envs)
Returns:
The name of the env, or the path to the env if base_dir is specified.
In either case, the return value should be valid to pass in to
`conda activate`.
"""
logger = get_hook_logger()
conda_path = get_conda_bin_executable("conda")
try:
exec_cmd([conda_path, "--help"], throw_on_error=False)
except EnvironmentError:
raise ValueError(
f"Could not find Conda executable at {conda_path}. "
"Ensure Conda is installed as per the instructions at "
"https://conda.io/projects/conda/en/latest/"
"user-guide/install/index.html. "
"You can also configure Ray to look for a specific "
f"Conda executable by setting the {RAY_CONDA_HOME} "
"environment variable to the path of the Conda executable.")
_, stdout, _ = exec_cmd([conda_path, "env", "list", "--json"])
envs = json.loads(stdout)["envs"]
env_name = _get_conda_env_name(conda_env_path)
if base_dir:
env_name = f"{base_dir}/{env_name}"
if env_name not in envs:
logger.info("Creating conda environment %s", env_name)
exec_cmd([
conda_path, "env", "create", "--file", conda_env_path,
"--prefix", env_name
],
stream_output=True)
return env_name
else:
env_names = [os.path.basename(env) for env in envs]
if env_name not in env_names:
logger.info("Creating conda environment %s", env_name)
exec_cmd([
conda_path, "env", "create", "-n", env_name, "--file",
conda_env_path
],
stream_output=True)
return env_name
def setup_runtime_env(runtime_env: dict, session_dir):
logger = get_hook_logger()
logger.debug(f"Setting up runtime environment {runtime_env}")
if runtime_env.get("conda") or runtime_env.get("pip"):
conda_dict = get_conda_dict(runtime_env, session_dir)
if isinstance(runtime_env.get("conda"), str):
conda_env_name = runtime_env["conda"]
else:
assert conda_dict is not None
ray_pip = current_ray_pip_specifier()
if ray_pip:
extra_pip_dependencies = [ray_pip, "ray[default]"]
elif runtime_env.get("_inject_current_ray"):
extra_pip_dependencies = (
_resolve_install_from_source_ray_dependencies())
else:
extra_pip_dependencies = []
conda_dict = inject_dependencies(conda_dict, _current_py_version(),
extra_pip_dependencies)
logger.info(f"Setting up conda environment with {runtime_env}")
# It is not safe for multiple processes to install conda envs
# concurrently, even if the envs are different, so use a global
# lock for all conda installs.
# See https://github.com/ray-project/ray/issues/17086
file_lock_name = "ray-conda-install.lock"
with FileLock(os.path.join(session_dir, file_lock_name)):
conda_dir = os.path.join(session_dir, "runtime_resources",
"conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(conda_dict, file, sort_keys=True)
conda_env_name = get_or_create_conda_env(
conda_yaml_path, conda_dir)
if runtime_env.get("_inject_current_ray"):
conda_path = os.path.join(conda_dir, conda_env_name)
_inject_ray_to_conda_site(conda_path)
logger.info(
f"Finished setting up runtime environment at {conda_env_name}")
return RuntimeEnvContext(conda_env_name)
return RuntimeEnvContext()
def _inject_ray_to_conda_site(conda_path):
"""Write the current Ray site package directory to a new site"""
python_binary = os.path.join(conda_path, "bin/python")
site_packages_path = subprocess.check_output(
[python_binary, "-c",
"import site; print(site.getsitepackages()[0])"]).decode().strip()
ray_path = _resolve_current_ray_path()
logger = get_hook_logger()
logger.warning(f"Injecting {ray_path} to environment {conda_path} "
"because _inject_current_ray flag is on.")
maybe_ray_dir = os.path.join(site_packages_path, "ray")
if os.path.isdir(maybe_ray_dir):
logger.warning(f"Replacing existing ray installation with {ray_path}")
shutil.rmtree(maybe_ray_dir)
# See usage of *.pth file at
# https://docs.python.org/3/library/site.html
with open(os.path.join(site_packages_path, "ray.pth"), "w") as f:
f.write(ray_path)
