def get_conda_dict(runtime_env, resources_dir) -> Optional[Dict[Any, Any]]:
""" Construct a conda dependencies dict from a runtime env.
This function does not inject Ray or Python into the conda dict.
If the runtime env does not specify pip or conda, or if it specifies
the name of a preinstalled conda environment, this function returns
None. If pip is specified, a conda dict is created containing the
pip dependencies. If conda is already given as a dict, this function
is the identity function.
"""
if runtime_env.has_conda():
if runtime_env.conda_config():
return json.loads(runtime_env.conda_config())
else:
return None
if runtime_env.has_pip():
requirements_txt = "\n".join(runtime_env.pip_packages()) + "\n"
pip_hash = hashlib.sha1(requirements_txt.encode("utf-8")).hexdigest()
pip_hash_str = f"pip-generated-{pip_hash}"
requirements_txt_path = os.path.join(
resources_dir, f"requirements-{pip_hash_str}.txt")
conda_dict = {
"name": pip_hash_str,
"dependencies": ["pip", {
"pip": [f"-r {requirements_txt_path}"]
}]
}
file_lock_name = f"ray-{pip_hash_str}.lock"
with FileLock(os.path.join(resources_dir, file_lock_name)):
try_to_create_directory(resources_dir)
with open(requirements_txt_path, "w") as file:
file.write(requirements_txt)
return conda_dict
return None
def __init__(self, resources_dir: str):
self._pip_resources_dir = os.path.join(resources_dir, "pip")
try_to_create_directory(self._pip_resources_dir)
# Concurrent pip installs are unsafe. This lock prevents concurrent
# installs (and deletions).
self._installs_and_deletions_file_lock = os.path.join(
self._pip_resources_dir, "ray-pip-installs-and-deletions.lock")
def _prepare_socket_file(self, socket_path, default_prefix):
"""Prepare the socket file for raylet and plasma.
This method helps to prepare a socket file.
1. Make the directory if the directory does not exist.
2. If the socket file exists, do nothing (this just means we aren't the
first worker on the node).
Args:
socket_path (string): the socket file to prepare.
"""
result = socket_path
is_mac = sys.platform.startswith("darwin")
if sys.platform == "win32":
if socket_path is None:
result = (f"tcp://{self._localhost}"
f":{self._get_unused_port()[0]}")
else:
if socket_path is None:
result = self._make_inc_temp(prefix=default_prefix,
directory_name=self._sockets_dir)
else:
try_to_create_directory(os.path.dirname(socket_path))
# Check socket path length to make sure it's short enough
maxlen = (104 if is_mac else 108) - 1 # sockaddr_un->sun_path
if len(result.split("://", 1)[-1].encode("utf-8")) > maxlen:
raise OSError("AF_UNIX path length cannot exceed "
"{} bytes: {!r}".format(maxlen, result))
return result
def __init__(self, resources_dir: str):
self._pip_resources_dir = os.path.join(resources_dir, "pip")
self._creating_task = {}
# Maps a URI to a lock that is used to prevent multiple concurrent
# installs of the same virtualenv, see #24513
self._create_locks: Dict[str, asyncio.Lock] = {}
try_to_create_directory(self._pip_resources_dir)
def _install_pip_list_to_dir(
pip_list: List[str],
target_dir: str,
logger: Optional[logging.Logger] = default_logger,
):
try_to_create_directory(target_dir)
try:
pip_requirements_file = os.path.join(target_dir, "requirements.txt")
with open(pip_requirements_file, "w") as file:
for line in pip_list:
file.write(line + "\n")
exit_code, output = exec_cmd_stream_to_logger(
[
"pip", "install", f"--target={target_dir}", "-r",
pip_requirements_file
],
logger,
)
if exit_code != 0:
shutil.rmtree(target_dir)
raise RuntimeError(
f"Failed to install pip requirements:\n{output}")
finally:
if os.path.exists(pip_requirements_file):
os.remove(pip_requirements_file)
def setup(input_args):
# remaining_args contains the arguments to the original worker command,
# minus the python executable, e.g. default_worker.py --node-ip-address=...
args, remaining_args = parser.parse_known_args(args=input_args)
# add worker-shim-pid argument
remaining_args.append("--worker-shim-pid={}".format(os.getpid()))
commands = []
runtime_env: dict = json.loads(args.serialized_runtime_env or "{}")
py_executable: str = sys.executable
if runtime_env.get("conda") or runtime_env.get("pip"):
conda_dict = get_conda_dict(runtime_env, args.session_dir)
py_executable = "python"
if isinstance(runtime_env.get("conda"), str):
conda_env_name = runtime_env["conda"]
else:
assert conda_dict is not None
py_version = ".".join(map(str,
sys.version_info[:3])) # like 3.6.10
ray_pip = current_ray_pip_specifier()
if ray_pip and not runtime_env.get("_skip_inject_ray"):
extra_pip_dependencies = [ray_pip, "ray[default]"]
else:
extra_pip_dependencies = []
conda_dict = inject_dependencies(conda_dict, py_version,
extra_pip_dependencies)
# Locking to avoid multiple processes installing concurrently
conda_hash = hashlib.sha1(
json.dumps(conda_dict,
sort_keys=True).encode("utf-8")).hexdigest()
conda_hash_str = f"conda-generated-{conda_hash}"
file_lock_name = f"ray-{conda_hash_str}.lock"
with FileLock(os.path.join(args.session_dir, file_lock_name)):
conda_dir = os.path.join(args.session_dir, "runtime_resources",
"conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(conda_dict, file, sort_keys=True)
conda_env_name = get_or_create_conda_env(
conda_yaml_path, conda_dir)
commands += get_conda_activate_commands(conda_env_name)
commands += [" ".join([f"exec {py_executable}"] + remaining_args)]
command_separator = " && "
command_str = command_separator.join(commands)
if runtime_env.get("env_vars"):
env_vars = runtime_env["env_vars"]
os.environ.update(env_vars)
os.execvp("bash", ["bash", "-c", command_str])
def _install_pip_list_to_dir(
pip_list: List[str],
target_dir: str,
logger: Optional[logging.Logger] = default_logger):
try_to_create_directory(target_dir)
exit_code, output = exec_cmd_stream_to_logger(
["pip", "install", f"--target={target_dir}"] + pip_list, logger)
if exit_code != 0:
shutil.rmtree(target_dir)
raise RuntimeError(f"Failed to install pip requirements:\n{output}")
def __init__(self, resources_dir: str):
self._resources_dir = os.path.join(resources_dir, "conda")
try_to_create_directory(self._resources_dir)
# It is not safe for multiple processes to install conda envs
# concurrently, even if the envs are different, so use a global
# lock for all conda installs and deletions.
# See https://github.com/ray-project/ray/issues/17086
self._installs_and_deletions_file_lock = os.path.join(
self._resources_dir, "ray-conda-installs-and-deletions.lock")
def setup_conda_or_pip(runtime_env: dict,
context: RuntimeEnvContext,
logger: Optional[logging.Logger] = None):
if logger is None:
logger = logging.getLogger(__name__)
if not runtime_env.get("conda") and not runtime_env.get("pip"):
return
logger.debug(f"Setting up conda or pip for runtime_env: {runtime_env}")
conda_dict = get_conda_dict(runtime_env, context.resources_dir)
if isinstance(runtime_env.get("conda"), str):
conda_env_name = runtime_env["conda"]
else:
assert conda_dict is not None
ray_pip = current_ray_pip_specifier(logger)
if ray_pip:
extra_pip_dependencies = [ray_pip, "ray[default]"]
elif runtime_env.get("_inject_current_ray"):
extra_pip_dependencies = (
_resolve_install_from_source_ray_dependencies())
else:
extra_pip_dependencies = []
conda_dict = inject_dependencies(conda_dict, _current_py_version(),
extra_pip_dependencies)
logger.info(f"Setting up conda environment with {runtime_env}")
# It is not safe for multiple processes to install conda envs
# concurrently, even if the envs are different, so use a global
# lock for all conda installs.
# See https://github.com/ray-project/ray/issues/17086
file_lock_name = "ray-conda-install.lock"
with FileLock(os.path.join(context.resources_dir, file_lock_name)):
conda_dir = os.path.join(context.resources_dir, "conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(conda_dict, file, sort_keys=True)
conda_env_name = get_or_create_conda_env(conda_yaml_path,
conda_dir,
logger=logger)
if runtime_env.get("_inject_current_ray"):
conda_path = os.path.join(conda_dir, conda_env_name)
_inject_ray_to_conda_site(conda_path, logger)
context.py_executable = "python"
context.command_prefix += get_conda_activate_commands(conda_env_name)
logger.info(f"Finished setting up runtime environment at {conda_env_name}")
def setup_runtime_env(runtime_env: dict, session_dir):
if runtime_env.get("conda") or runtime_env.get("pip"):
conda_dict = get_conda_dict(runtime_env, session_dir)
if isinstance(runtime_env.get("conda"), str):
conda_env_name = runtime_env["conda"]
else:
assert conda_dict is not None
ray_pip = current_ray_pip_specifier()
if ray_pip:
extra_pip_dependencies = [ray_pip, "ray[default]"]
elif runtime_env.get("_inject_current_ray"):
extra_pip_dependencies = (
_resolve_install_from_source_ray_dependencies())
print(extra_pip_dependencies)
else:
extra_pip_dependencies = []
conda_dict = inject_dependencies(conda_dict, _current_py_version(),
extra_pip_dependencies)
# It is not safe for multiple processes to install conda envs
# concurrently, even if the envs are different, so use a global
# lock for all conda installs.
# See https://github.com/ray-project/ray/issues/17086
file_lock_name = "ray-conda-install.lock"
with FileLock(os.path.join(session_dir, file_lock_name)):
conda_dir = os.path.join(session_dir, "runtime_resources",
"conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(conda_dict, file, sort_keys=True)
conda_env_name = get_or_create_conda_env(
conda_yaml_path, conda_dir)
if runtime_env.get("_inject_current_ray"):
conda_path = os.path.join(conda_dir, conda_env_name)
_inject_ray_to_conda_site(conda_path)
return RuntimeEnvContext(conda_env_name)
return RuntimeEnvContext()
def _setup_logging() -> None:
"""Log to autoscaler log file
(typically, /tmp/ray/session_latest/logs/monitor.*)
Also log to pod stdout (logs viewable with `kubectl logs <head-pod> -c autoscaler`).
"""
log_dir = os.path.join(
ray._private.utils.get_ray_temp_dir(), ray._private.node.SESSION_LATEST, "logs"
)
# The director should already exist, but try (safely) to create it just in case.
try_to_create_directory(log_dir)
# Write logs at info level to monitor.log.
setup_component_logger(
logging_level=ray_constants.LOGGER_LEVEL, # info
logging_format=ray_constants.LOGGER_FORMAT,
log_dir=log_dir,
filename=ray_constants.MONITOR_LOG_FILE_NAME, # monitor.log
max_bytes=ray_constants.LOGGING_ROTATE_BYTES,
backup_count=ray_constants.LOGGING_ROTATE_BACKUP_COUNT,
logger_name="ray", # Root of the logging hierarchy for Ray code.
)
def setup(input_args):
# remaining_args contains the arguments to the original worker command,
# minus the python executable, e.g. default_worker.py --node-ip-address=...
args, remaining_args = parser.parse_known_args(args=input_args)
commands = []
runtime_env: RuntimeEnvDict = json.loads(args.serialized_runtime_env
or "{}")
py_executable: str = sys.executable
if runtime_env.get("conda"):
py_executable = "python"
if isinstance(runtime_env["conda"], str):
commands += get_conda_activate_commands(runtime_env["conda"])
elif isinstance(runtime_env["conda"], dict):
# Locking to avoid multiple processes installing concurrently
with FileLock(
os.path.join(args.session_dir, "ray-conda-install.lock")):
conda_dir = os.path.join(args.session_dir, "runtime_resources",
"conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(runtime_env["conda"], file, sort_keys=True)
conda_env_name = get_or_create_conda_env(
conda_yaml_path, conda_dir)
if os.path.exists(conda_yaml_path):
os.remove(conda_yaml_path)
logger.error(conda_env_name)
commands += get_conda_activate_commands(conda_env_name)
commands += [" ".join([f"exec {py_executable}"] + remaining_args)]
command_separator = " && "
command_str = command_separator.join(commands)
os.execvp("bash", ["bash", "-c", command_str])
def setup_runtime_env(runtime_env: dict, session_dir):
if runtime_env.get("conda") or runtime_env.get("pip"):
conda_dict = get_conda_dict(runtime_env, session_dir)
if isinstance(runtime_env.get("conda"), str):
conda_env_name = runtime_env["conda"]
else:
assert conda_dict is not None
py_version = ".".join(map(str,
sys.version_info[:3])) # like 3.6.10
ray_pip = current_ray_pip_specifier()
if ray_pip and not runtime_env.get("_skip_inject_ray"):
extra_pip_dependencies = [ray_pip, "ray[default]"]
else:
extra_pip_dependencies = []
conda_dict = inject_dependencies(conda_dict, py_version,
extra_pip_dependencies)
# Locking to avoid multiple processes installing concurrently
conda_hash = hashlib.sha1(
json.dumps(conda_dict,
sort_keys=True).encode("utf-8")).hexdigest()
conda_hash_str = f"conda-generated-{conda_hash}"
file_lock_name = f"ray-{conda_hash_str}.lock"
with FileLock(os.path.join(session_dir, file_lock_name)):
conda_dir = os.path.join(session_dir, "runtime_resources",
"conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(conda_dict, file, sort_keys=True)
conda_env_name = get_or_create_conda_env(
conda_yaml_path, conda_dir)
return RuntimeEnvContext(conda_env_name)
return RuntimeEnvContext()
def test_pip_with_env_vars(start_cluster):
with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
TEST_ENV_NAME = "TEST_ENV_VARS"
TEST_ENV_VALUE = "TEST"
package_name = "test_package"
package_dir = os.path.join(tmpdir, package_name)
try_to_create_directory(package_dir)
setup_filename = os.path.join(package_dir, "setup.py")
setup_code = """import os
from setuptools import setup, find_packages
from setuptools.command.install import install
class InstallTestPackage(install):
# this function will be called when pip install this package
def run(self):
assert os.environ.get('{TEST_ENV_NAME}') == '{TEST_ENV_VALUE}'
setup(
name='test_package',
version='0.0.1',
packages=find_packages(),
cmdclass=dict(install=InstallTestPackage),
license="MIT",
zip_safe=False,
)
""".format(TEST_ENV_NAME=TEST_ENV_NAME, TEST_ENV_VALUE=TEST_ENV_VALUE)
with open(setup_filename, "wt") as f:
f.writelines(setup_code)
python_filename = os.path.join(package_dir, "test.py")
python_code = "import os; print(os.environ)"
with open(python_filename, "wt") as f:
f.writelines(python_code)
gz_filename = os.path.join(tmpdir, package_name + ".tar.gz")
subprocess.check_call(["tar", "-zcvf", gz_filename, package_name])
with pytest.raises(ray.exceptions.RuntimeEnvSetupError):
@ray.remote(runtime_env={
"env_vars": {
TEST_ENV_NAME: "failed"
},
"pip": [gz_filename],
})
def f1():
return True
ray.get(f1.remote())
@ray.remote(runtime_env={
"env_vars": {
TEST_ENV_NAME: TEST_ENV_VALUE
},
"pip": [gz_filename],
})
def f2():
return True
assert ray.get(f2.remote())
def __init__(self, resources_dir: str):
self._resources_dir = os.path.join(resources_dir, "working_dir_files")
try_to_create_directory(self._resources_dir)
assert _internal_kv_initialized()
def setup(input_args):
# remaining_args contains the arguments to the original worker command,
# minus the python executable, e.g. default_worker.py --node-ip-address=...
args, remaining_args = parser.parse_known_args(args=input_args)
commands = []
runtime_env: dict = json.loads(args.serialized_runtime_env or "{}")
py_executable: str = sys.executable
if runtime_env.get("conda"):
py_executable = "python"
if isinstance(runtime_env["conda"], str):
commands += get_conda_activate_commands(runtime_env["conda"])
elif isinstance(runtime_env["conda"], dict):
# Locking to avoid multiple processes installing concurrently
conda_hash = hashlib.sha1(
json.dumps(runtime_env["conda"],
sort_keys=True).encode("utf-8")).hexdigest()
conda_hash_str = f"conda-generated-{conda_hash}"
file_lock_name = f"ray-{conda_hash_str}.lock"
with FileLock(os.path.join(args.session_dir, file_lock_name)):
conda_dir = os.path.join(args.session_dir, "runtime_resources",
"conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(runtime_env["conda"], file, sort_keys=True)
conda_env_name = get_or_create_conda_env(
conda_yaml_path, conda_dir)
commands += get_conda_activate_commands(conda_env_name)
elif runtime_env.get("pip"):
# Install pip requirements into an empty conda env.
py_executable = "python"
requirements_txt = runtime_env["pip"]
pip_hash = hashlib.sha1(requirements_txt.encode("utf-8")).hexdigest()
pip_hash_str = f"pip-generated-{pip_hash}"
conda_dir = os.path.join(args.session_dir, "runtime_resources",
"conda")
requirements_txt_path = os.path.join(
conda_dir, f"requirements-{pip_hash_str}.txt")
py_version = ".".join(map(str, sys.version_info[:3])) # E.g. 3.6.13
conda_dict = {
"name": pip_hash_str,
"dependencies": [
f"python={py_version}", "pip", {
"pip": [f"-r {requirements_txt_path}"]
}
]
}
file_lock_name = f"ray-{pip_hash_str}.lock"
with FileLock(os.path.join(args.session_dir, file_lock_name)):
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir,
f"env-{pip_hash_str}.yml")
with open(conda_yaml_path, "w") as file:
yaml.dump(conda_dict, file, sort_keys=True)
with open(requirements_txt_path, "w") as file:
file.write(requirements_txt)
conda_env_name = get_or_create_conda_env(conda_yaml_path,
conda_dir)
commands += get_conda_activate_commands(conda_env_name)
commands += [" ".join([f"exec {py_executable}"] + remaining_args)]
command_separator = " && "
command_str = command_separator.join(commands)
os.execvp("bash", ["bash", "-c", command_str])
def __init__(self, resources_dir: str, gcs_aio_client: GcsAioClient):
self._resources_dir = os.path.join(resources_dir, "java_jars_files")
self._gcs_aio_client = gcs_aio_client
try_to_create_directory(self._resources_dir)
def _init_temp(self, redis_client):
# Create a dictionary to store temp file index.
self._incremental_dict = collections.defaultdict(lambda: 0)
if self.head:
self._temp_dir = self._ray_params.temp_dir
else:
temp_dir = _get_with_retry(redis_client, "temp_dir")
self._temp_dir = ray._private.utils.decode(temp_dir)
try_to_create_directory(self._temp_dir)
if self.head:
self._session_dir = os.path.join(self._temp_dir, self.session_name)
else:
session_dir = _get_with_retry(redis_client, "session_dir")
self._session_dir = ray._private.utils.decode(session_dir)
session_symlink = os.path.join(self._temp_dir, SESSION_LATEST)
# Send a warning message if the session exists.
try_to_create_directory(self._session_dir)
try_to_symlink(session_symlink, self._session_dir)
# Create a directory to be used for socket files.
self._sockets_dir = os.path.join(self._session_dir, "sockets")
try_to_create_directory(self._sockets_dir)
# Create a directory to be used for process log files.
self._logs_dir = os.path.join(self._session_dir, "logs")
try_to_create_directory(self._logs_dir)
old_logs_dir = os.path.join(self._logs_dir, "old")
try_to_create_directory(old_logs_dir)
# Create a directory to be used for runtime environment.
self._resource_dir = os.path.join(self._session_dir,
"runtime_resources")
try_to_create_directory(self._resource_dir)
import ray._private.runtime_env as runtime_env
runtime_env.PKG_DIR = self._resource_dir
def __init__(self, resources_dir: str):
self._pip_resources_dir = os.path.join(resources_dir, "pip")
self._creating_task = {}
try_to_create_directory(self._pip_resources_dir)
def __init__(self, resources_dir: str):
self._resources_dir = os.path.join(resources_dir, "conda")
try_to_create_directory(self._resources_dir)
self._created_envs: Set[str] = set()
def __init__(self, resources_dir: str):
self._resources_dir = os.path.join(resources_dir, "pip")
try_to_create_directory(self._resources_dir)
