def setup(input_args):
# remaining_args contains the arguments to the original worker command,
# minus the python executable, e.g. default_worker.py --node-ip-address=...
args, remaining_args = parser.parse_known_args(args=input_args)
# add worker-shim-pid argument
remaining_args.append("--worker-shim-pid={}".format(os.getpid()))
commands = []
runtime_env: dict = json.loads(args.serialized_runtime_env or "{}")
py_executable: str = sys.executable
if runtime_env.get("conda") or runtime_env.get("pip"):
conda_dict = get_conda_dict(runtime_env, args.session_dir)
py_executable = "python"
if isinstance(runtime_env.get("conda"), str):
conda_env_name = runtime_env["conda"]
else:
assert conda_dict is not None
py_version = ".".join(map(str,
sys.version_info[:3])) # like 3.6.10
ray_pip = current_ray_pip_specifier()
if ray_pip and not runtime_env.get("_skip_inject_ray"):
extra_pip_dependencies = [ray_pip, "ray[default]"]
else:
extra_pip_dependencies = []
conda_dict = inject_dependencies(conda_dict, py_version,
extra_pip_dependencies)
# Locking to avoid multiple processes installing concurrently
conda_hash = hashlib.sha1(
json.dumps(conda_dict,
sort_keys=True).encode("utf-8")).hexdigest()
conda_hash_str = f"conda-generated-{conda_hash}"
file_lock_name = f"ray-{conda_hash_str}.lock"
with FileLock(os.path.join(args.session_dir, file_lock_name)):
conda_dir = os.path.join(args.session_dir, "runtime_resources",
"conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(conda_dict, file, sort_keys=True)
conda_env_name = get_or_create_conda_env(
conda_yaml_path, conda_dir)
commands += get_conda_activate_commands(conda_env_name)
commands += [" ".join([f"exec {py_executable}"] + remaining_args)]
command_separator = " && "
command_str = command_separator.join(commands)
if runtime_env.get("env_vars"):
env_vars = runtime_env["env_vars"]
os.environ.update(env_vars)
os.execvp("bash", ["bash", "-c", command_str])
def setup(input_args):
# remaining_args contains the arguments to the original worker command,
# minus the python executable, e.g. default_worker.py --node-ip-address=...
args, remaining_args = parser.parse_known_args(args=input_args)
commands = []
runtime_env: RuntimeEnvDict = json.loads(args.serialized_runtime_env
or "{}")
py_executable: str = sys.executable
if runtime_env.get("conda"):
py_executable = "python"
if isinstance(runtime_env["conda"], str):
commands += get_conda_activate_commands(runtime_env["conda"])
elif isinstance(runtime_env["conda"], dict):
# Locking to avoid multiple processes installing concurrently
with FileLock(
os.path.join(args.session_dir, "ray-conda-install.lock")):
conda_dir = os.path.join(args.session_dir, "runtime_resources",
"conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(runtime_env["conda"], file, sort_keys=True)
conda_env_name = get_or_create_conda_env(
conda_yaml_path, conda_dir)
if os.path.exists(conda_yaml_path):
os.remove(conda_yaml_path)
logger.error(conda_env_name)
commands += get_conda_activate_commands(conda_env_name)
commands += [" ".join([f"exec {py_executable}"] + remaining_args)]
command_separator = " && "
command_str = command_separator.join(commands)
os.execvp("bash", ["bash", "-c", command_str])
def setup_worker(input_args):
# remaining_args contains the arguments to the original worker command,
# minus the python executable, e.g. default_worker.py --node-ip-address=...
args, remaining_args = parser.parse_known_args(args=input_args)
commands = []
py_executable: str = sys.executable
runtime_env: dict = json.loads(args.serialized_runtime_env or "{}")
runtime_env_context: RuntimeEnvContext = None
# Ray client server setups runtime env by itself instead of agent.
if runtime_env.get("conda") or runtime_env.get("pip"):
if not args.serialized_runtime_env_context:
runtime_env_context = setup_runtime_env(runtime_env,
args.session_dir)
else:
runtime_env_context = RuntimeEnvContext.deserialize(
args.serialized_runtime_env_context)
# activate conda
if runtime_env_context and runtime_env_context.conda_env_name:
py_executable = "python"
conda_activate_commands = get_conda_activate_commands(
runtime_env_context.conda_env_name)
if (conda_activate_commands):
commands += conda_activate_commands
elif runtime_env.get("conda"):
logger.warning(
"Conda env name is not found in context, "
"but conda exists in runtime env. The runtime env %s, "
"the context %s.", args.serialized_runtime_env,
args.serialized_runtime_env_context)
commands += [
" ".join(
[f"exec {py_executable}"] + remaining_args +
# Pass the runtime for working_dir setup.
# We can't do it in shim process here because it requires
# connection to gcs.
["--serialized-runtime-env", f"'{args.serialized_runtime_env}'"])
]
command_separator = " && "
command_str = command_separator.join(commands)
# update env vars
if runtime_env.get("env_vars"):
env_vars = runtime_env["env_vars"]
os.environ.update(env_vars)
os.execvp("bash", ["bash", "-c", command_str])
def setup(input_args):
# remaining_args contains the arguments to the original worker command,
# minus the python executable, e.g. default_worker.py --node-ip-address=...
args, remaining_args = parser.parse_known_args(args=input_args)
commands = []
if args.conda_env_name:
commands += get_conda_activate_commands(args.conda_env_name)
commands += [" ".join(["exec python"] + remaining_args)]
command_separator = " && "
command_str = command_separator.join(commands)
os.execvp("bash", ["bash", "-c", command_str])
def setup(input_args):
# remaining_args contains the arguments to the original worker command,
# minus the python executable, e.g. default_worker.py --node-ip-address=...
args, remaining_args = parser.parse_known_args(args=input_args)
commands = []
runtime_env: RuntimeEnvDict = json.loads(args.serialized_runtime_env
or "{}")
py_executable: str = sys.executable
if runtime_env.get("conda"):
if isinstance(runtime_env["conda"], str):
commands += get_conda_activate_commands(runtime_env["conda"])
py_executable = "python"
commands += [" ".join([f"exec {py_executable}"] + remaining_args)]
command_separator = " && "
command_str = command_separator.join(commands)
os.execvp("bash", ["bash", "-c", command_str])
def setup(input_args):
# remaining_args contains the arguments to the original worker command,
# minus the python executable, e.g. default_worker.py --node-ip-address=...
args, remaining_args = parser.parse_known_args(args=input_args)
commands = []
runtime_env: dict = json.loads(args.serialized_runtime_env or "{}")
py_executable: str = sys.executable
if runtime_env.get("conda"):
py_executable = "python"
if isinstance(runtime_env["conda"], str):
commands += get_conda_activate_commands(runtime_env["conda"])
elif isinstance(runtime_env["conda"], dict):
# Locking to avoid multiple processes installing concurrently
conda_hash = hashlib.sha1(
json.dumps(runtime_env["conda"],
sort_keys=True).encode("utf-8")).hexdigest()
conda_hash_str = f"conda-generated-{conda_hash}"
file_lock_name = f"ray-{conda_hash_str}.lock"
with FileLock(os.path.join(args.session_dir, file_lock_name)):
conda_dir = os.path.join(args.session_dir, "runtime_resources",
"conda")
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir, "environment.yml")
with open(conda_yaml_path, "w") as file:
# Sort keys because we hash based on the file contents,
# and we don't want the hash to depend on the order
# of the dependencies.
yaml.dump(runtime_env["conda"], file, sort_keys=True)
conda_env_name = get_or_create_conda_env(
conda_yaml_path, conda_dir)
commands += get_conda_activate_commands(conda_env_name)
elif runtime_env.get("pip"):
# Install pip requirements into an empty conda env.
py_executable = "python"
requirements_txt = runtime_env["pip"]
pip_hash = hashlib.sha1(requirements_txt.encode("utf-8")).hexdigest()
pip_hash_str = f"pip-generated-{pip_hash}"
conda_dir = os.path.join(args.session_dir, "runtime_resources",
"conda")
requirements_txt_path = os.path.join(
conda_dir, f"requirements-{pip_hash_str}.txt")
py_version = ".".join(map(str, sys.version_info[:3])) # E.g. 3.6.13
conda_dict = {
"name": pip_hash_str,
"dependencies": [
f"python={py_version}", "pip", {
"pip": [f"-r {requirements_txt_path}"]
}
]
}
file_lock_name = f"ray-{pip_hash_str}.lock"
with FileLock(os.path.join(args.session_dir, file_lock_name)):
try_to_create_directory(conda_dir)
conda_yaml_path = os.path.join(conda_dir,
f"env-{pip_hash_str}.yml")
with open(conda_yaml_path, "w") as file:
yaml.dump(conda_dict, file, sort_keys=True)
with open(requirements_txt_path, "w") as file:
file.write(requirements_txt)
conda_env_name = get_or_create_conda_env(conda_yaml_path,
conda_dir)
commands += get_conda_activate_commands(conda_env_name)
commands += [" ".join([f"exec {py_executable}"] + remaining_args)]
command_separator = " && "
command_str = command_separator.join(commands)
os.execvp("bash", ["bash", "-c", command_str])