def setup(input_args): # remaining_args contains the arguments to the original worker command, # minus the python executable, e.g. default_worker.py --node-ip-address=... args, remaining_args = parser.parse_known_args(args=input_args) # add worker-shim-pid argument remaining_args.append("--worker-shim-pid={}".format(os.getpid())) commands = [] runtime_env: dict = json.loads(args.serialized_runtime_env or "{}") py_executable: str = sys.executable if runtime_env.get("conda") or runtime_env.get("pip"): conda_dict = get_conda_dict(runtime_env, args.session_dir) py_executable = "python" if isinstance(runtime_env.get("conda"), str): conda_env_name = runtime_env["conda"] else: assert conda_dict is not None py_version = ".".join(map(str, sys.version_info[:3])) # like 3.6.10 ray_pip = current_ray_pip_specifier() if ray_pip and not runtime_env.get("_skip_inject_ray"): extra_pip_dependencies = [ray_pip, "ray[default]"] else: extra_pip_dependencies = [] conda_dict = inject_dependencies(conda_dict, py_version, extra_pip_dependencies) # Locking to avoid multiple processes installing concurrently conda_hash = hashlib.sha1( json.dumps(conda_dict, sort_keys=True).encode("utf-8")).hexdigest() conda_hash_str = f"conda-generated-{conda_hash}" file_lock_name = f"ray-{conda_hash_str}.lock" with FileLock(os.path.join(args.session_dir, file_lock_name)): conda_dir = os.path.join(args.session_dir, "runtime_resources", "conda") try_to_create_directory(conda_dir) conda_yaml_path = os.path.join(conda_dir, "environment.yml") with open(conda_yaml_path, "w") as file: # Sort keys because we hash based on the file contents, # and we don't want the hash to depend on the order # of the dependencies. yaml.dump(conda_dict, file, sort_keys=True) conda_env_name = get_or_create_conda_env( conda_yaml_path, conda_dir) commands += get_conda_activate_commands(conda_env_name) commands += [" ".join([f"exec {py_executable}"] + remaining_args)] command_separator = " && " command_str = command_separator.join(commands) if runtime_env.get("env_vars"): env_vars = runtime_env["env_vars"] os.environ.update(env_vars) os.execvp("bash", ["bash", "-c", command_str])
def setup(input_args): # remaining_args contains the arguments to the original worker command, # minus the python executable, e.g. default_worker.py --node-ip-address=... args, remaining_args = parser.parse_known_args(args=input_args) commands = [] runtime_env: RuntimeEnvDict = json.loads(args.serialized_runtime_env or "{}") py_executable: str = sys.executable if runtime_env.get("conda"): py_executable = "python" if isinstance(runtime_env["conda"], str): commands += get_conda_activate_commands(runtime_env["conda"]) elif isinstance(runtime_env["conda"], dict): # Locking to avoid multiple processes installing concurrently with FileLock( os.path.join(args.session_dir, "ray-conda-install.lock")): conda_dir = os.path.join(args.session_dir, "runtime_resources", "conda") try_to_create_directory(conda_dir) conda_yaml_path = os.path.join(conda_dir, "environment.yml") with open(conda_yaml_path, "w") as file: # Sort keys because we hash based on the file contents, # and we don't want the hash to depend on the order # of the dependencies. yaml.dump(runtime_env["conda"], file, sort_keys=True) conda_env_name = get_or_create_conda_env( conda_yaml_path, conda_dir) if os.path.exists(conda_yaml_path): os.remove(conda_yaml_path) logger.error(conda_env_name) commands += get_conda_activate_commands(conda_env_name) commands += [" ".join([f"exec {py_executable}"] + remaining_args)] command_separator = " && " command_str = command_separator.join(commands) os.execvp("bash", ["bash", "-c", command_str])
def setup_worker(input_args): # remaining_args contains the arguments to the original worker command, # minus the python executable, e.g. default_worker.py --node-ip-address=... args, remaining_args = parser.parse_known_args(args=input_args) commands = [] py_executable: str = sys.executable runtime_env: dict = json.loads(args.serialized_runtime_env or "{}") runtime_env_context: RuntimeEnvContext = None # Ray client server setups runtime env by itself instead of agent. if runtime_env.get("conda") or runtime_env.get("pip"): if not args.serialized_runtime_env_context: runtime_env_context = setup_runtime_env(runtime_env, args.session_dir) else: runtime_env_context = RuntimeEnvContext.deserialize( args.serialized_runtime_env_context) # activate conda if runtime_env_context and runtime_env_context.conda_env_name: py_executable = "python" conda_activate_commands = get_conda_activate_commands( runtime_env_context.conda_env_name) if (conda_activate_commands): commands += conda_activate_commands elif runtime_env.get("conda"): logger.warning( "Conda env name is not found in context, " "but conda exists in runtime env. The runtime env %s, " "the context %s.", args.serialized_runtime_env, args.serialized_runtime_env_context) commands += [ " ".join( [f"exec {py_executable}"] + remaining_args + # Pass the runtime for working_dir setup. # We can't do it in shim process here because it requires # connection to gcs. ["--serialized-runtime-env", f"'{args.serialized_runtime_env}'"]) ] command_separator = " && " command_str = command_separator.join(commands) # update env vars if runtime_env.get("env_vars"): env_vars = runtime_env["env_vars"] os.environ.update(env_vars) os.execvp("bash", ["bash", "-c", command_str])
def setup(input_args): # remaining_args contains the arguments to the original worker command, # minus the python executable, e.g. default_worker.py --node-ip-address=... args, remaining_args = parser.parse_known_args(args=input_args) commands = [] if args.conda_env_name: commands += get_conda_activate_commands(args.conda_env_name) commands += [" ".join(["exec python"] + remaining_args)] command_separator = " && " command_str = command_separator.join(commands) os.execvp("bash", ["bash", "-c", command_str])
def setup(input_args): # remaining_args contains the arguments to the original worker command, # minus the python executable, e.g. default_worker.py --node-ip-address=... args, remaining_args = parser.parse_known_args(args=input_args) commands = [] runtime_env: RuntimeEnvDict = json.loads(args.serialized_runtime_env or "{}") py_executable: str = sys.executable if runtime_env.get("conda"): if isinstance(runtime_env["conda"], str): commands += get_conda_activate_commands(runtime_env["conda"]) py_executable = "python" commands += [" ".join([f"exec {py_executable}"] + remaining_args)] command_separator = " && " command_str = command_separator.join(commands) os.execvp("bash", ["bash", "-c", command_str])
def setup(input_args): # remaining_args contains the arguments to the original worker command, # minus the python executable, e.g. default_worker.py --node-ip-address=... args, remaining_args = parser.parse_known_args(args=input_args) commands = [] runtime_env: dict = json.loads(args.serialized_runtime_env or "{}") py_executable: str = sys.executable if runtime_env.get("conda"): py_executable = "python" if isinstance(runtime_env["conda"], str): commands += get_conda_activate_commands(runtime_env["conda"]) elif isinstance(runtime_env["conda"], dict): # Locking to avoid multiple processes installing concurrently conda_hash = hashlib.sha1( json.dumps(runtime_env["conda"], sort_keys=True).encode("utf-8")).hexdigest() conda_hash_str = f"conda-generated-{conda_hash}" file_lock_name = f"ray-{conda_hash_str}.lock" with FileLock(os.path.join(args.session_dir, file_lock_name)): conda_dir = os.path.join(args.session_dir, "runtime_resources", "conda") try_to_create_directory(conda_dir) conda_yaml_path = os.path.join(conda_dir, "environment.yml") with open(conda_yaml_path, "w") as file: # Sort keys because we hash based on the file contents, # and we don't want the hash to depend on the order # of the dependencies. yaml.dump(runtime_env["conda"], file, sort_keys=True) conda_env_name = get_or_create_conda_env( conda_yaml_path, conda_dir) commands += get_conda_activate_commands(conda_env_name) elif runtime_env.get("pip"): # Install pip requirements into an empty conda env. py_executable = "python" requirements_txt = runtime_env["pip"] pip_hash = hashlib.sha1(requirements_txt.encode("utf-8")).hexdigest() pip_hash_str = f"pip-generated-{pip_hash}" conda_dir = os.path.join(args.session_dir, "runtime_resources", "conda") requirements_txt_path = os.path.join( conda_dir, f"requirements-{pip_hash_str}.txt") py_version = ".".join(map(str, sys.version_info[:3])) # E.g. 3.6.13 conda_dict = { "name": pip_hash_str, "dependencies": [ f"python={py_version}", "pip", { "pip": [f"-r {requirements_txt_path}"] } ] } file_lock_name = f"ray-{pip_hash_str}.lock" with FileLock(os.path.join(args.session_dir, file_lock_name)): try_to_create_directory(conda_dir) conda_yaml_path = os.path.join(conda_dir, f"env-{pip_hash_str}.yml") with open(conda_yaml_path, "w") as file: yaml.dump(conda_dict, file, sort_keys=True) with open(requirements_txt_path, "w") as file: file.write(requirements_txt) conda_env_name = get_or_create_conda_env(conda_yaml_path, conda_dir) commands += get_conda_activate_commands(conda_env_name) commands += [" ".join([f"exec {py_executable}"] + remaining_args)] command_separator = " && " command_str = command_separator.join(commands) os.execvp("bash", ["bash", "-c", command_str])