def upload_runtime_env_package_if_needed(job_config: JobConfig) -> None:
"""Upload runtime env if it's not there.
It'll check whether the runtime environment exists in the cluster or not.
If it doesn't exist, a package will be created based on the working
directory and modules defined in job config. The package will be
uploaded to the cluster after this.
Args:
job_config (JobConfig): The job config of driver.
"""
assert _internal_kv_initialized()
pkg_uris = job_config.get_runtime_env_uris()
for pkg_uri in pkg_uris:
if not package_exists(pkg_uri):
file_path = _get_local_path(pkg_uri)
pkg_file = Path(file_path)
working_dir = job_config.runtime_env.get("working_dir")
py_modules = job_config.runtime_env.get("py_modules")
excludes = job_config.runtime_env.get("excludes") or []
logger.info(f"{pkg_uri} doesn't exist. Create new package with"
f" {working_dir} and {py_modules}")
if not pkg_file.exists():
create_project_package(working_dir, py_modules, excludes,
file_path)
# Push the data to remote storage
pkg_size = push_package(pkg_uri, pkg_file)
logger.info(f"{pkg_uri} has been pushed with {pkg_size} bytes")
def upload_runtime_env_package_if_needed(
self,
job_config: JobConfig,
logger: Optional[logging.Logger] = default_logger):
"""Upload runtime env if it's not there.
It'll check whether the runtime environment exists in the cluster or
not. If it doesn't, a package will be created based on the working
directory and modules defined in job config. The package will be
uploaded to the cluster after this.
Args:
job_config (JobConfig): The job config of driver.
"""
if logger is None:
logger = default_logger
pkg_uris = job_config.get_runtime_env_uris()
if len(pkg_uris) == 0:
return # Return early to avoid internal kv check in this case.
for pkg_uri in pkg_uris:
if not package_exists(pkg_uri):
file_path = self._get_local_path(pkg_uri)
pkg_file = Path(file_path)
working_dir = job_config.runtime_env.get("working_dir")
py_modules = job_config.runtime_env.get("py_modules")
excludes = job_config.runtime_env.get("excludes") or []
logger.info(f"{pkg_uri} doesn't exist. Create new package with"
f" {working_dir} and {py_modules}")
if not pkg_file.exists():
create_project_package(working_dir,
py_modules,
excludes,
file_path,
logger=logger)
# Push the data to remote storage
pkg_size = push_package(pkg_uri, pkg_file)
logger.info(f"{pkg_uri} has been pushed with {pkg_size} bytes")
def start_specific_server(self, client_id: str,
job_config: JobConfig) -> bool:
"""
Start up a RayClient Server for an incoming client to
communicate with. Returns whether creation was successful.
"""
specific_server = self._get_server_for_client(client_id)
assert specific_server, f"Server has not been created for: {client_id}"
output, error = self.node.get_log_file_handles(
f"ray_client_server_{specific_server.port}", unique=True)
# Set up the working_dir for the server.
# TODO(edoakes): this should go be unified with the worker setup code
# by going through the runtime_env agent.
uris = job_config.get_runtime_env_uris() if job_config else []
if uris:
# Download and set up the working_dir locally.
working_dir = working_dir_pkg.ensure_runtime_env_setup(uris)
# Set PYTHONPATH in the environment variables so the working_dir
# is included in the module search path.
runtime_env = job_config.runtime_env
env_vars = runtime_env.get("env_vars", None) or {}
python_path = working_dir
if "PYTHONPATH" in env_vars:
python_path += (os.pathsep + runtime_env["PYTHONPATH"])
env_vars["PYTHONPATH"] = python_path
runtime_env["env_vars"] = env_vars
job_config.set_runtime_env(runtime_env)
serialized_runtime_env = job_config.get_serialized_runtime_env()
proc = start_ray_client_server(
self.redis_address,
specific_server.port,
stdout_file=output,
stderr_file=error,
fate_share=self.fate_share,
server_type="specific-server",
serialized_runtime_env=serialized_runtime_env,
session_dir=self.node.get_session_dir_path(),
redis_password=self._redis_password)
# Wait for the process being run transitions from the shim process
# to the actual RayClient Server.
pid = proc.process.pid
if sys.platform != "win32":
psutil_proc = psutil.Process(pid)
else:
psutil_proc = None
# Don't use `psutil` on Win32
while psutil_proc is not None:
if proc.process.poll() is not None:
logger.error(
f"SpecificServer startup failed for client: {client_id}")
break
cmd = psutil_proc.cmdline()
if _match_running_client_server(cmd):
break
logger.debug(
"Waiting for Process to reach the actual client server.")
time.sleep(0.5)
specific_server.set_result(proc)
logger.info(f"SpecificServer started on port: {specific_server.port} "
f"with PID: {pid} for client: {client_id}")
return proc.process.poll() is None