def upload_runtime_env_package_if_needed(job_config: JobConfig) -> None: """Upload runtime env if it's not there. It'll check whether the runtime environment exists in the cluster or not. If it doesn't exist, a package will be created based on the working directory and modules defined in job config. The package will be uploaded to the cluster after this. Args: job_config (JobConfig): The job config of driver. """ assert _internal_kv_initialized() pkg_uris = job_config.get_runtime_env_uris() for pkg_uri in pkg_uris: if not package_exists(pkg_uri): file_path = _get_local_path(pkg_uri) pkg_file = Path(file_path) working_dir = job_config.runtime_env.get("working_dir") py_modules = job_config.runtime_env.get("py_modules") excludes = job_config.runtime_env.get("excludes") or [] logger.info(f"{pkg_uri} doesn't exist. Create new package with" f" {working_dir} and {py_modules}") if not pkg_file.exists(): create_project_package(working_dir, py_modules, excludes, file_path) # Push the data to remote storage pkg_size = push_package(pkg_uri, pkg_file) logger.info(f"{pkg_uri} has been pushed with {pkg_size} bytes")
def upload_runtime_env_package_if_needed( self, job_config: JobConfig, logger: Optional[logging.Logger] = default_logger): """Upload runtime env if it's not there. It'll check whether the runtime environment exists in the cluster or not. If it doesn't, a package will be created based on the working directory and modules defined in job config. The package will be uploaded to the cluster after this. Args: job_config (JobConfig): The job config of driver. """ if logger is None: logger = default_logger pkg_uris = job_config.get_runtime_env_uris() if len(pkg_uris) == 0: return # Return early to avoid internal kv check in this case. for pkg_uri in pkg_uris: if not package_exists(pkg_uri): file_path = self._get_local_path(pkg_uri) pkg_file = Path(file_path) working_dir = job_config.runtime_env.get("working_dir") py_modules = job_config.runtime_env.get("py_modules") excludes = job_config.runtime_env.get("excludes") or [] logger.info(f"{pkg_uri} doesn't exist. Create new package with" f" {working_dir} and {py_modules}") if not pkg_file.exists(): create_project_package(working_dir, py_modules, excludes, file_path, logger=logger) # Push the data to remote storage pkg_size = push_package(pkg_uri, pkg_file) logger.info(f"{pkg_uri} has been pushed with {pkg_size} bytes")
def start_specific_server(self, client_id: str, job_config: JobConfig) -> bool: """ Start up a RayClient Server for an incoming client to communicate with. Returns whether creation was successful. """ specific_server = self._get_server_for_client(client_id) assert specific_server, f"Server has not been created for: {client_id}" output, error = self.node.get_log_file_handles( f"ray_client_server_{specific_server.port}", unique=True) # Set up the working_dir for the server. # TODO(edoakes): this should go be unified with the worker setup code # by going through the runtime_env agent. uris = job_config.get_runtime_env_uris() if job_config else [] if uris: # Download and set up the working_dir locally. working_dir = working_dir_pkg.ensure_runtime_env_setup(uris) # Set PYTHONPATH in the environment variables so the working_dir # is included in the module search path. runtime_env = job_config.runtime_env env_vars = runtime_env.get("env_vars", None) or {} python_path = working_dir if "PYTHONPATH" in env_vars: python_path += (os.pathsep + runtime_env["PYTHONPATH"]) env_vars["PYTHONPATH"] = python_path runtime_env["env_vars"] = env_vars job_config.set_runtime_env(runtime_env) serialized_runtime_env = job_config.get_serialized_runtime_env() proc = start_ray_client_server( self.redis_address, specific_server.port, stdout_file=output, stderr_file=error, fate_share=self.fate_share, server_type="specific-server", serialized_runtime_env=serialized_runtime_env, session_dir=self.node.get_session_dir_path(), redis_password=self._redis_password) # Wait for the process being run transitions from the shim process # to the actual RayClient Server. pid = proc.process.pid if sys.platform != "win32": psutil_proc = psutil.Process(pid) else: psutil_proc = None # Don't use `psutil` on Win32 while psutil_proc is not None: if proc.process.poll() is not None: logger.error( f"SpecificServer startup failed for client: {client_id}") break cmd = psutil_proc.cmdline() if _match_running_client_server(cmd): break logger.debug( "Waiting for Process to reach the actual client server.") time.sleep(0.5) specific_server.set_result(proc) logger.info(f"SpecificServer started on port: {specific_server.port} " f"with PID: {pid} for client: {client_id}") return proc.process.poll() is None