def _create_runtime_env(self, serialized_runtime_env: str,
specific_server: SpecificServer):
"""Creates the runtime_env by sending an RPC to the agent.
Includes retry logic to handle the case when the agent is
temporarily unreachable (e.g., hasn't been started up yet).
"""
if not check_dashboard_dependencies_installed():
raise RuntimeError("Not all required Ray dependencies for the "
"runtime_env feature were found on the "
"cluster. To install the required "
"dependencies, please run `pip install "
"\"ray[default]\"` on all cluster nodes.")
create_env_request = runtime_env_agent_pb2.CreateRuntimeEnvRequest(
serialized_runtime_env=serialized_runtime_env,
job_id=f"ray_client_server_{specific_server.port}".encode("utf-8"))
retries = 0
max_retries = 5
wait_time_s = 0.5
while retries <= max_retries:
try:
r = self._runtime_env_stub.CreateRuntimeEnv(create_env_request)
if (r.status ==
agent_manager_pb2.AgentRpcStatus.AGENT_RPC_STATUS_OK):
return r.serialized_runtime_env_context
elif (r.status == agent_manager_pb2.AgentRpcStatus.
AGENT_RPC_STATUS_FAILED):
raise RuntimeError(
"Failed to create runtime_env for Ray client "
f"server: {r.error_message}")
else:
assert False, f"Unknown status: {r.status}."
except grpc.RpcError as e:
# Whitelist of errors we consider transient.
# NOTE(edoakes): we can get UNIMPLEMENTED while the server
# starts up because the agent runs multiple gRPC services
# on the same port.
if e.code() not in [
grpc.StatusCode.UNAVAILABLE,
grpc.StatusCode.UNIMPLEMENTED
]:
raise e
logger.warning(f"CreateRuntimeEnv request failed: {e}. "
f"Retrying after {wait_time_s}s. "
f"{max_retries-retries} retries remaining.")
# Exponential backoff.
time.sleep(wait_time_s)
retries += 1
wait_time_s *= 2
raise TimeoutError(
f"CreateRuntimeEnv request failed after {max_retries} attempts.")
def get_all_modules(module_type):
"""
Get all importable modules that are subclass of a given module type.
"""
logger.info(f"Get all modules by type: {module_type.__name__}")
import ray.dashboard.modules
should_only_load_minimal_modules = not check_dashboard_dependencies_installed(
)
for module_loader, name, ispkg in pkgutil.walk_packages(
ray.dashboard.modules.__path__,
ray.dashboard.modules.__name__ + "."):
try:
importlib.import_module(name)
except ModuleNotFoundError as e:
logger.info(f"Module {name} cannot be loaded because "
"we cannot import all dependencies. Download "
"`pip install ray[default]` for the full "
f"dashboard functionality. Error: {e}")
if not should_only_load_minimal_modules:
logger.info(
"Although `pip install ray[default] is downloaded, "
"module couldn't be imported`")
raise e
imported_modules = []
# module_type.__subclasses__() should contain modules that
# we could successfully import.
for m in module_type.__subclasses__():
if not getattr(m, "__ray_dashboard_module_enable__", True):
continue
if should_only_load_minimal_modules and not m.is_minimal_module():
continue
imported_modules.append(m)
logger.info(f"Available modules: {imported_modules}")
return imported_modules