async def on_startup(): aiogrpc.init_grpc_aio() self.aiogrpc_gcs_channel = await ( make_gcs_grpc_channel(redis_client)) self.gcs_heartbeat_info_stub = ( gcs_service_pb2_grpc.HeartbeatInfoGcsServiceStub( self.aiogrpc_gcs_channel))
def check_health(address: str, timeout=2) -> bool: """Checks Ray cluster health, before / without actually connecting to the cluster via ray.init(). Args: address: Ray cluster / GCS address string, e.g. ip:port. timeout: request timeout. Returns: Returns True if the cluster is running and has matching Ray version. Returns False if no service is running. Raises an exception otherwise. """ req = gcs_service_pb2.CheckAliveRequest() try: channel = create_gcs_channel(address) stub = gcs_service_pb2_grpc.HeartbeatInfoGcsServiceStub(channel) resp = stub.CheckAlive(req, timeout=timeout) except grpc.RpcError: return False if resp.status.code != GcsCode.OK: raise RuntimeError( f"GCS running at {address} is unhealthy: {resp.status}") if resp.ray_version is None: resp.ray_version = "<= 1.12" if resp.ray_version != ray.__version__: raise RuntimeError(f"Ray cluster at {address} has version " f"{resp.ray_version}, but this process is running " f"Ray version {ray.__version__}.") return True
def __init__(self, gcs_address: str): self.grpc_gcs_channel = ray._private.utils.init_grpc_channel( gcs_address, options=GRPC_CHANNEL_OPTIONS) self.gcs_heartbeat_info_stub = gcs_service_pb2_grpc.HeartbeatInfoGcsServiceStub( self.grpc_gcs_channel) self.work_queue = Queue() super().__init__(daemon=True)
async def run(self): # Create an aioredis client for all modules. try: self.aioredis_client = await dashboard_utils.get_aioredis_client( self.redis_address, self.redis_password, dashboard_consts.CONNECT_REDIS_INTERNAL_SECONDS, dashboard_consts.RETRY_REDIS_CONNECTION_TIMES) except (socket.gaierror, ConnectionError): logger.error( "Dashboard head exiting: " "Failed to connect to redis at %s", self.redis_address) sys.exit(-1) # Create a http session for all modules. self.http_session = aiohttp.ClientSession( loop=asyncio.get_event_loop()) # Waiting for GCS is ready. while True: try: gcs_address = await self.aioredis_client.get( dashboard_consts.REDIS_KEY_GCS_SERVER_ADDRESS) if not gcs_address: raise Exception("GCS address not found.") logger.info("Connect to GCS at %s", gcs_address) options = (("grpc.enable_http_proxy", 0), ) channel = aiogrpc.insecure_channel( gcs_address, options=options) except Exception as ex: logger.error("Connect to GCS failed: %s, retry...", ex) await asyncio.sleep( dashboard_consts.GCS_RETRY_CONNECT_INTERVAL_SECONDS) else: self.aiogrpc_gcs_channel = channel break # Create a HeartbeatInfoGcsServiceStub. self._gcs_heartbeat_info_stub = \ gcs_service_pb2_grpc.HeartbeatInfoGcsServiceStub( self.aiogrpc_gcs_channel) # Start a grpc asyncio server. await self.server.start() async def _async_notify(): """Notify signals from queue.""" while True: co = await dashboard_utils.NotifyQueue.get() try: await co except Exception: logger.exception(f"Error notifying coroutine {co}") modules = self._load_modules() # Http server should be initialized after all modules loaded. app = aiohttp.web.Application() app.add_routes(routes=routes.bound_routes()) runner = aiohttp.web.AppRunner(app) await runner.setup() last_ex = None for i in range(1 + self.http_port_retries): try: site = aiohttp.web.TCPSite(runner, self.http_host, self.http_port) await site.start() break except OSError as e: last_ex = e self.http_port += 1 logger.warning("Try to use port %s: %s", self.http_port, e) else: raise Exception(f"Failed to find a valid port for dashboard after " f"{self.http_port_retries} retries: {last_ex}") http_host, http_port, *_ = site._server.sockets[0].getsockname() http_host = self.ip if ipaddress.ip_address( http_host).is_unspecified else http_host logger.info("Dashboard head http address: %s:%s", http_host, http_port) # Write the dashboard head port to redis. await self.aioredis_client.set(ray_constants.REDIS_KEY_DASHBOARD, f"{http_host}:{http_port}") await self.aioredis_client.set( dashboard_consts.REDIS_KEY_DASHBOARD_RPC, f"{self.ip}:{self.grpc_port}") # Dump registered http routes. dump_routes = [ r for r in app.router.routes() if r.method != hdrs.METH_HEAD ] for r in dump_routes: logger.info(r) logger.info("Registered %s routes.", len(dump_routes)) # Freeze signal after all modules loaded. dashboard_utils.SignalManager.freeze() concurrent_tasks = [ self._gcs_check_alive(), _async_notify(), DataOrganizer.purge(), DataOrganizer.organize(), ] await asyncio.gather(*concurrent_tasks, *(m.run(self.server) for m in modules)) await self.server.wait_for_termination()
def _connect(self): self._channel.connect() self._kv_stub = gcs_service_pb2_grpc.InternalKVGcsServiceStub( self._channel.channel()) self._heartbeat_info_stub = gcs_service_pb2_grpc.HeartbeatInfoGcsServiceStub( self._channel.channel())