def get_handle( self, endpoint_name: str, missing_ok: Optional[bool] = False, sync: bool = True) -> Union[RayServeHandle, RayServeSyncHandle]: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. sync (bool): If true, then Serve will return a ServeHandle that works everywhere. Otherwise, Serve will return a ServeHandle that's only usable in asyncio loop. Returns: RayServeHandle """ all_endpoints = ray.get(self._controller.get_all_endpoints.remote()) if not missing_ok and endpoint_name not in all_endpoints: raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") if asyncio.get_event_loop().is_running() and sync: logger.warning( "You are retrieving a sync handle inside an asyncio loop. " "Try getting client.get_handle(.., sync=False) to get better " "performance. Learn more at https://docs.ray.io/en/master/" "serve/http-servehandle.html#sync-and-async-handles") if not asyncio.get_event_loop().is_running() and not sync: logger.warning( "You are retrieving an async handle outside an asyncio loop. " "You should make sure client.get_handle is called inside a " "running event loop. Or call client.get_handle(.., sync=True) " "to create sync handle. Learn more at https://docs.ray.io/en/" "master/serve/http-servehandle.html#sync-and-async-handles") if endpoint_name in all_endpoints: this_endpoint = all_endpoints[endpoint_name] python_methods: List[str] = this_endpoint["python_methods"] else: # This can happen in the missing_ok=True case. # handle.method_name.remote won't work and user must # use the legacy handle.options(method).remote(). python_methods: List[str] = [] # NOTE(simon): this extra layer of router seems unnecessary # BUT it's needed still because of the shared asyncio thread. router = self._get_proxied_router(sync=sync, endpoint=endpoint_name) if sync: handle = RayServeSyncHandle(router, endpoint_name, known_python_methods=python_methods) else: handle = RayServeHandle(router, endpoint_name, known_python_methods=python_methods) return handle
def get_handle( self, endpoint_name: str, missing_ok: Optional[bool] = False, sync: bool = True) -> Union[RayServeHandle, RayServeSyncHandle]: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. sync (bool): If true, then Serve will return a ServeHandle that works everywhere. Otherwise, Serve will return a ServeHandle that's only usable in asyncio loop. Returns: RayServeHandle """ if not missing_ok and endpoint_name not in ray.get( self._controller.get_all_endpoints.remote()): raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") if asyncio.get_event_loop().is_running() and sync: logger.warning( "You are retrieving a sync handle inside an asyncio loop. " "Try getting client.get_handle(.., sync=False) to get better " "performance. Learn more at https://docs.ray.io/en/master/" "serve/advanced.html#sync-and-async-handles") if not asyncio.get_event_loop().is_running() and not sync: logger.warning( "You are retrieving an async handle outside an asyncio loop. " "You should make sure client.get_handle is called inside a " "running event loop. Or call client.get_handle(.., sync=True) " "to create sync handle. Learn more at https://docs.ray.io/en/" "master/serve/advanced.html#sync-and-async-handles") if sync: handle = RayServeSyncHandle(self._get_proxied_router(sync=sync), endpoint_name) else: handle = RayServeHandle(self._get_proxied_router(sync=sync), endpoint_name) return handle
def get_handle( self, deployment_name: str, missing_ok: Optional[bool] = False, sync: bool = True, _internal_pickled_http_request: bool = False, ) -> Union[RayServeHandle, RayServeSyncHandle]: """Retrieve RayServeHandle for service deployment to invoke it from Python. Args: deployment_name (str): A registered service deployment. missing_ok (bool): If true, then Serve won't check the deployment is registered. False by default. sync (bool): If true, then Serve will return a ServeHandle that works everywhere. Otherwise, Serve will return a ServeHandle that's only usable in asyncio loop. Returns: RayServeHandle """ cache_key = (deployment_name, missing_ok, sync) if cache_key in self.handle_cache: cached_handle = self.handle_cache[cache_key] if cached_handle.is_polling and cached_handle.is_same_loop: return cached_handle all_endpoints = ray.get(self._controller.get_all_endpoints.remote()) if not missing_ok and deployment_name not in all_endpoints: raise KeyError(f"Deployment '{deployment_name}' does not exist.") try: asyncio_loop_running = asyncio.get_event_loop().is_running() except RuntimeError as ex: if "There is no current event loop in thread" in str(ex): asyncio_loop_running = False else: raise ex if asyncio_loop_running and sync: logger.warning( "You are retrieving a sync handle inside an asyncio loop. " "Try getting client.get_handle(.., sync=False) to get better " "performance. Learn more at https://docs.ray.io/en/master/" "serve/http-servehandle.html#sync-and-async-handles") if not asyncio_loop_running and not sync: logger.warning( "You are retrieving an async handle outside an asyncio loop. " "You should make sure client.get_handle is called inside a " "running event loop. Or call client.get_handle(.., sync=True) " "to create sync handle. Learn more at https://docs.ray.io/en/" "master/serve/http-servehandle.html#sync-and-async-handles") if sync: handle = RayServeSyncHandle( self._controller, deployment_name, _internal_pickled_http_request=_internal_pickled_http_request, ) else: handle = RayServeHandle( self._controller, deployment_name, _internal_pickled_http_request=_internal_pickled_http_request, ) self.handle_cache[cache_key] = handle if cache_key in self._evicted_handle_keys: logger.warning( "You just got a ServeHandle that was evicted from internal " "cache. This means you are getting too many ServeHandles in " "the same process, this will bring down Serve's performance. " "Please post a github issue at " "https://github.com/ray-project/ray/issues to let the Serve " "team to find workaround for your use case.") if len(self.handle_cache) > MAX_CACHED_HANDLES: # Perform random eviction to keep the handle cache from growing # infinitely. We used use WeakValueDictionary but hit # https://github.com/ray-project/ray/issues/18980. evict_key = random.choice(list(self.handle_cache.keys())) self._evicted_handle_keys.add(evict_key) self.handle_cache.pop(evict_key) return handle
def get_handle( self, endpoint_name: str, missing_ok: Optional[bool] = False, sync: bool = True, _internal_pickled_http_request: bool = False, ) -> Union[RayServeHandle, RayServeSyncHandle]: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. sync (bool): If true, then Serve will return a ServeHandle that works everywhere. Otherwise, Serve will return a ServeHandle that's only usable in asyncio loop. Returns: RayServeHandle """ cache_key = (endpoint_name, missing_ok, sync) if cache_key in self.handle_cache: return self.handle_cache[cache_key] all_endpoints = ray.get(self._controller.get_all_endpoints.remote()) if not missing_ok and endpoint_name not in all_endpoints: raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") try: asyncio_loop_running = asyncio.get_event_loop().is_running() except RuntimeError as ex: if "There is no current event loop in thread" in str(ex): asyncio_loop_running = False else: raise ex if asyncio_loop_running and sync: logger.warning( "You are retrieving a sync handle inside an asyncio loop. " "Try getting client.get_handle(.., sync=False) to get better " "performance. Learn more at https://docs.ray.io/en/master/" "serve/http-servehandle.html#sync-and-async-handles") if not asyncio_loop_running and not sync: logger.warning( "You are retrieving an async handle outside an asyncio loop. " "You should make sure client.get_handle is called inside a " "running event loop. Or call client.get_handle(.., sync=True) " "to create sync handle. Learn more at https://docs.ray.io/en/" "master/serve/http-servehandle.html#sync-and-async-handles") if sync: handle = RayServeSyncHandle( self._controller, endpoint_name, _internal_pickled_http_request=_internal_pickled_http_request, ) else: handle = RayServeHandle( self._controller, endpoint_name, _internal_pickled_http_request=_internal_pickled_http_request, ) self.handle_cache[cache_key] = handle return handle