def get_handle(endpoint_name, relative_slo_ms=None, absolute_slo_ms=None, missing_ok=False): """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. relative_slo_ms(float): Specify relative deadline in milliseconds for queries fired using this handle. (Default: None) absolute_slo_ms(float): Specify absolute deadline in milliseconds for queries fired using this handle. (Default: None) missing_ok (bool): If true, skip the check for the endpoint existence. It can be useful when the endpoint has not been registered. Returns: RayServeHandle """ if not missing_ok: assert endpoint_name in retry_actor_failures( master_actor.get_all_endpoints) return RayServeHandle( retry_actor_failures(master_actor.get_router)[0], endpoint_name, relative_slo_ms, absolute_slo_ms, )
def get_handle(endpoint_name, relative_slo_ms=None, absolute_slo_ms=None, missing_ok=False): """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. relative_slo_ms(float): Specify relative deadline in milliseconds for queries fired using this handle. (Default: None) absolute_slo_ms(float): Specify absolute deadline in milliseconds for queries fired using this handle. (Default: None) missing_ok (bool): If true, skip the check for the endpoint existence. It can be useful when the endpoint has not been registered. Returns: RayServeHandle """ if not missing_ok: assert endpoint_name in expand( global_state.route_table.list_service( include_headless=True).values()) # Delay import due to it's dependency on global_state from ray.serve.handle import RayServeHandle return RayServeHandle( global_state.init_or_get_router(), endpoint_name, relative_slo_ms, absolute_slo_ms, )
def get_handle(self, endpoint_name: str, missing_ok: Optional[bool] = False) -> RayServeHandle: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. Returns: RayServeHandle """ if not missing_ok and endpoint_name not in ray.get( self._controller.get_all_endpoints.remote()): raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") routers = list(ray.get(self._controller.get_routers.remote()).values()) current_node_id = ray.get_runtime_context().node_id.hex() try: router_chosen = next( filter(lambda r: get_node_id_for_actor(r) == current_node_id, routers)) except StopIteration: logger.warning( f"When getting a handle for {endpoint_name}, Serve can't find " "a router on the same node. Serve will use a random router.") router_chosen = random.choice(routers) return RayServeHandle( router_chosen, endpoint_name, )
def get_handle(endpoint_name, relative_slo_ms=None, absolute_slo_ms=None, missing_ok=False): """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. relative_slo_ms(float): Specify relative deadline in milliseconds for queries fired using this handle. (Default: None) absolute_slo_ms(float): Specify absolute deadline in milliseconds for queries fired using this handle. (Default: None) missing_ok (bool): If true, skip the check for the endpoint existence. It can be useful when the endpoint has not been registered. Returns: RayServeHandle """ if not missing_ok: assert endpoint_name in ray.get(controller.get_all_endpoints.remote()) # TODO(edoakes): we should choose the router on the same node. routers = ray.get(controller.get_routers.remote()) return RayServeHandle( list(routers.values())[0], endpoint_name, relative_slo_ms, absolute_slo_ms, )
def get_handle(self, endpoint_name: str, missing_ok: Optional[bool] = False, sync: bool = True) -> RayServeHandle: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. sync (bool): If true, then Serve will return a ServeHandle that works everywhere. Otherwise, Serve will return a ServeHandle that's only usable in asyncio loop. Returns: RayServeHandle """ if not missing_ok and endpoint_name not in ray.get( self._controller.get_all_endpoints.remote()): raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") if asyncio.get_event_loop().is_running() and sync: logger.warning( "You are retrieving a ServeHandle inside an asyncio loop. " "Try getting client.get_handle(.., sync=False) to get better " "performance.") if endpoint_name not in self._handle_cache: handle = RayServeHandle(self._controller, endpoint_name, sync=sync) self._handle_cache[endpoint_name] = handle return self._handle_cache[endpoint_name]
def get_handle( self, endpoint_name: str, missing_ok: Optional[bool] = False, sync: bool = True) -> Union[RayServeHandle, RayServeSyncHandle]: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. sync (bool): If true, then Serve will return a ServeHandle that works everywhere. Otherwise, Serve will return a ServeHandle that's only usable in asyncio loop. Returns: RayServeHandle """ all_endpoints = ray.get(self._controller.get_all_endpoints.remote()) if not missing_ok and endpoint_name not in all_endpoints: raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") if asyncio.get_event_loop().is_running() and sync: logger.warning( "You are retrieving a sync handle inside an asyncio loop. " "Try getting client.get_handle(.., sync=False) to get better " "performance. Learn more at https://docs.ray.io/en/master/" "serve/http-servehandle.html#sync-and-async-handles") if not asyncio.get_event_loop().is_running() and not sync: logger.warning( "You are retrieving an async handle outside an asyncio loop. " "You should make sure client.get_handle is called inside a " "running event loop. Or call client.get_handle(.., sync=True) " "to create sync handle. Learn more at https://docs.ray.io/en/" "master/serve/http-servehandle.html#sync-and-async-handles") if endpoint_name in all_endpoints: this_endpoint = all_endpoints[endpoint_name] python_methods: List[str] = this_endpoint["python_methods"] else: # This can happen in the missing_ok=True case. # handle.method_name.remote won't work and user must # use the legacy handle.options(method).remote(). python_methods: List[str] = [] # NOTE(simon): this extra layer of router seems unnecessary # BUT it's needed still because of the shared asyncio thread. router = self._get_proxied_router(sync=sync, endpoint=endpoint_name) if sync: handle = RayServeSyncHandle(router, endpoint_name, known_python_methods=python_methods) else: handle = RayServeHandle(router, endpoint_name, known_python_methods=python_methods) return handle
def get_handle( self, endpoint_name: str, missing_ok: Optional[bool] = False, sync: bool = True) -> Union[RayServeHandle, RayServeSyncHandle]: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. sync (bool): If true, then Serve will return a ServeHandle that works everywhere. Otherwise, Serve will return a ServeHandle that's only usable in asyncio loop. Returns: RayServeHandle """ if not missing_ok and endpoint_name not in ray.get( self._controller.get_all_endpoints.remote()): raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") if asyncio.get_event_loop().is_running() and sync: logger.warning( "You are retrieving a sync handle inside an asyncio loop. " "Try getting client.get_handle(.., sync=False) to get better " "performance. Learn more at https://docs.ray.io/en/master/" "serve/advanced.html#sync-and-async-handles") if not asyncio.get_event_loop().is_running() and not sync: logger.warning( "You are retrieving an async handle outside an asyncio loop. " "You should make sure client.get_handle is called inside a " "running event loop. Or call client.get_handle(.., sync=True) " "to create sync handle. Learn more at https://docs.ray.io/en/" "master/serve/advanced.html#sync-and-async-handles") if sync: handle = RayServeSyncHandle(self._get_proxied_router(sync=sync), endpoint_name) else: handle = RayServeHandle(self._get_proxied_router(sync=sync), endpoint_name) return handle
def get_handle(self, endpoint_name: str) -> RayServeHandle: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. Returns: RayServeHandle """ if endpoint_name not in ray.get( self._controller.get_all_endpoints.remote()): raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") # TODO(edoakes): we should choose the router on the same node. routers = ray.get(self._controller.get_routers.remote()) return RayServeHandle( list(routers.values())[0], endpoint_name, )
def get_handle(endpoint_name: str, missing_ok: bool = False) -> RayServeHandle: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, skip the check for the endpoint existence. It can be useful when the endpoint has not been registered. Returns: RayServeHandle """ if not missing_ok: assert endpoint_name in ray.get(controller.get_all_endpoints.remote()) # TODO(edoakes): we should choose the router on the same node. routers = ray.get(controller.get_routers.remote()) return RayServeHandle( list(routers.values())[0], endpoint_name, )
def get_handle(self, endpoint_name: str, missing_ok: Optional[bool] = False) -> RayServeHandle: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. Returns: RayServeHandle """ if not missing_ok and endpoint_name not in ray.get( self._controller.get_all_endpoints.remote()): raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") if endpoint_name not in self._handle_cache: handle = RayServeHandle(self._controller, endpoint_name, sync=True) self._handle_cache[endpoint_name] = handle return self._handle_cache[endpoint_name]
def get_handle(endpoint_name, relative_slo_ms=None, absolute_slo_ms=None): """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. relative_slo_ms(float): Specify relative deadline in milliseconds for queries fired using this handle. (Default: None) absolute_slo_ms(float): Specify absolute deadline in milliseconds for queries fired using this handle. (Default: None) Returns: RayServeHandle """ assert endpoint_name in expand( global_state.route_table.list_service(include_headless=True).values()) # Delay import due to it's dependency on global_state from ray.serve.handle import RayServeHandle return RayServeHandle(global_state.init_or_get_router(), endpoint_name, relative_slo_ms, absolute_slo_ms)
def get_handle( self, deployment_name: str, missing_ok: Optional[bool] = False, sync: bool = True, _internal_pickled_http_request: bool = False, ) -> Union[RayServeHandle, RayServeSyncHandle]: """Retrieve RayServeHandle for service deployment to invoke it from Python. Args: deployment_name (str): A registered service deployment. missing_ok (bool): If true, then Serve won't check the deployment is registered. False by default. sync (bool): If true, then Serve will return a ServeHandle that works everywhere. Otherwise, Serve will return a ServeHandle that's only usable in asyncio loop. Returns: RayServeHandle """ cache_key = (deployment_name, missing_ok, sync) if cache_key in self.handle_cache: cached_handle = self.handle_cache[cache_key] if cached_handle.is_polling and cached_handle.is_same_loop: return cached_handle all_endpoints = ray.get(self._controller.get_all_endpoints.remote()) if not missing_ok and deployment_name not in all_endpoints: raise KeyError(f"Deployment '{deployment_name}' does not exist.") try: asyncio_loop_running = asyncio.get_event_loop().is_running() except RuntimeError as ex: if "There is no current event loop in thread" in str(ex): asyncio_loop_running = False else: raise ex if asyncio_loop_running and sync: logger.warning( "You are retrieving a sync handle inside an asyncio loop. " "Try getting client.get_handle(.., sync=False) to get better " "performance. Learn more at https://docs.ray.io/en/master/" "serve/http-servehandle.html#sync-and-async-handles") if not asyncio_loop_running and not sync: logger.warning( "You are retrieving an async handle outside an asyncio loop. " "You should make sure client.get_handle is called inside a " "running event loop. Or call client.get_handle(.., sync=True) " "to create sync handle. Learn more at https://docs.ray.io/en/" "master/serve/http-servehandle.html#sync-and-async-handles") if sync: handle = RayServeSyncHandle( self._controller, deployment_name, _internal_pickled_http_request=_internal_pickled_http_request, ) else: handle = RayServeHandle( self._controller, deployment_name, _internal_pickled_http_request=_internal_pickled_http_request, ) self.handle_cache[cache_key] = handle if cache_key in self._evicted_handle_keys: logger.warning( "You just got a ServeHandle that was evicted from internal " "cache. This means you are getting too many ServeHandles in " "the same process, this will bring down Serve's performance. " "Please post a github issue at " "https://github.com/ray-project/ray/issues to let the Serve " "team to find workaround for your use case.") if len(self.handle_cache) > MAX_CACHED_HANDLES: # Perform random eviction to keep the handle cache from growing # infinitely. We used use WeakValueDictionary but hit # https://github.com/ray-project/ray/issues/18980. evict_key = random.choice(list(self.handle_cache.keys())) self._evicted_handle_keys.add(evict_key) self.handle_cache.pop(evict_key) return handle
def get_handle( self, endpoint_name: str, missing_ok: Optional[bool] = False, sync: bool = True, _internal_pickled_http_request: bool = False, ) -> Union[RayServeHandle, RayServeSyncHandle]: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. sync (bool): If true, then Serve will return a ServeHandle that works everywhere. Otherwise, Serve will return a ServeHandle that's only usable in asyncio loop. Returns: RayServeHandle """ cache_key = (endpoint_name, missing_ok, sync) if cache_key in self.handle_cache: return self.handle_cache[cache_key] all_endpoints = ray.get(self._controller.get_all_endpoints.remote()) if not missing_ok and endpoint_name not in all_endpoints: raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") try: asyncio_loop_running = asyncio.get_event_loop().is_running() except RuntimeError as ex: if "There is no current event loop in thread" in str(ex): asyncio_loop_running = False else: raise ex if asyncio_loop_running and sync: logger.warning( "You are retrieving a sync handle inside an asyncio loop. " "Try getting client.get_handle(.., sync=False) to get better " "performance. Learn more at https://docs.ray.io/en/master/" "serve/http-servehandle.html#sync-and-async-handles") if not asyncio_loop_running and not sync: logger.warning( "You are retrieving an async handle outside an asyncio loop. " "You should make sure client.get_handle is called inside a " "running event loop. Or call client.get_handle(.., sync=True) " "to create sync handle. Learn more at https://docs.ray.io/en/" "master/serve/http-servehandle.html#sync-and-async-handles") if sync: handle = RayServeSyncHandle( self._controller, endpoint_name, _internal_pickled_http_request=_internal_pickled_http_request, ) else: handle = RayServeHandle( self._controller, endpoint_name, _internal_pickled_http_request=_internal_pickled_http_request, ) self.handle_cache[cache_key] = handle return handle