class ThreadProxiedRouter: def __init__(self, controller_handle, sync: bool): self.router = Router(controller_handle) if sync: self.async_loop = create_or_get_async_loop_in_thread() asyncio.run_coroutine_threadsafe( self.router.setup_in_async_loop(), self.async_loop, ) else: self.async_loop = asyncio.get_event_loop() self.async_loop.create_task(self.router.setup_in_async_loop()) def _remote(self, endpoint_name, handle_options, request_data, kwargs) -> Coroutine: request_metadata = RequestMetadata( get_random_letters(10), # Used for debugging. endpoint_name, TaskContext.Python, call_method=handle_options.method_name, shard_key=handle_options.shard_key, http_method=handle_options.http_method, http_headers=handle_options.http_headers, ) coro = self.router.assign_request(request_metadata, request_data, **kwargs) return coro
class ThreadProxiedRouter: def __init__(self, controller_handle, sync: bool, endpoint_tag: EndpointTag): self.controller_handle = controller_handle self.sync = sync self.endpoint_tag = endpoint_tag if sync: self._async_loop = create_or_get_async_loop_in_thread() else: self._async_loop = asyncio.get_event_loop() self.router = Router(controller_handle, endpoint_tag, self._async_loop) @property def async_loop(self): # called by handles return self._async_loop def _remote(self, endpoint_name, handle_options, request_data, kwargs) -> Coroutine: request_metadata = RequestMetadata( get_random_letters(10), # Used for debugging. endpoint_name, call_method=handle_options.method_name, shard_key=handle_options.shard_key, http_method=handle_options.http_method, http_headers=handle_options.http_headers, ) coro = self.router.assign_request(request_metadata, request_data, **kwargs) return coro def __reduce__(self): deserializer = ThreadProxiedRouter serialized_data = ( self.controller_handle, self.sync, self.endpoint_tag, ) return deserializer, serialized_data
class RayServeHandle: """A handle to a service endpoint. Invoking this endpoint with .remote is equivalent to pinging an HTTP endpoint. Example: >>> handle = serve.get_handle("my_endpoint") >>> handle RayServeHandle( Endpoint="my_endpoint", Traffic=... ) >>> handle.remote(my_request_content) ObjectRef(...) >>> ray.get(handle.remote(...)) # result >>> ray.get(handle.remote(let_it_crash_request)) # raises RayTaskError Exception """ def __init__( self, controller_handle, endpoint_name, sync: bool, *, method_name=None, shard_key=None, http_method=None, http_headers=None, ): self.controller_handle = controller_handle self.endpoint_name = endpoint_name self.method_name = method_name self.shard_key = shard_key self.http_method = http_method self.http_headers = http_headers self.router = Router(self.controller_handle) self.sync = sync # In the synchrounous mode, we create a new event loop in a separate # thread and run the Router.setup in that loop. In the async mode, we # can just use the current loop we are in right now. if self.sync: self.async_loop = create_or_get_async_loop_in_thread() asyncio.run_coroutine_threadsafe( self.router.setup_in_async_loop(), self.async_loop, ) else: # async self.async_loop = asyncio.get_event_loop() # create_task is not threadsafe. self.async_loop.create_task(self.router.setup_in_async_loop()) def _remote(self, request_data, kwargs) -> Coroutine: request_metadata = RequestMetadata( get_random_letters(10), # Used for debugging. self.endpoint_name, TaskContext.Python, call_method=self.method_name or "__call__", shard_key=self.shard_key, http_method=self.http_method or "GET", http_headers=self.http_headers or dict(), ) coro = self.router.assign_request(request_metadata, request_data, **kwargs) return coro def remote(self, request_data: Optional[Union[Dict, Any]] = None, **kwargs): """Issue an asynchrounous request to the endpoint. Returns a Ray ObjectRef whose results can be waited for or retrieved using ray.wait or ray.get, respectively. Returns: ray.ObjectRef Args: request_data(dict, Any): If it's a dictionary, the data will be available in ``request.json()`` or ``request.form()``. Otherwise, it will be available in ``request.data``. ``**kwargs``: All keyword arguments will be available in ``request.args``. """ if not self.sync: raise RayServeException( "You are trying to call handle.remote() with async handle. " "Please use `await handle.remote_async()` instead.") coro = self._remote(request_data, kwargs) future: concurrent.futures.Future = asyncio.run_coroutine_threadsafe( coro, self.async_loop) # Block until the result is ready. return future.result() async def remote_async(self, request_data: Optional[Union[Dict, Any]] = None, **kwargs) -> ray.ObjectRef: """Experimental API for enqueue a request in async context.""" if not asyncio.get_event_loop().is_running(): raise RayServeException( "remote_async must be called from a running event loop.") return await self._remote(request_data, kwargs) def options(self, method_name: Optional[str] = None, *, shard_key: Optional[str] = None, http_method: Optional[str] = None, http_headers: Optional[Dict[str, str]] = None): """Set options for this handle. Args: method_name(str): The method to invoke on the backend. http_method(str): The HTTP method to use for the request. shard_key(str): A string to use to deterministically map this request to a backend if there are multiple for this endpoint. """ # Don't override default non-null values. self.method_name = self.method_name or method_name self.shard_key = self.shard_key or shard_key self.http_method = self.http_method or http_method self.http_headers = self.http_headers or http_headers return self def __repr__(self): return f"RayServeHandle(endpoint='{self.endpoint_name}')"