def __init__(
self,
controller_handle,
endpoint_name,
sync: bool,
*,
method_name=None,
shard_key=None,
http_method=None,
http_headers=None,
):
self.controller_handle = controller_handle
self.endpoint_name = endpoint_name
self.method_name = method_name
self.shard_key = shard_key
self.http_method = http_method
self.http_headers = http_headers
self.router = Router(self.controller_handle)
self.sync = sync
# In the synchrounous mode, we create a new event loop in a separate
# thread and run the Router.setup in that loop. In the async mode, we
# can just use the current loop we are in right now.
if self.sync:
self.async_loop = create_or_get_async_loop_in_thread()
asyncio.run_coroutine_threadsafe(
self.router.setup_in_async_loop(),
self.async_loop,
)
else: # async
self.async_loop = asyncio.get_event_loop()
# create_task is not threadsafe.
self.async_loop.create_task(self.router.setup_in_async_loop())
class ThreadProxiedRouter:
def __init__(self, controller_handle, sync: bool):
self.router = Router(controller_handle)
if sync:
self.async_loop = create_or_get_async_loop_in_thread()
asyncio.run_coroutine_threadsafe(
self.router.setup_in_async_loop(),
self.async_loop,
)
else:
self.async_loop = asyncio.get_event_loop()
self.async_loop.create_task(self.router.setup_in_async_loop())
def _remote(self, endpoint_name, handle_options, request_data,
kwargs) -> Coroutine:
request_metadata = RequestMetadata(
get_random_letters(10), # Used for debugging.
endpoint_name,
TaskContext.Python,
call_method=handle_options.method_name,
shard_key=handle_options.shard_key,
http_method=handle_options.http_method,
http_headers=handle_options.http_headers,
)
coro = self.router.assign_request(request_metadata, request_data,
**kwargs)
return coro
def __init__(self, controller_handle, sync: bool,
endpoint_tag: EndpointTag):
self.controller_handle = controller_handle
self.sync = sync
self.endpoint_tag = endpoint_tag
if sync:
self._async_loop = create_or_get_async_loop_in_thread()
else:
self._async_loop = asyncio.get_event_loop()
self.router = Router(controller_handle, endpoint_tag, self._async_loop)
def __init__(self, controller_handle, sync: bool):
self.router = Router(controller_handle)
if sync:
self.async_loop = create_or_get_async_loop_in_thread()
asyncio.run_coroutine_threadsafe(
self.router.setup_in_async_loop(),
self.async_loop,
)
else:
self.async_loop = asyncio.get_event_loop()
self.async_loop.create_task(self.router.setup_in_async_loop())
def _make_router(self) -> Router:
# Delayed import because ray.serve.api depends on handles.
return Router(
self.controller_handle,
self.deployment_name,
event_loop=create_or_get_async_loop_in_thread(),
)
async def fetch_config_from_controller(self, name, instance_name=None):
assert ray.is_initialized()
controller = serve.api._get_controller()
self.route_table = await controller.get_router_config.remote()
self.request_counter = metrics.Count(
"num_http_requests", "The number of HTTP requests processed",
"requests", ["route"])
self.router = Router()
await self.router.setup(name, instance_name)
def __init__(self, controller_name):
controller = ray.get_actor(controller_name)
self.router = Router(controller)
self.long_poll_client = LongPollAsyncClient(
controller, {
LongPollKey.ROUTE_TABLE: self._update_route_table,
})
self.request_counter = metrics.Count(
"num_http_requests",
description="The number of HTTP requests processed",
tag_keys=("route", ))
async def fetch_config_from_controller(self, controller_name):
assert ray.is_initialized()
controller = ray.get_actor(controller_name)
self.route_table = await controller.get_router_config.remote()
self.request_counter = metrics.Count(
"num_http_requests",
description="The number of HTTP requests processed",
tag_keys=("route", ))
self.router = Router(controller)
await self.router.setup_in_async_loop()
async def fetch_config_from_controller(self, instance_name=None):
assert ray.is_initialized()
controller = serve.api._get_controller()
self.route_table = await controller.get_router_config.remote()
# The exporter is required to return results for /-/metrics endpoint.
[self.metric_exporter] = await controller.get_metric_exporter.remote()
self.metric_client = MetricClient(self.metric_exporter)
self.request_counter = self.metric_client.new_counter(
"num_http_requests",
description="The number of requests processed",
label_names=("route", ))
self.router = Router()
await self.router.setup(instance_name)
def __init__(self, controller_name):
# Set the controller name so that serve.connect() will connect to the
# controller instance this proxy is running in.
ray.serve.api._set_internal_controller_name(controller_name)
self.client = ray.serve.connect()
controller = ray.get_actor(controller_name)
self.route_table = {} # Should be updated via long polling.
self.router = Router(controller)
self.long_poll_client = LongPollAsyncClient(controller, {
LongPollKey.ROUTE_TABLE: self._update_route_table,
})
self.request_counter = metrics.Count(
"num_http_requests",
description="The number of HTTP requests processed",
tag_keys=("route", ))
class ThreadProxiedRouter:
def __init__(self, controller_handle, sync: bool,
endpoint_tag: EndpointTag):
self.controller_handle = controller_handle
self.sync = sync
self.endpoint_tag = endpoint_tag
if sync:
self._async_loop = create_or_get_async_loop_in_thread()
else:
self._async_loop = asyncio.get_event_loop()
self.router = Router(controller_handle, endpoint_tag, self._async_loop)
@property
def async_loop(self):
# called by handles
return self._async_loop
def _remote(self, endpoint_name, handle_options, request_data,
kwargs) -> Coroutine:
request_metadata = RequestMetadata(
get_random_letters(10), # Used for debugging.
endpoint_name,
call_method=handle_options.method_name,
shard_key=handle_options.shard_key,
http_method=handle_options.http_method,
http_headers=handle_options.http_headers,
)
coro = self.router.assign_request(request_metadata, request_data,
**kwargs)
return coro
def __reduce__(self):
deserializer = ThreadProxiedRouter
serialized_data = (
self.controller_handle,
self.sync,
self.endpoint_tag,
)
return deserializer, serialized_data
async def add_servable_to_router(
servable,
controller_name,
controller_actor,
**kwargs,
):
worker = setup_worker("backend",
servable,
controller_name=controller_name,
**kwargs)
await controller_actor.set_traffic.remote(
"endpoint",
TrafficPolicy({"backend": 1.0}),
)
await controller_actor.add_new_replica.remote(
"backend", worker, kwargs.get("backend_config", BackendConfig()))
router = Router(
controller_actor,
"endpoint",
asyncio.get_event_loop(),
)
return worker, router
def _make_router(self) -> Router:
return Router(
self.controller_handle,
self.deployment_name,
event_loop=asyncio.get_event_loop(),
)
class RayServeHandle:
"""A handle to a service endpoint.
Invoking this endpoint with .remote is equivalent to pinging
an HTTP endpoint.
Example:
>>> handle = serve.get_handle("my_endpoint")
>>> handle
RayServeHandle(
Endpoint="my_endpoint",
Traffic=...
)
>>> handle.remote(my_request_content)
ObjectRef(...)
>>> ray.get(handle.remote(...))
# result
>>> ray.get(handle.remote(let_it_crash_request))
# raises RayTaskError Exception
"""
def __init__(
self,
controller_handle,
endpoint_name,
sync: bool,
*,
method_name=None,
shard_key=None,
http_method=None,
http_headers=None,
):
self.controller_handle = controller_handle
self.endpoint_name = endpoint_name
self.method_name = method_name
self.shard_key = shard_key
self.http_method = http_method
self.http_headers = http_headers
self.router = Router(self.controller_handle)
self.sync = sync
# In the synchrounous mode, we create a new event loop in a separate
# thread and run the Router.setup in that loop. In the async mode, we
# can just use the current loop we are in right now.
if self.sync:
self.async_loop = create_or_get_async_loop_in_thread()
asyncio.run_coroutine_threadsafe(
self.router.setup_in_async_loop(),
self.async_loop,
)
else: # async
self.async_loop = asyncio.get_event_loop()
# create_task is not threadsafe.
self.async_loop.create_task(self.router.setup_in_async_loop())
def _remote(self, request_data, kwargs) -> Coroutine:
request_metadata = RequestMetadata(
get_random_letters(10), # Used for debugging.
self.endpoint_name,
TaskContext.Python,
call_method=self.method_name or "__call__",
shard_key=self.shard_key,
http_method=self.http_method or "GET",
http_headers=self.http_headers or dict(),
)
coro = self.router.assign_request(request_metadata, request_data,
**kwargs)
return coro
def remote(self,
request_data: Optional[Union[Dict, Any]] = None,
**kwargs):
"""Issue an asynchrounous request to the endpoint.
Returns a Ray ObjectRef whose results can be waited for or retrieved
using ray.wait or ray.get, respectively.
Returns:
ray.ObjectRef
Args:
request_data(dict, Any): If it's a dictionary, the data will be
available in ``request.json()`` or ``request.form()``.
Otherwise, it will be available in ``request.data``.
``**kwargs``: All keyword arguments will be available in
``request.args``.
"""
if not self.sync:
raise RayServeException(
"You are trying to call handle.remote() with async handle. "
"Please use `await handle.remote_async()` instead.")
coro = self._remote(request_data, kwargs)
future: concurrent.futures.Future = asyncio.run_coroutine_threadsafe(
coro, self.async_loop)
# Block until the result is ready.
return future.result()
async def remote_async(self,
request_data: Optional[Union[Dict, Any]] = None,
**kwargs) -> ray.ObjectRef:
"""Experimental API for enqueue a request in async context."""
if not asyncio.get_event_loop().is_running():
raise RayServeException(
"remote_async must be called from a running event loop.")
return await self._remote(request_data, kwargs)
def options(self,
method_name: Optional[str] = None,
*,
shard_key: Optional[str] = None,
http_method: Optional[str] = None,
http_headers: Optional[Dict[str, str]] = None):
"""Set options for this handle.
Args:
method_name(str): The method to invoke on the backend.
http_method(str): The HTTP method to use for the request.
shard_key(str): A string to use to deterministically map this
request to a backend if there are multiple for this endpoint.
"""
# Don't override default non-null values.
self.method_name = self.method_name or method_name
self.shard_key = self.shard_key or shard_key
self.http_method = self.http_method or http_method
self.http_headers = self.http_headers or http_headers
return self
def __repr__(self):
return f"RayServeHandle(endpoint='{self.endpoint_name}')"
