def init(name=None,
http_host=DEFAULT_HTTP_HOST,
http_port=DEFAULT_HTTP_PORT,
metric_exporter=InMemoryExporter,
_http_middlewares=[]):
"""Initialize or connect to a serve cluster.
If serve cluster is already initialized, this function will just return.
If `ray.init` has not been called in this process, it will be called with
no arguments. To specify kwargs to `ray.init`, it should be called
separately before calling `serve.init`.
Args:
name (str): A unique name for this serve instance. This allows
multiple serve instances to run on the same ray cluster. Must be
specified in all subsequent serve.init() calls.
http_host (str): Host for HTTP servers. Default to "0.0.0.0". Serve
starts one HTTP server per node in the Ray cluster.
http_port (int, List[int]): Port for HTTP server. Default to 8000.
metric_exporter(ExporterInterface): The class aggregates metrics from
all RayServe actors and optionally export them to external
services. Ray Serve has two options built in: InMemoryExporter and
PrometheusExporter
"""
if name is not None and not isinstance(name, str):
raise TypeError("name must be a string.")
# Initialize ray if needed.
if not ray.is_initialized():
ray.init()
# Try to get serve controller if it exists
global controller
controller_name = format_actor_name(SERVE_CONTROLLER_NAME, name)
try:
controller = ray.get_actor(controller_name)
return
except ValueError:
pass
controller = ServeController.options(
name=controller_name,
max_restarts=-1,
max_task_retries=-1,
).remote(name, http_host, http_port, metric_exporter, _http_middlewares)
futures = []
for node_id in ray.state.node_ids():
future = block_until_http_ready.options(
num_cpus=0, resources={
node_id: 0.01
}).remote(
"http://{}:{}/-/routes".format(http_host, http_port),
timeout=HTTP_PROXY_TIMEOUT)
futures.append(future)
ray.get(futures)
def start(detached: bool = False,
http_host: str = DEFAULT_HTTP_HOST,
http_port: int = DEFAULT_HTTP_PORT,
http_middlewares: List[Any] = []) -> Client:
"""Initialize a serve instance.
By default, the instance will be scoped to the lifetime of the returned
Client object (or when the script exits). If detached is set to True, the
instance will instead persist until client.shutdown() is called and clients
to it can be connected using serve.connect(). This is only relevant if
connecting to a long-running Ray cluster (e.g., with address="auto").
Args:
detached (bool): Whether not the instance should be detached from this
script.
http_host (str): Host for HTTP servers to listen on. Defaults to
"127.0.0.1". To expose Serve publicly, you probably want to set
this to "0.0.0.0". One HTTP server will be started on each node in
the Ray cluster.
http_port (int): Port for HTTP server. Defaults to 8000.
http_middlewares (list): A list of Starlette middlewares that will be
applied to the HTTP servers in the cluster.
"""
# Initialize ray if needed.
if not ray.is_initialized():
ray.init()
# Try to get serve controller if it exists
if detached:
controller_name = SERVE_CONTROLLER_NAME
try:
ray.get_actor(controller_name)
raise RayServeException("Called serve.start(detached=True) but a "
"detached instance is already running. "
"Please use serve.connect() to connect to "
"the running instance instead.")
except ValueError:
pass
else:
controller_name = format_actor_name(SERVE_CONTROLLER_NAME,
get_random_letters())
controller = ServeController.options(
name=controller_name,
lifetime="detached" if detached else None,
max_restarts=-1,
max_task_retries=-1,
).remote(controller_name,
http_host,
http_port,
http_middlewares,
detached=detached)
futures = []
for node_id in ray.state.node_ids():
future = block_until_http_ready.options(
num_cpus=0, resources={
node_id: 0.01
}).remote("http://{}:{}/-/routes".format(http_host, http_port),
timeout=HTTP_PROXY_TIMEOUT)
futures.append(future)
ray.get(futures)
return Client(controller, controller_name, detached=detached)
def create_endpoint(self,
endpoint_name: str,
*,
backend: str = None,
route: Optional[str] = None,
methods: List[str] = ["GET"]) -> None:
"""Create a service endpoint given route_expression.
Args:
endpoint_name (str): A name to associate to with the endpoint.
backend (str, required): The backend that will serve requests to
this endpoint. To change this or split traffic among backends,
use `serve.set_traffic`.
route (str, optional): A string begin with "/". HTTP server will
use the string to match the path.
methods(List[str], optional): The HTTP methods that are valid for
this endpoint.
"""
if backend is None:
raise TypeError("backend must be specified when creating "
"an endpoint.")
elif not isinstance(backend, str):
raise TypeError("backend must be a string, got {}.".format(
type(backend)))
if route is not None:
if not isinstance(route, str) or not route.startswith("/"):
raise TypeError("route must be a string starting with '/'.")
if not isinstance(methods, list):
raise TypeError(
"methods must be a list of strings, but got type {}".format(
type(methods)))
endpoints = self.list_endpoints()
if endpoint_name in endpoints:
methods_old = endpoints[endpoint_name]["methods"]
route_old = endpoints[endpoint_name]["route"]
if sorted(methods_old) == sorted(methods) and route_old == route:
raise ValueError(
"Route '{}' is already registered to endpoint '{}' "
"with methods '{}'. To set the backend for this "
"endpoint, please use serve.set_traffic().".format(
route, endpoint_name, methods))
upper_methods = []
for method in methods:
if not isinstance(method, str):
raise TypeError(
"methods must be a list of strings, but contained "
"an element of type {}".format(type(method)))
upper_methods.append(method.upper())
self._get_result(
self._controller.create_endpoint.remote(endpoint_name,
{backend: 1.0}, route,
upper_methods))
# Block until the route table has been propagated to all HTTP proxies.
if route is not None:
def check_ready(http_response):
return route in http_response.json()
futures = []
for node_id in ray.state.node_ids():
future = block_until_http_ready.options(
num_cpus=0, resources={
node_id: 0.01
}).remote("http://{}:{}/-/routes".format(
self._http_config.host, self._http_config.port),
check_ready=check_ready,
timeout=HTTP_PROXY_TIMEOUT)
futures.append(future)
try:
ray.get(futures)
except ray.exceptions.RayTaskError:
raise TimeoutError("Route not available at HTTP proxies "
"after {HTTP_PROXY_TIMEOUT}s.")
