def test_http_options():
HTTPOptions()
HTTPOptions(host="8.8.8.8", middlewares=[object()])
assert HTTPOptions(host=None).location == "NoServer"
assert HTTPOptions(location=None).location == "NoServer"
assert HTTPOptions(
location=DeploymentMode.EveryNode).location == "EveryNode"
def test_node_selection(patch_get_namespace):
def _make_http_state(http_options):
return HTTPState(
"mock_controller_name",
detached=True,
config=http_options,
_start_proxies_on_init=False,
)
all_nodes = [("node_id-index-head", "node-id-1")] + [
(f"node_idx-worker-{i}", f"node-id-{i}") for i in range(100)
]
with patch("ray.serve.http_state.get_all_node_ids") as func:
func.return_value = all_nodes
# Test NoServer
state = _make_http_state(HTTPOptions(location=DeploymentMode.NoServer))
assert state._get_target_nodes() == []
# Test HeadOnly
with patch("ray.serve.http_state.get_current_node_resource_key"
) as get_current_node:
get_current_node.return_value = "node-id-1"
state = _make_http_state(
HTTPOptions(location=DeploymentMode.HeadOnly))
assert state._get_target_nodes() == all_nodes[:1]
# Test EveryNode
state = _make_http_state(
HTTPOptions(location=DeploymentMode.EveryNode))
assert state._get_target_nodes() == all_nodes
# Test FixedReplica
state = _make_http_state(
HTTPOptions(location=DeploymentMode.FixedNumber,
fixed_number_replicas=5))
selected_nodes = state._get_target_nodes()
# it should have selection a subset of 5 nodes.
assert len(selected_nodes) == 5
assert set(all_nodes).issuperset(set(selected_nodes))
for _ in range(5):
# The selection should be deterministic.
assert selected_nodes == state._get_target_nodes()
another_seed = _make_http_state(
HTTPOptions(
location=DeploymentMode.FixedNumber,
fixed_number_replicas=5,
fixed_number_selection_seed=42,
))._get_target_nodes()
assert len(another_seed) == 5
assert set(all_nodes).issuperset(set(another_seed))
assert set(another_seed) != set(selected_nodes)
def _check_http_and_checkpoint_options(
client: ServeControllerClient,
http_options: Union[dict, HTTPOptions],
checkpoint_path: str,
) -> None:
if checkpoint_path and checkpoint_path != client.checkpoint_path:
logger.warning(
f"The new client checkpoint path '{checkpoint_path}' "
f"is different from the existing one '{client.checkpoint_path}'. "
"The new checkpoint path is ignored.")
if http_options:
client_http_options = client.http_config
new_http_options = (http_options
if isinstance(http_options, HTTPOptions) else
HTTPOptions.parse_obj(http_options))
different_fields = []
all_http_option_fields = new_http_options.__dict__
for field in all_http_option_fields:
if getattr(new_http_options, field) != getattr(
client_http_options, field):
different_fields.append(field)
if len(different_fields):
logger.warning(
"The new client HTTP config differs from the existing one "
f"in the following fields: {different_fields}. "
"The new HTTP config is ignored.")
def __init__(
self,
controller_name: str,
checkpoint_path: str,
detached: bool = False,
dedicated_cpu: bool = False,
http_proxy_port: int = 8000,
):
try:
self._controller = ray.get_actor(controller_name, namespace="serve")
except ValueError:
self._controller = None
if self._controller is None:
# Used for scheduling things to the head node explicitly.
head_node_id = ray.get_runtime_context().node_id.hex()
http_config = HTTPOptions()
http_config.port = http_proxy_port
self._controller = ServeController.options(
num_cpus=1 if dedicated_cpu else 0,
name=controller_name,
lifetime="detached" if detached else None,
max_restarts=-1,
max_task_retries=-1,
# Schedule the controller on the head node with a soft constraint. This
# prefers it to run on the head node in most cases, but allows it to be
# restarted on other nodes in an HA cluster.
scheduling_strategy=NodeAffinitySchedulingStrategy(
head_node_id, soft=True
),
namespace="serve",
max_concurrency=CONTROLLER_MAX_CONCURRENCY,
).remote(
controller_name,
http_config=http_config,
checkpoint_path=checkpoint_path,
head_node_id=head_node_id,
detached=detached,
)
def test_dedicated_cpu(controller_cpu, num_proxy_cpus, ray_cluster):
cluster = ray_cluster
num_cluster_cpus = 8
head_node = cluster.add_node(num_cpus=num_cluster_cpus)
ray.init(head_node.address)
wait_for_condition(lambda: ray.cluster_resources().get("CPU") == num_cluster_cpus)
num_cpus_used = int(controller_cpu) + num_proxy_cpus
serve.start(
dedicated_cpu=controller_cpu, http_options=HTTPOptions(num_cpus=num_proxy_cpus)
)
available_cpus = num_cluster_cpus - num_cpus_used
wait_for_condition(lambda: (ray.available_resources().get("CPU") == available_cpus))
serve.shutdown()
ray.shutdown()
def __init__(
self,
controller_name: str,
detached: bool,
config: HTTPOptions,
head_node_id: str,
# Used by unit testing
_start_proxies_on_init: bool = True,
):
self._controller_name = controller_name
self._detached = detached
if config is not None:
self._config = config
else:
self._config = HTTPOptions()
self._proxy_actors: Dict[NodeId, ActorHandle] = dict()
self._proxy_actor_names: Dict[NodeId, str] = dict()
self._head_node_id: str = head_node_id
assert isinstance(head_node_id, str)
# Will populate self.proxy_actors with existing actors.
if _start_proxies_on_init:
self._start_proxies_if_needed()
def start(
detached: bool = False,
http_options: Optional[Union[dict, HTTPOptions]] = None,
dedicated_cpu: bool = False,
_checkpoint_path: str = DEFAULT_CHECKPOINT_PATH,
**kwargs,
) -> ServeControllerClient:
"""Initialize a serve instance.
By default, the instance will be scoped to the lifetime of the returned
Client object (or when the script exits). If detached is set to True, the
instance will instead persist until serve.shutdown() is called. This is
only relevant if connecting to a long-running Ray cluster (e.g., with
ray.init(address="auto") or ray.init("ray://<remote_addr>")).
Args:
detached: Whether not the instance should be detached from this
script. If set, the instance will live on the Ray cluster until it is
explicitly stopped with serve.shutdown().
http_options (Optional[Dict, serve.HTTPOptions]): Configuration options
for HTTP proxy. You can pass in a dictionary or HTTPOptions object
with fields:
- host(str, None): Host for HTTP servers to listen on. Defaults to
"127.0.0.1". To expose Serve publicly, you probably want to set
this to "0.0.0.0".
- port(int): Port for HTTP server. Defaults to 8000.
- root_path(str): Root path to mount the serve application
(for example, "/serve"). All deployment routes will be prefixed
with this path. Defaults to "".
- middlewares(list): A list of Starlette middlewares that will be
applied to the HTTP servers in the cluster. Defaults to [].
- location(str, serve.config.DeploymentMode): The deployment
location of HTTP servers:
- "HeadOnly": start one HTTP server on the head node. Serve
assumes the head node is the node you executed serve.start
on. This is the default.
- "EveryNode": start one HTTP server per node.
- "NoServer" or None: disable HTTP server.
- num_cpus (int): The number of CPU cores to reserve for each
internal Serve HTTP proxy actor. Defaults to 0.
dedicated_cpu: Whether to reserve a CPU core for the internal
Serve controller actor. Defaults to False.
"""
usage_lib.record_library_usage("serve")
http_deprecated_args = ["http_host", "http_port", "http_middlewares"]
for key in http_deprecated_args:
if key in kwargs:
raise ValueError(
f"{key} is deprecated, please use serve.start(http_options="
f'{{"{key}": {kwargs[key]}}}) instead.')
# Initialize ray if needed.
ray._private.worker.global_worker.filter_logs_by_job = False
if not ray.is_initialized():
ray.init(namespace=SERVE_NAMESPACE)
try:
client = get_global_client(_health_check_controller=True)
logger.info(
f'Connecting to existing Serve app in namespace "{SERVE_NAMESPACE}".'
)
_check_http_and_checkpoint_options(client, http_options,
_checkpoint_path)
return client
except RayServeException:
pass
if detached:
controller_name = SERVE_CONTROLLER_NAME
else:
controller_name = format_actor_name(get_random_letters(),
SERVE_CONTROLLER_NAME)
if isinstance(http_options, dict):
http_options = HTTPOptions.parse_obj(http_options)
if http_options is None:
http_options = HTTPOptions()
controller = ServeController.options(
num_cpus=1 if dedicated_cpu else 0,
name=controller_name,
lifetime="detached" if detached else None,
max_restarts=-1,
max_task_retries=-1,
# Pin Serve controller on the head node.
resources={
get_current_node_resource_key(): 0.01
},
namespace=SERVE_NAMESPACE,
max_concurrency=CONTROLLER_MAX_CONCURRENCY,
).remote(
controller_name,
http_options,
_checkpoint_path,
detached=detached,
)
proxy_handles = ray.get(controller.get_http_proxies.remote())
if len(proxy_handles) > 0:
try:
ray.get(
[handle.ready.remote() for handle in proxy_handles.values()],
timeout=HTTP_PROXY_TIMEOUT,
)
except ray.exceptions.GetTimeoutError:
raise TimeoutError(
f"HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s.")
client = ServeControllerClient(
controller,
controller_name,
detached=detached,
)
set_global_client(client)
logger.info(f"Started{' detached ' if detached else ' '}Serve instance in "
f'namespace "{SERVE_NAMESPACE}".')
return client
def start(
detached: bool = False,
http_host: Optional[str] = DEFAULT_HTTP_HOST,
http_port: int = DEFAULT_HTTP_PORT,
http_middlewares: List[Any] = [],
http_options: Optional[Union[dict, HTTPOptions]] = None,
) -> Client:
"""Initialize a serve instance.
By default, the instance will be scoped to the lifetime of the returned
Client object (or when the script exits). If detached is set to True, the
instance will instead persist until client.shutdown() is called and clients
to it can be connected using serve.connect(). This is only relevant if
connecting to a long-running Ray cluster (e.g., with address="auto").
Args:
detached (bool): Whether not the instance should be detached from this
script.
http_host (Optional[str]): Deprecated, use http_options instead.
http_port (int): Deprecated, use http_options instead.
http_middlewares (list): Deprecated, use http_options instead.
http_options (Optional[Dict, serve.HTTPOptions]): Configuration options
for HTTP proxy. You can pass in a dictionary or HTTPOptions object
with fields:
- host(str, None): Host for HTTP servers to listen on. Defaults to
"127.0.0.1". To expose Serve publicly, you probably want to set
this to "0.0.0.0".
- port(int): Port for HTTP server. Defaults to 8000.
- middlewares(list): A list of Starlette middlewares that will be
applied to the HTTP servers in the cluster.
- location(str, serve.config.DeploymentMode): The deployment
location of HTTP servers:
- "HeadOnly": start one HTTP server on the head node. Serve
assumes the head node is the node you executed serve.start
on. This is the default.
- "EveryNode": start one HTTP server per node.
- "NoServer" or None: disable HTTP server.
"""
if ((http_host != DEFAULT_HTTP_HOST) or (http_port != DEFAULT_HTTP_PORT)
or (len(http_middlewares) != 0)):
if http_options is not None:
raise ValueError(
"You cannot specify both `http_options` and any of the "
"`http_host`, `http_port`, and `http_middlewares` arguments. "
"`http_options` is preferred.")
else:
warn(
"`http_host`, `http_port`, `http_middlewares` are deprecated. "
"Please use serve.start(http_options={'host': ..., "
"'port': ..., middlewares': ...}) instead.",
DeprecationWarning,
)
# Initialize ray if needed.
if not ray.is_initialized():
ray.init()
register_custom_serializers()
# Try to get serve controller if it exists
if detached:
controller_name = SERVE_CONTROLLER_NAME
try:
ray.get_actor(controller_name)
raise RayServeException("Called serve.start(detached=True) but a "
"detached instance is already running. "
"Please use serve.connect() to connect to "
"the running instance instead.")
except ValueError:
pass
else:
controller_name = format_actor_name(SERVE_CONTROLLER_NAME,
get_random_letters())
if isinstance(http_options, dict):
http_options = HTTPOptions.parse_obj(http_options)
if http_options is None:
http_options = HTTPOptions(
host=http_host, port=http_port, middlewares=http_middlewares)
controller = ServeController.options(
name=controller_name,
lifetime="detached" if detached else None,
max_restarts=-1,
max_task_retries=-1,
# Pin Serve controller on the head node.
resources={
get_current_node_resource_key(): 0.01
},
).remote(
controller_name,
http_options,
detached=detached,
)
proxy_handles = ray.get(controller.get_http_proxies.remote())
if len(proxy_handles) > 0:
try:
ray.get(
[handle.ready.remote() for handle in proxy_handles.values()],
timeout=HTTP_PROXY_TIMEOUT,
)
except ray.exceptions.GetTimeoutError:
raise TimeoutError(
"HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s.")
client = Client(controller, controller_name, detached=detached)
_set_global_client(client)
return client
