def test_refresh_controller_after_death(shutdown_ray, detached):
"""Check if serve.start() refreshes the controller handle if it's dead."""
ray_namespace = "ray_namespace"
controller_namespace = "controller_namespace"
ray.init(namespace=ray_namespace)
serve.shutdown() # Ensure serve isn't running before beginning the test
serve.start(detached=detached,
_override_controller_namespace=controller_namespace)
old_handle = internal_get_global_client()._controller
ray.kill(old_handle, no_restart=True)
def controller_died(handle):
try:
ray.get(handle.check_alive.remote())
return False
except RayActorError:
return True
wait_for_condition(controller_died, handle=old_handle, timeout=15)
# Call start again to refresh handle
serve.start(detached=detached,
_override_controller_namespace=controller_namespace)
new_handle = internal_get_global_client()._controller
assert new_handle is not old_handle
# Health check should not error
ray.get(new_handle.check_alive.remote())
serve.shutdown()
ray.shutdown()
async def put_all_deployments(self, req: Request) -> Response:
app = Application.from_dict(await req.json())
serve.run(app, _blocking=False)
new_names = set()
for deployment in app.deployments.values():
new_names.add(deployment.name)
all_deployments = serve.list_deployments()
all_names = set(all_deployments.keys())
names_to_delete = all_names.difference(new_names)
internal_get_global_client().delete_deployments(names_to_delete)
return Response()
def deploy(self, *init_args, _blocking=True, **init_kwargs):
"""Deploy or update this deployment.
Args:
init_args (optional): args to pass to the class __init__
method. Not valid if this deployment wraps a function.
init_kwargs (optional): kwargs to pass to the class __init__
method. Not valid if this deployment wraps a function.
"""
if len(init_args) == 0 and self._init_args is not None:
init_args = self._init_args
if len(init_kwargs) == 0 and self._init_kwargs is not None:
init_kwargs = self._init_kwargs
from ray.serve.api import internal_get_global_client
return internal_get_global_client().deploy(
self._name,
self._func_or_class,
init_args,
init_kwargs,
ray_actor_options=self._ray_actor_options,
config=self._config,
version=self._version,
prev_version=self._prev_version,
route_prefix=self.route_prefix,
url=self.url,
_blocking=_blocking,
)
def test_fixed_number_proxies(ray_cluster):
cluster = ray_cluster
head_node = cluster.add_node(num_cpus=4)
cluster.add_node(num_cpus=4)
cluster.add_node(num_cpus=4)
ray.init(head_node.address)
node_ids = ray.state.node_ids()
assert len(node_ids) == 3
with pytest.raises(
pydantic.ValidationError,
match="you must specify the `fixed_number_replicas` parameter.",
):
serve.start(http_options={
"location": "FixedNumber",
})
serve.start(
http_options={
"port": new_port(),
"location": "FixedNumber",
"fixed_number_replicas": 2,
})
# Only the controller and two http proxy should be started.
controller_handle = internal_get_global_client()._controller
node_to_http_actors = ray.get(controller_handle.get_http_proxies.remote())
assert len(node_to_http_actors) == 2
serve.shutdown()
ray.shutdown()
cluster.shutdown()
def url(self) -> Optional[str]:
"""Full HTTP url for this deployment."""
if self._route_prefix is None:
# this deployment is not exposed over HTTP
return None
from ray.serve.api import internal_get_global_client
return internal_get_global_client().root_url + self.route_prefix
def test_handle_cache_out_of_scope(serve_instance):
# https://github.com/ray-project/ray/issues/18980
initial_num_cached = len(internal_get_global_client().handle_cache)
@serve.deployment(name="f")
def f():
return "hi"
f.deploy()
handle = serve.get_deployment("f").get_handle()
handle_cache = internal_get_global_client().handle_cache
assert len(handle_cache) == initial_num_cached + 1
def sender_where_handle_goes_out_of_scope():
f = serve.get_deployment("f").get_handle()
assert f is handle
assert ray.get(f.remote()) == "hi"
[sender_where_handle_goes_out_of_scope() for _ in range(30)]
assert len(handle_cache) == initial_num_cached + 1
def test_override_namespace(shutdown_ray, detached):
"""Test the _override_controller_namespace flag in serve.start()."""
ray_namespace = "ray_namespace"
controller_namespace = "controller_namespace"
ray.init(namespace=ray_namespace)
serve.start(detached=detached, _override_controller_namespace=controller_namespace)
controller_name = internal_get_global_client()._controller_name
ray.get_actor(controller_name, namespace=controller_namespace)
serve.shutdown()
def deploy(self, blocking: bool = True):
"""Atomically deploys the Application's deployments to the Ray cluster.
The Application's deployments can carry handles to one another.
Args:
blocking (bool): If True, this function only returns after the
deployment is finished. If False, this function returns
immediately after requesting the deployment.
"""
if len(self._deployments) == 0:
return
parameter_group = []
for deployment in self._deployments.values():
if not isinstance(deployment, Deployment):
raise TypeError(
f"deploy_group only accepts Deployments, but got unexpected "
f"type {type(deployment)}."
)
deployment_parameters = {
"name": deployment._name,
"func_or_class": deployment._func_or_class,
"init_args": deployment.init_args,
"init_kwargs": deployment.init_kwargs,
"ray_actor_options": deployment._ray_actor_options,
"config": deployment._config,
"version": deployment._version,
"prev_version": deployment._prev_version,
"route_prefix": deployment.route_prefix,
"url": deployment.url,
}
parameter_group.append(deployment_parameters)
return internal_get_global_client().deploy_group(
parameter_group, _blocking=blocking
)
def get_handle(
self,
sync: Optional[bool] = True
) -> Union[RayServeHandle, RayServeSyncHandle]:
"""Get a ServeHandle to this deployment to invoke it from Python.
Args:
sync (bool): If true, then Serve will return a ServeHandle that
works everywhere. Otherwise, Serve will return an
asyncio-optimized ServeHandle that's only usable in an asyncio
loop.
Returns:
ServeHandle
"""
from ray.serve.api import internal_get_global_client
return internal_get_global_client().get_handle(self._name,
missing_ok=True,
sync=sync)
def delete(self):
"""Delete this deployment."""
from ray.serve.api import internal_get_global_client
return internal_get_global_client().delete_deployments([self._name])