def test_delete_endpoint(serve_instance, route):
def function():
return "hello"
backend_name = "delete-endpoint:v1"
serve.create_backend(backend_name, function)
endpoint_name = "delete_endpoint" + str(route)
serve.create_endpoint(endpoint_name, backend=backend_name, route=route)
serve.delete_endpoint(endpoint_name)
# Check that we can reuse a deleted endpoint name and route.
serve.create_endpoint(endpoint_name, backend=backend_name, route=route)
if route is not None:
assert requests.get(
"http://127.0.0.1:8000/delete-endpoint").text == "hello"
else:
handle = serve.get_handle(endpoint_name)
assert ray.get(handle.remote()) == "hello"
# Check that deleting the endpoint doesn't delete the backend.
serve.delete_endpoint(endpoint_name)
serve.create_endpoint(endpoint_name, backend=backend_name, route=route)
if route is not None:
assert requests.get(
"http://127.0.0.1:8000/delete-endpoint").text == "hello"
else:
handle = serve.get_handle(endpoint_name)
assert ray.get(handle.remote()) == "hello"
def serve_instance(_shared_serve_instance):
yield _shared_serve_instance
controller = serve.api._global_client._controller
# Clear all state between tests to avoid naming collisions.
for endpoint in ray.get(controller.get_all_endpoints.remote()):
serve.delete_endpoint(endpoint)
for backend in ray.get(controller.get_all_backends.remote()).keys():
serve.delete_backend(backend, force=True)
def serve_instance(_shared_serve_instance):
yield
master = serve.api._get_master_actor()
# Clear all state between tests to avoid naming collisions.
for endpoint in retry_actor_failures(master.get_all_endpoints):
serve.delete_endpoint(endpoint)
for backend in retry_actor_failures(master.get_all_backends):
serve.delete_backend(backend)
def serve_instance(_shared_serve_instance):
serve.init()
yield
master = serve.api._get_master_actor()
# Clear all state between tests to avoid naming collisions.
for endpoint in ray.get(master.get_all_endpoints.remote()):
serve.delete_endpoint(endpoint)
for backend in ray.get(master.get_all_backends.remote()):
serve.delete_backend(backend)
def serve_instance(_shared_serve_instance):
serve.init()
yield
# Re-init if necessary.
serve.init()
controller = serve.api._get_controller()
# Clear all state between tests to avoid naming collisions.
for endpoint in ray.get(controller.get_all_endpoints.remote()):
serve.delete_endpoint(endpoint)
for backend in ray.get(controller.get_all_backends.remote()):
serve.delete_backend(backend)
def create_endpoint(self):
if len(self.endpoints) == self.max_endpoints:
endpoint_to_delete = self.endpoints.pop()
serve.delete_endpoint(endpoint_to_delete)
serve.delete_backend(endpoint_to_delete)
new_endpoint = "".join(
[random.choice(string.ascii_letters) for _ in range(10)])
def handler(self, *args):
return new_endpoint
serve.create_backend(new_endpoint, handler)
serve.create_endpoint(new_endpoint, "/" + new_endpoint)
serve.set_traffic(new_endpoint, {new_endpoint: 1.0})
self.endpoints.append(new_endpoint)
def test_serve_forceful_shutdown(serve_instance):
def sleeper(_):
while True:
time.sleep(1000)
serve.create_backend(
"sleeper",
sleeper,
config=BackendConfig(experimental_graceful_shutdown_timeout_s=1))
serve.create_endpoint("sleeper", backend="sleeper")
handle = serve.get_handle("sleeper")
ref = handle.remote()
serve.delete_endpoint("sleeper")
serve.delete_backend("sleeper")
with pytest.raises(ray.exceptions.RayActorError):
ray.get(ref)
def test_list_endpoints(serve_instance):
serve.init()
def f():
pass
serve.create_backend("backend", f)
serve.create_backend("backend2", f)
serve.create_backend("backend3", f)
serve.create_endpoint("endpoint",
backend="backend",
route="/api",
methods=["GET", "POST"])
serve.create_endpoint("endpoint2", backend="backend2", methods=["POST"])
serve.shadow_traffic("endpoint", "backend3", 0.5)
endpoints = serve.list_endpoints()
assert "endpoint" in endpoints
assert endpoints["endpoint"] == {
"route": "/api",
"methods": ["GET", "POST"],
"traffic": {
"backend": 1.0
},
"shadows": {
"backend3": 0.5
}
}
assert "endpoint2" in endpoints
assert endpoints["endpoint2"] == {
"route": None,
"methods": ["POST"],
"traffic": {
"backend2": 1.0
},
"shadows": {}
}
serve.delete_endpoint("endpoint")
assert "endpoint2" in serve.list_endpoints()
serve.delete_endpoint("endpoint2")
assert len(serve.list_endpoints()) == 0
def test_cluster_name():
with pytest.raises(TypeError):
serve.init(cluster_name=1)
route = "/api"
backend = "backend"
endpoint = "endpoint"
serve.init(cluster_name="cluster1", blocking=True, http_port=8001)
serve.create_endpoint(endpoint, route=route)
def function():
return "hello1"
serve.create_backend(backend, function)
serve.set_traffic(endpoint, {backend: 1.0})
assert requests.get("http://127.0.0.1:8001" + route).text == "hello1"
# Create a second cluster on port 8002. Create an endpoint and backend with
# the same names and check that they don't collide.
serve.init(cluster_name="cluster2", blocking=True, http_port=8002)
serve.create_endpoint(endpoint, route=route)
def function():
return "hello2"
serve.create_backend(backend, function)
serve.set_traffic(endpoint, {backend: 1.0})
assert requests.get("http://127.0.0.1:8001" + route).text == "hello1"
assert requests.get("http://127.0.0.1:8002" + route).text == "hello2"
# Check that deleting the backend in the current cluster doesn't.
serve.delete_endpoint(endpoint)
serve.delete_backend(backend)
assert requests.get("http://127.0.0.1:8001" + route).text == "hello1"
# Check that we can re-connect to the first cluster.
serve.init(cluster_name="cluster1")
serve.delete_endpoint(endpoint)
serve.delete_backend(backend)
import sys
import ray
from ray import serve
ray.init(address="auto", ignore_reinit_error=True)
serve.init()
try:
serve.delete_endpoint("sentiment_endpoint")
except:
pass
try:
serve.delete_backend("pytorch_backend")
except:
pass
def do_blocking_delete():
serve.delete_endpoint("wait")
serve.delete_backend("wait")