def test_list_endpoints(serve_instance):
serve.init()
def f():
pass
serve.create_backend("backend", f)
serve.create_backend("backend2", f)
serve.create_backend("backend3", f)
serve.create_endpoint("endpoint",
backend="backend",
route="/api",
methods=["GET", "POST"])
serve.create_endpoint("endpoint2", backend="backend2", methods=["POST"])
serve.shadow_traffic("endpoint", "backend3", 0.5)
endpoints = serve.list_endpoints()
assert "endpoint" in endpoints
assert endpoints["endpoint"] == {
"route": "/api",
"methods": ["GET", "POST"],
"traffic": {
"backend": 1.0
},
"shadows": {
"backend3": 0.5
}
}
assert "endpoint2" in endpoints
assert endpoints["endpoint2"] == {
"route": None,
"methods": ["POST"],
"traffic": {
"backend2": 1.0
},
"shadows": {}
}
serve.delete_endpoint("endpoint")
assert "endpoint2" in serve.list_endpoints()
serve.delete_endpoint("endpoint2")
assert len(serve.list_endpoints()) == 0
def test_shadow_traffic(serve_instance):
@ray.remote
class RequestCounter:
def __init__(self):
self.requests = defaultdict(int)
def record(self, backend):
self.requests[backend] += 1
def get(self, backend):
return self.requests[backend]
counter = RequestCounter.remote()
def f():
ray.get(counter.record.remote("backend1"))
return "hello"
def f_shadow_1():
ray.get(counter.record.remote("backend2"))
return "oops"
def f_shadow_2():
ray.get(counter.record.remote("backend3"))
return "oops"
def f_shadow_3():
ray.get(counter.record.remote("backend4"))
return "oops"
serve.create_backend("backend1", f)
serve.create_backend("backend2", f_shadow_1)
serve.create_backend("backend3", f_shadow_2)
serve.create_backend("backend4", f_shadow_3)
serve.create_endpoint("endpoint", backend="backend1", route="/api")
serve.shadow_traffic("endpoint", "backend2", 1.0)
serve.shadow_traffic("endpoint", "backend3", 0.5)
serve.shadow_traffic("endpoint", "backend4", 0.1)
start = time.time()
num_requests = 100
for _ in range(num_requests):
assert requests.get("http://127.0.0.1:8000/api").text == "hello"
print("Finished 100 requests in {}s.".format(time.time() - start))
def requests_to_backend(backend):
return ray.get(counter.get.remote(backend))
def check_requests():
return all([
requests_to_backend("backend1") == num_requests,
requests_to_backend("backend2") == requests_to_backend("backend1"),
requests_to_backend("backend3") < requests_to_backend("backend2"),
requests_to_backend("backend4") < requests_to_backend("backend3"),
requests_to_backend("backend4") > 0,
])
wait_for_condition(check_requests)
def test_shadow_traffic(serve_instance):
def f():
return "hello"
def f_shadow():
return "oops"
serve.create_backend("backend1", f)
serve.create_backend("backend2", f_shadow)
serve.create_backend("backend3", f_shadow)
serve.create_backend("backend4", f_shadow)
serve.create_endpoint("endpoint", backend="backend1", route="/api")
serve.shadow_traffic("endpoint", "backend2", 1.0)
serve.shadow_traffic("endpoint", "backend3", 0.5)
serve.shadow_traffic("endpoint", "backend4", 0.1)
start = time.time()
num_requests = 100
for _ in range(num_requests):
assert requests.get("http://127.0.0.1:8000/api").text == "hello"
print("Finished 100 requests in {}s.".format(time.time() - start))
def requests_to_backend(backend):
for entry in serve.stat():
if entry["info"]["name"] == "backend_request_counter":
if entry["info"]["backend"] == backend:
return entry["value"]
return 0
def check_requests():
return all([
requests_to_backend("backend1") == num_requests,
requests_to_backend("backend2") == requests_to_backend("backend1"),
requests_to_backend("backend3") < requests_to_backend("backend2"),
requests_to_backend("backend4") < requests_to_backend("backend3"),
requests_to_backend("backend4") > 0,
])
assert wait_for_condition(check_requests)
