def test_list_endpoints(serve_instance): serve.init() def f(): pass serve.create_backend("backend", f) serve.create_backend("backend2", f) serve.create_backend("backend3", f) serve.create_endpoint("endpoint", backend="backend", route="/api", methods=["GET", "POST"]) serve.create_endpoint("endpoint2", backend="backend2", methods=["POST"]) serve.shadow_traffic("endpoint", "backend3", 0.5) endpoints = serve.list_endpoints() assert "endpoint" in endpoints assert endpoints["endpoint"] == { "route": "/api", "methods": ["GET", "POST"], "traffic": { "backend": 1.0 }, "shadows": { "backend3": 0.5 } } assert "endpoint2" in endpoints assert endpoints["endpoint2"] == { "route": None, "methods": ["POST"], "traffic": { "backend2": 1.0 }, "shadows": {} } serve.delete_endpoint("endpoint") assert "endpoint2" in serve.list_endpoints() serve.delete_endpoint("endpoint2") assert len(serve.list_endpoints()) == 0
def test_shadow_traffic(serve_instance): @ray.remote class RequestCounter: def __init__(self): self.requests = defaultdict(int) def record(self, backend): self.requests[backend] += 1 def get(self, backend): return self.requests[backend] counter = RequestCounter.remote() def f(): ray.get(counter.record.remote("backend1")) return "hello" def f_shadow_1(): ray.get(counter.record.remote("backend2")) return "oops" def f_shadow_2(): ray.get(counter.record.remote("backend3")) return "oops" def f_shadow_3(): ray.get(counter.record.remote("backend4")) return "oops" serve.create_backend("backend1", f) serve.create_backend("backend2", f_shadow_1) serve.create_backend("backend3", f_shadow_2) serve.create_backend("backend4", f_shadow_3) serve.create_endpoint("endpoint", backend="backend1", route="/api") serve.shadow_traffic("endpoint", "backend2", 1.0) serve.shadow_traffic("endpoint", "backend3", 0.5) serve.shadow_traffic("endpoint", "backend4", 0.1) start = time.time() num_requests = 100 for _ in range(num_requests): assert requests.get("http://127.0.0.1:8000/api").text == "hello" print("Finished 100 requests in {}s.".format(time.time() - start)) def requests_to_backend(backend): return ray.get(counter.get.remote(backend)) def check_requests(): return all([ requests_to_backend("backend1") == num_requests, requests_to_backend("backend2") == requests_to_backend("backend1"), requests_to_backend("backend3") < requests_to_backend("backend2"), requests_to_backend("backend4") < requests_to_backend("backend3"), requests_to_backend("backend4") > 0, ]) wait_for_condition(check_requests)
def test_shadow_traffic(serve_instance): def f(): return "hello" def f_shadow(): return "oops" serve.create_backend("backend1", f) serve.create_backend("backend2", f_shadow) serve.create_backend("backend3", f_shadow) serve.create_backend("backend4", f_shadow) serve.create_endpoint("endpoint", backend="backend1", route="/api") serve.shadow_traffic("endpoint", "backend2", 1.0) serve.shadow_traffic("endpoint", "backend3", 0.5) serve.shadow_traffic("endpoint", "backend4", 0.1) start = time.time() num_requests = 100 for _ in range(num_requests): assert requests.get("http://127.0.0.1:8000/api").text == "hello" print("Finished 100 requests in {}s.".format(time.time() - start)) def requests_to_backend(backend): for entry in serve.stat(): if entry["info"]["name"] == "backend_request_counter": if entry["info"]["backend"] == backend: return entry["value"] return 0 def check_requests(): return all([ requests_to_backend("backend1") == num_requests, requests_to_backend("backend2") == requests_to_backend("backend1"), requests_to_backend("backend3") < requests_to_backend("backend2"), requests_to_backend("backend4") < requests_to_backend("backend3"), requests_to_backend("backend4") > 0, ]) assert wait_for_condition(check_requests)