def echo_v1(_): return "v1" def echo_v2(_): return "v2" # specify the router policy as RoundRobin serve.init(blocking=True, queueing_policy=serve.RoutePolicy.RoundRobin) # create a service serve.create_endpoint("my_endpoint", "/echo", blocking=True) # create first backend serve.create_backend(echo_v1, "echo:v1") # create second backend serve.create_backend(echo_v2, "echo:v2") # link and split the service to two backends serve.split("my_endpoint", {"echo:v1": 0.5, "echo:v2": 0.5}) while True: resp = requests.get("http://127.0.0.1:8000/echo").json() print(pformat_color_json(resp)) print("...Sleeping for 2 seconds...") time.sleep(2)
def split(endpoint, traffic): ray.init(address="auto") serve.init() serve.split(endpoint, json.loads(traffic))