def test_not_killing_replicas(serve_instance):
class BatchSimple:
def __init__(self):
self.count = 0
@serve.accept_batch
def __call__(self, flask_request, temp=None):
batch_size = serve.context.batch_size
return [1] * batch_size
serve.create_endpoint("bsimple", "/bsimple")
b_config = BackendConfig(num_replicas=3, max_batch_size=2)
serve.create_backend(BatchSimple, "bsimple:v1", backend_config=b_config)
master_actor = serve.api._get_master_actor()
old_replica_tag_list = ray.get(
master_actor._list_replicas.remote("bsimple:v1"))
bnew_config = serve.get_backend_config("bsimple:v1")
# change the config
bnew_config.max_batch_size = 5
# set the config
serve.set_backend_config("bsimple:v1", bnew_config)
new_replica_tag_list = ray.get(
master_actor._list_replicas.remote("bsimple:v1"))
new_all_tag_list = []
for worker_dict in ray.get(
master_actor.get_all_worker_handles.remote()).values():
new_all_tag_list.extend(list(worker_dict.keys()))
# the old and new replica tag list should be identical
# and should be subset of all_tag_list
assert set(old_replica_tag_list) <= set(new_all_tag_list)
assert set(old_replica_tag_list) == set(new_replica_tag_list)
def test_batching(serve_instance):
class BatchingExample:
def __init__(self):
self.count = 0
@serve.accept_batch
def __call__(self, flask_request, temp=None):
self.count += 1
batch_size = serve.context.batch_size
return [self.count] * batch_size
serve.create_endpoint("counter1", "/increment")
# Keep checking the routing table until /increment is populated
while "/increment" not in requests.get(
"http://127.0.0.1:8000/-/routes").json():
time.sleep(0.2)
# set the max batch size
b_config = BackendConfig(max_batch_size=5)
serve.create_backend(
BatchingExample, "counter:v11", backend_config=b_config)
serve.link("counter1", "counter:v11")
future_list = []
handle = serve.get_handle("counter1")
for _ in range(20):
f = handle.remote(temp=1)
future_list.append(f)
counter_result = ray.get(future_list)
# since count is only updated per batch of queries
# If there atleast one __call__ fn call with batch size greater than 1
# counter result will always be less than 20
assert max(counter_result) < 20
def test_killing_replicas(serve_instance):
class Simple:
def __init__(self):
self.count = 0
def __call__(self, flask_request, temp=None):
return temp
serve.create_endpoint("simple", "/simple")
b_config = BackendConfig(num_replicas=3, num_cpus=2)
serve.create_backend(Simple, "simple:v1", backend_config=b_config)
master_actor = serve.api._get_master_actor()
old_replica_tag_list = ray.get(
master_actor._list_replicas.remote("simple:v1"))
bnew_config = serve.get_backend_config("simple:v1")
# change the config
bnew_config.num_cpus = 1
# set the config
serve.set_backend_config("simple:v1", bnew_config)
new_replica_tag_list = ray.get(
master_actor._list_replicas.remote("simple:v1"))
new_all_tag_list = []
for worker_dict in ray.get(
master_actor.get_all_worker_handles.remote()).values():
new_all_tag_list.extend(list(worker_dict.keys()))
# the new_replica_tag_list must be subset of all_tag_list
assert set(new_replica_tag_list) <= set(new_all_tag_list)
# the old_replica_tag_list must not be subset of all_tag_list
assert not set(old_replica_tag_list) <= set(new_all_tag_list)
def test_not_killing_replicas(serve_instance):
class BatchSimple:
def __init__(self):
self.count = 0
@serve.accept_batch
def __call__(self, flask_request, temp=None):
batch_size = serve.context.batch_size
return [1] * batch_size
serve.create_endpoint("bsimple", "/bsimple")
b_config = BackendConfig(num_replicas=3, max_batch_size=2)
serve.create_backend(BatchSimple, "bsimple:v1", backend_config=b_config)
global_state = serve.api._get_global_state()
old_replica_tag_list = global_state.backend_table.list_replicas(
"bsimple:v1")
bnew_config = serve.get_backend_config("bsimple:v1")
# change the config
bnew_config.max_batch_size = 5
# set the config
serve.set_backend_config("bsimple:v1", bnew_config)
new_replica_tag_list = global_state.backend_table.list_replicas(
"bsimple:v1")
global_state.refresh_actor_handle_cache()
new_all_tag_list = list(global_state.actor_handle_cache.keys())
# the old and new replica tag list should be identical
# and should be subset of all_tag_list
assert set(old_replica_tag_list) <= set(new_all_tag_list)
assert set(old_replica_tag_list) == set(new_replica_tag_list)
def test_killing_replicas(serve_instance):
class Simple:
def __init__(self):
self.count = 0
def __call__(self, flask_request, temp=None):
return temp
serve.create_endpoint("simple", "/simple")
b_config = BackendConfig(num_replicas=3, num_cpus=2)
serve.create_backend(Simple, "simple:v1", backend_config=b_config)
global_state = serve.api._get_global_state()
old_replica_tag_list = global_state.backend_table.list_replicas(
"simple:v1")
bnew_config = serve.get_backend_config("simple:v1")
# change the config
bnew_config.num_cpus = 1
# set the config
serve.set_backend_config("simple:v1", bnew_config)
new_replica_tag_list = global_state.backend_table.list_replicas(
"simple:v1")
global_state.refresh_actor_handle_cache()
new_all_tag_list = list(global_state.actor_handle_cache.keys())
# the new_replica_tag_list must be subset of all_tag_list
assert set(new_replica_tag_list) <= set(new_all_tag_list)
# the old_replica_tag_list must not be subset of all_tag_list
assert not set(old_replica_tag_list) <= set(new_all_tag_list)
async def run_test(num_replicas, num_forwarders, sync):
client = serve.start()
client.create_backend("worker",
worker,
config=BackendConfig(
num_replicas=num_replicas,
max_concurrent_queries=max_concurrent_queries,
))
client.create_endpoint("worker", backend="worker")
endpoint_name = "worker"
if num_forwarders > 0:
client.create_backend(
"ForwardActor",
ForwardActor,
sync,
config=BackendConfig(
num_replicas=num_forwarders,
max_concurrent_queries=max_concurrent_queries))
client.create_endpoint("ForwardActor", backend="ForwardActor")
endpoint_name = "ForwardActor"
handle = client.get_handle(endpoint_name, sync=sync)
# warmup - helpful to wait for gc.collect() and actors to start
start = time.time()
while time.time() - start < 1:
if sync:
ray.get(handle.remote())
else:
ray.get(await handle.remote_async())
# real test
start = time.time()
if sync:
ray.get([handle.remote() for _ in range(num_queries)])
else:
ray.get([(await handle.remote_async()) for _ in range(num_queries)])
qps = num_queries / (time.time() - start)
print(
f"Sync: {sync}, {num_forwarders} forwarders and {num_replicas} worker "
f"replicas: {int(qps)} requests/s")
client.shutdown()
def backend_updateconf(self,
backend_name,
scale_config=None,
user_config=None):
if scale_config:
self.client.update_backend_config(backend_name, scale_config)
if user_config:
from ray.serve import BackendConfig
backend_config = BackendConfig(user_config=user_config)
self.client.update_backend_config(backend_name, backend_config)
def test_scaling_replicas(serve_instance):
class Counter:
def __init__(self):
self.count = 0
def __call__(self, _):
self.count += 1
return self.count
serve.create_endpoint("counter", "/increment")
# Keep checking the routing table until /increment is populated
while "/increment" not in requests.get(
"http://127.0.0.1:8000/-/routes").json():
time.sleep(0.2)
b_config = BackendConfig(num_replicas=2)
serve.create_backend(Counter, "counter:v1", backend_config=b_config)
serve.set_traffic("counter", {"counter:v1": 1.0})
counter_result = []
for _ in range(10):
resp = requests.get("http://127.0.0.1:8000/increment").json()
counter_result.append(resp)
# If the load is shared among two replicas. The max result cannot be 10.
assert max(counter_result) < 10
b_config = serve.get_backend_config("counter:v1")
b_config.num_replicas = 1
serve.set_backend_config("counter:v1", b_config)
counter_result = []
for _ in range(10):
resp = requests.get("http://127.0.0.1:8000/increment").json()
counter_result.append(resp)
# Give some time for a replica to spin down. But majority of the request
# should be served by the only remaining replica.
assert max(counter_result) - min(counter_result) > 6
def run_test(num_replicas, num_forwarders):
replicas_config = BackendConfig(num_replicas=num_replicas)
client.update_backend_config("hello_world", replicas_config)
if (num_forwarders == 0):
handle = client.get_handle("hello_world")
else:
forwarders_config = BackendConfig(num_replicas=num_forwarders)
client.update_backend_config("ForwardActor", forwarders_config)
handle = client.get_handle("ForwardActor")
# warmup - helpful to wait for gc.collect() and actors to start
start = time.time()
while time.time() - start < 1:
ray.get(handle.remote())
# real test
start = time.time()
ray.get([handle.remote() for _ in range(num_queries)])
qps = num_queries / (time.time() - start)
logger.info("{} forwarders and {} worker replicas: {} requests/s".format(
num_forwarders, num_replicas, int(qps)))
def test_batching_exception(serve_instance):
class NoListReturned:
def __init__(self):
self.count = 0
@serve.accept_batch
def __call__(self, flask_request, temp=None):
batch_size = serve.context.batch_size
return batch_size
serve.create_endpoint("exception-test", "/noListReturned")
# set the max batch size
b_config = BackendConfig(max_batch_size=5)
serve.create_backend(
NoListReturned, "exception:v1", backend_config=b_config)
serve.link("exception-test", "exception:v1")
handle = serve.get_handle("exception-test")
with pytest.raises(ray.exceptions.RayTaskError):
assert ray.get(handle.remote(temp=1))
time_to_run = "10s"
pg = placement_group(
[{
"CPU": 1
} for _ in range(expected_num_nodes)], strategy="STRICT_SPREAD")
ray.get(pg.ready())
# The number of replicas is the number of cores remaining after accounting
# for the one HTTP proxy actor on each node, the "hey" requester task on each
# node, and the serve controller.
# num_replicas = expected_num_nodes * (cpus_per_node - 2) - 1
num_replicas = ray.available_resources()["CPU"]
logger.info("Starting %i replicas", num_replicas)
client.create_backend(
"hey", hey, config=BackendConfig(num_replicas=num_replicas))
client.create_endpoint("hey", backend="hey", route="/hey")
@ray.remote
def run_wrk():
logger.info("Warming up for ~3 seconds")
for _ in range(5):
resp = requests.get("http://127.0.0.1:8000/hey").text
logger.info("Received response \'" + resp + "\'")
time.sleep(0.5)
result = subprocess.run(
[
"wrk", "-c",
str(num_connections), "-t",
def hey(_):
time.sleep(0.01) # Sleep for 10ms
return b"hey"
pg = placement_group([{
"CPU": 1
} for _ in range(expected_num_nodes)],
strategy="STRICT_SPREAD")
ray.get(pg.ready())
logger.info("Starting %i replicas", num_replicas)
client.create_backend("hey",
hey,
config=BackendConfig(num_replicas=num_replicas))
client.create_endpoint("hey", backend="hey", route="/hey")
@ray.remote(num_cpus=0)
def run_wrk():
logger.info("Warming up")
for _ in range(10):
try:
resp = requests.get("http://127.0.0.1:8000/hey").text
logger.info("Received response '" + resp + "'")
time.sleep(0.5)
except Exception as e:
logger.info(f"Got exception {e}")
result = subprocess.run(
result = []
for flask_request in flask_request_list:
base_number = int(flask_request.args.get("base_number", "0"))
result.append(base_number)
return list(map(lambda x: x + self.increment, result))
else:
result = []
for b in base_number:
ans = b + self.increment
result.append(ans)
return result
serve.init(blocking=True)
serve.create_endpoint("magic_counter", "/counter")
b_config = BackendConfig(max_batch_size=5)
serve.create_backend(MagicCounter, "counter:v1", 42,
backend_config=b_config) # increment=42
serve.set_traffic("magic_counter", {"counter:v1": 1.0})
print("Sending ten queries via HTTP")
for i in range(10):
url = "http://127.0.0.1:8000/counter?base_number={}".format(i)
print("> Pinging {}".format(url))
resp = requests.get(url).json()
print(pformat_color_json(resp))
time.sleep(0.2)
print("Sending ten queries via Python")
handle = serve.get_handle("magic_counter")
import ray
from ray import serve
from ray.serve import BackendConfig
ray.init()
client = serve.start()
class Threshold:
def __init__(self):
# self.model won't be changed by reconfigure.
self.model = random.Random() # Imagine this is some heavyweight model.
def reconfigure(self, config):
# This will be called when the class is created and when
# the user_config field of BackendConfig is updated.
self.threshold = config["threshold"]
def __call__(self, request):
return self.model.random() > self.threshold
backend_config = BackendConfig(user_config={"threshold": 0.01})
client.create_backend("threshold", Threshold, config=backend_config)
client.create_endpoint("threshold", backend="threshold", route="/threshold")
print(requests.get("http://127.0.0.1:8000/threshold").text) # true, probably
backend_config = BackendConfig(user_config={"threshold": 0.99})
client.update_backend_config("threshold", backend_config)
print(requests.get("http://127.0.0.1:8000/threshold").text) # false, probably
