def test_backend_config_update():
b = BackendConfig({"num_replicas": 1, "max_batch_size": 1})
# Test updating a key works.
b.update({"num_replicas": 2})
assert b.num_replicas == 2
# Check that not specifying a key doesn't update it.
assert b.max_batch_size == 1
# Check that passing an invalid key fails.
with pytest.raises(ValueError):
b.update({"unknown": 1})
# Check that input is validated.
with pytest.raises(TypeError):
b.update({"num_replicas": "hello"})
with pytest.raises(ValueError):
b.update({"num_replicas": -1})
# Test batch validation.
b = BackendConfig({}, accepts_batches=False)
b.update({"max_batch_size": 1})
with pytest.raises(ValueError):
b.update({"max_batch_size": 2})
b = BackendConfig({}, accepts_batches=True)
b.update({"max_batch_size": 2})
def test_list_backends(serve_instance):
serve.init()
@serve.accept_batch
def f():
pass
serve.create_backend("backend", f, config=BackendConfig(max_batch_size=10))
backends = serve.list_backends()
assert len(backends) == 1
assert "backend" in backends
assert backends["backend"]["max_batch_size"] == 10
serve.create_backend("backend2", f, config=BackendConfig(num_replicas=10))
backends = serve.list_backends()
assert len(backends) == 2
assert backends["backend2"]["num_replicas"] == 10
serve.delete_backend("backend")
backends = serve.list_backends()
assert len(backends) == 1
assert "backend2" in backends
serve.delete_backend("backend2")
assert len(serve.list_backends()) == 0
def test_backend_user_config(serve_instance):
client = serve_instance
class Counter:
def __init__(self):
self.count = 10
def __call__(self, starlette_request):
return self.count, os.getpid()
def reconfigure(self, config):
self.count = config["count"]
config = BackendConfig(num_replicas=2, user_config={"count": 123, "b": 2})
client.create_backend("counter", Counter, config=config)
client.create_endpoint("counter", backend="counter")
handle = client.get_handle("counter")
def check(val, num_replicas):
pids_seen = set()
for i in range(100):
result = ray.get(handle.remote())
if str(result[0]) != val:
return False
pids_seen.add(result[1])
return len(pids_seen) == num_replicas
wait_for_condition(lambda: check("123", 2))
client.update_backend_config("counter", BackendConfig(num_replicas=3))
wait_for_condition(lambda: check("123", 3))
config = BackendConfig(user_config={"count": 456})
client.update_backend_config("counter", config)
wait_for_condition(lambda: check("456", 3))
def test_updating_config(serve_instance):
class BatchSimple:
def __init__(self):
self.count = 0
@serve.accept_batch
def __call__(self, flask_request, temp=None):
batch_size = serve.context.batch_size
return [1] * batch_size
serve.create_backend(
"bsimple:v1",
BatchSimple,
config=BackendConfig(max_batch_size=2, num_replicas=3))
serve.create_endpoint("bsimple", backend="bsimple:v1", route="/bsimple")
controller = serve.api._get_controller()
old_replica_tag_list = ray.get(
controller._list_replicas.remote("bsimple:v1"))
serve.update_backend_config("bsimple:v1", BackendConfig(max_batch_size=5))
new_replica_tag_list = ray.get(
controller._list_replicas.remote("bsimple:v1"))
new_all_tag_list = []
for worker_dict in ray.get(
controller.get_all_worker_handles.remote()).values():
new_all_tag_list.extend(list(worker_dict.keys()))
# the old and new replica tag list should be identical
# and should be subset of all_tag_list
assert set(old_replica_tag_list) <= set(new_all_tag_list)
assert set(old_replica_tag_list) == set(new_replica_tag_list)
def test_updating_config(serve_instance):
client = serve_instance
class BatchSimple:
def __init__(self):
self.count = 0
@serve.accept_batch
def __call__(self, request):
return [1] * len(request)
config = BackendConfig(max_batch_size=2, num_replicas=3)
client.create_backend("bsimple:v1", BatchSimple, config=config)
client.create_endpoint("bsimple", backend="bsimple:v1", route="/bsimple")
controller = client._controller
old_replica_tag_list = list(
ray.get(controller._all_replica_handles.remote())["bsimple:v1"].keys())
update_config = BackendConfig(max_batch_size=5)
client.update_backend_config("bsimple:v1", update_config)
new_replica_tag_list = list(
ray.get(controller._all_replica_handles.remote())["bsimple:v1"].keys())
new_all_tag_list = []
for worker_dict in ray.get(
controller._all_replica_handles.remote()).values():
new_all_tag_list.extend(list(worker_dict.keys()))
# the old and new replica tag list should be identical
# and should be subset of all_tag_list
assert set(old_replica_tag_list) <= set(new_all_tag_list)
assert set(old_replica_tag_list) == set(new_replica_tag_list)
def test_imported_backend(serve_instance):
config = BackendConfig(user_config="config")
serve.create_backend("imported",
"ray.serve.utils.MockImportedBackend",
"input_arg",
config=config)
serve.create_endpoint("imported", backend="imported")
# Basic sanity check.
handle = serve.get_handle("imported")
assert ray.get(handle.remote()) == {"arg": "input_arg", "config": "config"}
# Check that updating backend config works.
serve.update_backend_config("imported",
BackendConfig(user_config="new_config"))
assert ray.get(handle.remote()) == {
"arg": "input_arg",
"config": "new_config"
}
# Check that other call methods work.
handle = handle.options(method_name="other_method")
assert ray.get(handle.remote("hello")) == "hello"
# Check that functions work as well.
serve.create_backend("imported_func",
"ray.serve.utils.mock_imported_function")
serve.create_endpoint("imported_func", backend="imported_func")
handle = serve.get_handle("imported_func")
assert ray.get(handle.remote("hello")) == "hello"
def test_list_backends(serve_instance):
client = serve_instance
@serve.accept_batch
def f():
pass
config1 = BackendConfig(max_batch_size=10)
client.create_backend("backend", f, config=config1)
backends = client.list_backends()
assert len(backends) == 1
assert "backend" in backends
assert backends["backend"].max_batch_size == 10
config2 = BackendConfig(num_replicas=10)
client.create_backend("backend2", f, config=config2)
backends = client.list_backends()
assert len(backends) == 2
assert backends["backend2"].num_replicas == 10
client.delete_backend("backend")
backends = client.list_backends()
assert len(backends) == 1
assert "backend2" in backends
client.delete_backend("backend2")
assert len(client.list_backends()) == 0
def test_list_backends(serve_instance, use_legacy_config):
client = serve_instance
@serve.accept_batch
def f():
pass
config1 = {
"max_batch_size": 10
} if use_legacy_config else BackendConfig(max_batch_size=10)
client.create_backend("backend", f, config=config1)
backends = client.list_backends()
assert len(backends) == 1
assert "backend" in backends
assert backends["backend"]["max_batch_size"] == 10
config2 = {
"num_replicas": 10
} if use_legacy_config else BackendConfig(num_replicas=10)
client.create_backend("backend2", f, config=config2)
backends = client.list_backends()
assert len(backends) == 2
assert backends["backend2"]["num_replicas"] == 10
client.delete_backend("backend")
backends = client.list_backends()
assert len(backends) == 1
assert "backend2" in backends
client.delete_backend("backend2")
assert len(client.list_backends()) == 0
def test_updating_config(serve_instance):
@serve.deployment(
"bsimple", config=BackendConfig(max_batch_size=2, num_replicas=2))
class BatchSimple:
def __init__(self):
self.count = 0
@serve.accept_batch
def __call__(self, request):
return [1] * len(request)
BatchSimple.deploy()
controller = serve.api._global_client._controller
old_replica_tag_list = list(
ray.get(controller._all_replica_handles.remote())["bsimple"].keys())
BatchSimple.options(
config=BackendConfig(max_batch_size=5, num_replicas=2)).deploy()
new_replica_tag_list = list(
ray.get(controller._all_replica_handles.remote())["bsimple"].keys())
new_all_tag_list = []
for worker_dict in ray.get(
controller._all_replica_handles.remote()).values():
new_all_tag_list.extend(list(worker_dict.keys()))
# the old and new replica tag list should be identical
# and should be subset of all_tag_list
assert set(old_replica_tag_list) <= set(new_all_tag_list)
assert set(old_replica_tag_list) == set(new_replica_tag_list)
def test_scaling_replicas(serve_instance):
@serve.deployment("counter", config=BackendConfig(num_replicas=2))
class Counter:
def __init__(self):
self.count = 0
def __call__(self, _):
self.count += 1
return self.count
Counter.deploy()
counter_result = []
for _ in range(10):
resp = requests.get("http://127.0.0.1:8000/counter").json()
counter_result.append(resp)
# If the load is shared among two replicas. The max result cannot be 10.
assert max(counter_result) < 10
Counter.options(config=BackendConfig(num_replicas=1)).deploy()
counter_result = []
for _ in range(10):
resp = requests.get("http://127.0.0.1:8000/counter").json()
counter_result.append(resp)
# Give some time for a replica to spin down. But majority of the request
# should be served by the only remaining replica.
assert max(counter_result) - min(counter_result) > 6
def test_backend_user_config(serve_instance):
client = serve_instance
class Counter:
def __init__(self):
self.count = 10
def __call__(self, flask_request):
return self.count, os.getpid()
def reconfigure(self, config):
self.count = config["count"]
config = BackendConfig(num_replicas=2, user_config={"count": 123, "b": 2})
client.create_backend("counter", Counter, config=config)
client.create_endpoint("counter", backend="counter", route="/counter")
handle = client.get_handle("counter")
def check(val, num_replicas):
pids_seen = set()
for i in range(100):
result = ray.get(handle.remote())
assert (str(result[0]) == val), result[0]
pids_seen.add(result[1])
assert (len(pids_seen) == num_replicas)
check("123", 2)
client.update_backend_config("counter", BackendConfig(num_replicas=3))
check("123", 3)
config = BackendConfig(user_config={"count": 456})
client.update_backend_config("counter", config)
check("456", 3)
def test_scaling_replicas(serve_instance):
client = serve_instance
class Counter:
def __init__(self):
self.count = 0
def __call__(self, _):
self.count += 1
return self.count
config = BackendConfig(num_replicas=2)
client.create_backend("counter:v1", Counter, config=config)
client.create_endpoint("counter", backend="counter:v1", route="/increment")
counter_result = []
for _ in range(10):
resp = requests.get("http://127.0.0.1:8000/increment").json()
counter_result.append(resp)
# If the load is shared among two replicas. The max result cannot be 10.
assert max(counter_result) < 10
update_config = BackendConfig(num_replicas=1)
client.update_backend_config("counter:v1", update_config)
counter_result = []
for _ in range(10):
resp = requests.get("http://127.0.0.1:8000/increment").json()
counter_result.append(resp)
# Give some time for a replica to spin down. But majority of the request
# should be served by the only remaining replica.
assert max(counter_result) - min(counter_result) > 6
def test_with_proto():
# Test roundtrip
config = BackendConfig(num_replicas=100, max_concurrent_queries=16)
assert config == BackendConfig.from_proto_bytes(config.to_proto_bytes())
# Test user_config object
config = BackendConfig(user_config={"python": ("native", ["objects"])})
assert config == BackendConfig.from_proto_bytes(config.to_proto_bytes())
def test_backend_config_validation():
# Test unknown key.
with pytest.raises(ValidationError):
BackendConfig(unknown_key=-1)
# Test num_replicas validation.
BackendConfig(num_replicas=1)
with pytest.raises(ValidationError, match="type_error"):
BackendConfig(num_replicas="hello")
with pytest.raises(ValidationError, match="value_error"):
BackendConfig(num_replicas=-1)
# Test dynamic default for max_concurrent_queries.
assert BackendConfig().max_concurrent_queries == 100
def setup_worker(name,
backend_def,
init_args=None,
backend_config=BackendConfig(),
controller_name=""):
if init_args is None:
init_args = ()
@ray.remote
class WorkerActor:
async def __init__(self):
self.worker = object.__new__(create_backend_replica(backend_def))
await self.worker.__init__(name, name + ":tag", init_args,
backend_config, controller_name)
def ready(self):
pass
@ray.method(num_returns=2)
async def handle_request(self, *args, **kwargs):
return await self.worker.handle_request(*args, **kwargs)
def update_config(self, new_config):
return self.worker.update_config(new_config)
async def drain_pending_queries(self):
return await self.worker.drain_pending_queries()
worker = WorkerActor.remote()
ray.get(worker.ready.remote())
return worker
def test_scaling_replicas(serve_instance):
class Counter:
def __init__(self):
self.count = 0
def __call__(self, _):
self.count += 1
return self.count
serve.create_backend(
"counter:v1", Counter, config=BackendConfig(num_replicas=2))
serve.create_endpoint("counter", backend="counter:v1", route="/increment")
# Keep checking the routing table until /increment is populated
while "/increment" not in requests.get(
"http://127.0.0.1:8000/-/routes").json():
time.sleep(0.2)
counter_result = []
for _ in range(10):
resp = requests.get("http://127.0.0.1:8000/increment").json()
counter_result.append(resp)
# If the load is shared among two replicas. The max result cannot be 10.
assert max(counter_result) < 10
serve.update_backend_config("counter:v1", {"num_replicas": 1})
counter_result = []
for _ in range(10):
resp = requests.get("http://127.0.0.1:8000/increment").json()
counter_result.append(resp)
# Give some time for a replica to spin down. But majority of the request
# should be served by the only remaining replica.
assert max(counter_result) - min(counter_result) > 6
def test_batching(serve_instance):
client = serve_instance
class BatchingExample:
def __init__(self):
self.count = 0
@serve.accept_batch
def __call__(self, requests):
self.count += 1
batch_size = len(requests)
return [self.count] * batch_size
# set the max batch size
config = BackendConfig(max_batch_size=5, batch_wait_timeout=1)
client.create_backend("counter:v11", BatchingExample, config=config)
client.create_endpoint("counter1",
backend="counter:v11",
route="/increment2")
future_list = []
handle = client.get_handle("counter1")
for _ in range(20):
f = handle.remote(temp=1)
future_list.append(f)
counter_result = ray.get(future_list)
# since count is only updated per batch of queries
# If there atleast one __call__ fn call with batch size greater than 1
# counter result will always be less than 20
assert max(counter_result) < 20
async def test_servable_batch_error(serve_instance, router,
mock_controller_with_name):
@serve.accept_batch
class ErrorBatcher:
def error_different_size(self, requests):
return [""] * (len(requests) + 10)
def error_non_iterable(self, _):
return 42
def return_np_array(self, requests):
return np.array([1] * len(requests)).astype(np.int32)
backend_config = BackendConfig(
max_batch_size=4,
internal_metadata=BackendMetadata(accepts_batches=True))
await add_servable_to_router(ErrorBatcher,
router,
mock_controller_with_name[0],
backend_config=backend_config)
with pytest.raises(RayServeException, match="doesn't preserve batch size"):
different_size = make_request_param("error_different_size")
await (await router.assign_request.remote(different_size))
with pytest.raises(RayServeException, match="iterable"):
non_iterable = make_request_param("error_non_iterable")
await (await router.assign_request.remote(non_iterable))
np_array = make_request_param("return_np_array")
result_np_value = await (await router.assign_request.remote(np_array))
assert isinstance(result_np_value, np.int32)
async def test_task_runner_custom_method_batch(serve_instance, router,
mock_controller_with_name):
@serve.accept_batch
class Batcher:
def a(self, requests):
return ["a-{}".format(i) for i in range(len(requests))]
def b(self, requests):
return ["b-{}".format(i) for i in range(len(requests))]
backend_config = BackendConfig(
max_batch_size=4,
batch_wait_timeout=10,
internal_metadata=BackendMetadata(accepts_batches=True))
await add_servable_to_router(Batcher,
router,
mock_controller_with_name[0],
backend_config=backend_config)
a_query_param = make_request_param("a")
b_query_param = make_request_param("b")
futures = [
await router.assign_request.remote(a_query_param) for _ in range(2)
]
futures += [
await router.assign_request.remote(b_query_param) for _ in range(2)
]
gathered = await asyncio.gather(*futures)
assert set(gathered) == {"a-0", "a-1", "b-0", "b-1"}
async def test_task_runner_custom_method_batch(serve_instance):
q = RoundRobinPolicyQueueActor.remote()
@serve.accept_batch
class Batcher:
def a(self, _):
return ["a-{}".format(i) for i in range(serve.context.batch_size)]
def b(self, _):
return ["b-{}".format(i) for i in range(serve.context.batch_size)]
CONSUMER_NAME = "runner"
PRODUCER_NAME = "producer"
worker = setup_worker(CONSUMER_NAME, Batcher)
await q.set_traffic.remote(PRODUCER_NAME, {CONSUMER_NAME: 1.0})
await q.set_backend_config.remote(
CONSUMER_NAME,
BackendConfig({
"max_batch_size": 10
}, accepts_batches=True))
a_query_param = RequestMetadata(
PRODUCER_NAME, context.TaskContext.Python, call_method="a")
b_query_param = RequestMetadata(
PRODUCER_NAME, context.TaskContext.Python, call_method="b")
futures = [q.enqueue_request.remote(a_query_param) for _ in range(2)]
futures += [q.enqueue_request.remote(b_query_param) for _ in range(2)]
await q.add_new_worker.remote(CONSUMER_NAME, "replica1", worker)
gathered = await asyncio.gather(*futures)
assert set(gathered) == {"a-0", "a-1", "b-0", "b-1"}
def test_create_infeasible_error(serve_instance):
client = serve_instance
def f():
pass
# Non existent resource should be infeasible.
with pytest.raises(RayServeException, match="Cannot scale backend"):
client.create_backend(
"f:1",
f,
ray_actor_options={"resources": {
"MagicMLResource": 100
}})
# Even though each replica might be feasible, the total might not be.
current_cpus = int(ray.nodes()[0]["Resources"]["CPU"])
num_replicas = current_cpus + 20
config = BackendConfig(num_replicas=num_replicas)
with pytest.raises(RayServeException, match="Cannot scale backend"):
client.create_backend("f:1",
f,
ray_actor_options={"resources": {
"CPU": 1,
}},
config=config)
def test_batching(serve_instance):
class BatchingExample:
def __init__(self):
self.count = 0
@serve.accept_batch
def __call__(self, flask_request, temp=None):
self.count += 1
batch_size = serve.context.batch_size
return [self.count] * batch_size
# set the max batch size
serve.create_backend(
"counter:v11",
BatchingExample,
config=BackendConfig(max_batch_size=5, batch_wait_timeout=1))
serve.create_endpoint(
"counter1", backend="counter:v11", route="/increment2")
# Keep checking the routing table until /increment is populated
while "/increment2" not in requests.get(
"http://127.0.0.1:8000/-/routes").json():
time.sleep(0.2)
future_list = []
handle = serve.get_handle("counter1")
for _ in range(20):
f = handle.remote(temp=1)
future_list.append(f)
counter_result = ray.get(future_list)
# since count is only updated per batch of queries
# If there atleast one __call__ fn call with batch size greater than 1
# counter result will always be less than 20
assert max(counter_result) < 20
def test_parallel_start(serve_instance):
# Test the ability to start multiple replicas in parallel.
# In the past, when Serve scale up a backend, it does so one by one and
# wait for each replica to initialize. This test avoid this by preventing
# the first replica to finish initialization unless the second replica is
# also started.
@ray.remote
class Barrier:
def __init__(self, release_on):
self.release_on = release_on
self.current_waiters = 0
self.event = asyncio.Event()
async def wait(self):
self.current_waiters += 1
if self.current_waiters == self.release_on:
self.event.set()
else:
await self.event.wait()
barrier = Barrier.remote(release_on=2)
class LongStartingServable:
def __init__(self):
ray.get(barrier.wait.remote(), timeout=10)
def __call__(self, _):
return "Ready"
serve.create_backend(
"p:v0", LongStartingServable, config=BackendConfig(num_replicas=2))
serve.create_endpoint("test-parallel", backend="p:v0")
handle = serve.get_handle("test-parallel")
ray.get(handle.remote(), timeout=10)
async def test_task_runner_perform_async(serve_instance,
mock_controller_with_name):
@ray.remote
class Barrier:
def __init__(self, release_on):
self.release_on = release_on
self.current_waiters = 0
self.event = asyncio.Event()
async def wait(self):
self.current_waiters += 1
if self.current_waiters == self.release_on:
self.event.set()
else:
await self.event.wait()
barrier = Barrier.remote(release_on=10)
async def wait_and_go(*args, **kwargs):
await barrier.wait.remote()
return "done!"
config = BackendConfig(max_concurrent_queries=10)
worker, router = await add_servable_to_router(
wait_and_go, *mock_controller_with_name, backend_config=config)
query_param = make_request_param()
done, not_done = await asyncio.wait(
[(await router.assign_request(query_param)) for _ in range(10)],
timeout=10)
assert len(done) == 10
for item in done:
assert await item == "done!"
def test_create_infeasible_error(serve_instance):
client = serve_instance
def f():
pass
# Non existent resource should be infeasible.
with pytest.raises(RayServeException, match="Cannot scale backend"):
client.create_backend(
"f:1",
f,
ray_actor_options={"resources": {
"MagicMLResource": 100
}})
# Even each replica might be feasible, the total might not be.
current_cpus = int(ray.nodes()[0]["Resources"]["CPU"])
with pytest.raises(RayServeException, match="Cannot scale backend"):
client.create_backend("f:1",
f,
ray_actor_options={"resources": {
"CPU": 1,
}},
config=BackendConfig(num_replicas=(current_cpus +
20)))
# No replica should be created!
replicas = ray.get(client._controller._list_replicas.remote("f1"))
assert len(replicas) == 0
async def test_task_runner_perform_batch(serve_instance):
q = ray.remote(Router).remote()
await q.setup.remote()
def batcher(*args, **kwargs):
return [serve.context.batch_size] * serve.context.batch_size
CONSUMER_NAME = "runner"
PRODUCER_NAME = "producer"
config = BackendConfig({
"max_batch_size": 2,
"batch_wait_timeout": 10
},
accepts_batches=True)
worker = setup_worker(CONSUMER_NAME, batcher, backend_config=config)
await q.add_new_worker.remote(CONSUMER_NAME, "replica1", worker)
await q.set_backend_config.remote(CONSUMER_NAME, config)
await q.set_traffic.remote(PRODUCER_NAME,
TrafficPolicy({CONSUMER_NAME: 1.0}))
query_param = RequestMetadata(PRODUCER_NAME, context.TaskContext.Python)
my_batch_sizes = await asyncio.gather(
*[q.enqueue_request.remote(query_param) for _ in range(3)])
assert my_batch_sizes == [2, 2, 1]
def create_backend(backend_tag,
func_or_class,
*actor_init_args,
ray_actor_options=None,
config=None):
"""Create a backend with the provided tag.
The backend will serve requests with func_or_class.
Args:
backend_tag (str): a unique tag assign to identify this backend.
func_or_class (callable, class): a function or a class implementing
__call__.
actor_init_args (optional): the arguments to pass to the class.
initialization method.
ray_actor_options (optional): options to be passed into the
@ray.remote decorator for the backend actor.
config: (optional) configuration options for this backend.
"""
if config is None:
config = {}
if not isinstance(config, dict):
raise TypeError("config must be a dictionary.")
replica_config = ReplicaConfig(func_or_class,
*actor_init_args,
ray_actor_options=ray_actor_options)
backend_config = BackendConfig(config, replica_config.accepts_batches)
retry_actor_failures(master_actor.create_backend, backend_tag,
backend_config, replica_config)
def setup_worker(name,
func_or_class,
init_args=None,
backend_config=BackendConfig({})):
if init_args is None:
init_args = ()
@ray.remote
class WorkerActor:
def __init__(self):
self.worker = create_backend_worker(func_or_class)(name,
name + ":tag",
init_args,
backend_config)
def ready(self):
pass
async def handle_request(self, *args, **kwargs):
return await self.worker.handle_request(*args, **kwargs)
def update_config(self, new_config):
return self.worker.update_config(new_config)
worker = WorkerActor.remote()
ray.get(worker.ready.remote())
return worker
def test_worker_replica_failure(serve_instance):
@ray.remote
class Counter:
def __init__(self):
self.count = 0
def inc_and_get(self):
self.count += 1
return self.count
class Worker:
# Assumes that two replicas are started. Will hang forever in the
# constructor for any workers that are restarted.
def __init__(self, counter):
self.should_hang = False
self.index = ray.get(counter.inc_and_get.remote())
if self.index > 2:
while True:
pass
def __call__(self, *args):
return self.index
counter = Counter.remote()
serve.create_backend("replica_failure", Worker, counter)
serve.update_backend_config("replica_failure",
BackendConfig(num_replicas=2))
serve.create_endpoint("replica_failure",
backend="replica_failure",
route="/replica_failure")
# Wait until both replicas have been started.
responses = set()
start = time.time()
while time.time() - start < 30:
time.sleep(0.1)
response = request_with_retries("/replica_failure", timeout=1).text
assert response in ["1", "2"]
responses.add(response)
if len(responses) > 1:
break
else:
raise TimeoutError("Timed out waiting for replicas after 30s.")
# Kill one of the replicas.
handles = _get_worker_handles("replica_failure")
assert len(handles) == 2
ray.kill(handles[0], no_restart=False)
# Check that the other replica still serves requests.
for _ in range(10):
while True:
try:
# The timeout needs to be small here because the request to
# the restarting worker will hang.
request_with_retries("/replica_failure", timeout=0.1)
break
except TimeoutError:
time.sleep(0.1)
def test_worker_replica_failure(serve_instance):
serve.http_proxy.MAX_ACTOR_DEAD_RETRIES = 0
serve.init()
class Worker:
# Assumes that two replicas are started. Will hang forever in the
# constructor for any workers that are restarted.
def __init__(self, path):
self.should_hang = False
if not os.path.exists(path):
with open(path, "w") as f:
f.write("1")
else:
with open(path, "r") as f:
num = int(f.read())
with open(path, "w") as f:
if num == 2:
self.should_hang = True
else:
f.write(str(num + 1))
if self.should_hang:
while True:
pass
def __call__(self):
pass
temp_path = os.path.join(tempfile.gettempdir(),
serve.utils.get_random_letters())
serve.create_backend("replica_failure", Worker, temp_path)
serve.update_backend_config("replica_failure",
BackendConfig(num_replicas=2))
serve.create_endpoint("replica_failure",
backend="replica_failure",
route="/replica_failure")
# Wait until both replicas have been started.
responses = set()
while len(responses) == 1:
responses.add(request_with_retries("/replica_failure", timeout=1).text)
time.sleep(0.1)
# Kill one of the replicas.
handles = _get_worker_handles("replica_failure")
assert len(handles) == 2
ray.kill(handles[0], no_restart=False)
# Check that the other replica still serves requests.
for _ in range(10):
while True:
try:
# The timeout needs to be small here because the request to
# the restarting worker will hang.
request_with_retries("/replica_failure", timeout=0.1)
break
except TimeoutError:
time.sleep(0.1)