def test_reconfigure_multiple_replicas(serve_instance, use_handle):
# Tests that updating the user_config with multiple replicas performs a
# rolling update.
client = serve_instance
name = "test"
@ray.remote(num_cpus=0)
def call():
if use_handle:
handle = serve.get_deployment(name).get_handle()
ret = ray.get(handle.handler.remote())
else:
ret = requests.get(f"http://*****:*****@serve.deployment(name=name, version="1", num_replicas=2)
class V1:
def __init__(self):
self.config = None
async def reconfigure(self, config):
# Don't block when the replica is first created.
if self.config is not None:
signal = ray.get_actor(signal_name)
ray.get(signal.wait.remote())
self.config = config
async def handler(self):
return f"{self.config}|{os.getpid()}"
async def __call__(self, request):
return await self.handler()
def make_nonblocking_calls(expected, expect_blocking=False):
# Returns dict[val, set(pid)].
blocking = []
responses = defaultdict(set)
start = time.time()
while time.time() - start < 30:
refs = [call.remote() for _ in range(10)]
ready, not_ready = ray.wait(refs, timeout=5)
for ref in ready:
val, pid = ray.get(ref)
responses[val].add(pid)
for ref in not_ready:
blocking.extend(not_ready)
if (all(
len(responses[val]) == num
for val, num in expected.items())
and (expect_blocking is False or len(blocking) > 0)):
break
else:
assert False, f"Timed out, responses: {responses}."
return responses, blocking
V1.options(user_config="1").deploy()
responses1, _ = make_nonblocking_calls({"1": 2})
pids1 = responses1["1"]
# Reconfigure should block one replica until the signal is sent. Check that
# some requests are now blocking.
goal_ref = V1.options(user_config="2").deploy(_blocking=False)
responses2, blocking2 = make_nonblocking_calls({"1": 1},
expect_blocking=True)
assert list(responses2["1"])[0] in pids1
# Signal reconfigure to finish. Now the goal should complete and both
# replicas should have the updated config.
ray.get(signal.send.remote())
assert client._wait_for_goal(goal_ref)
make_nonblocking_calls({"2": 2})
def test_redeploy_single_replica(serve_instance, use_handle):
# Tests that redeploying a deployment with a single replica waits for the
# replica to completely shut down before starting a new one.
client = serve_instance
name = "test"
@ray.remote
def call(block=False):
if use_handle:
handle = serve.get_deployment(name).get_handle()
ret = ray.get(handle.handler.remote(block))
else:
ret = requests.get(f"http://*****:*****@serve.deployment(name=name, version="1")
class V1:
async def handler(self, block: bool):
if block:
signal = ray.get_actor(signal_name)
await signal.wait.remote()
return f"1|{os.getpid()}"
async def __call__(self, request):
return await self.handler(request.query_params["block"] == "True")
class V2:
async def handler(self, *args):
return f"2|{os.getpid()}"
async def __call__(self, request):
return await self.handler()
V1.deploy()
ref1 = call.remote(block=False)
val1, pid1 = ray.get(ref1)
assert val1 == "1"
# ref2 will block until the signal is sent.
ref2 = call.remote(block=True)
assert len(ray.wait([ref2], timeout=0.1)[0]) == 0
# Redeploy new version. This should not go through until the old version
# replica completely stops.
V2 = V1.options(func_or_class=V2, version="2")
goal_ref = V2.deploy(_blocking=False)
assert not client._wait_for_goal(goal_ref, timeout=0.1)
# It may take some time for the handle change to propagate and requests
# to get sent to the new version. Repeatedly send requests until they
# start blocking
start = time.time()
new_version_ref = None
while time.time() - start < 30:
ready, not_ready = ray.wait([call.remote(block=False)], timeout=5)
if len(ready) == 1:
# If the request doesn't block, it must have been the old version.
val, pid = ray.get(ready[0])
assert val == "1"
assert pid == pid1
elif len(not_ready) == 1:
# If the request blocks, it must have been the new version.
new_version_ref = not_ready[0]
break
else:
assert False, "Timed out waiting for new version to be called."
# Signal the original call to exit.
ray.get(signal.send.remote())
val2, pid2 = ray.get(ref2)
assert val2 == "1"
assert pid2 == pid1
# Now the goal and request to the new version should complete.
assert client._wait_for_goal(goal_ref)
new_version_val, new_version_pid = ray.get(new_version_ref)
assert new_version_val == "2"
assert new_version_pid != pid2
def test_redeploy_multiple_replicas(serve_instance, use_handle):
# Tests that redeploying a deployment with multiple replicas performs
# a rolling update.
client = serve_instance
name = "test"
@ray.remote(num_cpus=0)
def call(block=False):
if use_handle:
handle = serve.get_deployment(name).get_handle()
ret = ray.get(handle.handler.remote(block))
else:
ret = requests.get(f"http://*****:*****@serve.deployment(name=name, version="1", num_replicas=2)
class V1:
async def handler(self, block: bool):
if block:
signal = ray.get_actor(signal_name)
await signal.wait.remote()
return f"1|{os.getpid()}"
async def __call__(self, request):
return await self.handler(request.query_params["block"] == "True")
class V2:
async def handler(self, *args):
return f"2|{os.getpid()}"
async def __call__(self, request):
return await self.handler()
def make_nonblocking_calls(expected, expect_blocking=False):
# Returns dict[val, set(pid)].
blocking = []
responses = defaultdict(set)
start = time.time()
while time.time() - start < 30:
refs = [call.remote(block=False) for _ in range(10)]
ready, not_ready = ray.wait(refs, timeout=5)
for ref in ready:
val, pid = ray.get(ref)
responses[val].add(pid)
for ref in not_ready:
blocking.extend(not_ready)
if (all(
len(responses[val]) == num
for val, num in expected.items())
and (expect_blocking is False or len(blocking) > 0)):
break
else:
assert False, f"Timed out, responses: {responses}."
return responses, blocking
V1.deploy()
responses1, _ = make_nonblocking_calls({"1": 2})
pids1 = responses1["1"]
# ref2 will block a single replica until the signal is sent. Check that
# some requests are now blocking.
ref2 = call.remote(block=True)
responses2, blocking2 = make_nonblocking_calls({"1": 1},
expect_blocking=True)
assert list(responses2["1"])[0] in pids1
# Redeploy new version. Since there is one replica blocking, only one new
# replica should be started up.
V2 = V1.options(func_or_class=V2, version="2")
goal_ref = V2.deploy(_blocking=False)
assert not client._wait_for_goal(goal_ref, timeout=0.1)
responses3, blocking3 = make_nonblocking_calls({"1": 1},
expect_blocking=True)
# Signal the original call to exit.
ray.get(signal.send.remote())
val, pid = ray.get(ref2)
assert val == "1"
assert pid in responses1["1"]
# Now the goal and requests to the new version should complete.
# We should have two running replicas of the new version.
assert client._wait_for_goal(goal_ref)
make_nonblocking_calls({"2": 2})
def test_recover_rolling_update_from_replica_actor_names(serve_instance):
"""Test controller is able to recover starting -> updating -> running
replicas from actor names, with right replica versions during rolling
update.
"""
client = serve_instance
name = "test"
@ray.remote(num_cpus=0)
def call(block=False):
handle = serve.get_deployment(name).get_handle()
ret = ray.get(handle.handler.remote(block))
return ret.split("|")[0], ret.split("|")[1]
signal_name = f"signal#{get_random_letters()}"
signal = SignalActor.options(name=signal_name).remote()
@serve.deployment(name=name, version="1", num_replicas=2)
class V1:
async def handler(self, block: bool):
if block:
signal = ray.get_actor(signal_name)
await signal.wait.remote()
return f"1|{os.getpid()}"
async def __call__(self, request):
return await self.handler(request.query_params["block"] == "True")
class V2:
async def handler(self, *args):
return f"2|{os.getpid()}"
async def __call__(self, request):
return await self.handler()
def make_nonblocking_calls(expected, expect_blocking=False, num_returns=1):
# Returns dict[val, set(pid)].
blocking = []
responses = defaultdict(set)
start = time.time()
timeout_value = 60 if sys.platform == "win32" else 30
while time.time() - start < timeout_value:
refs = [call.remote(block=False) for _ in range(10)]
ready, not_ready = ray.wait(refs,
timeout=5,
num_returns=num_returns)
for ref in ready:
val, pid = ray.get(ref)
responses[val].add(pid)
for ref in not_ready:
blocking.extend(not_ready)
if all(
len(responses[val]) >= num
for val, num in expected.items()) and (
expect_blocking is False or len(blocking) > 0):
break
else:
assert False, f"Timed out, responses: {responses}."
return responses, blocking
V1.deploy()
responses1, _ = make_nonblocking_calls({"1": 2}, num_returns=2)
pids1 = responses1["1"]
# ref2 will block a single replica until the signal is sent. Check that
# some requests are now blocking.
ref2 = call.remote(block=True)
responses2, blocking2 = make_nonblocking_calls({"1": 1},
expect_blocking=True)
assert list(responses2["1"])[0] in pids1
ray.kill(serve.context._global_client._controller, no_restart=False)
# Redeploy new version. Since there is one replica blocking, only one new
# replica should be started up.
V2 = V1.options(func_or_class=V2, version="2")
V2.deploy(_blocking=False)
with pytest.raises(TimeoutError):
client._wait_for_deployment_healthy(V2.name, timeout_s=0.1)
responses3, blocking3 = make_nonblocking_calls({"1": 1},
expect_blocking=True)
ray.kill(serve.context._global_client._controller, no_restart=False)
# Signal the original call to exit.
ray.get(signal.send.remote())
val, pid = ray.get(ref2)
assert val == "1"
assert pid in responses1["1"]
# Now the goal and requests to the new version should complete.
# We should have two running replicas of the new version.
client._wait_for_deployment_healthy(V2.name)
make_nonblocking_calls({"2": 2}, num_returns=2)
