def test_wide_fanout_deployment_graph(fanout_degree,
init_delay_secs=0,
compute_delay_secs=0):
"""
Test that focuses on wide fanout of deployment graph
-> Node_1
/ \
INPUT --> Node_2 --> combine -> OUTPUT
\ ... /
-> Node_10
1) Intermediate blob size can be large / small
2) Compute time each node can be long / short
3) Init time can be long / short
"""
nodes = [
Node.bind(i, init_delay_secs=init_delay_secs)
for i in range(0, fanout_degree)
]
outputs = []
with InputNode() as user_input:
for i in range(0, fanout_degree):
outputs.append(nodes[i].compute.bind(
user_input, compute_delay_secs=compute_delay_secs))
dag = combine.bind(outputs)
serve_dag = DAGDriver.bind(dag)
return serve_dag
def test_autoscaling_0_replica(serve_instance):
autoscaling_config = {
"metrics_interval_s": 0.1,
"min_replicas": 0,
"max_replicas": 2,
"look_back_period_s": 0.4,
"downscale_delay_s": 0,
"upscale_delay_s": 0,
}
@serve.deployment(
autoscaling_config=autoscaling_config, )
class Model:
def __init__(self, weight):
self.weight = weight
def forward(self, input):
return input + self.weight
with InputNode() as user_input:
model = Model.bind(1)
output = model.forward.bind(user_input)
serve_dag = DAGDriver.options(
route_prefix="/my-dag",
autoscaling_config=autoscaling_config,
).bind(output)
dag_handle = serve.run(serve_dag)
assert 2 == ray.get(dag_handle.predict.remote(1))
def test_long_chain_deployment_graph(
chain_length, init_delay_secs=0, compute_delay_secs=0
):
"""
Test that focuses on long chain of deployment graph
INPUT -> Node_1 -> Node_2 -> ... -> Node_10 -> OUTPUT
1) Intermediate blob size can be large / small
2) Compute time each node can be long / short
3) Init time can be long / short
"""
nodes = [Node.bind(i, init_delay_secs=init_delay_secs) for i in range(chain_length)]
prev_outputs = [None for _ in range(chain_length)]
with InputNode() as user_input:
for i in range(chain_length):
if i == 0:
prev_outputs[i] = nodes[i].compute.bind(
user_input, compute_delay_secs=compute_delay_secs
)
else:
prev_outputs[i] = nodes[i].compute.bind(
prev_outputs[i - 1], compute_delay_secs=compute_delay_secs
)
serve_dag = DAGDriver.bind(prev_outputs[-1])
return serve_dag
def test_chained_function(serve_instance, use_build):
@serve.deployment
def func_1(input):
return input
@serve.deployment
def func_2(input):
return input * 2
@serve.deployment
def func_3(input):
return input * 3
with InputNode() as dag_input:
output_1 = func_1.bind(dag_input)
output_2 = func_2.bind(dag_input)
output_3 = func_3.bind(output_2)
ray_dag = combine.bind(output_1, output_2, kwargs_output=output_3)
with pytest.raises(ValueError, match="Please provide a driver class"):
_ = serve.run(ray_dag)
serve_dag = DAGDriver.bind(ray_dag, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(2)) == 18 # 2 + 2*2 + (2*2) * 3
assert requests.post("http://127.0.0.1:8000/", json=2).json() == 18
def test_single_func_deployment_dag(serve_instance, use_build):
with InputNode() as dag_input:
dag = combine.bind(dag_input[0], dag_input[1], kwargs_output=1)
serve_dag = DAGDriver.bind(dag, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote([1, 2])) == 4
assert requests.post("http://127.0.0.1:8000/", json=[1, 2]).json() == 4
def test_serve_pipeline_func_class_with_class_method_plot():
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.bind(m1_output,
m2_output,
kwargs_output=dag_input[2])
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE" in to_string
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE_1" in to_string
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE_2" in to_string
assert "Model -> forward" in to_string
assert "INPUT_ATTRIBUTE_NODE -> forward" in to_string
assert "Model_1 -> forward_1" in to_string
assert "INPUT_ATTRIBUTE_NODE_1 -> forward_1" in to_string
assert "INPUT_ATTRIBUTE_NODE_2 -> combine" in to_string
assert "forward -> combine" in to_string
assert "forward_1 -> combine" in to_string
assert "combine -> DAGDriver" in to_string
def test_passing_handle(serve_instance, use_build):
child = Adder.bind(1)
parent = TakeHandle.bind(child)
driver = DAGDriver.bind(parent, http_adapter=json_resolver)
handle = serve.run(driver)
assert ray.get(handle.predict.remote(1)) == 2
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 2
def test_np_in_composed_model(serve_instance):
# https://github.com/ray-project/ray/issues/9441
# AttributeError: 'bytes' object has no attribute 'readonly'
# in cloudpickle _from_numpy_buffer
@serve.deployment
class Sum:
def __call__(self, data):
return np.sum(data)
@serve.deployment(name="model")
class ComposedModel:
def __init__(self, handle):
self.model = handle
async def __call__(self):
data = np.ones((10, 10))
return await self.model.remote(data)
sum_d = Sum.bind()
cm_d = ComposedModel.bind(sum_d)
dag = DAGDriver.bind(cm_d)
serve.run(dag)
result = requests.get("http://127.0.0.1:8000/")
assert result.status_code == 200
assert result.json() == 100.0
def test_suprious_call(serve_instance):
# https://github.com/ray-project/ray/issues/24116
@serve.deployment
class CallTracker:
def __init__(self):
self.records = []
def __call__(self, inp):
self.records.append("__call__")
def predict(self, inp):
self.records.append("predict")
def get(self):
return self.records
tracker = CallTracker.bind()
with InputNode() as inp:
dag = DAGDriver.bind(tracker.predict.bind(inp))
handle = serve.run(dag)
ray.get(handle.predict.remote(1))
call_tracker = CallTracker.get_handle()
assert ray.get(call_tracker.get.remote()) == ["predict"]
def test_simple_class_with_class_method(serve_instance, use_build):
with InputNode() as dag_input:
model = Model.bind(2, ratio=0.3)
dag = model.forward.bind(dag_input)
serve_dag = DAGDriver.bind(dag, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 0.6
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 0.6
def test_shared_deployment_handle(serve_instance, use_build):
with InputNode() as dag_input:
m = Model.bind(2)
combine = Combine.bind(m, m2=m)
combine_output = combine.__call__.bind(dag_input)
serve_dag = DAGDriver.bind(combine_output, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 4
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 4
def test_single_node_driver_sucess(serve_instance, use_build):
m1 = Adder.bind(1)
m2 = Adder.bind(2)
with InputNode() as input_node:
out = m1.forward.bind(input_node)
out = m2.forward.bind(out)
driver = DAGDriver.bind(out, http_adapter=json_resolver)
handle = serve.run(driver)
assert ray.get(handle.predict.remote(39)) == 42
assert requests.post("http://127.0.0.1:8000/", json=39).json() == 42
def test_multi_instantiation_class_deployment_in_init_args(serve_instance):
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2=m2)
combine_output = combine.bind(dag_input)
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 5
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def test_dag_driver_custom_schema(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
handle = serve.run(DAGDriver.bind(dag, input_schema=resolver))
assert ray.get(handle.predict.remote(42)) == 42
resp = requests.get("http://127.0.0.1:8000/?my_custom_param=100")
print(resp.text)
resp.raise_for_status()
assert resp.json() == 100
def test_multi_instantiation_class_nested_deployment_arg_dag(serve_instance, use_build):
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True)
output = combine.bind(dag_input)
serve_dag = DAGDriver.bind(output, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 5
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def test_func_class_with_class_method(serve_instance, use_build):
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.bind(m1_output, m2_output, kwargs_output=dag_input[2])
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote([1, 2, 3])) == 8
assert requests.post("http://127.0.0.1:8000/", json=[1, 2, 3]).json() == 8
def test_dag_driver_default(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
handle = serve.run(DAGDriver.bind(dag))
assert ray.get(handle.predict.remote(42)) == 42
resp = requests.post("http://127.0.0.1:8000/", json={"array": [1]})
print(resp.text)
resp.raise_for_status()
assert resp.json() == "starlette!"
def test_deploy_nullify_route_prefix(serve_instance, prefixes):
@serve.deployment
def f(*args):
return "got me"
for prefix in prefixes:
dag = DAGDriver.options(route_prefix=prefix).bind(f.bind())
handle = serve.run(dag)
if prefix is None:
assert requests.get("http://localhost:8000/f").status_code == 404
else:
assert requests.get("http://localhost:8000/f").text == '"got me"'
assert ray.get(handle.predict.remote()) == "got me"
def test_dag_driver_custom_pydantic_schema(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
handle = serve.run(DAGDriver.bind(dag, http_adapter=MyType))
assert ray.get(handle.predict.remote(MyType(a=1,
b="str"))) == MyType(a=1,
b="str")
resp = requests.post("http://127.0.0.1:8000/", json={"a": 1, "b": "str"})
print(resp.text)
resp.raise_for_status()
assert resp.json() == {"a": 1, "b": "str"}
def test_dag_driver_partial_input(serve_instance):
with InputNode() as inp:
dag = DAGDriver.bind(
combine.bind(echo.bind(inp[0]), echo.bind(inp[1]), echo.bind(inp[2])),
input_schema=json_request,
)
handle = serve.run(dag)
assert ray.get(handle.predict.remote([1, 2, [3, 4]])) == [1, 2, [3, 4]]
assert ray.get(handle.predict.remote(1, 2, [3, 4])) == [1, 2, [3, 4]]
resp = requests.post("http://127.0.0.1:8000/", json=[1, 2, [3, 4]])
print(resp.text)
resp.raise_for_status()
assert resp.json() == [1, 2, [3, 4]]
def test_serve_pipeline_class_with_class_method_plot():
with InputNode() as dag_input:
model = Model.bind(2, ratio=0.3)
dag = model.forward.bind(dag_input)
serve_dag = DAGDriver.bind(dag, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "Model -> forward" in to_string
assert "INPUT_NODE -> forward" in to_string
assert "forward -> DAGDriver" in to_string
def test_serve_pipeline_single_func_deployment_dag_plot():
with InputNode() as dag_input:
dag = combine.bind(dag_input[0], dag_input[1], kwargs_output=1)
serve_dag = DAGDriver.bind(dag, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE" in to_string
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE_1" in to_string
assert "INPUT_ATTRIBUTE_NODE -> combine" in to_string
assert "INPUT_ATTRIBUTE_NODE_1 -> combine" in to_string
assert "combine -> DAGDriver" in to_string
def test_serve_pipeline_test_shared_deployment_handle_plot():
with InputNode() as dag_input:
m = Model.bind(2)
combine = Combine.bind(m, m2=m)
combine_output = combine.__call__.bind(dag_input)
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "Model -> Combine" in to_string
assert "Combine -> __call__" in to_string
assert "INPUT_NODE -> __call__" in to_string
assert "__call__ -> DAGDriver" in to_string
def test_chained_function(serve_instance, use_build):
@serve.deployment
def func_1(input):
return input
@serve.deployment
def func_2(input):
return input * 2
with InputNode() as dag_input:
output_1 = func_1.bind(dag_input)
output_2 = func_2.bind(dag_input)
serve_dag = combine.bind(output_1, output_2)
with pytest.raises(ValueError, match="Please provide a driver class"):
_ = serve.run(serve_dag)
handle = serve.run(DAGDriver.bind(serve_dag, input_schema=json_resolver))
assert ray.get(handle.predict.remote(2)) == 6 # 2 + 2*2
assert requests.post("http://127.0.0.1:8000/", json=2).json() == 6
def test_dag_driver_sync_warning(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
log_file = io.StringIO()
with contextlib.redirect_stderr(log_file):
handle = serve.run(DAGDriver.bind(dag))
assert ray.get(handle.predict.remote(42)) == 42
def wait_for_request_success_log():
lines = log_file.getvalue().splitlines()
for line in lines:
if "DAGDriver" in line and "HANDLE predict OK" in line:
return True
return False
wait_for_condition(wait_for_request_success_log)
assert ("You are retrieving a sync handle inside an asyncio loop."
not in log_file.getvalue())
def test_serve_pipeline_multi_instantiation_class_nested_deployment_arg_dag_plot(
):
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True)
output = combine.__call__.bind(dag_input)
serve_dag = DAGDriver.bind(output, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "Model -> Combine" in to_string
assert "Model_1 -> Combine" in to_string
assert "Combine -> __call__" in to_string
assert "INPUT_NODE -> __call__" in to_string
assert "__call__ -> DAGDriver" in to_string
def test_autoscaling_with_chain_nodes(min_replicas, serve_instance):
signal = SignalActor.remote()
autoscaling_config = {
"metrics_interval_s": 0.1,
"min_replicas": min_replicas,
"max_replicas": 2,
"look_back_period_s": 0.4,
"downscale_delay_s": 30,
"upscale_delay_s": 0,
}
@serve.deployment(
autoscaling_config=autoscaling_config,
_graceful_shutdown_timeout_s=1,
)
class Model1:
def __init__(self, weight):
self.weight = weight
def forward(self, input):
ray.get(signal.wait.remote())
return input + self.weight
@serve.deployment(
autoscaling_config=autoscaling_config,
_graceful_shutdown_timeout_s=1,
)
class Model2:
def __init__(self, weight):
self.weight = weight
def forward(self, input):
return input + self.weight
with InputNode() as user_input:
model1 = Model1.bind(0)
model2 = Model2.bind(1)
output = model1.forward.bind(user_input)
output2 = model2.forward.bind(output)
serve_dag = DAGDriver.options(
route_prefix="/my-dag",
autoscaling_config=autoscaling_config,
_graceful_shutdown_timeout_s=1,
).bind(output2)
dag_handle = serve.run(serve_dag)
controller = serve_instance._controller
# upscaling
[dag_handle.predict.remote(0) for _ in range(10)]
wait_for_condition(
lambda: get_num_running_replicas(controller, DAGDriver.name) >= 1)
[dag_handle.predict.remote(0) for _ in range(10)]
wait_for_condition(
lambda: get_num_running_replicas(controller, DAGDriver.name) >= 2)
wait_for_condition(
lambda: get_num_running_replicas(controller, Model1.name) >= 1,
timeout=40)
wait_for_condition(
lambda: get_num_running_replicas(controller, Model1.name) >= 2,
timeout=40)
signal.send.remote()
wait_for_condition(
lambda: get_num_running_replicas(controller, Model2.name) >= 1,
timeout=40)
# downscaling
wait_for_condition(
lambda: get_num_running_replicas(controller, DAGDriver.name) ==
min_replicas,
timeout=60,
)
wait_for_condition(
lambda: get_num_running_replicas(controller, Model1.name) ==
min_replicas,
timeout=60,
)
wait_for_condition(
lambda: get_num_running_replicas(controller, Model2.name) ==
min_replicas,
timeout=60,
)
@serve.deployment(
ray_actor_options={
"num_cpus": 0.1,
}
)
class Router:
def __init__(self, adder: RayHandleLike, subtractor: RayHandleLike):
self.adder = adder
self.subtractor = subtractor
def route(self, op: Operation, input: int) -> int:
if op == Operation.ADD:
return ray.get(self.adder.add.remote(input))
elif op == Operation.SUBTRACT:
return ray.get(self.subtractor.subtract.remote(input))
async def json_resolver(request: starlette.requests.Request) -> List:
return await request.json()
with InputNode() as inp:
operation, amount_input = inp[0], inp[1]
adder = Add.bind()
subtractor = Subtract.bind()
router = Router.bind(adder, subtractor)
amount = router.route.bind(operation, amount_input)
serve_dag = DAGDriver.bind(amount, http_adapter=json_resolver)
def test_autoscaling_with_ensemble_nodes(serve_instance):
signal = SignalActor.remote()
autoscaling_config = {
"metrics_interval_s": 0.1,
"min_replicas": 0,
"max_replicas": 2,
"look_back_period_s": 0.4,
"downscale_delay_s": 30,
"upscale_delay_s": 0,
}
@serve.deployment(
_autoscaling_config=autoscaling_config,
_graceful_shutdown_timeout_s=1,
)
class Model:
def __init__(self, weight):
self.weight = weight
def forward(self, input):
return input + self.weight
@serve.deployment(
_autoscaling_config=autoscaling_config,
_graceful_shutdown_timeout_s=1,
)
def combine(value_refs):
ray.get(signal.wait.remote())
return sum(ray.get(value_refs))
with InputNode() as user_input:
model1 = Model.bind(0)
model2 = Model.bind(1)
output1 = model1.forward.bind(user_input)
output2 = model2.forward.bind(user_input)
output = combine.bind([output1, output2])
serve_dag = DAGDriver.options(
route_prefix="/my-dag",
_autoscaling_config=autoscaling_config,
_graceful_shutdown_timeout_s=1,
).bind(output)
dag_handle = serve.run(serve_dag)
controller = serve_instance._controller
assert get_num_running_replicas(controller, "Model") == 0
assert get_num_running_replicas(controller, "Model_1") == 0
assert get_num_running_replicas(controller, "combine") == 0
# upscaling
[dag_handle.predict.remote(0) for _ in range(10)]
wait_for_condition(
lambda: get_num_running_replicas(controller, DAGDriver.name) >= 1)
wait_for_condition(
lambda: get_num_running_replicas(controller, "Model") >= 1, timeout=40)
wait_for_condition(
lambda: get_num_running_replicas(controller, "Model_1") >= 1,
timeout=40)
wait_for_condition(
lambda: get_num_running_replicas(controller, "combine") >= 2,
timeout=40)
signal.send.remote()
# downscaling
wait_for_condition(
lambda: get_num_running_replicas(controller, DAGDriver.name) == 0,
timeout=60,
)
wait_for_condition(
lambda: get_num_running_replicas(controller, "Model") == 0,
timeout=60,
)
wait_for_condition(
lambda: get_num_running_replicas(controller, "Model_1") == 0,
timeout=60,
)
wait_for_condition(
lambda: get_num_running_replicas(controller, "combine") == 0,
timeout=60)
async def json_resolver(request: Request) -> List:
return await request.json()
with InputNode() as query:
fruit, amount = query[0], query[1]
mango_stand = MangoStand.bind()
orange_stand = OrangeStand.bind()
pear_stand = PearStand.bind()
fruit_market = FruitMarket.bind(mango_stand, orange_stand, pear_stand)
net_price = fruit_market.check_price.bind(fruit, amount)
deployment_graph = DAGDriver.bind(net_price, http_adapter=json_request)
# __fruit_example_end__
# Test example's behavior
import requests # noqa: E402
from ray.serve.schema import ServeApplicationSchema # noqa: E402
from ray.serve.api import build # noqa: E402
from ray._private.test_utils import wait_for_condition # noqa: E402
def check_fruit_deployment_graph():
"""Checks the fruit deployment graph from this example."""
assert requests.post("http://localhost:8000/", json=["MANGO",
1]).json() == 3
assert requests.post("http://localhost:8000/", json=["ORANGE",
