def test_two_dags_shared_instance(serve_instance, use_build):
"""Test classmethod chain behavior is consistent across core and serve dag.
Note this only works if serve also has one replica given each class method
call mutates its internal state, but forming class method call chains that
mutate replica state should be considered anti-pattern in serve, given
request could be routed to different replicas each time.
"""
counter = Counter.bind(0)
with InputNode() as input_1:
# Will be carried over to second dag if counter reused
counter.inc.bind(2)
# Only applicable to current execution
counter.inc.bind(input_1)
dag = counter.get.bind()
serve_dag = DAGDriver.options(route_prefix="/serve_dag").bind(dag)
with InputNode() as _:
counter.inc.bind(10)
counter.inc.bind(20)
other_dag = counter.get.bind()
other_serve_dag = DAGDriver.options(
route_prefix="/other_serve_dag").bind(other_dag)
# First DAG
assert ray.get(dag.execute(3)) == 5 # 0 + 2 + input(3)
serve_handle = serve.run(maybe_build(serve_dag, use_build))
assert ray.get(serve_handle.predict.remote(3)) == 5 # 0 + 2 + input(3)
# Second DAG with shared counter ClassNode
assert ray.get(other_dag.execute(0)) == 32 # 0 + 2 + 10 + 20
other_serve_handle = serve.run(maybe_build(other_serve_dag, use_build))
assert ray.get(
other_serve_handle.predict.remote(0)) == 32 # 0 + 2 + 10 + 20
def test_single_func_deployment_dag(serve_instance, use_build):
with InputNode() as dag_input:
dag = combine.bind(dag_input[0], dag_input[1], kwargs_output=1)
serve_dag = DAGDriver.bind(dag, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote([1, 2])) == 4
assert requests.post("http://127.0.0.1:8000/", json=[1, 2]).json() == 4
def test_chained_function(serve_instance, use_build):
@serve.deployment
def func_1(input):
return input
@serve.deployment
def func_2(input):
return input * 2
@serve.deployment
def func_3(input):
return input * 3
with InputNode() as dag_input:
output_1 = func_1.bind(dag_input)
output_2 = func_2.bind(dag_input)
output_3 = func_3.bind(output_2)
ray_dag = combine.bind(output_1, output_2, kwargs_output=output_3)
with pytest.raises(ValueError, match="Please provide a driver class"):
_ = serve.run(ray_dag)
serve_dag = DAGDriver.bind(ray_dag, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(2)) == 18 # 2 + 2*2 + (2*2) * 3
assert requests.post("http://127.0.0.1:8000/", json=2).json() == 18
def test_serve_pipeline_func_class_with_class_method_plot():
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.bind(m1_output,
m2_output,
kwargs_output=dag_input[2])
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE" in to_string
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE_1" in to_string
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE_2" in to_string
assert "Model -> forward" in to_string
assert "INPUT_ATTRIBUTE_NODE -> forward" in to_string
assert "Model_1 -> forward_1" in to_string
assert "INPUT_ATTRIBUTE_NODE_1 -> forward_1" in to_string
assert "INPUT_ATTRIBUTE_NODE_2 -> combine" in to_string
assert "forward -> combine" in to_string
assert "forward_1 -> combine" in to_string
assert "combine -> DAGDriver" in to_string
def test_serve_pipeline_chained_function_plot():
@serve.deployment
def func_1(input):
return input
@serve.deployment
def func_2(input):
return input * 2
with InputNode() as dag_input:
output_1 = func_1.bind(dag_input)
output_2 = func_2.bind(dag_input)
serve_dag = combine.bind(output_1, output_2)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "INPUT_NODE -> func_1" in to_string
assert "INPUT_NODE -> func_2" in to_string
assert "func_1 -> combine" in to_string
assert "func_2 -> combine" in to_string
def test_suprious_call(serve_instance):
# https://github.com/ray-project/ray/issues/24116
@serve.deployment
class CallTracker:
def __init__(self):
self.records = []
def __call__(self, inp):
self.records.append("__call__")
def predict(self, inp):
self.records.append("predict")
def get(self):
return self.records
tracker = CallTracker.bind()
with InputNode() as inp:
dag = DAGDriver.bind(tracker.predict.bind(inp))
handle = serve.run(dag)
ray.get(handle.predict.remote(1))
call_tracker = CallTracker.get_handle()
assert ray.get(call_tracker.get.remote()) == ["predict"]
def test_simple_class_with_class_method(serve_instance, use_build):
with InputNode() as dag_input:
model = Model.bind(2, ratio=0.3)
dag = model.forward.bind(dag_input)
serve_dag = DAGDriver.bind(dag, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 0.6
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 0.6
def test_class_factory(serve_instance):
with InputNode() as _:
instance = serve.deployment(class_factory()).bind(3)
output = instance.get.bind()
serve_dag = NoargDriver.bind(output)
handle = serve.run(serve_dag)
assert ray.get(handle.remote()) == 3
assert requests.get("http://127.0.0.1:8000/").text == "3"
def test_shared_deployment_handle(serve_instance, use_build):
with InputNode() as dag_input:
m = Model.bind(2)
combine = Combine.bind(m, m2=m)
combine_output = combine.__call__.bind(dag_input)
serve_dag = DAGDriver.bind(combine_output, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 4
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 4
def test_single_node_driver_sucess(serve_instance, use_build):
m1 = Adder.bind(1)
m2 = Adder.bind(2)
with InputNode() as input_node:
out = m1.forward.bind(input_node)
out = m2.forward.bind(out)
driver = DAGDriver.bind(out, http_adapter=json_resolver)
handle = serve.run(driver)
assert ray.get(handle.predict.remote(39)) == 42
assert requests.post("http://127.0.0.1:8000/", json=39).json() == 42
def test_multi_instantiation_class_nested_deployment_arg_dag(serve_instance, use_build):
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True)
output = combine.__call__.bind(dag_input)
serve_dag = DAGDriver.bind(output, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 5
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def test_func_class_with_class_method(serve_instance, use_build):
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.bind(m1_output, m2_output, kwargs_output=dag_input[2])
serve_dag = DAGDriver.bind(combine_output, http_adapter=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote([1, 2, 3])) == 8
assert requests.post("http://127.0.0.1:8000/", json=[1, 2, 3]).json() == 8
def test_multi_instantiation_class_deployment_in_init_args(
serve_instance, use_build):
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2=m2)
combine_output = combine.__call__.bind(dag_input)
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 5
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def test_serve_pipeline_class_factory_plot():
with InputNode() as _:
instance = serve.deployment(class_factory()).bind(3)
output = instance.get.bind()
serve_dag = NoargDriver.bind(output)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "MyInlineClass -> get" in to_string
assert "get -> NoargDriver" in to_string
def test_serve_pipeline_class_with_class_method_plot():
with InputNode() as dag_input:
model = Model.bind(2, ratio=0.3)
dag = model.forward.bind(dag_input)
serve_dag = DAGDriver.bind(dag, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "Model -> forward" in to_string
assert "INPUT_NODE -> forward" in to_string
assert "forward -> DAGDriver" in to_string
def test_serve_pipeline_single_func_deployment_dag_plot():
with InputNode() as dag_input:
dag = combine.bind(dag_input[0], dag_input[1], kwargs_output=1)
serve_dag = DAGDriver.bind(dag, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE" in to_string
assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE_1" in to_string
assert "INPUT_ATTRIBUTE_NODE -> combine" in to_string
assert "INPUT_ATTRIBUTE_NODE_1 -> combine" in to_string
assert "combine -> DAGDriver" in to_string
def test_chained_function(serve_instance, use_build):
@serve.deployment
def func_1(input):
return input
@serve.deployment
def func_2(input):
return input * 2
with InputNode() as dag_input:
output_1 = func_1.bind(dag_input)
output_2 = func_2.bind(dag_input)
serve_dag = combine.bind(output_1, output_2)
with pytest.raises(ValueError, match="Please provide a driver class"):
_ = serve.run(serve_dag)
handle = serve.run(DAGDriver.bind(serve_dag, input_schema=json_resolver))
assert ray.get(handle.predict.remote(2)) == 6 # 2 + 2*2
assert requests.post("http://127.0.0.1:8000/", json=2).json() == 6
def test_serve_pipeline_test_shared_deployment_handle_plot():
with InputNode() as dag_input:
m = Model.bind(2)
combine = Combine.bind(m, m2=m)
combine_output = combine.__call__.bind(dag_input)
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "Model -> Combine" in to_string
assert "Combine -> __call__" in to_string
assert "INPUT_NODE -> __call__" in to_string
assert "__call__ -> DAGDriver" in to_string
def test_serve_pipeline_multi_instantiation_class_nested_deployment_arg_dag_plot(
):
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True)
output = combine.__call__.bind(dag_input)
serve_dag = DAGDriver.bind(output, input_schema=json_resolver)
serve_dag = ray_dag_to_serve_dag(serve_dag)
with tempfile.TemporaryDirectory() as tmpdir:
to_file = os.path.join(tmpdir, "tmp.png")
ray.experimental.dag.plot(serve_dag, to_file)
assert os.path.isfile(to_file)
graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag)
to_string = graph.to_string()
assert "Model -> Combine" in to_string
assert "Model_1 -> Combine" in to_string
assert "Combine -> __call__" in to_string
assert "INPUT_NODE -> __call__" in to_string
assert "__call__ -> DAGDriver" in to_string
import ray
from ray import serve
from ray.serve.deployment_graph import InputNode
ray.init()
@serve.deployment
class Model:
def __init__(self, weight):
self.weight = weight
def forward(self, input):
return input + self.weight
# 3 nodes chain in a line
num_nodes = 3
nodes = [Model.bind(w) for w in range(num_nodes)]
outputs = [None] * num_nodes
with InputNode() as dag_input:
for i in range(num_nodes):
if i == 0:
# first node
outputs[i] = nodes[i].forward.bind(dag_input)
else:
outputs[i] = nodes[i].forward.bind(outputs[i - 1])
print(ray.get(outputs[-1].execute(0)))
# This default price is overwritten by the one specified in the
# user_config through the reconfigure() method.
self.price = self.DEFAULT_PRICE
def reconfigure(self, config: Dict):
self.price = config.get("price", self.DEFAULT_PRICE)
def check_price(self, amount: float) -> float:
return self.price * amount
async def json_resolver(request: Request) -> List:
return await request.json()
with InputNode() as query:
fruit, amount = query[0], query[1]
mango_stand = MangoStand.bind()
orange_stand = OrangeStand.bind()
pear_stand = PearStand.bind()
fruit_market = FruitMarket.bind(mango_stand, orange_stand, pear_stand)
net_price = fruit_market.check_price.bind(fruit, amount)
deployment_graph = DAGDriver.bind(net_price, http_adapter=json_request)
# __fruit_example_end__
# Test example's behavior
import requests # noqa: E402
@serve.deployment
class Model:
def __init__(self, weight):
self.weight = weight
def forward(self, input):
return input + self.weight
@serve.deployment
def combine(output_1, output_2, kwargs_output=0):
return output_1 + output_2 + kwargs_output
with InputNode() as user_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(user_input[0])
m2_output = m2.forward.bind(user_input[1])
dag = combine.bind(m1_output, m2_output, kwargs_output=user_input[2])
# Partial DAG visualization
graph = ray.dag.vis_utils._dag_to_dot(m1_output)
to_string = graph.to_string()
print(to_string)
# Entire DAG visualization
graph = ray.dag.vis_utils._dag_to_dot(dag)
to_string = graph.to_string()
print(to_string)
@serve.deployment(
ray_actor_options={
"num_cpus": 0.1,
}
)
class Router:
def __init__(self, adder: RayHandleLike, subtractor: RayHandleLike):
self.adder = adder
self.subtractor = subtractor
def route(self, op: Operation, input: int) -> int:
if op == Operation.ADD:
return ray.get(self.adder.add.remote(input))
elif op == Operation.SUBTRACT:
return ray.get(self.subtractor.subtract.remote(input))
async def json_resolver(request: starlette.requests.Request) -> List:
return await request.json()
with InputNode() as inp:
operation, amount_input = inp[0], inp[1]
adder = Add.bind()
subtractor = Subtract.bind()
router = Router.bind(adder, subtractor)
amount = router.route.bind(operation, amount_input)
serve_dag = DAGDriver.bind(amount, http_adapter=json_resolver)
