def test_nested_building():
with InputNode() as inp:
out = func.bind(inp)
out = Driver.bind().__call__.bind(out)
out = func.bind(out)
dag = Driver.bind(out, func.bind())
assert len(pipeline_build(dag)) == 5
def test_deploment_options_func_class_with_class_method():
with InputNode() as dag_input:
counter = Counter.bind()
m1 = Model.options(name="m1", max_concurrent_queries=3).bind(1)
m2 = Model.options(name="m2", max_concurrent_queries=5).bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.options(num_replicas=3, max_concurrent_queries=7).bind(
m1_output, m2_output, kwargs_output=dag_input[2]
)
dag = counter.__call__.bind(combine_output)
serve_dag = Driver.bind(dag)
deployments = pipeline_build(serve_dag)
hit_count = 0
for deployment in deployments:
if deployment.name == "counter":
assert deployment.num_replicas == 2
assert deployment.user_config == {"count": 123, "b": 2}
hit_count += 1
elif deployment.name == "m1":
assert deployment.max_concurrent_queries == 3
hit_count += 1
elif deployment.name == "m2":
assert deployment.max_concurrent_queries == 5
hit_count += 1
elif deployment.name == "combine":
assert deployment.num_replicas == 3
assert deployment.max_concurrent_queries == 7
hit_count += 1
assert hit_count == 4, "Not all deployments with expected name were found."
def test_air_integrations_reconfigure(serve_instance):
path = tempfile.mkdtemp()
uri = f"file://{path}/test_uri"
Checkpoint.from_dict({"increment": 2}).to_uri(uri)
predictor_cls = "ray.serve.tests.test_air_integrations.AdderPredictor"
additional_config = {
"checkpoint": {"increment": 5},
"predictor_cls": "ray.serve.tests.test_air_integrations.AdderPredictor",
}
with InputNode() as dag_input:
m1 = PredictorDeployment.options(user_config=additional_config).bind(
predictor_cls=predictor_cls,
checkpoint=uri,
)
dag = m1.predict.bind(dag_input)
deployments = build(Ingress.bind(dag))
for d in deployments:
d.deploy()
resp = requests.post("http://127.0.0.1:8000/ingress", json={"array": [40]})
print(resp.text)
resp.raise_for_status()
return resp.json() == {"value": [45], "batch_size": 1}
def test_model_wrappers_in_pipeline(serve_instance):
_, path = tempfile.mkstemp()
with open(path, "w") as f:
json.dump(2, f)
predictor_cls = "ray.serve.tests.test_model_wrappers.AdderPredictor"
checkpoint_cls = "ray.serve.tests.test_model_wrappers.AdderCheckpoint"
with InputNode() as dag_input:
m1 = ModelWrapperDeployment.bind(
predictor_cls=predictor_cls, # TODO: can't be the raw class right now?
checkpoint={ # TODO: can't be the raw object right now?
"checkpoint_cls": checkpoint_cls,
"uri": path,
},
)
dag = m1.predict.bind(dag_input)
deployments = build(Ingress.bind(dag))
for d in deployments:
d.deploy()
resp = requests.post("http://127.0.0.1:8000/ingress", json={"array": [40]})
print(resp.text)
resp.raise_for_status()
return resp.json() == {"value": [42], "batch_size": 1}
def test_no_input_node_as_init_args():
"""
User should NOT directly create instances of Deployment or DeploymentNode.
"""
with pytest.raises(
ValueError,
match="cannot be used as args, kwargs, or other_args_to_resolve",
):
_ = DeploymentNode(
Actor,
"test",
(InputNode()),
{},
{},
other_args_to_resolve={USE_SYNC_HANDLE_KEY: True},
)
with pytest.raises(
ValueError,
match="cannot be used as args, kwargs, or other_args_to_resolve",
):
_ = DeploymentNode(
Actor,
"test",
(),
{"a": InputNode()},
{},
other_args_to_resolve={USE_SYNC_HANDLE_KEY: True},
)
with pytest.raises(
ValueError,
match="cannot be used as args, kwargs, or other_args_to_resolve",
):
_ = DeploymentNode(
Actor,
"test",
(),
{},
{},
other_args_to_resolve={"arg": {
"options_a": InputNode()
}},
)
def test_dag_driver_custom_schema(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
handle = serve.run(DAGDriver.bind(dag, http_adapter=resolver))
assert ray.get(handle.predict.remote(42)) == 42
resp = requests.get("http://127.0.0.1:8000/?my_custom_param=100")
print(resp.text)
resp.raise_for_status()
assert resp.json() == 100
def test_dag_driver_default(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
handle = serve.run(DAGDriver.bind(dag))
assert ray.get(handle.predict.remote(42)) == 42
resp = requests.post("http://127.0.0.1:8000/", json={"array": [1]})
print(resp.text)
resp.raise_for_status()
assert resp.json() == "starlette!"
def test_dag_driver_custom_pydantic_schema(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
handle = serve.run(DAGDriver.bind(dag, http_adapter=MyType))
assert ray.get(handle.predict.remote(MyType(a=1,
b="str"))) == MyType(a=1,
b="str")
resp = requests.post("http://127.0.0.1:8000/", json={"a": 1, "b": "str"})
print(resp.text)
resp.raise_for_status()
assert resp.json() == {"a": 1, "b": "str"}
def test_http_reconfigure_non_default_route_prefix_on_root(serve_instance):
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.bind(combine_output)
deployments = pipeline_build(serve_dag)
non_root_deployment = deployments[-1].options(route_prefix="/yoo")
deployments[-1] = non_root_deployment
_ = get_and_validate_ingress_deployment(deployments)
def test_http_user_bring_own_driver_route_prefix(serve_instance):
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[0])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.options(route_prefix="/hello").bind(combine_output)
deployments = pipeline_build(serve_dag)
ingress_deployment = get_and_validate_ingress_deployment(deployments)
assert ingress_deployment.route_prefix == "/hello"
for deployment in deployments[:-1]:
assert deployment.route_prefix is None
def test_http_no_non_ingress_deployment_rout_prefix(serve_instance):
with InputNode() as dag_input:
m1 = Model.options(route_prefix="/should-fail").bind(1)
m2 = Model.bind(1)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[0])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.bind(combine_output)
with pytest.raises(
ValueError,
match="Route prefix is only configurable on the ingress deployment",
):
_ = pipeline_build(serve_dag)
def test_dag_driver_partial_input(serve_instance):
with InputNode() as inp:
dag = DAGDriver.bind(
combine.bind(echo.bind(inp[0]), echo.bind(inp[1]),
echo.bind(inp[2])),
http_adapter=json_request,
)
handle = serve.run(dag)
assert ray.get(handle.predict.remote([1, 2, [3, 4]])) == [1, 2, [3, 4]]
assert ray.get(handle.predict.remote(1, 2, [3, 4])) == [1, 2, [3, 4]]
resp = requests.post("http://127.0.0.1:8000/", json=[1, 2, [3, 4]])
print(resp.text)
resp.raise_for_status()
assert resp.json() == [1, 2, [3, 4]]
def test_http_we_provide_default_route_prefix_cls(serve_instance):
"""Ensure the default ingress deployment route is '/' instead of driver
class name
"""
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(1)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[0])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.bind(combine_output)
deployments = pipeline_build(serve_dag)
ingress_deployment = get_and_validate_ingress_deployment(deployments)
assert ingress_deployment.route_prefix == "/"
for deployment in deployments[:-1]:
assert deployment.route_prefix is None
def test_http_only_one_ingress_deployment(serve_instance):
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(1)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[0])
combine_output = combine.bind(m1_output, m2_output)
serve_dag = Driver.bind(combine_output)
deployments = pipeline_build(serve_dag)
non_root_deployment = deployments[0].options(route_prefix="/")
deployments[0] = non_root_deployment
with pytest.raises(
ValueError,
match=(
"Only one deployment in an Serve Application or DAG can have "
"non-None route prefix"),
):
_ = get_and_validate_ingress_deployment(deployments)
def test_dag_driver_sync_warning(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
log_file = io.StringIO()
with contextlib.redirect_stderr(log_file):
handle = serve.run(DAGDriver.bind(dag))
assert ray.get(handle.predict.remote(42)) == 42
def wait_for_request_success_log():
lines = log_file.getvalue().splitlines()
for line in lines:
if "DAGDriver" in line and "HANDLE predict OK" in line:
return True
return False
wait_for_condition(wait_for_request_success_log)
assert ("You are retrieving a sync handle inside an asyncio loop."
not in log_file.getvalue())
def test_model_wrappers_in_pipeline(serve_instance):
path = tempfile.mkdtemp()
uri = f"file://{path}/test_uri"
Checkpoint.from_dict({"increment": 2}).to_uri(uri)
predictor_cls = "ray.serve.tests.test_model_wrappers.AdderPredictor"
with InputNode() as dag_input:
m1 = ModelWrapperDeployment.bind(
predictor_cls=predictor_cls,
checkpoint=uri,
)
dag = m1.predict.bind(dag_input)
deployments = build(Ingress.bind(dag))
for d in deployments:
d.deploy()
resp = requests.post("http://127.0.0.1:8000/ingress", json={"array": [40]})
print(resp.text)
resp.raise_for_status()
return resp.json() == {"value": [42], "batch_size": 1}
def test_driver_np_serializer(serve_instance):
# https://github.com/ray-project/ray/pull/24215#issuecomment-1115237058
with InputNode() as inp:
dag = DAGDriver.bind(return_np_int.bind(inp))
serve.run(dag)
assert requests.get("http://127.0.0.1:8000/").json() == [42]
import ray
from ray import serve
from ray.serve.dag import InputNode
from ray.serve.drivers import DAGDriver
@serve.deployment
def preprocess(inp: int):
return inp + 1
@serve.deployment
class Model:
def __init__(self, increment: int):
self.increment = increment
def predict(self, inp: int):
return inp + self.increment
with InputNode() as inp:
model = Model.bind(increment=2)
output = model.predict.bind(preprocess.bind(inp))
serve_dag = DAGDriver.bind(output)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 4
