def test_nested_building(): with InputNode() as inp: out = func.bind(inp) out = Driver.bind().__call__.bind(out) out = func.bind(out) dag = Driver.bind(out, func.bind()) assert len(pipeline_build(dag)) == 5
def test_deploment_options_func_class_with_class_method(): with InputNode() as dag_input: counter = Counter.bind() m1 = Model.options(name="m1", max_concurrent_queries=3).bind(1) m2 = Model.options(name="m2", max_concurrent_queries=5).bind(2) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[1]) combine_output = combine.options(num_replicas=3, max_concurrent_queries=7).bind( m1_output, m2_output, kwargs_output=dag_input[2] ) dag = counter.__call__.bind(combine_output) serve_dag = Driver.bind(dag) deployments = pipeline_build(serve_dag) hit_count = 0 for deployment in deployments: if deployment.name == "counter": assert deployment.num_replicas == 2 assert deployment.user_config == {"count": 123, "b": 2} hit_count += 1 elif deployment.name == "m1": assert deployment.max_concurrent_queries == 3 hit_count += 1 elif deployment.name == "m2": assert deployment.max_concurrent_queries == 5 hit_count += 1 elif deployment.name == "combine": assert deployment.num_replicas == 3 assert deployment.max_concurrent_queries == 7 hit_count += 1 assert hit_count == 4, "Not all deployments with expected name were found."
def test_air_integrations_reconfigure(serve_instance): path = tempfile.mkdtemp() uri = f"file://{path}/test_uri" Checkpoint.from_dict({"increment": 2}).to_uri(uri) predictor_cls = "ray.serve.tests.test_air_integrations.AdderPredictor" additional_config = { "checkpoint": {"increment": 5}, "predictor_cls": "ray.serve.tests.test_air_integrations.AdderPredictor", } with InputNode() as dag_input: m1 = PredictorDeployment.options(user_config=additional_config).bind( predictor_cls=predictor_cls, checkpoint=uri, ) dag = m1.predict.bind(dag_input) deployments = build(Ingress.bind(dag)) for d in deployments: d.deploy() resp = requests.post("http://127.0.0.1:8000/ingress", json={"array": [40]}) print(resp.text) resp.raise_for_status() return resp.json() == {"value": [45], "batch_size": 1}
def test_model_wrappers_in_pipeline(serve_instance): _, path = tempfile.mkstemp() with open(path, "w") as f: json.dump(2, f) predictor_cls = "ray.serve.tests.test_model_wrappers.AdderPredictor" checkpoint_cls = "ray.serve.tests.test_model_wrappers.AdderCheckpoint" with InputNode() as dag_input: m1 = ModelWrapperDeployment.bind( predictor_cls=predictor_cls, # TODO: can't be the raw class right now? checkpoint={ # TODO: can't be the raw object right now? "checkpoint_cls": checkpoint_cls, "uri": path, }, ) dag = m1.predict.bind(dag_input) deployments = build(Ingress.bind(dag)) for d in deployments: d.deploy() resp = requests.post("http://127.0.0.1:8000/ingress", json={"array": [40]}) print(resp.text) resp.raise_for_status() return resp.json() == {"value": [42], "batch_size": 1}
def test_no_input_node_as_init_args(): """ User should NOT directly create instances of Deployment or DeploymentNode. """ with pytest.raises( ValueError, match="cannot be used as args, kwargs, or other_args_to_resolve", ): _ = DeploymentNode( Actor, "test", (InputNode()), {}, {}, other_args_to_resolve={USE_SYNC_HANDLE_KEY: True}, ) with pytest.raises( ValueError, match="cannot be used as args, kwargs, or other_args_to_resolve", ): _ = DeploymentNode( Actor, "test", (), {"a": InputNode()}, {}, other_args_to_resolve={USE_SYNC_HANDLE_KEY: True}, ) with pytest.raises( ValueError, match="cannot be used as args, kwargs, or other_args_to_resolve", ): _ = DeploymentNode( Actor, "test", (), {}, {}, other_args_to_resolve={"arg": { "options_a": InputNode() }}, )
def test_dag_driver_custom_schema(serve_instance): with InputNode() as inp: dag = echo.bind(inp) handle = serve.run(DAGDriver.bind(dag, http_adapter=resolver)) assert ray.get(handle.predict.remote(42)) == 42 resp = requests.get("http://127.0.0.1:8000/?my_custom_param=100") print(resp.text) resp.raise_for_status() assert resp.json() == 100
def test_dag_driver_default(serve_instance): with InputNode() as inp: dag = echo.bind(inp) handle = serve.run(DAGDriver.bind(dag)) assert ray.get(handle.predict.remote(42)) == 42 resp = requests.post("http://127.0.0.1:8000/", json={"array": [1]}) print(resp.text) resp.raise_for_status() assert resp.json() == "starlette!"
def test_dag_driver_custom_pydantic_schema(serve_instance): with InputNode() as inp: dag = echo.bind(inp) handle = serve.run(DAGDriver.bind(dag, http_adapter=MyType)) assert ray.get(handle.predict.remote(MyType(a=1, b="str"))) == MyType(a=1, b="str") resp = requests.post("http://127.0.0.1:8000/", json={"a": 1, "b": "str"}) print(resp.text) resp.raise_for_status() assert resp.json() == {"a": 1, "b": "str"}
def test_http_reconfigure_non_default_route_prefix_on_root(serve_instance): with InputNode() as dag_input: m1 = Model.bind(1) m2 = Model.bind(2) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[1]) combine_output = combine.bind(m1_output, m2_output) serve_dag = Driver.bind(combine_output) deployments = pipeline_build(serve_dag) non_root_deployment = deployments[-1].options(route_prefix="/yoo") deployments[-1] = non_root_deployment _ = get_and_validate_ingress_deployment(deployments)
def test_http_user_bring_own_driver_route_prefix(serve_instance): with InputNode() as dag_input: m1 = Model.bind(1) m2 = Model.bind(2) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[0]) combine_output = combine.bind(m1_output, m2_output) serve_dag = Driver.options(route_prefix="/hello").bind(combine_output) deployments = pipeline_build(serve_dag) ingress_deployment = get_and_validate_ingress_deployment(deployments) assert ingress_deployment.route_prefix == "/hello" for deployment in deployments[:-1]: assert deployment.route_prefix is None
def test_http_no_non_ingress_deployment_rout_prefix(serve_instance): with InputNode() as dag_input: m1 = Model.options(route_prefix="/should-fail").bind(1) m2 = Model.bind(1) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[0]) combine_output = combine.bind(m1_output, m2_output) serve_dag = Driver.bind(combine_output) with pytest.raises( ValueError, match="Route prefix is only configurable on the ingress deployment", ): _ = pipeline_build(serve_dag)
def test_dag_driver_partial_input(serve_instance): with InputNode() as inp: dag = DAGDriver.bind( combine.bind(echo.bind(inp[0]), echo.bind(inp[1]), echo.bind(inp[2])), http_adapter=json_request, ) handle = serve.run(dag) assert ray.get(handle.predict.remote([1, 2, [3, 4]])) == [1, 2, [3, 4]] assert ray.get(handle.predict.remote(1, 2, [3, 4])) == [1, 2, [3, 4]] resp = requests.post("http://127.0.0.1:8000/", json=[1, 2, [3, 4]]) print(resp.text) resp.raise_for_status() assert resp.json() == [1, 2, [3, 4]]
def test_http_we_provide_default_route_prefix_cls(serve_instance): """Ensure the default ingress deployment route is '/' instead of driver class name """ with InputNode() as dag_input: m1 = Model.bind(1) m2 = Model.bind(1) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[0]) combine_output = combine.bind(m1_output, m2_output) serve_dag = Driver.bind(combine_output) deployments = pipeline_build(serve_dag) ingress_deployment = get_and_validate_ingress_deployment(deployments) assert ingress_deployment.route_prefix == "/" for deployment in deployments[:-1]: assert deployment.route_prefix is None
def test_http_only_one_ingress_deployment(serve_instance): with InputNode() as dag_input: m1 = Model.bind(1) m2 = Model.bind(1) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[0]) combine_output = combine.bind(m1_output, m2_output) serve_dag = Driver.bind(combine_output) deployments = pipeline_build(serve_dag) non_root_deployment = deployments[0].options(route_prefix="/") deployments[0] = non_root_deployment with pytest.raises( ValueError, match=( "Only one deployment in an Serve Application or DAG can have " "non-None route prefix"), ): _ = get_and_validate_ingress_deployment(deployments)
def test_dag_driver_sync_warning(serve_instance): with InputNode() as inp: dag = echo.bind(inp) log_file = io.StringIO() with contextlib.redirect_stderr(log_file): handle = serve.run(DAGDriver.bind(dag)) assert ray.get(handle.predict.remote(42)) == 42 def wait_for_request_success_log(): lines = log_file.getvalue().splitlines() for line in lines: if "DAGDriver" in line and "HANDLE predict OK" in line: return True return False wait_for_condition(wait_for_request_success_log) assert ("You are retrieving a sync handle inside an asyncio loop." not in log_file.getvalue())
def test_model_wrappers_in_pipeline(serve_instance): path = tempfile.mkdtemp() uri = f"file://{path}/test_uri" Checkpoint.from_dict({"increment": 2}).to_uri(uri) predictor_cls = "ray.serve.tests.test_model_wrappers.AdderPredictor" with InputNode() as dag_input: m1 = ModelWrapperDeployment.bind( predictor_cls=predictor_cls, checkpoint=uri, ) dag = m1.predict.bind(dag_input) deployments = build(Ingress.bind(dag)) for d in deployments: d.deploy() resp = requests.post("http://127.0.0.1:8000/ingress", json={"array": [40]}) print(resp.text) resp.raise_for_status() return resp.json() == {"value": [42], "batch_size": 1}
def test_driver_np_serializer(serve_instance): # https://github.com/ray-project/ray/pull/24215#issuecomment-1115237058 with InputNode() as inp: dag = DAGDriver.bind(return_np_int.bind(inp)) serve.run(dag) assert requests.get("http://127.0.0.1:8000/").json() == [42]
import ray from ray import serve from ray.serve.dag import InputNode from ray.serve.drivers import DAGDriver @serve.deployment def preprocess(inp: int): return inp + 1 @serve.deployment class Model: def __init__(self, increment: int): self.increment = increment def predict(self, inp: int): return inp + self.increment with InputNode() as inp: model = Model.bind(increment=2) output = model.predict.bind(preprocess.bind(inp)) serve_dag = DAGDriver.bind(output) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 4