def test_np_in_composed_model(serve_instance): # https://github.com/ray-project/ray/issues/9441 # AttributeError: 'bytes' object has no attribute 'readonly' # in cloudpickle _from_numpy_buffer @serve.deployment class Sum: def __call__(self, data): return np.sum(data) @serve.deployment(name="model") class ComposedModel: def __init__(self, handle): self.model = handle async def __call__(self): data = np.ones((10, 10)) return await self.model.remote(data) sum_d = Sum.bind() cm_d = ComposedModel.bind(sum_d) dag = DAGDriver.bind(cm_d) serve.run(dag) result = requests.get("http://127.0.0.1:8000/") assert result.status_code == 200 assert result.json() == 100.0
def test_passing_handle(serve_instance, use_build): child = Adder.bind(1) parent = TakeHandle.bind(child) driver = DAGDriver.bind(parent, http_adapter=json_resolver) handle = serve.run(driver) assert ray.get(handle.predict.remote(1)) == 2 assert requests.post("http://127.0.0.1:8000/", json=1).json() == 2
def test_wide_fanout_deployment_graph(fanout_degree, init_delay_secs=0, compute_delay_secs=0): """ Test that focuses on wide fanout of deployment graph -> Node_1 / \ INPUT --> Node_2 --> combine -> OUTPUT \ ... / -> Node_10 1) Intermediate blob size can be large / small 2) Compute time each node can be long / short 3) Init time can be long / short """ nodes = [ Node.bind(i, init_delay_secs=init_delay_secs) for i in range(0, fanout_degree) ] outputs = [] with InputNode() as user_input: for i in range(0, fanout_degree): outputs.append(nodes[i].compute.bind( user_input, compute_delay_secs=compute_delay_secs)) dag = combine.bind(outputs) serve_dag = DAGDriver.bind(dag) return serve_dag
def test_serve_pipeline_func_class_with_class_method_plot(): with InputNode() as dag_input: m1 = Model.bind(1) m2 = Model.bind(2) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[1]) combine_output = combine.bind(m1_output, m2_output, kwargs_output=dag_input[2]) serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver) serve_dag = ray_dag_to_serve_dag(serve_dag) with tempfile.TemporaryDirectory() as tmpdir: to_file = os.path.join(tmpdir, "tmp.png") ray.experimental.dag.plot(serve_dag, to_file) assert os.path.isfile(to_file) graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag) to_string = graph.to_string() assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE" in to_string assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE_1" in to_string assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE_2" in to_string assert "Model -> forward" in to_string assert "INPUT_ATTRIBUTE_NODE -> forward" in to_string assert "Model_1 -> forward_1" in to_string assert "INPUT_ATTRIBUTE_NODE_1 -> forward_1" in to_string assert "INPUT_ATTRIBUTE_NODE_2 -> combine" in to_string assert "forward -> combine" in to_string assert "forward_1 -> combine" in to_string assert "combine -> DAGDriver" in to_string
def test_single_func_deployment_dag(serve_instance, use_build): with InputNode() as dag_input: dag = combine.bind(dag_input[0], dag_input[1], kwargs_output=1) serve_dag = DAGDriver.bind(dag, http_adapter=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote([1, 2])) == 4 assert requests.post("http://127.0.0.1:8000/", json=[1, 2]).json() == 4
def test_chained_function(serve_instance, use_build): @serve.deployment def func_1(input): return input @serve.deployment def func_2(input): return input * 2 @serve.deployment def func_3(input): return input * 3 with InputNode() as dag_input: output_1 = func_1.bind(dag_input) output_2 = func_2.bind(dag_input) output_3 = func_3.bind(output_2) ray_dag = combine.bind(output_1, output_2, kwargs_output=output_3) with pytest.raises(ValueError, match="Please provide a driver class"): _ = serve.run(ray_dag) serve_dag = DAGDriver.bind(ray_dag, http_adapter=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(2)) == 18 # 2 + 2*2 + (2*2) * 3 assert requests.post("http://127.0.0.1:8000/", json=2).json() == 18
def test_long_chain_deployment_graph( chain_length, init_delay_secs=0, compute_delay_secs=0 ): """ Test that focuses on long chain of deployment graph INPUT -> Node_1 -> Node_2 -> ... -> Node_10 -> OUTPUT 1) Intermediate blob size can be large / small 2) Compute time each node can be long / short 3) Init time can be long / short """ nodes = [Node.bind(i, init_delay_secs=init_delay_secs) for i in range(chain_length)] prev_outputs = [None for _ in range(chain_length)] with InputNode() as user_input: for i in range(chain_length): if i == 0: prev_outputs[i] = nodes[i].compute.bind( user_input, compute_delay_secs=compute_delay_secs ) else: prev_outputs[i] = nodes[i].compute.bind( prev_outputs[i - 1], compute_delay_secs=compute_delay_secs ) serve_dag = DAGDriver.bind(prev_outputs[-1]) return serve_dag
def test_suprious_call(serve_instance): # https://github.com/ray-project/ray/issues/24116 @serve.deployment class CallTracker: def __init__(self): self.records = [] def __call__(self, inp): self.records.append("__call__") def predict(self, inp): self.records.append("predict") def get(self): return self.records tracker = CallTracker.bind() with InputNode() as inp: dag = DAGDriver.bind(tracker.predict.bind(inp)) handle = serve.run(dag) ray.get(handle.predict.remote(1)) call_tracker = CallTracker.get_handle() assert ray.get(call_tracker.get.remote()) == ["predict"]
def test_simple_class_with_class_method(serve_instance, use_build): with InputNode() as dag_input: model = Model.bind(2, ratio=0.3) dag = model.forward.bind(dag_input) serve_dag = DAGDriver.bind(dag, http_adapter=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 0.6 assert requests.post("http://127.0.0.1:8000/", json=1).json() == 0.6
def test_single_node_driver_sucess(serve_instance, use_build): m1 = Adder.bind(1) m2 = Adder.bind(2) with InputNode() as input_node: out = m1.forward.bind(input_node) out = m2.forward.bind(out) driver = DAGDriver.bind(out, http_adapter=json_resolver) handle = serve.run(driver) assert ray.get(handle.predict.remote(39)) == 42 assert requests.post("http://127.0.0.1:8000/", json=39).json() == 42
def test_shared_deployment_handle(serve_instance, use_build): with InputNode() as dag_input: m = Model.bind(2) combine = Combine.bind(m, m2=m) combine_output = combine.__call__.bind(dag_input) serve_dag = DAGDriver.bind(combine_output, http_adapter=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 4 assert requests.post("http://127.0.0.1:8000/", json=1).json() == 4
def test_multi_instantiation_class_nested_deployment_arg_dag(serve_instance, use_build): with InputNode() as dag_input: m1 = Model.bind(2) m2 = Model.bind(3) combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True) output = combine.bind(dag_input) serve_dag = DAGDriver.bind(output, input_schema=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 5 assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def test_dag_driver_custom_schema(serve_instance): with InputNode() as inp: dag = echo.bind(inp) handle = serve.run(DAGDriver.bind(dag, input_schema=resolver)) assert ray.get(handle.predict.remote(42)) == 42 resp = requests.get("http://127.0.0.1:8000/?my_custom_param=100") print(resp.text) resp.raise_for_status() assert resp.json() == 100
def test_multi_instantiation_class_deployment_in_init_args(serve_instance): with InputNode() as dag_input: m1 = Model.bind(2) m2 = Model.bind(3) combine = Combine.bind(m1, m2=m2) combine_output = combine.bind(dag_input) serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 5 assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def test_func_class_with_class_method(serve_instance, use_build): with InputNode() as dag_input: m1 = Model.bind(1) m2 = Model.bind(2) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[1]) combine_output = combine.bind(m1_output, m2_output, kwargs_output=dag_input[2]) serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote([1, 2, 3])) == 8 assert requests.post("http://127.0.0.1:8000/", json=[1, 2, 3]).json() == 8
def test_dag_driver_default(serve_instance): with InputNode() as inp: dag = echo.bind(inp) handle = serve.run(DAGDriver.bind(dag)) assert ray.get(handle.predict.remote(42)) == 42 resp = requests.post("http://127.0.0.1:8000/", json={"array": [1]}) print(resp.text) resp.raise_for_status() assert resp.json() == "starlette!"
def test_dag_driver_custom_pydantic_schema(serve_instance): with InputNode() as inp: dag = echo.bind(inp) handle = serve.run(DAGDriver.bind(dag, http_adapter=MyType)) assert ray.get(handle.predict.remote(MyType(a=1, b="str"))) == MyType(a=1, b="str") resp = requests.post("http://127.0.0.1:8000/", json={"a": 1, "b": "str"}) print(resp.text) resp.raise_for_status() assert resp.json() == {"a": 1, "b": "str"}
def test_dag_driver_partial_input(serve_instance): with InputNode() as inp: dag = DAGDriver.bind( combine.bind(echo.bind(inp[0]), echo.bind(inp[1]), echo.bind(inp[2])), input_schema=json_request, ) handle = serve.run(dag) assert ray.get(handle.predict.remote([1, 2, [3, 4]])) == [1, 2, [3, 4]] assert ray.get(handle.predict.remote(1, 2, [3, 4])) == [1, 2, [3, 4]] resp = requests.post("http://127.0.0.1:8000/", json=[1, 2, [3, 4]]) print(resp.text) resp.raise_for_status() assert resp.json() == [1, 2, [3, 4]]
def test_serve_pipeline_class_with_class_method_plot(): with InputNode() as dag_input: model = Model.bind(2, ratio=0.3) dag = model.forward.bind(dag_input) serve_dag = DAGDriver.bind(dag, input_schema=json_resolver) serve_dag = ray_dag_to_serve_dag(serve_dag) with tempfile.TemporaryDirectory() as tmpdir: to_file = os.path.join(tmpdir, "tmp.png") ray.experimental.dag.plot(serve_dag, to_file) assert os.path.isfile(to_file) graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag) to_string = graph.to_string() assert "Model -> forward" in to_string assert "INPUT_NODE -> forward" in to_string assert "forward -> DAGDriver" in to_string
def test_serve_pipeline_single_func_deployment_dag_plot(): with InputNode() as dag_input: dag = combine.bind(dag_input[0], dag_input[1], kwargs_output=1) serve_dag = DAGDriver.bind(dag, input_schema=json_resolver) serve_dag = ray_dag_to_serve_dag(serve_dag) with tempfile.TemporaryDirectory() as tmpdir: to_file = os.path.join(tmpdir, "tmp.png") ray.experimental.dag.plot(serve_dag, to_file) assert os.path.isfile(to_file) graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag) to_string = graph.to_string() assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE" in to_string assert "INPUT_NODE -> INPUT_ATTRIBUTE_NODE_1" in to_string assert "INPUT_ATTRIBUTE_NODE -> combine" in to_string assert "INPUT_ATTRIBUTE_NODE_1 -> combine" in to_string assert "combine -> DAGDriver" in to_string
def test_chained_function(serve_instance, use_build): @serve.deployment def func_1(input): return input @serve.deployment def func_2(input): return input * 2 with InputNode() as dag_input: output_1 = func_1.bind(dag_input) output_2 = func_2.bind(dag_input) serve_dag = combine.bind(output_1, output_2) with pytest.raises(ValueError, match="Please provide a driver class"): _ = serve.run(serve_dag) handle = serve.run(DAGDriver.bind(serve_dag, input_schema=json_resolver)) assert ray.get(handle.predict.remote(2)) == 6 # 2 + 2*2 assert requests.post("http://127.0.0.1:8000/", json=2).json() == 6
def test_serve_pipeline_test_shared_deployment_handle_plot(): with InputNode() as dag_input: m = Model.bind(2) combine = Combine.bind(m, m2=m) combine_output = combine.__call__.bind(dag_input) serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver) serve_dag = ray_dag_to_serve_dag(serve_dag) with tempfile.TemporaryDirectory() as tmpdir: to_file = os.path.join(tmpdir, "tmp.png") ray.experimental.dag.plot(serve_dag, to_file) assert os.path.isfile(to_file) graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag) to_string = graph.to_string() assert "Model -> Combine" in to_string assert "Combine -> __call__" in to_string assert "INPUT_NODE -> __call__" in to_string assert "__call__ -> DAGDriver" in to_string
def test_dag_driver_sync_warning(serve_instance): with InputNode() as inp: dag = echo.bind(inp) log_file = io.StringIO() with contextlib.redirect_stderr(log_file): handle = serve.run(DAGDriver.bind(dag)) assert ray.get(handle.predict.remote(42)) == 42 def wait_for_request_success_log(): lines = log_file.getvalue().splitlines() for line in lines: if "DAGDriver" in line and "HANDLE predict OK" in line: return True return False wait_for_condition(wait_for_request_success_log) assert ("You are retrieving a sync handle inside an asyncio loop." not in log_file.getvalue())
def test_serve_pipeline_multi_instantiation_class_nested_deployment_arg_dag_plot( ): with InputNode() as dag_input: m1 = Model.bind(2) m2 = Model.bind(3) combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True) output = combine.__call__.bind(dag_input) serve_dag = DAGDriver.bind(output, input_schema=json_resolver) serve_dag = ray_dag_to_serve_dag(serve_dag) with tempfile.TemporaryDirectory() as tmpdir: to_file = os.path.join(tmpdir, "tmp.png") ray.experimental.dag.plot(serve_dag, to_file) assert os.path.isfile(to_file) graph = ray.experimental.dag.vis_utils.dag_to_dot(serve_dag) to_string = graph.to_string() assert "Model -> Combine" in to_string assert "Model_1 -> Combine" in to_string assert "Combine -> __call__" in to_string assert "INPUT_NODE -> __call__" in to_string assert "__call__ -> DAGDriver" in to_string
import ray from ray import serve from ray.serve.dag import InputNode from ray.serve.drivers import DAGDriver @serve.deployment def preprocess(inp: int): return inp + 1 @serve.deployment class Model: def __init__(self, increment: int): self.increment = increment def predict(self, inp: int): return inp + self.increment with InputNode() as inp: model = Model.bind(increment=2) output = model.predict.bind(preprocess.bind(inp)) serve_dag = DAGDriver.bind(output) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 4
@serve.deployment( ray_actor_options={ "num_cpus": 0.1, } ) class Router: def __init__(self, adder: RayHandleLike, subtractor: RayHandleLike): self.adder = adder self.subtractor = subtractor def route(self, op: Operation, input: int) -> int: if op == Operation.ADD: return ray.get(self.adder.add.remote(input)) elif op == Operation.SUBTRACT: return ray.get(self.subtractor.subtract.remote(input)) async def json_resolver(request: starlette.requests.Request) -> List: return await request.json() with InputNode() as inp: operation, amount_input = inp[0], inp[1] adder = Add.bind() subtractor = Subtract.bind() router = Router.bind(adder, subtractor) amount = router.route.bind(operation, amount_input) serve_dag = DAGDriver.bind(amount, http_adapter=json_resolver)
def test_driver_np_serializer(serve_instance): # https://github.com/ray-project/ray/pull/24215#issuecomment-1115237058 with InputNode() as inp: dag = DAGDriver.bind(return_np_int.bind(inp)) serve.run(dag) assert requests.get("http://127.0.0.1:8000/").json() == [42]
async def json_resolver(request: Request) -> List: return await request.json() with InputNode() as query: fruit, amount = query[0], query[1] mango_stand = MangoStand.bind() orange_stand = OrangeStand.bind() pear_stand = PearStand.bind() fruit_market = FruitMarket.bind(mango_stand, orange_stand, pear_stand) net_price = fruit_market.check_price.bind(fruit, amount) deployment_graph = DAGDriver.bind(net_price, http_adapter=json_request) # __fruit_example_end__ # Test example's behavior import requests # noqa: E402 from ray.serve.schema import ServeApplicationSchema # noqa: E402 from ray.serve.api import build # noqa: E402 from ray._private.test_utils import wait_for_condition # noqa: E402 def check_fruit_deployment_graph(): """Checks the fruit deployment graph from this example.""" assert requests.post("http://localhost:8000/", json=["MANGO", 1]).json() == 3 assert requests.post("http://localhost:8000/", json=["ORANGE",
if os.getenv("override_increment") is not None: return input + int(os.getenv("override_increment")) return input + self.increment @serve.deployment(ray_actor_options={ "num_cpus": 0.1, }) def create_order(amount: int) -> str: return f"{amount} pizzas please!" async def json_resolver(request: starlette.requests.Request) -> List: return await request.json() # Overwritten by user_config ORIGINAL_INCREMENT = 1 ORIGINAL_FACTOR = 1 with InputNode() as inp: operation, amount_input = inp[0], inp[1] multiplier = Multiplier.bind(ORIGINAL_FACTOR) adder = Adder.bind(ORIGINAL_INCREMENT) router = Router.bind(multiplier, adder) amount = router.route.bind(operation, amount_input) order = create_order.bind(amount) serve_dag = DAGDriver.bind(order, http_adapter=json_resolver)