def test_invalid_input_node_in_class_method_node(shared_ray_instance): @ray.remote class Actor: def __init__(self, val): self.val = val def get(self, input1, input2): return self.val + input1 + input2 actor = Actor._bind(1) with pytest.raises( ValueError, match="ensure InputNode is the only input to a ClassMethodNode", ): actor.get._bind([[{"nested": InputNode()}]]) with pytest.raises( ValueError, match="ensure InputNode is the only input to a ClassMethodNode", ): actor.get._bind(InputNode(), 1, 2) with pytest.raises( ValueError, match="ensure InputNode is the only input to a ClassMethodNode", ): actor.get._bind(1, 2, key=InputNode()) with pytest.raises( ValueError, match="ensure InputNode is the only input to a ClassMethodNode", ): actor.get._bind(InputNode(), key=123)
def test_func_class_mixed_input(shared_ray_instance): """ Test both class method and function are used as input in the same dag. """ @ray.remote class Model: def __init__(self, weight: int): self.weight = weight def forward(self, input: int): return self.weight * input @ray.remote def model_func(input: int): return input * 2 @ray.remote def combine(m1: "RayHandleLike", m2: "RayHandleLike"): return m1 + m2 m1 = Model._bind(3) m1_output = m1.forward._bind(InputNode()) m2_output = model_func._bind(InputNode()) dag = combine._bind(m1_output, m2_output) print(dag) # 2*3 + 2*2 assert ray.get(dag.execute(2)) == 10 # 3*3 + 3*2 assert ray.get(dag.execute(3)) == 15
def test_ensure_in_context_manager(shared_ray_instance): # No enforcement on creation given __enter__ executes after __init__ input = InputNode() with pytest.raises( AssertionError, match=( "InputNode is a singleton instance that should be only used " "in context manager" ), ): input.execute() @ray.remote def f(input): return input # No enforcement on creation given __enter__ executes after __init__ dag = f.bind(InputNode()) with pytest.raises( AssertionError, match=( "InputNode is a singleton instance that should be only used " "in context manager" ), ): dag.execute()
def test_multi_class_method_input(shared_ray_instance): """ Test a multiple class methods can all be used as inputs in a dag. """ @ray.remote class Model: def __init__(self, weight: int): self.weight = weight def forward(self, input: int): return self.weight * input @ray.remote def combine(m1: "RayHandleLike", m2: "RayHandleLike"): return m1 + m2 m1 = Model._bind(2) m2 = Model._bind(3) m1_output = m1.forward._bind(InputNode()) m2_output = m2.forward._bind(InputNode()) dag = combine._bind(m1_output, m2_output) print(dag) # 1*2 + 1*3 assert ray.get(dag.execute(1)) == 5 # 2*2 + 2*3 assert ray.get(dag.execute(2)) == 10
def test_no_args_to_input_node(shared_ray_instance): @ray.remote def f(input): return input with pytest.raises(ValueError, match="InputNode should not take any args or kwargs"): f._bind(InputNode(0)) with pytest.raises(ValueError, match="InputNode should not take any args or kwargs"): f._bind(InputNode(key=1))
def test_deploment_options_func_class_with_class_method(): with InputNode() as dag_input: counter = Counter.bind() m1 = Model.options(name="m1", max_concurrent_queries=3).bind(1) m2 = Model.options(name="m2", max_concurrent_queries=5).bind(2) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[1]) combine_output = combine.options(num_replicas=3, max_concurrent_queries=7).bind( m1_output, m2_output, kwargs_output=dag_input[2] ) dag = counter.__call__.bind(combine_output) serve_dag = Driver.bind(dag) deployments = pipeline_build(serve_dag) hit_count = 0 for deployment in deployments: if deployment.name == "counter": assert deployment.num_replicas == 2 assert deployment.user_config == {"count": 123, "b": 2} hit_count += 1 elif deployment.name == "m1": assert deployment.max_concurrent_queries == 3 hit_count += 1 elif deployment.name == "m2": assert deployment.max_concurrent_queries == 5 hit_count += 1 elif deployment.name == "combine": assert deployment.num_replicas == 3 assert deployment.max_concurrent_queries == 7 hit_count += 1 assert hit_count == 4, "Not all deployments with expected name were found."
def test_func_dag(shared_ray_instance): @ray.remote def a(user_input): return user_input @ray.remote def b(x): return x * 2 @ray.remote def c(x): return x + 1 @ray.remote def d(x, y): return x + y a_ref = a._bind(InputNode()) b_ref = b._bind(a_ref) c_ref = c._bind(a_ref) d_ref = d._bind(b_ref, c_ref) d1_ref = d._bind(d_ref, d_ref) d2_ref = d._bind(d1_ref, d_ref) dag = d._bind(d2_ref, d_ref) print(dag) # [(2*2 + 2+1) + (2*2 + 2+1)] + [(2*2 + 2+1) + (2*2 + 2+1)] assert ray.get(dag.execute(2)) == 28 # [(3*2 + 3+1) + (3*2 + 3+1)] + [(3*2 + 3+1) + (3*2 + 3+1)] assert ray.get(dag.execute(3)) == 40
def test_class_method_input(shared_ray_instance): @ray.remote class Model: def __init__(self, weight: int): self.weight = weight def forward(self, input: "RayHandleLike"): return self.weight * input @ray.remote class FeatureProcessor: def __init__(self, scale): self.scale = scale def process(self, input: int): return input * self.scale preprocess = FeatureProcessor._bind(0.5) feature = preprocess.process._bind(InputNode()) model = Model._bind(4) dag = model.forward._bind(feature) print(dag) # 2 * 0.5 * 4 assert ray.get(dag.execute(2)) == 4 # 6 * 0.5 * 4 assert ray.get(dag.execute(6)) == 12
def get_shared_deployment_handle_dag(): with InputNode() as dag_input: m = Model.bind(2) combine = Combine.bind(m, m2=m) ray_dag = combine.__call__.bind(dag_input) return ray_dag, dag_input
def test_autoscaling_0_replica(serve_instance): autoscaling_config = { "metrics_interval_s": 0.1, "min_replicas": 0, "max_replicas": 2, "look_back_period_s": 0.4, "downscale_delay_s": 0, "upscale_delay_s": 0, } @serve.deployment( _autoscaling_config=autoscaling_config, ) class Model: def __init__(self, weight): self.weight = weight def forward(self, input): return input + self.weight with InputNode() as user_input: model = Model.bind(1) output = model.forward.bind(user_input) serve_dag = DAGDriver.options( route_prefix="/my-dag", _autoscaling_config=autoscaling_config, ).bind(output) dag_handle = serve.run(serve_dag) assert 2 == ray.get(dag_handle.predict.remote(1))
def test_model_wrappers_in_pipeline(serve_instance): _, path = tempfile.mkstemp() with open(path, "w") as f: json.dump(2, f) predictor_cls = "ray.serve.tests.test_model_wrappers.AdderPredictor" checkpoint_cls = "ray.serve.tests.test_model_wrappers.AdderCheckpoint" with InputNode() as dag_input: m1 = ModelWrapperDeployment.bind( predictor_cls=predictor_cls, # TODO: can't be the raw class right now? checkpoint={ # TODO: can't be the raw object right now? "checkpoint_cls": checkpoint_cls, "uri": path, }, ) dag = m1.predict.bind(dag_input) deployments = build(Ingress.bind(dag)) for d in deployments: d.deploy() resp = requests.post("http://127.0.0.1:8000/ingress", json={"array": [40]}) print(resp.text) resp.raise_for_status() return resp.json() == {"value": [42], "batch_size": 1}
def test_single_func_deployment_dag(serve_instance, use_build): with InputNode() as dag_input: dag = combine.bind(dag_input[0], dag_input[1], kwargs_output=1) serve_dag = DAGDriver.bind(dag, input_schema=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote([1, 2])) == 4 assert requests.post("http://127.0.0.1:8000/", json=[1, 2]).json() == 4
def test_nested_building(): with InputNode() as inp: out = func.bind(inp) out = Driver.bind().__call__.bind(out) out = func.bind(out) dag = Driver.bind(out, func.bind()) assert len(pipeline_build(dag)) == 5
def get_multi_instantiation_class_nested_deployment_arg_dag(): with InputNode() as dag_input: m1 = Model.bind(2) m2 = Model.bind(3) combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True) ray_dag = combine.__call__.bind(dag_input) return ray_dag, dag_input
def get_multi_instantiation_class_deployment_in_init_args_dag(): with InputNode() as dag_input: m1 = Model.bind(2) m2 = Model.bind(3) combine = Combine.bind(m1, m2=m2) ray_dag = combine.__call__.bind(dag_input) return ray_dag, dag_input
def test_simple_class_with_class_method(serve_instance, use_build): with InputNode() as dag_input: model = Model.bind(2, ratio=0.3) dag = model.forward.bind(dag_input) serve_dag = DAGDriver.bind(dag, input_schema=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 0.6 assert requests.post("http://127.0.0.1:8000/", json=1).json() == 0.6
def test_class_factory(serve_instance): with InputNode() as _: instance = serve.deployment(class_factory()).bind(3) output = instance.get.bind() serve_dag = NoargDriver.bind(output) handle = serve.run(serve_dag) assert ray.get(handle.remote()) == 3 assert requests.get("http://127.0.0.1:8000/").text == "3"
def test_ensure_input_node_singleton(shared_ray_instance): @ray.remote def f(input): return input @ray.remote def combine(a, b): return a + b with InputNode() as input_1: a = f.bind(input_1) with InputNode() as input_2: b = f.bind(input_2) dag = combine.bind(a, b) with pytest.raises( AssertionError, match="Each DAG should only have one unique InputNode" ): _ = ray.get(dag.execute(2))
def test_single_node_driver_sucess(serve_instance, use_build): m1 = Adder.bind(1) m2 = Adder.bind(2) with InputNode() as input_node: out = m1.forward.bind(input_node) out = m2.forward.bind(out) driver = DAGDriver.bind(out, input_schema=json_resolver) handle = serve.run(driver) assert ray.get(handle.predict.remote(39)) == 42 assert requests.post("http://127.0.0.1:8000/", json=39).json() == 42
def test_shared_deployment_handle(serve_instance, use_build): with InputNode() as dag_input: m = Model.bind(2) combine = Combine.bind(m, m2=m) combine_output = combine.bind(dag_input) serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 4 assert requests.post("http://127.0.0.1:8000/", json=1).json() == 4
def test_no_input_node_as_init_args(): """ User should NOT directly create instances of Deployment or DeploymentNode. """ with pytest.raises( ValueError, match="cannot be used as args, kwargs, or other_args_to_resolve", ): _ = DeploymentNode( Actor, "test", (InputNode()), {}, {}, other_args_to_resolve={USE_SYNC_HANDLE_KEY: True}, ) with pytest.raises( ValueError, match="cannot be used as args, kwargs, or other_args_to_resolve", ): _ = DeploymentNode( Actor, "test", (), {"a": InputNode()}, {}, other_args_to_resolve={USE_SYNC_HANDLE_KEY: True}, ) with pytest.raises( ValueError, match="cannot be used as args, kwargs, or other_args_to_resolve", ): _ = DeploymentNode( Actor, "test", (), {}, {}, other_args_to_resolve={"arg": { "options_a": InputNode() }}, )
def test_multi_instantiation_class_nested_deployment_arg_dag(serve_instance, use_build): with InputNode() as dag_input: m1 = Model.bind(2) m2 = Model.bind(3) combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True) output = combine.bind(dag_input) serve_dag = DAGDriver.bind(output, input_schema=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 5 assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def get_func_class_with_class_method_dag(): with InputNode() as dag_input: m1 = Model.bind(1) m2 = Model.bind(2) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[1]) ray_dag = combine.bind(m1_output, m2_output, kwargs_output=dag_input[2]) return ray_dag, dag_input
def test_multi_instantiation_class_deployment_in_init_args(serve_instance): with InputNode() as dag_input: m1 = Model.bind(2) m2 = Model.bind(3) combine = Combine.bind(m1, m2=m2) combine_output = combine.bind(dag_input) serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote(1)) == 5 assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def test_dag_driver_custom_schema(serve_instance): with InputNode() as inp: dag = echo.bind(inp) handle = serve.run(DAGDriver.bind(dag, input_schema=resolver)) assert ray.get(handle.predict.remote(42)) == 42 resp = requests.get("http://127.0.0.1:8000/?my_custom_param=100") print(resp.text) resp.raise_for_status() assert resp.json() == 100
def test_chain_of_values(): with InputNode() as dag_input: out = fn.bind(1) out_2 = fn.bind(out, incr=2) out_val = fn.bind(out_2, incr=3) model = Model.bind(out_val) ray_dag = model.forward.bind(dag_input) json_serialized = json.dumps(ray_dag, cls=DAGNodeEncoder) deserialized_dag_node = json.loads(json_serialized, object_hook=dagnode_from_json) assert ray.get(deserialized_dag_node.execute(2)) == ray.get(ray_dag.execute(2))
def test_func_class_with_class_method(serve_instance, use_build): with InputNode() as dag_input: m1 = Model.bind(1) m2 = Model.bind(2) m1_output = m1.forward.bind(dag_input[0]) m2_output = m2.forward.bind(dag_input[1]) combine_output = combine.bind(m1_output, m2_output, kwargs_output=dag_input[2]) serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver) handle = serve.run(serve_dag) assert ray.get(handle.predict.remote([1, 2, 3])) == 8 assert requests.post("http://127.0.0.1:8000/", json=[1, 2, 3]).json() == 8
def test_invalid_input_node_in_function_node(shared_ray_instance): @ray.remote def f(input): return input with pytest.raises( ValueError, match="ensure InputNode is the only input to a FunctionNode" ): f._bind([[{"nested": InputNode()}]]) with pytest.raises( ValueError, match="ensure InputNode is the only input to a FunctionNode" ): f._bind(InputNode(), 1, 2) with pytest.raises( ValueError, match="ensure InputNode is the only input to a FunctionNode" ): f._bind(1, 2, key=InputNode()) with pytest.raises( ValueError, match="ensure InputNode is the only input to a FunctionNode" ): f._bind(InputNode(), key=123)
def test_dag_driver_default(serve_instance): with InputNode() as inp: dag = echo.bind(inp) handle = serve.run(DAGDriver.bind(dag)) assert ray.get(handle.predict.remote(42)) == 42 resp = requests.post("http://127.0.0.1:8000/", json={"array": [1]}) print(resp.text) resp.raise_for_status() assert resp.json() == "starlette!"
def test_multi_input_func_dag(shared_ray_instance): @ray.remote def a(user_input): return user_input * 2 @ray.remote def b(user_input): return user_input + 1 @ray.remote def c(x, y): return x + y a_ref = a._bind(InputNode()) b_ref = b._bind(InputNode()) dag = c._bind(a_ref, b_ref) print(dag) # (2*2) + (2*1) assert ray.get(dag.execute(2)) == 7 # (3*2) + (3*1) assert ray.get(dag.execute(3)) == 10