def test_invalid_input_node_in_class_method_node(shared_ray_instance):
@ray.remote
class Actor:
def __init__(self, val):
self.val = val
def get(self, input1, input2):
return self.val + input1 + input2
actor = Actor._bind(1)
with pytest.raises(
ValueError,
match="ensure InputNode is the only input to a ClassMethodNode",
):
actor.get._bind([[{"nested": InputNode()}]])
with pytest.raises(
ValueError,
match="ensure InputNode is the only input to a ClassMethodNode",
):
actor.get._bind(InputNode(), 1, 2)
with pytest.raises(
ValueError,
match="ensure InputNode is the only input to a ClassMethodNode",
):
actor.get._bind(1, 2, key=InputNode())
with pytest.raises(
ValueError,
match="ensure InputNode is the only input to a ClassMethodNode",
):
actor.get._bind(InputNode(), key=123)
def test_func_class_mixed_input(shared_ray_instance):
"""
Test both class method and function are used as input in the
same dag.
"""
@ray.remote
class Model:
def __init__(self, weight: int):
self.weight = weight
def forward(self, input: int):
return self.weight * input
@ray.remote
def model_func(input: int):
return input * 2
@ray.remote
def combine(m1: "RayHandleLike", m2: "RayHandleLike"):
return m1 + m2
m1 = Model._bind(3)
m1_output = m1.forward._bind(InputNode())
m2_output = model_func._bind(InputNode())
dag = combine._bind(m1_output, m2_output)
print(dag)
# 2*3 + 2*2
assert ray.get(dag.execute(2)) == 10
# 3*3 + 3*2
assert ray.get(dag.execute(3)) == 15
def test_ensure_in_context_manager(shared_ray_instance):
# No enforcement on creation given __enter__ executes after __init__
input = InputNode()
with pytest.raises(
AssertionError,
match=(
"InputNode is a singleton instance that should be only used "
"in context manager"
),
):
input.execute()
@ray.remote
def f(input):
return input
# No enforcement on creation given __enter__ executes after __init__
dag = f.bind(InputNode())
with pytest.raises(
AssertionError,
match=(
"InputNode is a singleton instance that should be only used "
"in context manager"
),
):
dag.execute()
def test_multi_class_method_input(shared_ray_instance):
"""
Test a multiple class methods can all be used as inputs in a dag.
"""
@ray.remote
class Model:
def __init__(self, weight: int):
self.weight = weight
def forward(self, input: int):
return self.weight * input
@ray.remote
def combine(m1: "RayHandleLike", m2: "RayHandleLike"):
return m1 + m2
m1 = Model._bind(2)
m2 = Model._bind(3)
m1_output = m1.forward._bind(InputNode())
m2_output = m2.forward._bind(InputNode())
dag = combine._bind(m1_output, m2_output)
print(dag)
# 1*2 + 1*3
assert ray.get(dag.execute(1)) == 5
# 2*2 + 2*3
assert ray.get(dag.execute(2)) == 10
def test_no_args_to_input_node(shared_ray_instance):
@ray.remote
def f(input):
return input
with pytest.raises(ValueError,
match="InputNode should not take any args or kwargs"):
f._bind(InputNode(0))
with pytest.raises(ValueError,
match="InputNode should not take any args or kwargs"):
f._bind(InputNode(key=1))
def test_deploment_options_func_class_with_class_method():
with InputNode() as dag_input:
counter = Counter.bind()
m1 = Model.options(name="m1", max_concurrent_queries=3).bind(1)
m2 = Model.options(name="m2", max_concurrent_queries=5).bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.options(num_replicas=3, max_concurrent_queries=7).bind(
m1_output, m2_output, kwargs_output=dag_input[2]
)
dag = counter.__call__.bind(combine_output)
serve_dag = Driver.bind(dag)
deployments = pipeline_build(serve_dag)
hit_count = 0
for deployment in deployments:
if deployment.name == "counter":
assert deployment.num_replicas == 2
assert deployment.user_config == {"count": 123, "b": 2}
hit_count += 1
elif deployment.name == "m1":
assert deployment.max_concurrent_queries == 3
hit_count += 1
elif deployment.name == "m2":
assert deployment.max_concurrent_queries == 5
hit_count += 1
elif deployment.name == "combine":
assert deployment.num_replicas == 3
assert deployment.max_concurrent_queries == 7
hit_count += 1
assert hit_count == 4, "Not all deployments with expected name were found."
def test_func_dag(shared_ray_instance):
@ray.remote
def a(user_input):
return user_input
@ray.remote
def b(x):
return x * 2
@ray.remote
def c(x):
return x + 1
@ray.remote
def d(x, y):
return x + y
a_ref = a._bind(InputNode())
b_ref = b._bind(a_ref)
c_ref = c._bind(a_ref)
d_ref = d._bind(b_ref, c_ref)
d1_ref = d._bind(d_ref, d_ref)
d2_ref = d._bind(d1_ref, d_ref)
dag = d._bind(d2_ref, d_ref)
print(dag)
# [(2*2 + 2+1) + (2*2 + 2+1)] + [(2*2 + 2+1) + (2*2 + 2+1)]
assert ray.get(dag.execute(2)) == 28
# [(3*2 + 3+1) + (3*2 + 3+1)] + [(3*2 + 3+1) + (3*2 + 3+1)]
assert ray.get(dag.execute(3)) == 40
def test_class_method_input(shared_ray_instance):
@ray.remote
class Model:
def __init__(self, weight: int):
self.weight = weight
def forward(self, input: "RayHandleLike"):
return self.weight * input
@ray.remote
class FeatureProcessor:
def __init__(self, scale):
self.scale = scale
def process(self, input: int):
return input * self.scale
preprocess = FeatureProcessor._bind(0.5)
feature = preprocess.process._bind(InputNode())
model = Model._bind(4)
dag = model.forward._bind(feature)
print(dag)
# 2 * 0.5 * 4
assert ray.get(dag.execute(2)) == 4
# 6 * 0.5 * 4
assert ray.get(dag.execute(6)) == 12
def get_shared_deployment_handle_dag():
with InputNode() as dag_input:
m = Model.bind(2)
combine = Combine.bind(m, m2=m)
ray_dag = combine.__call__.bind(dag_input)
return ray_dag, dag_input
def test_autoscaling_0_replica(serve_instance):
autoscaling_config = {
"metrics_interval_s": 0.1,
"min_replicas": 0,
"max_replicas": 2,
"look_back_period_s": 0.4,
"downscale_delay_s": 0,
"upscale_delay_s": 0,
}
@serve.deployment(
_autoscaling_config=autoscaling_config, )
class Model:
def __init__(self, weight):
self.weight = weight
def forward(self, input):
return input + self.weight
with InputNode() as user_input:
model = Model.bind(1)
output = model.forward.bind(user_input)
serve_dag = DAGDriver.options(
route_prefix="/my-dag",
_autoscaling_config=autoscaling_config,
).bind(output)
dag_handle = serve.run(serve_dag)
assert 2 == ray.get(dag_handle.predict.remote(1))
def test_model_wrappers_in_pipeline(serve_instance):
_, path = tempfile.mkstemp()
with open(path, "w") as f:
json.dump(2, f)
predictor_cls = "ray.serve.tests.test_model_wrappers.AdderPredictor"
checkpoint_cls = "ray.serve.tests.test_model_wrappers.AdderCheckpoint"
with InputNode() as dag_input:
m1 = ModelWrapperDeployment.bind(
predictor_cls=predictor_cls, # TODO: can't be the raw class right now?
checkpoint={ # TODO: can't be the raw object right now?
"checkpoint_cls": checkpoint_cls,
"uri": path,
},
)
dag = m1.predict.bind(dag_input)
deployments = build(Ingress.bind(dag))
for d in deployments:
d.deploy()
resp = requests.post("http://127.0.0.1:8000/ingress", json={"array": [40]})
print(resp.text)
resp.raise_for_status()
return resp.json() == {"value": [42], "batch_size": 1}
def test_single_func_deployment_dag(serve_instance, use_build):
with InputNode() as dag_input:
dag = combine.bind(dag_input[0], dag_input[1], kwargs_output=1)
serve_dag = DAGDriver.bind(dag, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote([1, 2])) == 4
assert requests.post("http://127.0.0.1:8000/", json=[1, 2]).json() == 4
def test_nested_building():
with InputNode() as inp:
out = func.bind(inp)
out = Driver.bind().__call__.bind(out)
out = func.bind(out)
dag = Driver.bind(out, func.bind())
assert len(pipeline_build(dag)) == 5
def get_multi_instantiation_class_nested_deployment_arg_dag():
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True)
ray_dag = combine.__call__.bind(dag_input)
return ray_dag, dag_input
def get_multi_instantiation_class_deployment_in_init_args_dag():
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2=m2)
ray_dag = combine.__call__.bind(dag_input)
return ray_dag, dag_input
def test_simple_class_with_class_method(serve_instance, use_build):
with InputNode() as dag_input:
model = Model.bind(2, ratio=0.3)
dag = model.forward.bind(dag_input)
serve_dag = DAGDriver.bind(dag, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 0.6
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 0.6
def test_class_factory(serve_instance):
with InputNode() as _:
instance = serve.deployment(class_factory()).bind(3)
output = instance.get.bind()
serve_dag = NoargDriver.bind(output)
handle = serve.run(serve_dag)
assert ray.get(handle.remote()) == 3
assert requests.get("http://127.0.0.1:8000/").text == "3"
def test_ensure_input_node_singleton(shared_ray_instance):
@ray.remote
def f(input):
return input
@ray.remote
def combine(a, b):
return a + b
with InputNode() as input_1:
a = f.bind(input_1)
with InputNode() as input_2:
b = f.bind(input_2)
dag = combine.bind(a, b)
with pytest.raises(
AssertionError, match="Each DAG should only have one unique InputNode"
):
_ = ray.get(dag.execute(2))
def test_single_node_driver_sucess(serve_instance, use_build):
m1 = Adder.bind(1)
m2 = Adder.bind(2)
with InputNode() as input_node:
out = m1.forward.bind(input_node)
out = m2.forward.bind(out)
driver = DAGDriver.bind(out, input_schema=json_resolver)
handle = serve.run(driver)
assert ray.get(handle.predict.remote(39)) == 42
assert requests.post("http://127.0.0.1:8000/", json=39).json() == 42
def test_shared_deployment_handle(serve_instance, use_build):
with InputNode() as dag_input:
m = Model.bind(2)
combine = Combine.bind(m, m2=m)
combine_output = combine.bind(dag_input)
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 4
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 4
def test_no_input_node_as_init_args():
"""
User should NOT directly create instances of Deployment or DeploymentNode.
"""
with pytest.raises(
ValueError,
match="cannot be used as args, kwargs, or other_args_to_resolve",
):
_ = DeploymentNode(
Actor,
"test",
(InputNode()),
{},
{},
other_args_to_resolve={USE_SYNC_HANDLE_KEY: True},
)
with pytest.raises(
ValueError,
match="cannot be used as args, kwargs, or other_args_to_resolve",
):
_ = DeploymentNode(
Actor,
"test",
(),
{"a": InputNode()},
{},
other_args_to_resolve={USE_SYNC_HANDLE_KEY: True},
)
with pytest.raises(
ValueError,
match="cannot be used as args, kwargs, or other_args_to_resolve",
):
_ = DeploymentNode(
Actor,
"test",
(),
{},
{},
other_args_to_resolve={"arg": {
"options_a": InputNode()
}},
)
def test_multi_instantiation_class_nested_deployment_arg_dag(serve_instance, use_build):
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True)
output = combine.bind(dag_input)
serve_dag = DAGDriver.bind(output, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 5
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def get_func_class_with_class_method_dag():
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
ray_dag = combine.bind(m1_output,
m2_output,
kwargs_output=dag_input[2])
return ray_dag, dag_input
def test_multi_instantiation_class_deployment_in_init_args(serve_instance):
with InputNode() as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2=m2)
combine_output = combine.bind(dag_input)
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote(1)) == 5
assert requests.post("http://127.0.0.1:8000/", json=1).json() == 5
def test_dag_driver_custom_schema(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
handle = serve.run(DAGDriver.bind(dag, input_schema=resolver))
assert ray.get(handle.predict.remote(42)) == 42
resp = requests.get("http://127.0.0.1:8000/?my_custom_param=100")
print(resp.text)
resp.raise_for_status()
assert resp.json() == 100
def test_chain_of_values():
with InputNode() as dag_input:
out = fn.bind(1)
out_2 = fn.bind(out, incr=2)
out_val = fn.bind(out_2, incr=3)
model = Model.bind(out_val)
ray_dag = model.forward.bind(dag_input)
json_serialized = json.dumps(ray_dag, cls=DAGNodeEncoder)
deserialized_dag_node = json.loads(json_serialized, object_hook=dagnode_from_json)
assert ray.get(deserialized_dag_node.execute(2)) == ray.get(ray_dag.execute(2))
def test_func_class_with_class_method(serve_instance, use_build):
with InputNode() as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
combine_output = combine.bind(m1_output, m2_output, kwargs_output=dag_input[2])
serve_dag = DAGDriver.bind(combine_output, input_schema=json_resolver)
handle = serve.run(serve_dag)
assert ray.get(handle.predict.remote([1, 2, 3])) == 8
assert requests.post("http://127.0.0.1:8000/", json=[1, 2, 3]).json() == 8
def test_invalid_input_node_in_function_node(shared_ray_instance):
@ray.remote
def f(input):
return input
with pytest.raises(
ValueError, match="ensure InputNode is the only input to a FunctionNode"
):
f._bind([[{"nested": InputNode()}]])
with pytest.raises(
ValueError, match="ensure InputNode is the only input to a FunctionNode"
):
f._bind(InputNode(), 1, 2)
with pytest.raises(
ValueError, match="ensure InputNode is the only input to a FunctionNode"
):
f._bind(1, 2, key=InputNode())
with pytest.raises(
ValueError, match="ensure InputNode is the only input to a FunctionNode"
):
f._bind(InputNode(), key=123)
def test_dag_driver_default(serve_instance):
with InputNode() as inp:
dag = echo.bind(inp)
handle = serve.run(DAGDriver.bind(dag))
assert ray.get(handle.predict.remote(42)) == 42
resp = requests.post("http://127.0.0.1:8000/", json={"array": [1]})
print(resp.text)
resp.raise_for_status()
assert resp.json() == "starlette!"
def test_multi_input_func_dag(shared_ray_instance):
@ray.remote
def a(user_input):
return user_input * 2
@ray.remote
def b(user_input):
return user_input + 1
@ray.remote
def c(x, y):
return x + y
a_ref = a._bind(InputNode())
b_ref = b._bind(InputNode())
dag = c._bind(a_ref, b_ref)
print(dag)
# (2*2) + (2*1)
assert ray.get(dag.execute(2)) == 7
# (3*2) + (3*1)
assert ray.get(dag.execute(3)) == 10