def test_multi_instantiation_class_nested_deployment_arg(serve_instance):
"""
Test we can pass deployments with **nested** init_arg or init_kwarg,
instantiated multiple times for the same class, and we can still correctly
replace args with deployment handle and parse correct deployment instances.
"""
m1 = Model._bind(2)
m2 = Model._bind(3)
combine = Combine._bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True)
ray_dag = combine.__call__._bind(InputNode())
print(f"Ray DAG: \n{ray_dag}")
serve_root_dag = ray_dag._apply_recursive(transform_ray_dag_to_serve_dag)
print(f"Serve DAG: \n{serve_root_dag}")
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 3
# Ensure Deployments with other deployment nodes in init arg are replaced
# with correct handle
combine_deployment = deployments[2]
init_arg_handle = combine_deployment.init_args[0]
assert isinstance(init_arg_handle, RayServeSyncHandle)
assert init_arg_handle.deployment_name == "Model"
init_kwarg_handle = combine_deployment.init_kwargs["m2"][NESTED_HANDLE_KEY]
assert isinstance(init_kwarg_handle, RayServeSyncHandle)
assert init_kwarg_handle.deployment_name == "Model_1"
for deployment in deployments:
deployment.deploy()
_validate_consistent_output(deployments[2],
ray_dag,
"Combine",
input=1,
output=5)
def test_wide_fanout_deployment_graph(fanout_degree,
init_delay_secs=0,
compute_delay_secs=0):
"""
Test that focuses on wide fanout of deployment graph
-> Node_1
/ \
INPUT --> Node_2 --> combine -> OUTPUT
\ ... /
-> Node_10
1) Intermediate blob size can be large / small
2) Compute time each node can be long / short
3) Init time can be long / short
"""
nodes = [
Node.bind(i, init_delay_secs=init_delay_secs)
for i in range(0, fanout_degree)
]
outputs = []
with InputNode() as user_input:
for i in range(0, fanout_degree):
outputs.append(nodes[i].compute.bind(
user_input, compute_delay_secs=compute_delay_secs))
dag = combine.bind(outputs)
serve_dag = DAGDriver.bind(dag)
return serve_dag
def test_long_chain_deployment_graph(
chain_length, init_delay_secs=0, compute_delay_secs=0
):
"""
Test that focuses on long chain of deployment graph
INPUT -> Node_1 -> Node_2 -> ... -> Node_10 -> OUTPUT
1) Intermediate blob size can be large / small
2) Compute time each node can be long / short
3) Init time can be long / short
"""
nodes = [Node.bind(i, init_delay_secs=init_delay_secs) for i in range(chain_length)]
prev_outputs = [None for _ in range(chain_length)]
with InputNode() as user_input:
for i in range(chain_length):
if i == 0:
prev_outputs[i] = nodes[i].compute.bind(
user_input, compute_delay_secs=compute_delay_secs
)
else:
prev_outputs[i] = nodes[i].compute.bind(
prev_outputs[i - 1], compute_delay_secs=compute_delay_secs
)
serve_dag = DAGDriver.bind(prev_outputs[-1])
return serve_dag
def test_dag_to_workflow_execution(workflow_start_regular_shared):
"""This test constructs a DAG with complex dependencies
and turns it into a workflow."""
@ray.remote
def begin(x, pos, a):
return x * a + pos # 23.14
@ray.remote
def left(x, c, a):
return f"left({x}, {c}, {a})"
@ray.remote
def right(x, b, pos):
return f"right({x}, {b}, {pos})"
@ray.remote
def end(lf, rt, b):
return f"{lf},{rt};{b}"
with pytest.raises(TypeError):
workflow.create(begin.remote(1, 2, 3))
with InputNode() as dag_input:
f = begin.bind(2, dag_input[1], a=dag_input.a)
lf = left.bind(f, "hello", dag_input.a)
rt = right.bind(f, b=dag_input.b, pos=dag_input[0])
b = end.bind(lf, rt, b=dag_input.b)
wf = workflow.create(b, 2, 3.14, a=10, b="ok")
assert len(list(wf._iter_workflows_in_dag())) == 4, "incorrect amount of steps"
assert wf.run() == "left(23.14, hello, 10),right(23.14, ok, 2);ok"
def test_get_pipeline_input_node():
# 1) No PipelineInputNode found
ray_dag = combine.bind(1, 2)
serve_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
with pytest.raises(
AssertionError,
match="There should be one and only one PipelineInputNode"):
get_pipeline_input_node(serve_dag)
# 2) More than one PipelineInputNode found
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
a = combine.bind(dag_input[0], dag_input[1])
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input_2:
b = combine.bind(dag_input_2[0], dag_input_2[1])
ray_dag = combine.bind(a, b)
serve_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
with pytest.raises(
AssertionError,
match="There should be one and only one PipelineInputNode"):
get_pipeline_input_node(serve_dag)
# 3) User forgot to change InputNode to PipelineInputNode
with InputNode() as dag_input:
ray_dag = combine.bind(dag_input[0], dag_input[1])
serve_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
with pytest.raises(
ValueError,
match="Please change Ray DAG InputNode to PipelineInputNode"):
get_pipeline_input_node(serve_dag)
def test_get_pipeline_input_node():
# 1) No InputNode found
ray_dag = combine.bind(1, 2)
serve_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
with pytest.raises(AssertionError,
match="There should be one and only one InputNode"):
get_pipeline_input_node(serve_dag)
# 2) More than one InputNode found
with InputNode() as dag_input:
a = combine.bind(dag_input[0], dag_input[1])
with InputNode() as dag_input_2:
b = combine.bind(dag_input_2[0], dag_input_2[1])
ray_dag = combine.bind(a, b)
with pytest.raises(AssertionError,
match="Each DAG should only have one unique InputNode"):
serve_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
get_pipeline_input_node(serve_dag)
def dagnode_from_json(input_json: Any) -> Union[DAGNode, RayServeHandle, Any]:
"""
Decode a DAGNode from given input json dictionary. JSON serialization is
only used and enforced in ray serve from ray core API authored DAGNode(s).
Covers both RayServeHandle and DAGNode types.
Assumptions:
- User object's JSON dict does not have keys that collide with our
reserved DAGNODE_TYPE_KEY
- RayServeHandle and Deployment can be re-constructed without losing
states needed for their functionality or correctness.
- DAGNode type can be re-constructed with new stable_uuid upon each
deserialization without effective correctness of execution.
- Only exception is ClassNode used as parent of ClassMethodNode
that we perserve the same parent node.
- .options() does not contain any DAGNode type
"""
# Deserialize RayServeHandle type
if SERVE_HANDLE_JSON_KEY in input_json:
return serve_handle_from_json_dict(input_json)
# Base case for plain objects
elif DAGNODE_TYPE_KEY not in input_json:
return input_json
elif input_json[DAGNODE_TYPE_KEY] == RayServeDAGHandle.__name__:
return RayServeDAGHandle(input_json["dag_node_json"])
elif input_json[DAGNODE_TYPE_KEY] == "DeploymentSchema":
return DeploymentSchema.parse_obj(input_json["schema"])
elif input_json[DAGNODE_TYPE_KEY] == RayServeLazySyncHandle.__name__:
return RayServeLazySyncHandle(
input_json["deployment_name"],
HandleOptions(input_json["handle_options_method_name"]),
)
# Deserialize DAGNode type
elif input_json[DAGNODE_TYPE_KEY] == InputNode.__name__:
return InputNode.from_json(input_json)
elif input_json[DAGNODE_TYPE_KEY] == InputAttributeNode.__name__:
return InputAttributeNode.from_json(input_json)
elif input_json[DAGNODE_TYPE_KEY] == ClassMethodNode.__name__:
return ClassMethodNode.from_json(input_json)
elif input_json[DAGNODE_TYPE_KEY] == DeploymentNode.__name__:
return DeploymentNode.from_json(input_json)
elif input_json[DAGNODE_TYPE_KEY] == DeploymentMethodNode.__name__:
return DeploymentMethodNode.from_json(input_json)
elif input_json[DAGNODE_TYPE_KEY] == DeploymentFunctionNode.__name__:
return DeploymentFunctionNode.from_json(input_json)
else:
# Class and Function nodes require original module as body.
module_name, attr_name = parse_import_path(input_json["import_path"])
module = getattr(import_module(module_name), attr_name)
if input_json[DAGNODE_TYPE_KEY] == FunctionNode.__name__:
return FunctionNode.from_json(input_json, module)
elif input_json[DAGNODE_TYPE_KEY] == ClassNode.__name__:
return ClassNode.from_json(input_json, module)
def test_single_class_with_invalid_deployment_options(serve_instance):
model = Model.options(name="my_deployment")._bind(2, ratio=0.3)
ray_dag = model.forward._bind(InputNode())
serve_root_dag = ray_dag._apply_recursive(transform_ray_dag_to_serve_dag)
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 1
with pytest.raises(
ValueError,
match="Specifying name in ray_actor_options is not allowed"):
deployments[0].deploy()
def test_simple_single_class(serve_instance):
# Assert converting both arg and kwarg
model = Model._bind(2, ratio=0.3)
ray_dag = model.forward._bind(InputNode())
serve_root_dag = ray_dag._apply_recursive(transform_ray_dag_to_serve_dag)
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 1
deployments[0].deploy()
_validate_consistent_output(deployments[0],
ray_dag,
"Model",
input=1,
output=0.6)
def test_single_class_with_invalid_deployment_options(serve_instance):
with InputNode() as dag_input:
model = Model.options(name="my_deployment").bind(2, ratio=0.3)
ray_dag = model.forward.bind(dag_input)
with DAGNodeNameGenerator() as node_name_generator:
serve_root_dag = ray_dag.apply_recursive(
lambda node: transform_ray_dag_to_serve_dag(
node, node_name_generator))
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 1
with pytest.raises(
ValueError,
match="Specifying 'name' in ray_actor_options is not allowed"):
deployments[0].deploy()
def test_get_pipeline_input_node():
# 1) No InputNode found
ray_dag = combine.bind(1, 2)
with DeploymentNameGenerator() as deployment_name_generator:
serve_dag = ray_dag.apply_recursive(
lambda node: transform_ray_dag_to_serve_dag(
node, deployment_name_generator))
with pytest.raises(AssertionError,
match="There should be one and only one InputNode"):
get_pipeline_input_node(serve_dag)
# 2) More than one InputNode found
with InputNode() as dag_input:
a = combine.bind(dag_input[0], dag_input[1])
with InputNode() as dag_input_2:
b = combine.bind(dag_input_2[0], dag_input_2[1])
ray_dag = combine.bind(a, b)
with pytest.raises(AssertionError,
match="Each DAG should only have one unique InputNode"):
with DeploymentNameGenerator() as deployment_name_generator:
serve_dag = ray_dag.apply_recursive(
lambda node: transform_ray_dag_to_serve_dag(
node, deployment_name_generator))
get_pipeline_input_node(serve_dag)
def test_single_class_with_valid_ray_options(serve_instance):
model = Model.options(num_cpus=1, memory=1000)._bind(2, ratio=0.3)
ray_dag = model.forward._bind(InputNode())
serve_root_dag = ray_dag._apply_recursive(transform_ray_dag_to_serve_dag)
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 1
deployments[0].deploy()
_validate_consistent_output(deployments[0],
ray_dag,
deployments[0].name,
input=1,
output=0.6)
deployment = serve.get_deployment(deployments[0].name)
assert deployment.ray_actor_options.get("num_cpus") == 1
assert deployment.ray_actor_options.get("memory") == 1000
assert deployment.ray_actor_options.get("runtime_env") == {}
def test_same_object_many_dags(workflow_start_regular_shared):
"""Ensure that when we dedupe uploads, we upload the object once per DAG,
since different DAGs shouldn't look in each others object directories.
"""
@ray.remote
def f(a):
return [a[0]]
x = {0: ray.put(10)}
result1 = workflow.create(f.bind(x)).run()
result2 = workflow.create(f.bind(x)).run()
with InputNode() as dag_input:
result3 = workflow.create(f.bind(dag_input.x), x=x).run()
assert ray.get(*result1) == 10
assert ray.get(*result2) == 10
assert ray.get(*result3) == 10
def test_shared_deployment_handle(serve_instance):
"""
Test we can re-use the same deployment handle multiple times or in
multiple places, without incorrectly parsing duplicated deployments.
"""
m = Model._bind(2)
combine = Combine._bind(m, m2=m)
ray_dag = combine.__call__._bind(InputNode())
print(f"Ray DAG: \n{ray_dag}")
serve_root_dag = ray_dag._apply_recursive(transform_ray_dag_to_serve_dag)
print(f"Serve DAG: \n{serve_root_dag}")
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 2
for deployment in deployments:
deployment.deploy()
_validate_consistent_output(deployments[1],
ray_dag,
"Combine",
input=1,
output=4)
def test_single_class_with_valid_ray_options(serve_instance):
with InputNode() as dag_input:
model = Model.options(num_cpus=1, memory=1000).bind(2, ratio=0.3)
ray_dag = model.forward.bind(dag_input)
with DAGNodeNameGenerator() as node_name_generator:
serve_root_dag = ray_dag.apply_recursive(
lambda node: transform_ray_dag_to_serve_dag(
node, node_name_generator))
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 1
deployments[0].deploy()
_validate_consistent_python_output(deployments[0],
ray_dag,
deployments[0].name,
input=1,
output=0.6)
deployment = serve.get_deployment(deployments[0].name)
assert deployment.ray_actor_options.get("num_cpus") == 1
assert deployment.ray_actor_options.get("memory") == 1000
assert deployment.ray_actor_options.get("runtime_env") == {}
def test_multi_instantiation_class_deployment_in_init_args(serve_instance):
"""
Test we can pass deployments as init_arg or init_kwarg, instantiated
multiple times for the same class, and we can still correctly replace
args with deployment handle and parse correct deployment instances.
"""
m1 = Model._bind(2)
m2 = Model._bind(3)
combine = Combine._bind(m1, m2=m2)
ray_dag = combine.__call__._bind(InputNode())
print(f"Ray DAG: \n{ray_dag}")
serve_root_dag = ray_dag._apply_recursive(transform_ray_dag_to_serve_dag)
print(f"Serve DAG: \n{serve_root_dag}")
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 3
for deployment in deployments:
deployment.deploy()
_validate_consistent_output(deployments[2],
ray_dag,
"Combine",
input=1,
output=5)
