def test_get_pipeline_input_node():
# 1) No PipelineInputNode found
ray_dag = combine.bind(1, 2)
serve_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
with pytest.raises(
AssertionError,
match="There should be one and only one PipelineInputNode"):
get_pipeline_input_node(serve_dag)
# 2) More than one PipelineInputNode found
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
a = combine.bind(dag_input[0], dag_input[1])
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input_2:
b = combine.bind(dag_input_2[0], dag_input_2[1])
ray_dag = combine.bind(a, b)
serve_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
with pytest.raises(
AssertionError,
match="There should be one and only one PipelineInputNode"):
get_pipeline_input_node(serve_dag)
# 3) User forgot to change InputNode to PipelineInputNode
with InputNode() as dag_input:
ray_dag = combine.bind(dag_input[0], dag_input[1])
serve_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
with pytest.raises(
ValueError,
match="Please change Ray DAG InputNode to PipelineInputNode"):
get_pipeline_input_node(serve_dag)
def test_no_inline_class_or_func(serve_instance):
# 1) Inline function
@ray.remote
def inline_func(val):
return val
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
ray_dag = inline_func.bind(dag_input)
assert ray.get(ray_dag.execute(1)) == 1
with pytest.raises(
AssertionError,
match="Function used in DAG should not be in-line defined",
):
_ = json.dumps(ray_dag, cls=DAGNodeEncoder)
# 2) Inline class
@ray.remote
class InlineClass:
def __init__(self, val):
self.val = val
def get(self, input):
return self.val + input
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
node = InlineClass.bind(1)
ray_dag = node.get.bind(dag_input)
with pytest.raises(
AssertionError,
match="Class used in DAG should not be in-line defined",
):
_ = json.dumps(ray_dag, cls=DAGNodeEncoder)
# 3) Inline preprocessor fn
def inline_preprocessor_fn(input):
return input
with PipelineInputNode(preprocessor=inline_preprocessor_fn) as dag_input:
ray_dag = combine.bind(dag_input[0], 2)
with pytest.raises(
AssertionError,
match="Preprocessor used in DAG should not be in-line defined",
):
_ = json.dumps(ray_dag, cls=DAGNodeEncoder)
# 4) Class factory that function returns class object
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
instance = ray.remote(class_factory()).bind()
ray_dag = instance.get.bind()
with pytest.raises(
AssertionError,
match="Class used in DAG should not be in-line defined",
):
_ = json.dumps(ray_dag, cls=DAGNodeEncoder)
def get_shared_deployment_handle_dag():
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
m = Model.bind(2)
combine = Combine.bind(m, m2=m)
ray_dag = combine.__call__.bind(dag_input)
return ray_dag, dag_input
def test_model_wrappers_in_pipeline(serve_instance):
_, path = tempfile.mkstemp()
with open(path, "w") as f:
json.dump(2, f)
predictor_cls = "ray.serve.tests.test_model_wrappers.AdderPredictor"
checkpoint_cls = "ray.serve.tests.test_model_wrappers.AdderCheckpoint"
schema_cls = "ray.serve.http_adapters.array_to_databatch"
with PipelineInputNode(preprocessor=schema_cls) as dag_input:
m1 = ray.remote(ModelWrapper).bind(
predictor_cls=predictor_cls, # TODO: can't be the raw class right now?
checkpoint={ # TODO: can't be the raw object right now?
"checkpoint_cls": checkpoint_cls,
"uri": path,
},
)
dag = m1.predict.bind(dag_input)
deployments = build(dag)
for d in deployments:
d.deploy()
resp = requests.post("http://127.0.0.1:8000/ingress", json={"array": [40]})
resp.raise_for_status()
return resp.json() == {"value": [42], "batch_size": 1}
def get_multi_instantiation_class_nested_deployment_arg_dag():
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True)
ray_dag = combine.__call__.bind(dag_input)
return ray_dag, dag_input
def get_multi_instantiation_class_deployment_in_init_args_dag():
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2=m2)
ray_dag = combine.__call__.bind(dag_input)
return ray_dag, dag_input
def get_func_class_with_class_method_dag():
with PipelineInputNode(preprocessor=request_to_data_obj) as dag_input:
m1 = Model.bind(1)
m2 = Model.bind(2)
m1_output = m1.forward.bind(dag_input[0])
m2_output = m2.forward.bind(dag_input[1])
ray_dag = combine.bind(m1_output,
m2_output,
kwargs_output=dag_input[2])
return ray_dag, dag_input
def dagnode_from_json(input_json: Any) -> Union[DAGNode, RayServeHandle, Any]:
"""
Decode a DAGNode from given input json dictionary. JSON serialization is
only used and enforced in ray serve from ray core API authored DAGNode(s).
Covers both RayServeHandle and DAGNode types.
Assumptions:
- User object's JSON dict does not have keys that collide with our
reserved DAGNODE_TYPE_KEY
- RayServeHandle and Deployment can be re-constructed without losing
states needed for their functionality or correctness.
- DAGNode type can be re-constructed with new stable_uuid upon each
deserialization without effective correctness of execution.
- Only exception is ClassNode used as parent of ClassMethodNode
that we perserve the same parent node.
- .options() does not contain any DAGNode type
"""
# Deserialize RayServeHandle type
if SERVE_HANDLE_JSON_KEY in input_json:
return json.loads(input_json, object_hook=serve_handle_object_hook)
# Base case for plain objects
elif DAGNODE_TYPE_KEY not in input_json:
try:
return json.loads(input_json)
except Exception:
return input_json
# Deserialize DAGNode type
elif input_json[DAGNODE_TYPE_KEY] == InputNode.__name__:
return InputNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == InputAtrributeNode.__name__:
return InputAtrributeNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == PipelineInputNode.__name__:
return PipelineInputNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == ClassMethodNode.__name__:
return ClassMethodNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == DeploymentNode.__name__:
return DeploymentNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == DeploymentMethodNode.__name__:
return DeploymentMethodNode.from_json(input_json, object_hook=dagnode_from_json)
else:
# Class and Function nodes require original module as body.
print(f"import_path: {input_json['import_path']}")
module_name, attr_name = parse_import_path(input_json["import_path"])
module = getattr(import_module(module_name), attr_name)
if input_json[DAGNODE_TYPE_KEY] == FunctionNode.__name__:
return FunctionNode.from_json(
input_json, module, object_hook=dagnode_from_json
)
elif input_json[DAGNODE_TYPE_KEY] == ClassNode.__name__:
return ClassNode.from_json(
input_json, module, object_hook=dagnode_from_json
)
def test_single_class_with_invalid_deployment_options(serve_instance):
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
model = Model.options(name="my_deployment").bind(2, ratio=0.3)
ray_dag = model.forward.bind(dag_input)
serve_root_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 1
with pytest.raises(
ValueError,
match="Specifying name in ray_actor_options is not allowed"):
deployments[0].deploy()
def test_multi_instantiation_class_nested_deployment_arg(serve_instance):
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
combine = Combine.bind(m1, m2={NESTED_HANDLE_KEY: m2}, m2_nested=True)
ray_dag = combine.__call__.bind(dag_input)
(
serve_root_dag,
deserialized_serve_root_dag_node,
) = _test_deployment_json_serde_helper(ray_dag,
input=1,
expected_num_deployments=3)
assert ray.get(serve_root_dag.execute(1)) == ray.get(
deserialized_serve_root_dag_node.execute(1))
def test_nested_deployment_node_json_serde(serve_instance):
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
m1 = Model.bind(2)
m2 = Model.bind(3)
m1_output = m1.forward.bind(dag_input)
m2_output = m2.forward.bind(dag_input)
ray_dag = combine.bind(m1_output, m2_output)
(
serve_root_dag,
deserialized_serve_root_dag_node,
) = _test_deployment_json_serde_helper(ray_dag,
input=1,
expected_num_deployments=2)
assert ray.get(serve_root_dag.execute(1)) == ray.get(
deserialized_serve_root_dag_node.execute(1))
def test_single_class_with_valid_ray_options(serve_instance):
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
model = Model.options(num_cpus=1, memory=1000).bind(2, ratio=0.3)
ray_dag = model.forward.bind(dag_input)
serve_root_dag = ray_dag.apply_recursive(transform_ray_dag_to_serve_dag)
deployments = extract_deployments_from_serve_dag(serve_root_dag)
assert len(deployments) == 1
deployments[0].deploy()
_validate_consistent_python_output(deployments[0],
ray_dag,
deployments[0].name,
input=1,
output=0.6)
deployment = serve.get_deployment(deployments[0].name)
assert deployment.ray_actor_options.get("num_cpus") == 1
assert deployment.ray_actor_options.get("memory") == 1000
assert deployment.ray_actor_options.get("runtime_env") == {}
def get_ingress_deployment(
serve_dag_root_node: DAGNode, pipeline_input_node: PipelineInputNode
) -> Deployment:
"""Return an Ingress deployment to handle user HTTP inputs.
Args:
serve_dag_root_node (DAGNode): Transformed as serve DAG's root. User
inputs are translated to serve_dag_root_node.execute().
pipeline_input_node (DAGNode): Singleton PipelineInputNode instance that
contains input preprocessor info.
Returns:
ingress (Deployment): Generated pipeline ingress deployment to serve
user HTTP requests.
"""
serve_dag_root_json = json.dumps(serve_dag_root_node, cls=DAGNodeEncoder)
preprocessor_import_path = pipeline_input_node.get_preprocessor_import_path()
serve_dag_root_deployment = make_ingress_deployment(
DEFAULT_INGRESS_DEPLOYMENT_NAME,
serve_dag_root_json,
preprocessor_import_path,
)
return serve_dag_root_deployment
def get_simple_class_with_class_method_dag():
with PipelineInputNode(preprocessor=request_to_data_int) as dag_input:
model = Model.bind(2, ratio=0.3)
ray_dag = model.forward.bind(dag_input)
return ray_dag, dag_input
def get_simple_func_dag():
with PipelineInputNode(preprocessor=request_to_data_obj) as dag_input:
ray_dag = combine.bind(dag_input[0], dag_input[1], kwargs_output=1)
return ray_dag, dag_input