def get_pipeline_input_node(serve_dag_root_node: DAGNode):
"""Return the InputNode singleton node from serve dag, and throw
exceptions if we didn't find any, or found more than one.
Args:
ray_dag_root_node: DAGNode acting as root of a Ray authored DAG. It
should be executable via `ray_dag_root_node.execute(user_input)`
and should have `InputNode` in it.
Returns
pipeline_input_node: Singleton input node for the serve pipeline.
"""
input_nodes = []
def extractor(dag_node):
if isinstance(dag_node, InputNode):
input_nodes.append(dag_node)
serve_dag_root_node.apply_recursive(extractor)
assert len(input_nodes) == 1, (
"There should be one and only one InputNode in the DAG. "
f"Found {len(input_nodes)} InputNode(s) instead."
)
return input_nodes[0]
def get_node_name(self, node: DAGNode):
if isinstance(node, InputNode):
node_name = "INPUT_NODE"
elif isinstance(node, InputAttributeNode):
node_name = "INPUT_ATTRIBUTE_NODE"
elif isinstance(node, ClassMethodNode):
node_name = node.get_options().get("name",
None) or node._method_name
elif isinstance(node, (ClassNode, FunctionNode)):
node_name = node.get_options().get("name",
None) or node._body.__name__
# we use instance class name check here to avoid importing ServeNodes as
# serve components are not included in Ray Core.
elif type(node).__name__ in ("DeploymentNode",
"DeploymentFunctionNode"):
node_name = node.get_deployment_name()
elif type(node).__name__ == "DeploymentMethodNode":
node_name = node.get_deployment_method_name()
else:
raise ValueError(
"get_node_name() should only be called on DAGNode instances.")
if node_name not in self.name_to_suffix:
self.name_to_suffix[node_name] = 0
return node_name
else:
self.name_to_suffix[node_name] += 1
suffix_num = self.name_to_suffix[node_name]
return f"{node_name}_{suffix_num}"
def extract_deployments_from_serve_dag(
serve_dag_root: DAGNode,
) -> List[Deployment]:
"""Extract deployment python objects from a transformed serve DAG. Should
only be called after `transform_ray_dag_to_serve_dag`, otherwise nothing
to return.
Args:
serve_dag_root (DAGNode): Transformed serve dag root node.
Returns:
deployments (List[Deployment]): List of deployment python objects
fetched from serve dag.
"""
deployments = OrderedDict()
def extractor(dag_node):
if isinstance(dag_node, (DeploymentNode, DeploymentFunctionNode)):
deployment = dag_node._deployment
# In case same deployment is used in multiple DAGNodes
deployments[deployment.name] = deployment
return dag_node
serve_dag_root.apply_recursive(extractor)
return list(deployments.values())
def get_pipeline_input_node(serve_dag_root_node: DAGNode):
"""Return the PipelineInputNode singleton node from serve dag, and throw
exceptions if we didn't find any, or found more than one.
Args:
ray_dag_root_node: DAGNode acting as root of a Ray authored DAG. It
should be executable via `ray_dag_root_node.execute(user_input)`
and should have `PipelineInputNode` in it.
Returns
pipeline_input_node: Singleton input node for the serve pipeline.
"""
input_nodes = []
def extractor(dag_node):
if isinstance(dag_node, PipelineInputNode):
input_nodes.append(dag_node)
elif isinstance(dag_node, InputNode):
raise ValueError(
"Please change Ray DAG InputNode to PipelineInputNode in order "
"to build serve application. See docstring of "
"PipelineInputNode for examples."
)
serve_dag_root_node.apply_recursive(extractor)
assert len(input_nodes) == 1, (
"There should be one and only one PipelineInputNode in the DAG. "
f"Found {len(input_nodes)} PipelineInputNode(s) instead."
)
return input_nodes[0]
def generate_executor_dag_driver_deployment(
serve_executor_dag_root_node: DAGNode, original_driver_deployment: Deployment
):
"""Given a transformed minimal execution serve dag, and original DAGDriver
deployment, generate new DAGDriver deployment that uses new serve executor
dag as init_args.
Args:
serve_executor_dag_root_node (DeploymentExecutorNode): Transformed
executor serve dag with only barebone deployment handles.
original_driver_deployment (Deployment): User's original DAGDriver
deployment that wrapped Ray DAG as init args.
Returns:
executor_dag_driver_deployment (Deployment): New DAGDriver deployment
with executor serve dag as init args.
"""
def replace_with_handle(node):
if isinstance(node, DeploymentExecutorNode):
return node._deployment_handle
elif isinstance(
node,
(
DeploymentMethodExecutorNode,
DeploymentFunctionExecutorNode,
),
):
serve_dag_root_json = json.dumps(node, cls=DAGNodeEncoder)
return RayServeDAGHandle(serve_dag_root_json)
(
replaced_deployment_init_args,
replaced_deployment_init_kwargs,
) = serve_executor_dag_root_node.apply_functional(
[
serve_executor_dag_root_node.get_args(),
serve_executor_dag_root_node.get_kwargs(),
],
predictate_fn=lambda node: isinstance(
node,
(
DeploymentExecutorNode,
DeploymentFunctionExecutorNode,
DeploymentMethodExecutorNode,
),
),
apply_fn=replace_with_handle,
)
return original_driver_deployment.options(
init_args=replaced_deployment_init_args,
init_kwargs=replaced_deployment_init_kwargs,
)
def transform_ray_dag_to_workflow(dag_node: DAGNode,
input_context: DAGInputData):
"""
Transform a Ray DAG to a workflow. Map FunctionNode to workflow step with
the workflow decorator.
Args:
dag_node: The DAG to be converted to a workflow.
input_context: The input data that wraps varibles for the input node of the DAG.
"""
if not isinstance(dag_node, FunctionNode):
raise TypeError(
"Currently workflow does not support classes as DAG inputs.")
def _node_visitor(node: Any) -> Any:
if isinstance(node, FunctionNode):
# "_resolve_like_object_ref_in_args" indicates we should resolve the
# workflow like an ObjectRef, when included in the arguments of
# another workflow.
workflow_step = workflow.step(
node._body).options(**node._bound_options,
_resolve_like_object_ref_in_args=True)
wf = workflow_step.step(*node._bound_args, **node._bound_kwargs)
return wf
if isinstance(node, InputAtrributeNode):
return node._execute_impl() # get data from input node
if isinstance(node, InputNode):
return input_context # replace input node with input data
if not isinstance(node, DAGNode):
return node # return normal objects
raise TypeError(f"Unsupported DAG node: {node}")
return dag_node.apply_recursive(_node_visitor)
def transform_serve_dag_to_serve_executor_dag(serve_dag_root_node: DAGNode):
"""Given a runnable serve dag with deployment init args and options
processed, transform into an equivalent, but minimal dag optimized for
execution.
"""
if isinstance(serve_dag_root_node, DeploymentNode):
return DeploymentExecutorNode(
serve_dag_root_node._deployment_handle,
serve_dag_root_node.get_args(),
serve_dag_root_node.get_kwargs(),
)
elif isinstance(serve_dag_root_node, DeploymentMethodNode):
return DeploymentMethodExecutorNode(
# Deployment method handle
serve_dag_root_node._deployment_method_name,
serve_dag_root_node.get_args(),
serve_dag_root_node.get_kwargs(),
other_args_to_resolve=serve_dag_root_node.get_other_args_to_resolve(),
)
elif isinstance(serve_dag_root_node, DeploymentFunctionNode):
return DeploymentFunctionExecutorNode(
serve_dag_root_node._deployment_handle,
serve_dag_root_node.get_args(),
serve_dag_root_node.get_kwargs(),
)
else:
return serve_dag_root_node
def build(ray_dag_root_node: DAGNode):
"""Do all the DAG transformation, extraction and generation needed to
produce a runnable and deployable serve pipeline application from a valid
DAG authored with Ray DAG API.
This should be the only user facing API that user interacts with.
Assumptions:
Following enforcements are only applied at generating and applying
pipeline artifact, but not blockers for local development and testing.
- ALL args and kwargs used in DAG building should be JSON serializable.
This means in order to ensure your pipeline application can run on
a remote cluster potentially with different runtime environment,
among all options listed:
1) binding in-memory objects
2) Rely on pickling
3) Enforce JSON serialibility on all args used
We believe both 1) & 2) rely on unstable in-memory objects or
cross version pickling / closure capture, where JSON serialization
provides the right contract needed for proper deployment.
- ALL classes and methods used should be visible on top of the file and
importable via a fully qualified name. Thus no inline class or
function definitions should be used.
Args:
ray_dag_root_node: DAGNode acting as root of a Ray authored DAG. It
should be executable via `ray_dag_root_node.execute(user_input)`
and should have `PipelineInputNode` in it.
Returns:
app: The Ray Serve application object that wraps all deployments needed
along with ingress deployment for an e2e runnable serve pipeline,
accessible via python .remote() call and HTTP.
Examples:
>>> with ServeInputNode(preprocessor=request_to_data_int) as dag_input:
... m1 = Model.bind(1)
... m2 = Model.bind(2)
... m1_output = m1.forward.bind(dag_input[0])
... m2_output = m2.forward.bind(dag_input[1])
... ray_dag = ensemble.bind(m1_output, m2_output)
Assuming we have non-JSON serializable or inline defined class or
function in local pipeline development.
>>> app = serve.pipeline.build(ray_dag) # This works
>>> handle = app.deploy()
>>> # This also works, we're simply executing the transformed serve_dag.
>>> ray.get(handle.remote(data)
>>> # This will fail where enforcements are applied.
>>> deployment_yaml = app.to_yaml()
"""
serve_root_dag = ray_dag_root_node.apply_recursive(transform_ray_dag_to_serve_dag)
deployments = extract_deployments_from_serve_dag(serve_root_dag)
pipeline_input_node = get_pipeline_input_node(serve_root_dag)
ingress_deployment = get_ingress_deployment(serve_root_dag, pipeline_input_node)
deployments.insert(0, ingress_deployment)
# TODO (jiaodong): Call into Application once Shreyas' PR is merged
# TODO (jiaodong): Apply enforcements at serve app to_yaml level
return deployments
def test_serialize_warning():
node = DAGNode([], {}, {}, {})
with pytest.raises(ValueError):
pickle.dumps(node)
def transform_ray_dag_to_serve_dag(
dag_node: DAGNode, deployment_name_generator: DeploymentNameGenerator):
"""
Transform a Ray DAG to a Serve DAG. Map ClassNode to DeploymentNode with
ray decorated body passed in, and ClassMethodNode to DeploymentMethodNode.
"""
if isinstance(dag_node, ClassNode):
deployment_name = deployment_name_generator.get_deployment_name(
dag_node)
return DeploymentNode(
dag_node._body,
deployment_name,
dag_node.get_args(),
dag_node.get_kwargs(),
dag_node.get_options(),
# TODO: (jiaodong) Support .options(metadata=xxx) for deployment
other_args_to_resolve=dag_node.get_other_args_to_resolve(),
)
elif isinstance(dag_node, ClassMethodNode):
other_args_to_resolve = dag_node.get_other_args_to_resolve()
# TODO: (jiaodong) Need to capture DAGNodes in the parent node
parent_deployment_node = other_args_to_resolve[PARENT_CLASS_NODE_KEY]
return DeploymentMethodNode(
parent_deployment_node._deployment,
dag_node._method_name,
dag_node.get_args(),
dag_node.get_kwargs(),
dag_node.get_options(),
other_args_to_resolve=dag_node.get_other_args_to_resolve(),
)
elif isinstance(
dag_node, FunctionNode
# TODO (jiaodong): We do not convert ray function to deployment function
# yet, revisit this later
) and dag_node.get_other_args_to_resolve().get("is_from_serve_deployment"):
deployment_name = deployment_name_generator.get_deployment_name(
dag_node)
return DeploymentFunctionNode(
dag_node._body,
deployment_name,
dag_node.get_args(),
dag_node.get_kwargs(),
dag_node.get_options(),
other_args_to_resolve=dag_node.get_other_args_to_resolve(),
)
else:
# TODO: (jiaodong) Support FunctionNode or leave it as ray task
return dag_node
def ray_dag_to_serve_dag(dag: DAGNode):
with DAGNodeNameGenerator() as deployment_name_generator:
serve_dag = dag.apply_recursive(
lambda node: transform_ray_dag_to_serve_dag(
node, deployment_name_generator))
return serve_dag
def build(ray_dag_root_node: DAGNode) -> List[Deployment]:
"""Do all the DAG transformation, extraction and generation needed to
produce a runnable and deployable serve pipeline application from a valid
DAG authored with Ray DAG API.
This should be the only user facing API that user interacts with.
Assumptions:
Following enforcements are only applied at generating and applying
pipeline artifact, but not blockers for local development and testing.
- ALL args and kwargs used in DAG building should be JSON serializable.
This means in order to ensure your pipeline application can run on
a remote cluster potentially with different runtime environment,
among all options listed:
1) binding in-memory objects
2) Rely on pickling
3) Enforce JSON serialibility on all args used
We believe both 1) & 2) rely on unstable in-memory objects or
cross version pickling / closure capture, where JSON serialization
provides the right contract needed for proper deployment.
- ALL classes and methods used should be visible on top of the file and
importable via a fully qualified name. Thus no inline class or
function definitions should be used.
Args:
ray_dag_root_node: DAGNode acting as root of a Ray authored DAG. It
should be executable via `ray_dag_root_node.execute(user_input)`
and should have `InputNode` in it.
Returns:
deployments: All deployments needed for an e2e runnable serve pipeline,
accessible via python .remote() call.
Examples:
>>> with InputNode() as dag_input:
... m1 = Model.bind(1)
... m2 = Model.bind(2)
... m1_output = m1.forward.bind(dag_input[0])
... m2_output = m2.forward.bind(dag_input[1])
... ray_dag = ensemble.bind(m1_output, m2_output)
Assuming we have non-JSON serializable or inline defined class or
function in local pipeline development.
>>> from ray.serve.api import build as build_app
>>> deployments = build_app(ray_dag) # it can be method node
>>> deployments = build_app(m1) # or just a regular node.
"""
with DAGNodeNameGenerator() as node_name_generator:
serve_root_dag = ray_dag_root_node.apply_recursive(
lambda node: transform_ray_dag_to_serve_dag(node, node_name_generator)
)
deployments = extract_deployments_from_serve_dag(serve_root_dag)
# After Ray DAG is transformed to Serve DAG with deployments and their init
# args filled, generate a minimal weight executor serve dag for perf
serve_executor_root_dag = serve_root_dag.apply_recursive(
transform_serve_dag_to_serve_executor_dag
)
root_driver_deployment = deployments[-1]
new_driver_deployment = generate_executor_dag_driver_deployment(
serve_executor_root_dag, root_driver_deployment
)
# Replace DAGDriver deployment with executor DAGDriver deployment
deployments[-1] = new_driver_deployment
# Validate and only expose HTTP for the endpoint
deployments_with_http = process_ingress_deployment_in_serve_dag(deployments)
return deployments_with_http
def __init__(self, func_body, func_args, func_kwargs, func_options):
self._body = func_body
DAGNode.__init__(self, func_args, func_kwargs, func_options)