def extract_deployments_from_serve_dag(
serve_dag_root: DAGNode,
) -> List[Deployment]:
"""Extract deployment python objects from a transformed serve DAG. Should
only be called after `transform_ray_dag_to_serve_dag`, otherwise nothing
to return.
Args:
serve_dag_root (DAGNode): Transformed serve dag root node.
Returns:
deployments (List[Deployment]): List of deployment python objects
fetched from serve dag.
"""
deployments = OrderedDict()
def extractor(dag_node):
if isinstance(dag_node, (DeploymentNode, DeploymentFunctionNode)):
deployment = dag_node._deployment
# In case same deployment is used in multiple DAGNodes
deployments[deployment.name] = deployment
return dag_node
serve_dag_root.apply_recursive(extractor)
return list(deployments.values())
def get_pipeline_input_node(serve_dag_root_node: DAGNode):
"""Return the InputNode singleton node from serve dag, and throw
exceptions if we didn't find any, or found more than one.
Args:
ray_dag_root_node: DAGNode acting as root of a Ray authored DAG. It
should be executable via `ray_dag_root_node.execute(user_input)`
and should have `InputNode` in it.
Returns
pipeline_input_node: Singleton input node for the serve pipeline.
"""
input_nodes = []
def extractor(dag_node):
if isinstance(dag_node, InputNode):
input_nodes.append(dag_node)
serve_dag_root_node.apply_recursive(extractor)
assert len(input_nodes) == 1, (
"There should be one and only one InputNode in the DAG. "
f"Found {len(input_nodes)} InputNode(s) instead."
)
return input_nodes[0]
def get_pipeline_input_node(serve_dag_root_node: DAGNode):
"""Return the PipelineInputNode singleton node from serve dag, and throw
exceptions if we didn't find any, or found more than one.
Args:
ray_dag_root_node: DAGNode acting as root of a Ray authored DAG. It
should be executable via `ray_dag_root_node.execute(user_input)`
and should have `PipelineInputNode` in it.
Returns
pipeline_input_node: Singleton input node for the serve pipeline.
"""
input_nodes = []
def extractor(dag_node):
if isinstance(dag_node, PipelineInputNode):
input_nodes.append(dag_node)
elif isinstance(dag_node, InputNode):
raise ValueError(
"Please change Ray DAG InputNode to PipelineInputNode in order "
"to build serve application. See docstring of "
"PipelineInputNode for examples."
)
serve_dag_root_node.apply_recursive(extractor)
assert len(input_nodes) == 1, (
"There should be one and only one PipelineInputNode in the DAG. "
f"Found {len(input_nodes)} PipelineInputNode(s) instead."
)
return input_nodes[0]
def transform_ray_dag_to_workflow(dag_node: DAGNode,
input_context: DAGInputData):
"""
Transform a Ray DAG to a workflow. Map FunctionNode to workflow step with
the workflow decorator.
Args:
dag_node: The DAG to be converted to a workflow.
input_context: The input data that wraps varibles for the input node of the DAG.
"""
if not isinstance(dag_node, FunctionNode):
raise TypeError(
"Currently workflow does not support classes as DAG inputs.")
def _node_visitor(node: Any) -> Any:
if isinstance(node, FunctionNode):
# "_resolve_like_object_ref_in_args" indicates we should resolve the
# workflow like an ObjectRef, when included in the arguments of
# another workflow.
workflow_step = workflow.step(
node._body).options(**node._bound_options,
_resolve_like_object_ref_in_args=True)
wf = workflow_step.step(*node._bound_args, **node._bound_kwargs)
return wf
if isinstance(node, InputAtrributeNode):
return node._execute_impl() # get data from input node
if isinstance(node, InputNode):
return input_context # replace input node with input data
if not isinstance(node, DAGNode):
return node # return normal objects
raise TypeError(f"Unsupported DAG node: {node}")
return dag_node.apply_recursive(_node_visitor)
def build(ray_dag_root_node: DAGNode):
"""Do all the DAG transformation, extraction and generation needed to
produce a runnable and deployable serve pipeline application from a valid
DAG authored with Ray DAG API.
This should be the only user facing API that user interacts with.
Assumptions:
Following enforcements are only applied at generating and applying
pipeline artifact, but not blockers for local development and testing.
- ALL args and kwargs used in DAG building should be JSON serializable.
This means in order to ensure your pipeline application can run on
a remote cluster potentially with different runtime environment,
among all options listed:
1) binding in-memory objects
2) Rely on pickling
3) Enforce JSON serialibility on all args used
We believe both 1) & 2) rely on unstable in-memory objects or
cross version pickling / closure capture, where JSON serialization
provides the right contract needed for proper deployment.
- ALL classes and methods used should be visible on top of the file and
importable via a fully qualified name. Thus no inline class or
function definitions should be used.
Args:
ray_dag_root_node: DAGNode acting as root of a Ray authored DAG. It
should be executable via `ray_dag_root_node.execute(user_input)`
and should have `PipelineInputNode` in it.
Returns:
app: The Ray Serve application object that wraps all deployments needed
along with ingress deployment for an e2e runnable serve pipeline,
accessible via python .remote() call and HTTP.
Examples:
>>> with ServeInputNode(preprocessor=request_to_data_int) as dag_input:
... m1 = Model.bind(1)
... m2 = Model.bind(2)
... m1_output = m1.forward.bind(dag_input[0])
... m2_output = m2.forward.bind(dag_input[1])
... ray_dag = ensemble.bind(m1_output, m2_output)
Assuming we have non-JSON serializable or inline defined class or
function in local pipeline development.
>>> app = serve.pipeline.build(ray_dag) # This works
>>> handle = app.deploy()
>>> # This also works, we're simply executing the transformed serve_dag.
>>> ray.get(handle.remote(data)
>>> # This will fail where enforcements are applied.
>>> deployment_yaml = app.to_yaml()
"""
serve_root_dag = ray_dag_root_node.apply_recursive(transform_ray_dag_to_serve_dag)
deployments = extract_deployments_from_serve_dag(serve_root_dag)
pipeline_input_node = get_pipeline_input_node(serve_root_dag)
ingress_deployment = get_ingress_deployment(serve_root_dag, pipeline_input_node)
deployments.insert(0, ingress_deployment)
# TODO (jiaodong): Call into Application once Shreyas' PR is merged
# TODO (jiaodong): Apply enforcements at serve app to_yaml level
return deployments
def ray_dag_to_serve_dag(dag: DAGNode):
with DAGNodeNameGenerator() as deployment_name_generator:
serve_dag = dag.apply_recursive(
lambda node: transform_ray_dag_to_serve_dag(
node, deployment_name_generator))
return serve_dag
def build(ray_dag_root_node: DAGNode) -> List[Deployment]:
"""Do all the DAG transformation, extraction and generation needed to
produce a runnable and deployable serve pipeline application from a valid
DAG authored with Ray DAG API.
This should be the only user facing API that user interacts with.
Assumptions:
Following enforcements are only applied at generating and applying
pipeline artifact, but not blockers for local development and testing.
- ALL args and kwargs used in DAG building should be JSON serializable.
This means in order to ensure your pipeline application can run on
a remote cluster potentially with different runtime environment,
among all options listed:
1) binding in-memory objects
2) Rely on pickling
3) Enforce JSON serialibility on all args used
We believe both 1) & 2) rely on unstable in-memory objects or
cross version pickling / closure capture, where JSON serialization
provides the right contract needed for proper deployment.
- ALL classes and methods used should be visible on top of the file and
importable via a fully qualified name. Thus no inline class or
function definitions should be used.
Args:
ray_dag_root_node: DAGNode acting as root of a Ray authored DAG. It
should be executable via `ray_dag_root_node.execute(user_input)`
and should have `InputNode` in it.
Returns:
deployments: All deployments needed for an e2e runnable serve pipeline,
accessible via python .remote() call.
Examples:
>>> with InputNode() as dag_input:
... m1 = Model.bind(1)
... m2 = Model.bind(2)
... m1_output = m1.forward.bind(dag_input[0])
... m2_output = m2.forward.bind(dag_input[1])
... ray_dag = ensemble.bind(m1_output, m2_output)
Assuming we have non-JSON serializable or inline defined class or
function in local pipeline development.
>>> from ray.serve.api import build as build_app
>>> deployments = build_app(ray_dag) # it can be method node
>>> deployments = build_app(m1) # or just a regular node.
"""
with DAGNodeNameGenerator() as node_name_generator:
serve_root_dag = ray_dag_root_node.apply_recursive(
lambda node: transform_ray_dag_to_serve_dag(node, node_name_generator)
)
deployments = extract_deployments_from_serve_dag(serve_root_dag)
# After Ray DAG is transformed to Serve DAG with deployments and their init
# args filled, generate a minimal weight executor serve dag for perf
serve_executor_root_dag = serve_root_dag.apply_recursive(
transform_serve_dag_to_serve_executor_dag
)
root_driver_deployment = deployments[-1]
new_driver_deployment = generate_executor_dag_driver_deployment(
serve_executor_root_dag, root_driver_deployment
)
# Replace DAGDriver deployment with executor DAGDriver deployment
deployments[-1] = new_driver_deployment
# Validate and only expose HTTP for the endpoint
deployments_with_http = process_ingress_deployment_in_serve_dag(deployments)
return deployments_with_http