def execute_workflow(workflow: "Workflow") -> "WorkflowExecutionResult":
"""Execute workflow.
Returns:
An object ref that represent the result.
"""
if workflow.executed:
return workflow.result
workflow_data = workflow.data
baked_inputs = _BakedWorkflowInputs.from_workflow_inputs(
workflow_data.inputs)
persisted_output, volatile_output = _workflow_step_executor.options(
**workflow_data.ray_options).remote(
workflow_data.step_type, workflow_data.func_body,
workflow_context.get_workflow_step_context(), workflow.step_id,
baked_inputs, workflow_data.catch_exceptions,
workflow_data.max_retries)
if not isinstance(persisted_output, WorkflowOutputType):
raise TypeError("Unexpected return type of the workflow.")
if workflow_data.step_type != StepType.READONLY_ACTOR_METHOD:
_record_step_status(workflow.step_id, WorkflowStatus.RUNNING,
[volatile_output])
result = WorkflowExecutionResult(persisted_output, volatile_output)
workflow._result = result
workflow._executed = True
return result
def resume_workflow_step(
job_id: str,
workflow_id: str,
step_id: "StepID",
current_output: Optional[ray.ObjectRef],
) -> WorkflowExecutionResult:
"""Resume a step of a workflow.
Args:
job_id: The ID of the job that submits the workflow execution. The ID
is used to identify the submitter of the workflow.
workflow_id: The ID of the workflow job. The ID is used to identify
the workflow.
step_id: The step to resume in the workflow.
Raises:
WorkflowNotResumableException: fail to resume the workflow.
Returns:
The execution result of the workflow, represented by Ray ObjectRef.
"""
if current_output is None:
current_output = []
else:
current_output = [current_output]
persisted_output, volatile_output = _resume_workflow_step_executor.remote(
job_id, workflow_id, step_id, current_output)
persisted_output = WorkflowStaticRef.from_output(step_id, persisted_output)
volatile_output = WorkflowStaticRef.from_output(step_id, volatile_output)
return WorkflowExecutionResult(persisted_output, volatile_output)
def resume_workflow_step(
workflow_id: str, step_id: "StepID", store_url: str,
current_output: Optional[ray.ObjectRef]) -> WorkflowExecutionResult:
"""Resume a step of a workflow.
Args:
workflow_id: The ID of the workflow job. The ID is used to identify
the workflow.
step_id: The step to resume in the workflow.
store_url: The url of the storage to access the workflow.
Raises:
WorkflowNotResumableException: fail to resume the workflow.
Returns:
The execution result of the workflow, represented by Ray ObjectRef.
"""
if current_output is None:
current_output = []
else:
current_output = [current_output]
persisted_output, volatile_output = _resume_workflow_step_executor.remote(
workflow_id, step_id, store_url, current_output)
return WorkflowExecutionResult(persisted_output, volatile_output)
def execute_workflow(
workflow: "Workflow",
outer_most_step_id: Optional[str] = None,
last_step_of_workflow: bool = False) -> "WorkflowExecutionResult":
"""Execute workflow.
To fully explain what we are doing, we need to introduce some syntax first.
The syntax for dependencies between workflow steps
"B.step(A.step())" is "A - B"; the syntax for nested workflow steps
"def A(): return B.step()" is "A / B".
In a chain/DAG of step dependencies, the "output step" is the step of last
(topological) order. For example, in "A - B - C", C is the output step.
In a chain of nested workflow steps, the initial "output step" is
called the "outer most step" for other "output steps". For example, in
"A / B / C / D", "A" is the outer most step for "B", "C", "D";
in the hybrid workflow "((A - B) / C / D) - (E / (F - G) / H)",
"B" is the outer most step for "C", "D"; "E" is the outer most step
for "G", "H".
Args:
workflow: The workflow to be executed.
outer_most_step_id: The ID of the outer most workflow. None if it
does not exists. See "step_executor.execute_workflow" for detailed
explanation.
last_step_of_workflow: The step that generates the output of the
workflow (including nested steps).
Returns:
An object ref that represent the result.
"""
if workflow.executed:
return workflow.result
workflow_data = workflow.data
baked_inputs = _BakedWorkflowInputs.from_workflow_inputs(
workflow_data.inputs)
persisted_output, volatile_output = _workflow_step_executor.options(
**workflow_data.ray_options).remote(
workflow_data.step_type, workflow_data.func_body,
workflow_context.get_workflow_step_context(), workflow.step_id,
baked_inputs, outer_most_step_id, workflow_data.catch_exceptions,
workflow_data.max_retries, last_step_of_workflow)
if not isinstance(persisted_output, WorkflowOutputType):
raise TypeError("Unexpected return type of the workflow.")
if workflow_data.step_type != StepType.READONLY_ACTOR_METHOD:
_record_step_status(workflow.step_id, WorkflowStatus.RUNNING,
[volatile_output])
result = WorkflowExecutionResult(persisted_output, volatile_output)
workflow._result = result
workflow._executed = True
return result
def execute_workflow(workflow: "Workflow") -> "WorkflowExecutionResult":
"""Execute workflow.
Args:
workflow: The workflow to be executed.
Returns:
An object ref that represent the result.
"""
if workflow.executed:
return workflow.result
# Stage 1: prepare inputs
workflow_data = workflow.data
inputs = workflow_data.inputs
workflow_outputs = []
with workflow_context.fork_workflow_step_context(
outer_most_step_id=None, last_step_of_workflow=False):
for w in inputs.workflows:
static_ref = w.ref
if static_ref is None:
# The input workflow is not a reference to an executed
# workflow .
output = execute_workflow(w).persisted_output
static_ref = WorkflowStaticRef(step_id=w.step_id, ref=output)
workflow_outputs.append(static_ref)
baked_inputs = _BakedWorkflowInputs(
args=workflow_data.inputs.args,
workflow_outputs=workflow_outputs,
workflow_refs=inputs.workflow_refs,
)
# Stage 2: match executors
step_options = workflow_data.step_options
if step_options.allow_inplace:
# TODO(suquark): For inplace execution, it is impossible
# to get the ObjectRef of the output before execution.
# Here we use a dummy ObjectRef, because _record_step_status does not
# even use it (?!).
_record_step_status(workflow.step_id, WorkflowStatus.RUNNING,
[ray.put(None)])
# Note: we need to be careful about workflow context when
# calling the executor directly.
# TODO(suquark): We still have recursive Python calls.
# This would cause stack overflow if we have a really
# deep recursive call. We should fix it later.
if step_options.step_type == StepType.WAIT:
executor = _workflow_wait_executor
else:
executor = _workflow_step_executor
else:
if step_options.step_type == StepType.WAIT:
# This is very important to set "num_cpus=0" to
# ensure "workflow.wait" is not blocked by other
# tasks.
executor = _workflow_wait_executor_remote.options(
num_cpus=0).remote
else:
executor = _workflow_step_executor_remote.options(
**step_options.ray_options).remote
# Stage 3: execution
persisted_output, volatile_output = executor(
workflow_data.func_body,
workflow_context.get_workflow_step_context(),
workflow.step_id,
baked_inputs,
workflow_data.step_options,
)
# Stage 4: post processing outputs
if not isinstance(persisted_output, WorkflowOutputType):
persisted_output = ray.put(persisted_output)
if not isinstance(persisted_output, WorkflowOutputType):
volatile_output = ray.put(volatile_output)
if step_options.step_type != StepType.READONLY_ACTOR_METHOD:
if not step_options.allow_inplace:
# TODO: [Possible flaky bug] Here the RUNNING state may
# be recorded earlier than SUCCESSFUL. This caused some
# confusion during development.
_record_step_status(workflow.step_id, WorkflowStatus.RUNNING,
[volatile_output])
result = WorkflowExecutionResult(persisted_output, volatile_output)
workflow._result = result
workflow._executed = True
return result
def execute_workflow(workflow: "Workflow") -> "WorkflowExecutionResult":
"""Execute workflow.
Args:
workflow: The workflow to be executed.
Returns:
An object ref that represent the result.
"""
if workflow.executed:
return workflow.result
# Stage 1: prepare inputs
workflow_data = workflow.data
inputs = workflow_data.inputs
# Here A is the outer workflow step, B & C are the inner steps.
# C is the output step for A, because C produces the output for A.
#
# @workflow.step
# def A():
# b = B.step()
# return C.step(b)
#
# If the outer workflow step skips checkpointing, it would
# update the checkpoint context of all inner steps except
# the output step, marking them "detached" from the DAG.
# Output step is not detached from the DAG because once
# completed, it replaces the output of the outer step.
step_context = workflow_context.get_workflow_step_context()
checkpoint_context = step_context.checkpoint_context.copy()
# "detached" could be defined recursively:
# detached := already detached or the outer step skips checkpointing
checkpoint_context.detached_from_dag = (
checkpoint_context.detached_from_dag
or not step_context.checkpoint_context.checkpoint)
# Apply checkpoint context to input steps. Since input steps
# further apply them to their inputs, this would eventually
# apply to all steps except the output step. This avoids
# detaching the output step.
workflow_outputs = []
with workflow_context.fork_workflow_step_context(
outer_most_step_id=None,
last_step_of_workflow=False,
checkpoint_context=checkpoint_context,
):
for w in inputs.workflows:
static_ref = w.ref
if static_ref is None:
# The input workflow is not a reference to an executed
# workflow .
output = execute_workflow(w).persisted_output
static_ref = WorkflowStaticRef(step_id=w.step_id, ref=output)
workflow_outputs.append(static_ref)
baked_inputs = _BakedWorkflowInputs(
args=inputs.args,
workflow_outputs=workflow_outputs,
workflow_refs=inputs.workflow_refs,
)
# Stage 2: match executors
step_options = workflow_data.step_options
if step_options.allow_inplace:
# TODO(suquark): For inplace execution, it is impossible
# to get the ObjectRef of the output before execution.
# Here we use a dummy ObjectRef, because _record_step_status does not
# even use it (?!).
_record_step_status(workflow.step_id, WorkflowStatus.RUNNING,
[ray.put(None)])
# Note: we need to be careful about workflow context when
# calling the executor directly.
# TODO(suquark): We still have recursive Python calls.
# This would cause stack overflow if we have a really
# deep recursive call. We should fix it later.
if step_options.step_type == StepType.WAIT:
executor = _workflow_wait_executor
else:
executor = _workflow_step_executor
else:
if step_options.step_type == StepType.WAIT:
# This is very important to set "num_cpus=0" to
# ensure "workflow.wait" is not blocked by other
# tasks.
executor = _workflow_wait_executor_remote.options(
num_cpus=0).remote
else:
executor = _workflow_step_executor_remote.options(
**step_options.ray_options).remote
# Stage 3: execution
persisted_output, volatile_output = executor(
workflow_data.func_body,
step_context,
workflow.step_id,
baked_inputs,
workflow_data.step_options,
)
# Stage 4: post processing outputs
if not isinstance(persisted_output, WorkflowOutputType):
persisted_output = ray.put(persisted_output)
if not isinstance(persisted_output, WorkflowOutputType):
volatile_output = ray.put(volatile_output)
if step_options.step_type != StepType.READONLY_ACTOR_METHOD:
if not step_options.allow_inplace:
# TODO: [Possible flaky bug] Here the RUNNING state may
# be recorded earlier than SUCCESSFUL. This caused some
# confusion during development.
_record_step_status(workflow.step_id, WorkflowStatus.RUNNING,
[volatile_output])
result = WorkflowExecutionResult(persisted_output, volatile_output)
workflow._result = result
workflow._executed = True
return result
def _execute_workflow(job_id,
workflow: "Workflow") -> "WorkflowExecutionResult":
"""Internal function of workflow execution."""
if workflow.executed:
return workflow.result
# Stage 1: prepare inputs
workflow_data = workflow.data
inputs = workflow_data.inputs
# Here A is the outer workflow step, B & C are the inner steps.
# C is the output step for A, because C produces the output for A.
#
# @workflow.step
# def A():
# b = B.step()
# return C.step(b)
#
# If the outer workflow step skips checkpointing, it would
# update the checkpoint context of all inner steps except
# the output step, marking them "detached" from the DAG.
# Output step is not detached from the DAG because once
# completed, it replaces the output of the outer step.
step_context = workflow_context.get_workflow_step_context()
checkpoint_context = step_context.checkpoint_context.copy()
# "detached" could be defined recursively:
# detached := already detached or the outer step skips checkpointing
checkpoint_context.detached_from_dag = (
checkpoint_context.detached_from_dag
or not step_context.checkpoint_context.checkpoint)
# Apply checkpoint context to input steps. Since input steps
# further apply them to their inputs, this would eventually
# apply to all steps except the output step. This avoids
# detaching the output step.
workflow_outputs = []
with workflow_context.fork_workflow_step_context(
outer_most_step_id=None,
last_step_of_workflow=False,
checkpoint_context=checkpoint_context,
):
for w in inputs.workflows:
static_ref = w.ref
if static_ref is None:
extra_options = w.data.step_options.ray_options
# The input workflow is not a reference to an executed
# workflow.
static_ref = execute_workflow(job_id, w).output
static_ref._resolve_like_object_ref_in_args = extra_options.get(
"_resolve_like_object_ref_in_args", False)
workflow_outputs.append(static_ref)
baked_inputs = _BakedWorkflowInputs(
args=inputs.args,
workflow_outputs=workflow_outputs,
workflow_refs=inputs.workflow_refs,
job_id=job_id,
)
# Stage 2: match executors
step_options = workflow_data.step_options
if step_options.allow_inplace:
# TODO(suquark): For inplace execution, it is impossible
# to get the ObjectRef of the output before execution.
# Here we use a dummy ObjectRef, because _record_step_status does not
# even use it (?!).
_record_step_status(workflow.step_id, WorkflowStatus.RUNNING,
[ray.put(None)])
# Note: we need to be careful about workflow context when
# calling the executor directly.
# TODO(suquark): We still have recursive Python calls.
# This would cause stack overflow if we have a really
# deep recursive call. We should fix it later.
if step_options.step_type == StepType.WAIT:
executor = _workflow_wait_executor
else:
# Tell the executor that we are running inplace. This enables
# tail-recursion optimization.
executor = functools.partial(_workflow_step_executor, inplace=True)
else:
if step_options.step_type == StepType.WAIT:
# This is very important to set "num_cpus=0" to
# ensure "workflow.wait" is not blocked by other
# tasks.
executor = _workflow_wait_executor_remote.options(
num_cpus=0).remote
else:
ray_options = step_options.ray_options.copy()
# cleanup the "_resolve_like_object_ref_in_args" option, it is not for Ray.
ray_options.pop("_resolve_like_object_ref_in_args", None)
executor = _workflow_step_executor_remote.options(
**ray_options).remote
# Stage 3: execution
output = executor(
workflow_data.func_body,
step_context,
job_id,
workflow.step_id,
baked_inputs,
workflow_data.step_options,
)
# Stage 4: post processing outputs
if not step_options.allow_inplace:
# TODO: [Possible flaky bug] Here the RUNNING state may
# be recorded earlier than SUCCESSFUL. This caused some
# confusion during development.
# convert into workflow static ref for step status record.
_record_step_status(workflow.step_id, WorkflowStatus.RUNNING, [None])
result = WorkflowExecutionResult(output)
workflow._result = result
workflow._executed = True
return result