def _workflow_step_executor(step_type: StepType, func: Callable,
context: workflow_context.WorkflowStepContext,
step_id: "StepID",
step_inputs: "StepInputTupleToResolve",
outer_most_step_id: "StepID",
catch_exceptions: bool, max_retries: int,
last_step_of_workflow: bool) -> Any:
"""Executor function for workflow step.
Args:
step_type: The type of workflow step.
func: The workflow step function.
context: Workflow step context. Used to access correct storage etc.
step_id: The ID of the step.
step_inputs: The inputs tuple of the step.
outer_most_step_id: See "step_executor.execute_workflow" for
explanation.
catch_exceptions: If set to be true, return
(Optional[Result], Optional[Error]) instead of Result.
max_retries: Max number of retries encounter of a failure.
last_step_of_workflow: The step that generates the output of the
workflow (including nested steps).
Returns:
Workflow step output.
"""
workflow_context.update_workflow_step_context(context, step_id)
args, kwargs = _resolve_step_inputs(step_inputs)
state, output = _wrap_run(func, step_type, step_id, catch_exceptions,
max_retries, *args, **kwargs)
if step_type != StepType.READONLY_ACTOR_METHOD:
store = workflow_storage.get_workflow_storage()
# Save workflow output
commit_step(store, step_id, state, outer_most_step_id)
# We MUST execute the workflow after saving the output.
if isinstance(state, Workflow):
if step_type == StepType.FUNCTION:
# execute sub-workflow
state = execute_workflow(state, outer_most_step_id,
last_step_of_workflow)
else:
# TODO(suquark): Support returning a workflow inside
# a virtual actor.
raise TypeError("Only a workflow step function "
"can return a workflow.")
elif last_step_of_workflow:
# advance the progress of the workflow
store.advance_progress(step_id)
_record_step_status(step_id, WorkflowStatus.SUCCESSFUL)
logger.info(get_step_status_info(WorkflowStatus.SUCCESSFUL))
return state, output
def _workflow_step_executor(step_type: StepType, func: Callable,
context: workflow_context.WorkflowStepContext,
step_id: "StepID",
baked_inputs: "_BakedWorkflowInputs",
outer_most_step_id: "StepID",
catch_exceptions: bool, max_retries: int,
last_step_of_workflow: bool) -> Any:
"""Executor function for workflow step.
Args:
step_type: The type of workflow step.
func: The workflow step function.
context: Workflow step context. Used to access correct storage etc.
step_id: The ID of the step.
baked_inputs: The processed inputs for the step.
outer_most_step_id: See "step_executor.execute_workflow" for
explanation.
catch_exceptions: If set to be true, return
(Optional[Result], Optional[Error]) instead of Result.
max_retries: Max number of retries encounter of a failure.
last_step_of_workflow: The step that generates the output of the
workflow (including nested steps).
Returns:
Workflow step output.
"""
workflow_context.update_workflow_step_context(context, step_id)
args, kwargs = _resolve_step_inputs(baked_inputs)
persisted_output, volatile_output = _wrap_run(func, step_type, step_id,
catch_exceptions,
max_retries, *args, **kwargs)
if step_type != StepType.READONLY_ACTOR_METHOD:
store = workflow_storage.get_workflow_storage()
# Save workflow output
commit_step(store, step_id, persisted_output, outer_most_step_id)
# We MUST execute the workflow after saving the output.
if isinstance(persisted_output, Workflow):
if step_type == StepType.FUNCTION:
# Passing down outer most step so inner nested steps would
# access the same outer most step.
if not outer_most_step_id:
# The current workflow step returns a nested workflow, and
# there is no outer step for the current step. So the
# current step is the outer most step for the inner nested
# workflow steps.
outer_most_step_id = workflow_context.get_current_step_id()
# execute sub-workflow
persisted_output = execute_workflow(
persisted_output, outer_most_step_id,
last_step_of_workflow).persisted_output
else:
# TODO(suquark): Support returning a workflow inside
# a virtual actor.
raise TypeError("Only a workflow step function "
"can return a workflow.")
elif last_step_of_workflow:
# advance the progress of the workflow
store.advance_progress(step_id)
_record_step_status(step_id, WorkflowStatus.SUCCESSFUL)
logger.info(get_step_status_info(WorkflowStatus.SUCCESSFUL))
return persisted_output, volatile_output
def _wrap_run(func: Callable, step_type: StepType, step_id: "StepID",
catch_exceptions: bool, max_retries: int, *args,
**kwargs) -> Tuple[Any, Any]:
"""Wrap the function and execute it.
It returns two parts, persisted_output (p-out) and volatile_output (v-out).
P-out is the part of result to persist in a storage and pass to the
next step. V-out is the part of result to return to the user but does not
require persistence.
This table describes their relationships
+-----------------------------+-------+--------+----------------------+
| Step Type | p-out | v-out | catch exception into |
+-----------------------------+-------+--------+----------------------+
| Function Step | Y | N | p-out |
+-----------------------------+-------+--------+----------------------+
| Virtual Actor Step | Y | Y | v-out |
+-----------------------------+-------+--------+----------------------+
| Readonly Virtual Actor Step | N | Y | v-out |
+-----------------------------+-------+--------+----------------------+
Args:
step_type: The type of the step producing the result.
catch_exceptions: True if we would like to catch the exception.
max_retries: Max retry times for failure.
Returns:
State and output.
"""
exception = None
result = None
# max_retries are for application level failure.
# For ray failure, we should use max_retries.
for i in range(max_retries):
logger.info(f"{get_step_status_info(WorkflowStatus.RUNNING)}"
f"\t[{i+1}/{max_retries}]")
try:
result = func(*args, **kwargs)
exception = None
break
except BaseException as e:
if i + 1 == max_retries:
retry_msg = "Maximum retry reached, stop retry."
else:
retry_msg = "The step will be retried."
logger.error(
f"{workflow_context.get_step_name()} failed with error message"
f" {e}. {retry_msg}")
exception = e
if catch_exceptions:
if step_type == StepType.FUNCTION:
persisted_output, volatile_output = (result, exception), None
elif step_type == StepType.ACTOR_METHOD:
# virtual actors do not persist exception
persisted_output, volatile_output = result[0], (result[1],
exception)
elif step_type == StepType.READONLY_ACTOR_METHOD:
persisted_output, volatile_output = None, (result, exception)
else:
raise ValueError(f"Unknown StepType '{step_type}'")
else:
if exception is not None:
if step_type != StepType.READONLY_ACTOR_METHOD:
status = WorkflowStatus.FAILED
_record_step_status(step_id, status)
logger.info(get_step_status_info(status))
raise exception
if step_type == StepType.FUNCTION:
persisted_output, volatile_output = result, None
elif step_type == StepType.ACTOR_METHOD:
persisted_output, volatile_output = result
elif step_type == StepType.READONLY_ACTOR_METHOD:
persisted_output, volatile_output = None, result
else:
raise ValueError(f"Unknown StepType '{step_type}'")
is_nested = isinstance(persisted_output, Workflow)
if step_type != StepType.FUNCTION and is_nested:
# TODO(suquark): Support returning a workflow inside
# a virtual actor.
raise TypeError("Only a workflow step function "
"can return a workflow.")
return persisted_output, volatile_output
def _workflow_step_executor(
step_type: StepType, func: Callable,
context: workflow_context.WorkflowStepContext, step_id: "StepID",
baked_inputs: "_BakedWorkflowInputs", outer_most_step_id: "StepID",
catch_exceptions: bool, max_retries: int,
last_step_of_workflow: bool) -> Any:
"""Executor function for workflow step.
Args:
step_type: The type of workflow step.
func: The workflow step function.
context: Workflow step context. Used to access correct storage etc.
step_id: The ID of the step.
baked_inputs: The processed inputs for the step.
outer_most_step_id: See "step_executor.execute_workflow" for
explanation.
catch_exceptions: If set to be true, return
(Optional[Result], Optional[Error]) instead of Result.
max_retries: Max number of retries encounter of a failure.
last_step_of_workflow: The step that generates the output of the
workflow (including nested steps).
Returns:
Workflow step output.
"""
workflow_context.update_workflow_step_context(context, step_id)
args, kwargs = _resolve_step_inputs(baked_inputs)
store = workflow_storage.get_workflow_storage()
try:
persisted_output, volatile_output = _wrap_run(
func, step_type, step_id, catch_exceptions, max_retries, *args,
**kwargs)
except Exception as e:
commit_step(store, step_id, None, e, outer_most_step_id)
raise e
if step_type == StepType.READONLY_ACTOR_METHOD:
if isinstance(volatile_output, Workflow):
raise TypeError(
"Returning a Workflow from a readonly virtual actor "
"is not allowed.")
assert not isinstance(persisted_output, Workflow)
else:
store = workflow_storage.get_workflow_storage()
commit_step(store, step_id, persisted_output, None, outer_most_step_id)
if isinstance(persisted_output, Workflow):
if step_type == StepType.FUNCTION:
# Passing down outer most step so inner nested steps would
# access the same outer most step.
if not outer_most_step_id:
# The current workflow step returns a nested workflow, and
# there is no outer step for the current step. So the
# current step is the outer most step for the inner nested
# workflow steps.
outer_most_step_id = workflow_context.get_current_step_id()
assert volatile_output is None
# execute sub-workflow
result = execute_workflow(persisted_output, outer_most_step_id,
last_step_of_workflow)
# When virtual actor returns a workflow in the method,
# the volatile_output and persisted_output will be put together
persisted_output = result.persisted_output
volatile_output = result.volatile_output
elif last_step_of_workflow:
# advance the progress of the workflow
store.advance_progress(step_id)
_record_step_status(step_id, WorkflowStatus.SUCCESSFUL)
logger.info(get_step_status_info(WorkflowStatus.SUCCESSFUL))
if isinstance(volatile_output, Workflow):
# This is the case where a step method is called in the virtual actor.
# We need to run the method to get the final result.
assert step_type == StepType.ACTOR_METHOD
volatile_output = volatile_output.run_async(
workflow_context.get_current_workflow_id())
return persisted_output, volatile_output