def commit_step(
store: workflow_storage.WorkflowStorage,
step_id: "StepID",
ret: Union["Workflow", Any],
*,
exception: Optional[Exception],
):
"""Checkpoint the step output.
Args:
store: The storage the current workflow is using.
step_id: The ID of the step.
ret: The returned object of the workflow step.
exception: The exception caught by the step.
"""
from ray.workflow.common import Workflow
if isinstance(ret, Workflow):
assert not ret.executed
tasks = []
for w in ret._iter_workflows_in_dag():
# If this is a reference to a workflow, do not checkpoint
# its input (again).
if w.ref is None:
tasks.append(_write_step_inputs(store, w.step_id, w.data))
asyncio.get_event_loop().run_until_complete(asyncio.gather(*tasks))
context = workflow_context.get_workflow_step_context()
store.save_step_output(step_id,
ret,
exception=exception,
outer_most_step_id=context.outer_most_step_id)
def step(method_name, method, *args, **kwargs):
readonly = getattr(method, "__virtual_actor_readonly__", False)
flattened_args = self.flatten_args(method_name, args, kwargs)
actor_id = workflow_context.get_current_workflow_id()
if not readonly:
if method_name == "__init__":
state_ref = None
else:
ws = WorkflowStorage(actor_id, get_global_storage())
state_ref = WorkflowRef(ws.get_entrypoint_step_id())
# This is a hack to insert a positional argument.
flattened_args = [signature.DUMMY_TYPE, state_ref
] + flattened_args
workflow_inputs = serialization_context.make_workflow_inputs(
flattened_args)
if readonly:
_actor_method = _wrap_readonly_actor_method(
actor_id, self.cls, method_name)
step_type = StepType.READONLY_ACTOR_METHOD
else:
_actor_method = _wrap_actor_method(self.cls, method_name)
step_type = StepType.ACTOR_METHOD
# TODO(suquark): Support actor options.
workflow_data = WorkflowData(
func_body=_actor_method,
step_type=step_type,
inputs=workflow_inputs,
max_retries=1,
catch_exceptions=False,
ray_options={},
name=None,
)
wf = Workflow(workflow_data)
return wf
def delete(workflow_id: str) -> None:
"""Delete a workflow, its checkpoints, and other information it may have
persisted to storage. To stop a running workflow, see
`workflow.cancel()`.
NOTE: The caller should ensure that the workflow is not currently
running before deleting it.
Args:
workflow_id: The workflow to delete.
Examples:
>>> from ray import workflow
>>> some_job = ... # doctest: +SKIP
>>> workflow_step = some_job.step() # doctest: +SKIP
>>> output = workflow_step.run_async(workflow_id="some_job") # doctest: +SKIP
>>> workflow.delete(workflow_id="some_job") # doctest: +SKIP
>>> assert [] == workflow.list_all() # doctest: +SKIP
Returns:
None
"""
_ensure_workflow_initialized()
try:
status = get_status(workflow_id)
if status == WorkflowStatus.RUNNING:
raise WorkflowRunningError("DELETE", workflow_id)
except ValueError:
raise WorkflowNotFoundError(workflow_id)
wf_storage = WorkflowStorage(workflow_id)
wf_storage.delete_workflow()
def _create(self, args: Tuple[Any], kwargs: Dict[str, Any]):
workflow_storage = WorkflowStorage(self._actor_id, self._storage)
workflow_storage.save_actor_class_body(self._metadata.cls)
ref = self._actor_method_call("__init__", args, kwargs)
workflow_manager = get_or_create_management_actor()
# keep the ref in a list to prevent dereference
ray.get(workflow_manager.init_actor.remote(self._actor_id, [ref]))
def step(self, *args, **kwargs):
flattened_args = signature.flatten_args(self._signature, args, kwargs)
actor_id = workflow_context.get_current_workflow_id()
if not self.readonly:
if self._method_name == "__init__":
state_ref = None
else:
ws = WorkflowStorage(actor_id, get_global_storage())
state_ref = WorkflowRef(ws.get_entrypoint_step_id())
# This is a hack to insert a positional argument.
flattened_args = [signature.DUMMY_TYPE, state_ref] + flattened_args
workflow_inputs = serialization_context.make_workflow_inputs(
flattened_args)
if self.readonly:
_actor_method = _wrap_readonly_actor_method(
actor_id, self._original_class, self._method_name)
else:
_actor_method = _wrap_actor_method(self._original_class,
self._method_name)
workflow_data = WorkflowData(
func_body=_actor_method,
inputs=workflow_inputs,
name=self._name,
step_options=self._options,
user_metadata=self._user_metadata,
)
wf = Workflow(workflow_data)
return wf
def get_output_async(workflow_id: str,
*,
name: Optional[str] = None) -> ray.ObjectRef:
"""Get the output of a running workflow asynchronously.
Args:
workflow_id: The workflow to get the output of.
name: If set, fetch the specific step instead of the output of the
workflow.
Returns:
An object reference that can be used to retrieve the workflow task result.
"""
_ensure_workflow_initialized()
try:
workflow_manager = workflow_access.get_management_actor()
except ValueError as e:
raise ValueError(
"Failed to connect to the workflow management "
"actor. The workflow could have already failed. You can use "
"workflow.resume() to resume the workflow.") from e
try:
# check storage first
wf_store = WorkflowStorage(workflow_id)
tid = wf_store.inspect_output(name)
if tid is not None:
return workflow_access.load_step_output_from_storage.remote(
workflow_id, name)
except ValueError:
pass
return workflow_manager.get_output.remote(workflow_id, name)
def _construct_resume_workflow_from_step(
reader: workflow_storage.WorkflowStorage, step_id: StepID,
input_map: Dict[StepID, Any]) -> Union[Workflow, StepID]:
"""Try to construct a workflow (step) that recovers the workflow step.
If the workflow step already has an output checkpointing file, we return
the workflow step id instead.
Args:
reader: The storage reader for inspecting the step.
step_id: The ID of the step we want to recover.
input_map: This is a context storing the input which has been loaded.
This context is important for dedupe
Returns:
A workflow that recovers the step, or a ID of a step
that contains the output checkpoint file.
"""
result: workflow_storage.StepInspectResult = reader.inspect_step(step_id)
if result.output_object_valid:
# we already have the output
return step_id
if isinstance(result.output_step_id, str):
return _construct_resume_workflow_from_step(reader,
result.output_step_id,
input_map)
# output does not exists or not valid. try to reconstruct it.
if not result.is_recoverable():
raise WorkflowStepNotRecoverableError(step_id)
with serialization.objectref_cache():
input_workflows = []
for i, _step_id in enumerate(result.workflows):
# Check whether the step has been loaded or not to avoid
# duplication
if _step_id in input_map:
r = input_map[_step_id]
else:
r = _construct_resume_workflow_from_step(
reader, _step_id, input_map)
input_map[_step_id] = r
if isinstance(r, Workflow):
input_workflows.append(r)
else:
assert isinstance(r, StepID)
# TODO (Alex): We should consider caching these outputs too.
input_workflows.append(reader.load_step_output(r))
workflow_refs = list(map(WorkflowRef, result.workflow_refs))
args, kwargs = reader.load_step_args(step_id, input_workflows,
workflow_refs)
recovery_workflow: Workflow = _recover_workflow_step.options(
max_retries=result.max_retries,
catch_exceptions=result.catch_exceptions,
**result.ray_options).step(args, kwargs, input_workflows,
workflow_refs)
recovery_workflow._step_id = step_id
recovery_workflow.data.step_type = result.step_type
return recovery_workflow
def _create(self, args: Tuple[Any], kwargs: Dict[str, Any]):
workflow_storage = WorkflowStorage(self._actor_id)
workflow_storage.save_actor_class_body(self._metadata.cls)
method_helper = self._metadata.methods["__init__"]
job_id = ray.get_runtime_context().job_id.hex()
ref = self._actor_method_call(job_id, method_helper, args, kwargs)
workflow_manager = get_or_create_management_actor()
# keep the ref in a list to prevent dereference
ray.get(workflow_manager.init_actor.remote(self._actor_id, [ref]))
def get_metadata(workflow_id: str,
name: Optional[str] = None) -> Dict[str, Any]:
"""Get the metadata of the workflow.
This will return a dict of metadata of either the workflow (
if only workflow_id is given) or a specific workflow step (if
both workflow_id and step name are given). Exception will be
raised if the given workflow id or step name does not exist.
If only workflow id is given, this will return metadata on
workflow level, which includes running status, workflow-level
user metadata and workflow-level running stats (e.g. the
start time and end time of the workflow).
If both workflow id and step name are given, this will return
metadata on workflow step level, which includes step inputs,
step-level user metadata and step-level running stats (e.g.
the start time and end time of the step).
Args:
workflow_id: The workflow to get the metadata of.
name: If set, fetch the metadata of the specific step instead of
the metadata of the workflow.
Examples:
>>> from ray import workflow
>>> trip = ... # doctest: +SKIP
>>> workflow_step = trip.options( # doctest: +SKIP
... name="trip", metadata={"k1": "v1"}).step()
>>> workflow_step.run( # doctest: +SKIP
... workflow_id="trip1", metadata={"k2": "v2"})
>>> workflow_metadata = workflow.get_metadata("trip1") # doctest: +SKIP
>>> assert workflow_metadata["status"] == "SUCCESSFUL" # doctest: +SKIP
>>> assert workflow_metadata["user_metadata"] == {"k2": "v2"} # doctest: +SKIP
>>> assert "start_time" in workflow_metadata["stats"] # doctest: +SKIP
>>> assert "end_time" in workflow_metadata["stats"] # doctest: +SKIP
>>> step_metadata = workflow.get_metadata("trip1", "trip") # doctest: +SKIP
>>> assert step_metadata["step_type"] == "FUNCTION" # doctest: +SKIP
>>> assert step_metadata["user_metadata"] == {"k1": "v1"} # doctest: +SKIP
>>> assert "start_time" in step_metadata["stats"] # doctest: +SKIP
>>> assert "end_time" in step_metadata["stats"] # doctest: +SKIP
Returns:
A dictionary containing the metadata of the workflow.
Raises:
ValueError: if given workflow or workflow step does not exist.
"""
_ensure_workflow_initialized()
store = WorkflowStorage(workflow_id)
if name is None:
return store.load_workflow_metadata()
else:
return store.load_step_metadata(name)
def get_actor(actor_id: str, storage: Storage) -> VirtualActor:
"""Get an virtual actor.
Args:
actor_id: The ID of the actor.
storage: The storage of the actor.
Returns:
A virtual actor.
"""
ws = WorkflowStorage(actor_id, storage)
cls = ws.load_actor_class_body()
v_cls = VirtualActorClass._from_class(cls)
return v_cls._construct(actor_id, storage)
def _reconstruct_wait_step(
reader: workflow_storage.WorkflowStorage,
result: workflow_storage.StepInspectResult,
input_map: Dict[StepID, Any],
):
input_workflows = []
step_options = result.step_options
wait_options = step_options.ray_options.get("wait_options", {})
for i, _step_id in enumerate(result.workflows):
# Check whether the step has been loaded or not to avoid
# duplication
if _step_id in input_map:
r = input_map[_step_id]
else:
r = _construct_resume_workflow_from_step(reader, _step_id, input_map)
input_map[_step_id] = r
if isinstance(r, Workflow):
input_workflows.append(r)
else:
assert isinstance(r, StepID)
# TODO (Alex): We should consider caching these outputs too.
output = reader.load_step_output(r)
# Simulate a workflow with a workflow reference so it could be
# used directly by 'workflow.wait'.
static_ref = WorkflowStaticRef(step_id=r, ref=ray.put(output))
wf = Workflow.from_ref(static_ref)
input_workflows.append(wf)
from ray import workflow
return workflow.wait(input_workflows, **wait_options)
def resolve(self, store: workflow_storage.WorkflowStorage) -> Tuple[List, Dict]:
"""
This function resolves the inputs for the code inside
a workflow step (works on the callee side). For outputs from other
workflows, we resolve them into object instances inplace.
For each ObjectRef argument, the function returns both the ObjectRef
and the object instance. If the ObjectRef is a chain of nested
ObjectRefs, then we resolve it recursively until we get the
object instance, and we return the *direct* ObjectRef of the
instance. This function does not resolve ObjectRef
inside another object (e.g. list of ObjectRefs) to give users some
flexibility.
Returns:
Instances of arguments.
"""
workflow_ref_mapping = []
for r in self.workflow_refs:
if r.ref is None:
workflow_ref_mapping.append(store.load_step_output(r.task_id))
else:
workflow_ref_mapping.append(r.ref)
with serialization_context.workflow_args_resolving_context(
workflow_ref_mapping
):
# reconstruct input arguments under correct serialization context
flattened_args: List[Any] = ray.get(self.args)
# dereference arguments like Ray remote functions
flattened_args = [
ray.get(a) if isinstance(a, ObjectRef) else a for a in flattened_args
]
return signature.recover_args(flattened_args)
def commit_step(store: workflow_storage.WorkflowStorage,
step_id: "StepID",
ret: Union["Workflow", Any],
exception: Optional[Exception],
outer_most_step_id: Optional[str] = None):
"""Checkpoint the step output.
Args:
store: The storage the current workflow is using.
step_id: The ID of the step.
ret: The returned object of the workflow step.
outer_most_step_id: The ID of the outer most workflow. None if it
does not exists. See "step_executor.execute_workflow" for detailed
explanation.
"""
from ray.workflow.common import Workflow
if isinstance(ret, Workflow):
store.save_subworkflow(ret)
store.save_step_output(step_id, ret, exception, outer_most_step_id)
async def _write_step_inputs(wf_storage: workflow_storage.WorkflowStorage,
step_id: StepID, inputs: WorkflowData) -> None:
"""Save workflow inputs."""
metadata = inputs.to_metadata()
with serialization_context.workflow_args_keeping_context():
# TODO(suquark): in the future we should write to storage directly
# with plasma store object in memory.
args_obj = ray.get(inputs.inputs.args)
workflow_id = wf_storage._workflow_id
storage = wf_storage._storage
save_tasks = [
# TODO (Alex): Handle the json case better?
wf_storage._put(
wf_storage._key_step_input_metadata(step_id), metadata, True),
serialization.dump_to_storage(
wf_storage._key_step_function_body(step_id), inputs.func_body,
workflow_id, storage),
serialization.dump_to_storage(
wf_storage._key_step_args(step_id), args_obj, workflow_id, storage)
]
await asyncio.gather(*save_tasks)
def commit_step(store: workflow_storage.WorkflowStorage, step_id: "StepID",
ret: Union["Workflow", Any], exception: Optional[Exception]):
"""Checkpoint the step output.
Args:
store: The storage the current workflow is using.
step_id: The ID of the step.
ret: The returned object of the workflow step.
"""
from ray.workflow.common import Workflow
if isinstance(ret, Workflow):
assert not ret.executed
tasks = [
_write_step_inputs(store, w.step_id, w.data)
for w in ret._iter_workflows_in_dag()
]
asyncio.get_event_loop().run_until_complete(asyncio.gather(*tasks))
context = workflow_context.get_workflow_step_context()
store.save_step_output(step_id,
ret,
exception=exception,
outer_most_step_id=context.outer_most_step_id)
def commit_step(store: workflow_storage.WorkflowStorage,
step_id: "StepID",
ret: Union["Workflow", Any],
exception: Optional[Exception],
outer_most_step_id: Optional[str] = None):
"""Checkpoint the step output.
Args:
store: The storage the current workflow is using.
step_id: The ID of the step.
ret: The returned object of the workflow step.
outer_most_step_id: The ID of the outer most workflow. None if it
does not exists. See "step_executor.execute_workflow" for detailed
explanation.
"""
from ray.workflow.common import Workflow
if isinstance(ret, Workflow):
assert not ret.executed
tasks = [
_write_step_inputs(store, w.step_id, w.data)
for w in ret._iter_workflows_in_dag()
]
asyncio.get_event_loop().run_until_complete(asyncio.gather(*tasks))
store.save_step_output(step_id, ret, exception, outer_most_step_id)
def list_all(
status_filter: Optional[Union[Union[WorkflowStatus, str],
Set[Union[WorkflowStatus, str]]]] = None
) -> List[Tuple[str, WorkflowStatus]]:
"""List all workflows matching a given status filter. When returning "RESUMEABLE"
workflows, the workflows that was running ranks before the workflow that was pending
in the result list.
Args:
status_filter: If given, only returns workflow with that status. This can
be a single status or set of statuses. The string form of the
status is also acceptable, i.e.,
"RUNNING"/"FAILED"/"SUCCESSFUL"/"CANCELED"/"RESUMABLE"/"PENDING".
Examples:
>>> from ray import workflow
>>> long_running_job = ... # doctest: +SKIP
>>> workflow_step = long_running_job.step() # doctest: +SKIP
>>> wf = workflow_step.run_async( # doctest: +SKIP
... workflow_id="long_running_job")
>>> jobs = workflow.list_all() # doctest: +SKIP
>>> assert jobs == [ ("long_running_job", workflow.RUNNING) ] # doctest: +SKIP
>>> ray.get(wf) # doctest: +SKIP
>>> jobs = workflow.list_all({workflow.RUNNING}) # doctest: +SKIP
>>> assert jobs == [] # doctest: +SKIP
>>> jobs = workflow.list_all(workflow.SUCCESSFUL) # doctest: +SKIP
>>> assert jobs == [ # doctest: +SKIP
... ("long_running_job", workflow.SUCCESSFUL)]
Returns:
A list of tuple with workflow id and workflow status
"""
_ensure_workflow_initialized()
if isinstance(status_filter, str):
status_filter = set({WorkflowStatus(status_filter)})
elif isinstance(status_filter, WorkflowStatus):
status_filter = set({status_filter})
elif isinstance(status_filter, set):
if all(isinstance(s, str) for s in status_filter):
status_filter = {WorkflowStatus(s) for s in status_filter}
elif not all(isinstance(s, WorkflowStatus) for s in status_filter):
raise TypeError("status_filter contains element which is not"
" a type of `WorkflowStatus or str`."
f" {status_filter}")
elif status_filter is None:
status_filter = set(WorkflowStatus)
status_filter.discard(WorkflowStatus.NONE)
else:
raise TypeError(
"status_filter must be WorkflowStatus or a set of WorkflowStatus.")
try:
workflow_manager = workflow_access.get_management_actor()
except ValueError:
workflow_manager = None
if workflow_manager is None:
non_terminating_workflows = {}
else:
non_terminating_workflows = ray.get(
workflow_manager.list_non_terminating_workflows.remote())
ret = []
if set(non_terminating_workflows.keys()).issuperset(status_filter):
for status, workflows in non_terminating_workflows.items():
if status in status_filter:
for w in workflows:
ret.append((w, status))
return ret
ret = []
# Here we don't have workflow id, so use empty one instead
store = WorkflowStorage("")
modified_status_filter = status_filter.copy()
# Here we have to add non-terminating status to the status filter, because some
# "RESUMABLE" workflows are converted from non-terminating workflows below.
# This is the tricky part: the status "RESUMABLE" neither come from
# the workflow management actor nor the storage. It is the status where
# the storage says it is non-terminating but the workflow management actor
# is not running it. This usually happened when there was a sudden crash
# of the whole Ray runtime or the workflow management actor
# (due to cluster etc.). So we includes non terminating status in the storage
# filter to get "RESUMABLE" candidates.
modified_status_filter.update(WorkflowStatus.non_terminating_status())
status_from_storage = store.list_workflow(modified_status_filter)
non_terminating_workflows = {
k: set(v)
for k, v in non_terminating_workflows.items()
}
resume_running = []
resume_pending = []
for (k, s) in status_from_storage:
if s in non_terminating_workflows and k not in non_terminating_workflows[
s]:
if s == WorkflowStatus.RUNNING:
resume_running.append(k)
elif s == WorkflowStatus.PENDING:
resume_pending.append(k)
else:
assert False, "This line of code should not be reachable."
continue
if s in status_filter:
ret.append((k, s))
if WorkflowStatus.RESUMABLE in status_filter:
# The running workflows ranks before the pending workflows.
for w in resume_running:
ret.append((w, WorkflowStatus.RESUMABLE))
for w in resume_pending:
ret.append((w, WorkflowStatus.RESUMABLE))
return ret
def _construct_resume_workflow_from_step(
reader: workflow_storage.WorkflowStorage,
step_id: StepID,
input_map: Dict[StepID, Any],
) -> Union[Workflow, StepID]:
"""Try to construct a workflow (step) that recovers the workflow step.
If the workflow step already has an output checkpointing file, we return
the workflow step id instead.
Args:
reader: The storage reader for inspecting the step.
step_id: The ID of the step we want to recover.
input_map: This is a context storing the input which has been loaded.
This context is important for dedupe
Returns:
A workflow that recovers the step, or a ID of a step
that contains the output checkpoint file.
"""
result: workflow_storage.StepInspectResult = reader.inspect_step(step_id)
if result.output_object_valid:
# we already have the output
return step_id
if isinstance(result.output_step_id, str):
return _construct_resume_workflow_from_step(
reader, result.output_step_id, input_map
)
# output does not exists or not valid. try to reconstruct it.
if not result.is_recoverable():
raise WorkflowStepNotRecoverableError(step_id)
step_options = result.step_options
# Process the wait step as a special case.
if step_options.step_type == StepType.WAIT:
return _reconstruct_wait_step(reader, step_id, result, input_map)
with serialization.objectref_cache():
input_workflows = []
for i, _step_id in enumerate(result.workflows):
# Check whether the step has been loaded or not to avoid
# duplication
if _step_id in input_map:
r = input_map[_step_id]
else:
r = _construct_resume_workflow_from_step(reader, _step_id, input_map)
input_map[_step_id] = r
if isinstance(r, Workflow):
input_workflows.append(r)
else:
assert isinstance(r, StepID)
# TODO (Alex): We should consider caching these outputs too.
input_workflows.append(reader.load_step_output(r))
workflow_refs = list(map(WorkflowRef, result.workflow_refs))
args, kwargs = reader.load_step_args(step_id, input_workflows, workflow_refs)
# Note: we must uppack args and kwargs, so the refs in the args/kwargs can get
# resolved consistently like in Ray.
recovery_workflow: Workflow = _recover_workflow_step.step(
input_workflows,
workflow_refs,
*args,
**kwargs,
)
recovery_workflow._step_id = step_id
# override step_options
recovery_workflow.data.step_options = step_options
return recovery_workflow
def _construct_resume_workflow_from_step(
reader: workflow_storage.WorkflowStorage,
step_id: StepID,
objectref_cache: Dict[str, Any] = None) -> Union[Workflow, StepID]:
"""Try to construct a workflow (step) that recovers the workflow step.
If the workflow step already has an output checkpointing file, we return
the workflow step id instead.
Args:
reader: The storage reader for inspecting the step.
step_id: The ID of the step we want to recover.
Returns:
A workflow that recovers the step, or a ID of a step
that contains the output checkpoint file.
"""
if objectref_cache is None:
objectref_cache = {}
result: workflow_storage.StepInspectResult = reader.inspect_step(step_id)
if result.output_object_valid:
# we already have the output
return step_id
if isinstance(result.output_step_id, str):
return _construct_resume_workflow_from_step(
reader, result.output_step_id, objectref_cache=objectref_cache)
# output does not exists or not valid. try to reconstruct it.
if not result.is_recoverable():
raise WorkflowStepNotRecoverableError(step_id)
input_workflows = []
instant_workflow_outputs: Dict[int, str] = {}
for i, _step_id in enumerate(result.workflows):
r = _construct_resume_workflow_from_step(
reader, _step_id, objectref_cache=objectref_cache)
if isinstance(r, Workflow):
input_workflows.append(r)
else:
input_workflows.append(None)
instant_workflow_outputs[i] = r
workflow_refs = list(map(WorkflowRef, result.workflow_refs))
# TODO (Alex): Refactor to remove this special case handling of object refs
resolved_object_refs = []
identifiers_to_await = []
promises_to_await = []
for identifier in result.object_refs:
if identifier not in objectref_cache:
paths = reader._key_step_args(identifier)
promise = reader._get(paths)
promises_to_await.append(promise)
identifiers_to_await.append(identifier)
loop = asyncio.get_event_loop()
object_refs_to_cache = loop.run_until_complete(
asyncio.gather(*promises_to_await))
for identifier, object_ref in zip(identifiers_to_await,
object_refs_to_cache):
objectref_cache[identifier] = object_ref
for identifier in result.object_refs:
resolved_object_refs.append(objectref_cache[identifier])
recovery_workflow: Workflow = _recover_workflow_step.options(
max_retries=result.max_retries,
catch_exceptions=result.catch_exceptions,
**result.ray_options).step(resolved_object_refs, input_workflows,
workflow_refs, instant_workflow_outputs)
recovery_workflow._step_id = step_id
recovery_workflow.data.step_type = result.step_type
return recovery_workflow
def run_async(
dag: DAGNode,
*args,
workflow_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
**kwargs,
) -> ray.ObjectRef:
"""Run a workflow asynchronously.
If the workflow with the given id already exists, it will be resumed.
Args:
workflow_id: A unique identifier that can be used to resume the
workflow. If not specified, a random id will be generated.
metadata: The metadata to add to the workflow. It has to be able
to serialize to json.
Returns:
The running result as ray.ObjectRef.
"""
_ensure_workflow_initialized()
if not isinstance(dag, DAGNode):
raise TypeError("Input should be a DAG.")
input_data = DAGInputData(*args, **kwargs)
validate_user_metadata(metadata)
metadata = metadata or {}
if workflow_id is None:
# Workflow ID format: {Entry workflow UUID}.{Unix time to nanoseconds}
workflow_id = f"{str(uuid.uuid4())}.{time.time():.9f}"
state = workflow_state_from_dag(dag, input_data, workflow_id)
logger.info(f'Workflow job created. [id="{workflow_id}"].')
context = workflow_context.WorkflowStepContext(workflow_id=workflow_id)
with workflow_context.workflow_step_context(context):
# checkpoint the workflow
ws = WorkflowStorage(workflow_id)
ws.save_workflow_user_metadata(metadata)
job_id = ray.get_runtime_context().job_id.hex()
try:
ws.get_entrypoint_step_id()
wf_exists = True
except Exception:
# The workflow does not exist. We must checkpoint entry workflow.
ws.save_workflow_execution_state("", state)
wf_exists = False
workflow_manager = workflow_access.get_management_actor()
if ray.get(
workflow_manager.is_workflow_non_terminating.remote(
workflow_id)):
raise RuntimeError(
f"Workflow '{workflow_id}' is already running or pending.")
if wf_exists:
return resume_async(workflow_id)
ignore_existing = ws.load_workflow_status() == WorkflowStatus.NONE
ray.get(
workflow_manager.submit_workflow.remote(
workflow_id, state, ignore_existing=ignore_existing))
return workflow_manager.execute_workflow.remote(job_id, context)