def load_workflow_status(self, workflow_id: str):
"""Load the committed workflow status."""
raw_data = self._storage.get(self._key_workflow_metadata(workflow_id))
if raw_data is not None:
metadata = json.loads(raw_data)
return WorkflowStatus(metadata["status"])
return WorkflowStatus.NONE
async def _list_workflow(self) -> List[Tuple[str, WorkflowStatus]]:
prefix = self._storage.make_key("")
workflow_ids = await self._storage.scan_prefix(prefix)
metadata = await asyncio.gather(*[
self._get([workflow_id, WORKFLOW_META], True)
for workflow_id in workflow_ids
])
return [(wid, WorkflowStatus(meta["status"]) if meta else None)
for (wid, meta) in zip(workflow_ids, metadata)]
def list_all(
status_filter: Optional[Union[Union[WorkflowStatus, str],
Set[Union[WorkflowStatus, str]]]] = None
) -> List[Tuple[str, WorkflowStatus]]:
"""List all workflows matching a given status filter.
Args:
status: If given, only returns workflow with that status. This can
be a single status or set of statuses. The string form of the
status is also acceptable, i.e.,
"RUNNING"/"FAILED"/"SUCCESSFUL"/"CANCELED"/"RESUMABLE".
Examples:
>>> workflow_step = long_running_job.step()
>>> wf = workflow_step.run_async(workflow_id="long_running_job")
>>> jobs = workflow.list_all()
>>> assert jobs == [ ("long_running_job", workflow.RUNNING) ]
>>> ray.get(wf)
>>> jobs = workflow.list_all({workflow.RUNNING})
>>> assert jobs == []
>>> jobs = workflow.list_all(workflow.SUCCESSFUL)
>>> assert jobs == [ ("long_running_job", workflow.SUCCESSFUL) ]
Returns:
A list of tuple with workflow id and workflow status
"""
ensure_ray_initialized()
if isinstance(status_filter, str):
status_filter = set({WorkflowStatus(status_filter)})
elif isinstance(status_filter, WorkflowStatus):
status_filter = set({status_filter})
elif isinstance(status_filter, set):
if all(isinstance(s, str) for s in status_filter):
status_filter = {WorkflowStatus(s) for s in status_filter}
elif not all(isinstance(s, WorkflowStatus) for s in status_filter):
raise TypeError("status_filter contains element which is not"
" a type of `WorkflowStatus or str`."
f" {status_filter}")
elif status_filter is None:
status_filter = set(WorkflowStatus.__members__.keys())
else:
raise TypeError(
"status_filter must be WorkflowStatus or a set of WorkflowStatus.")
return execution.list_all(status_filter)
def _list_workflow(self) -> List[Tuple[str, WorkflowStatus]]:
results = []
for workflow_id in self._scan("", ignore_errors=True):
try:
metadata = self._get(os.path.join(workflow_id, WORKFLOW_META),
True)
results.append(
(workflow_id, WorkflowStatus(metadata["status"])))
except KeyNotFoundError:
pass
return results
def load_workflow_meta(self) -> Optional[WorkflowMetaData]:
"""Load the metadata of the current workflow.
Returns:
The metadata of the current workflow. If it doesn't exist,
return None.
"""
try:
metadata = asyncio_run(self._get(self._key_workflow_metadata(), True))
return WorkflowMetaData(status=WorkflowStatus(metadata["status"]))
except KeyNotFoundError:
return None
def get_workflow_status(self, workflow_id: str) -> WorkflowStatus:
"""Get the status of the workflow."""
if workflow_id in self._workflow_executors:
if workflow_id in self._queued_workflows:
return WorkflowStatus.PENDING
return WorkflowStatus.RUNNING
store = workflow_storage.get_workflow_storage(workflow_id)
status = store.load_workflow_status()
if status == WorkflowStatus.NONE:
raise WorkflowNotFoundError(workflow_id)
elif status in WorkflowStatus.non_terminating_status():
return WorkflowStatus.RESUMABLE
return status
def list_all(
status_filter: Optional[Union[Union[WorkflowStatus, str],
Set[Union[WorkflowStatus, str]]]] = None
) -> List[Tuple[str, WorkflowStatus]]:
"""List all workflows matching a given status filter. When returning "RESUMEABLE"
workflows, the workflows that was running ranks before the workflow that was pending
in the result list.
Args:
status_filter: If given, only returns workflow with that status. This can
be a single status or set of statuses. The string form of the
status is also acceptable, i.e.,
"RUNNING"/"FAILED"/"SUCCESSFUL"/"CANCELED"/"RESUMABLE"/"PENDING".
Examples:
>>> from ray import workflow
>>> long_running_job = ... # doctest: +SKIP
>>> workflow_step = long_running_job.step() # doctest: +SKIP
>>> wf = workflow_step.run_async( # doctest: +SKIP
... workflow_id="long_running_job")
>>> jobs = workflow.list_all() # doctest: +SKIP
>>> assert jobs == [ ("long_running_job", workflow.RUNNING) ] # doctest: +SKIP
>>> ray.get(wf) # doctest: +SKIP
>>> jobs = workflow.list_all({workflow.RUNNING}) # doctest: +SKIP
>>> assert jobs == [] # doctest: +SKIP
>>> jobs = workflow.list_all(workflow.SUCCESSFUL) # doctest: +SKIP
>>> assert jobs == [ # doctest: +SKIP
... ("long_running_job", workflow.SUCCESSFUL)]
Returns:
A list of tuple with workflow id and workflow status
"""
_ensure_workflow_initialized()
if isinstance(status_filter, str):
status_filter = set({WorkflowStatus(status_filter)})
elif isinstance(status_filter, WorkflowStatus):
status_filter = set({status_filter})
elif isinstance(status_filter, set):
if all(isinstance(s, str) for s in status_filter):
status_filter = {WorkflowStatus(s) for s in status_filter}
elif not all(isinstance(s, WorkflowStatus) for s in status_filter):
raise TypeError("status_filter contains element which is not"
" a type of `WorkflowStatus or str`."
f" {status_filter}")
elif status_filter is None:
status_filter = set(WorkflowStatus)
status_filter.discard(WorkflowStatus.NONE)
else:
raise TypeError(
"status_filter must be WorkflowStatus or a set of WorkflowStatus.")
try:
workflow_manager = workflow_access.get_management_actor()
except ValueError:
workflow_manager = None
if workflow_manager is None:
non_terminating_workflows = {}
else:
non_terminating_workflows = ray.get(
workflow_manager.list_non_terminating_workflows.remote())
ret = []
if set(non_terminating_workflows.keys()).issuperset(status_filter):
for status, workflows in non_terminating_workflows.items():
if status in status_filter:
for w in workflows:
ret.append((w, status))
return ret
ret = []
# Here we don't have workflow id, so use empty one instead
store = WorkflowStorage("")
modified_status_filter = status_filter.copy()
# Here we have to add non-terminating status to the status filter, because some
# "RESUMABLE" workflows are converted from non-terminating workflows below.
# This is the tricky part: the status "RESUMABLE" neither come from
# the workflow management actor nor the storage. It is the status where
# the storage says it is non-terminating but the workflow management actor
# is not running it. This usually happened when there was a sudden crash
# of the whole Ray runtime or the workflow management actor
# (due to cluster etc.). So we includes non terminating status in the storage
# filter to get "RESUMABLE" candidates.
modified_status_filter.update(WorkflowStatus.non_terminating_status())
status_from_storage = store.list_workflow(modified_status_filter)
non_terminating_workflows = {
k: set(v)
for k, v in non_terminating_workflows.items()
}
resume_running = []
resume_pending = []
for (k, s) in status_from_storage:
if s in non_terminating_workflows and k not in non_terminating_workflows[
s]:
if s == WorkflowStatus.RUNNING:
resume_running.append(k)
elif s == WorkflowStatus.PENDING:
resume_pending.append(k)
else:
assert False, "This line of code should not be reachable."
continue
if s in status_filter:
ret.append((k, s))
if WorkflowStatus.RESUMABLE in status_filter:
# The running workflows ranks before the pending workflows.
for w in resume_running:
ret.append((w, WorkflowStatus.RESUMABLE))
for w in resume_pending:
ret.append((w, WorkflowStatus.RESUMABLE))
return ret
