def _resume_workflow_step_executor(workflow_id: str, step_id: "StepID",
store_url: str, current_output: [
ray.ObjectRef
]) -> Tuple[ray.ObjectRef, ray.ObjectRef]:
# TODO (yic): We need better dependency management for virtual actor
# The current output will always be empty for normal workflow
# For virtual actor, if it's not empty, it means the previous job is
# running. This is a really bad one.
for ref in current_output:
try:
while isinstance(ref, ray.ObjectRef):
ref = ray.get(ref)
except Exception:
pass
try:
store = storage.create_storage(store_url)
wf_store = workflow_storage.WorkflowStorage(workflow_id, store)
r = _construct_resume_workflow_from_step(wf_store, step_id)
except Exception as e:
raise WorkflowNotResumableError(workflow_id) from e
if isinstance(r, Workflow):
with workflow_context.workflow_step_context(workflow_id,
store.storage_url):
from ray.workflow.step_executor import (execute_workflow)
result = execute_workflow(r, last_step_of_workflow=True)
return result.persisted_output, result.volatile_output
assert isinstance(r, StepID)
return wf_store.load_step_output(r), None
def update_workflow_step_context(context: Optional[WorkflowStepContext],
step_id: str):
global _context
_context = context
_context.workflow_scope.append(step_id)
# avoid cyclic import
from ray.workflow import storage
# TODO(suquark): [optimization] if the original storage has the same URL,
# skip creating the new one
storage.set_global_storage(storage.create_storage(context.storage_url))
def init(storage: "Optional[Union[str, Storage]]" = None) -> None:
"""Initialize workflow.
Args:
storage: The external storage URL or a custom storage class. If not
specified, ``/tmp/ray/workflow_data`` will be used.
"""
if storage is None:
storage = os.environ.get("RAY_WORKFLOW_STORAGE")
if storage is None:
# We should use get_temp_dir_path, but for ray client, we don't
# have this one. We need a flag to tell whether it's a client
# or a driver to use the right dir.
# For now, just use /tmp/ray/workflow_data
storage = "file:///tmp/ray/workflow_data"
if isinstance(storage, str):
logger.info(f"Using storage: {storage}")
storage = storage_base.create_storage(storage)
elif not isinstance(storage, Storage):
raise TypeError("'storage' should be None, str, or Storage type.")
try:
_storage = storage_base.get_global_storage()
except RuntimeError:
pass
else:
# we have to use the 'else' branch because we would raise a
# runtime error, but we do not want to be captured by 'except'
if _storage.storage_url == storage.storage_url:
logger.warning("Calling 'workflow.init()' again with the same "
"storage.")
else:
raise RuntimeError("Calling 'workflow.init()' again with a "
"different storage")
storage_base.set_global_storage(storage)
workflow_access.init_management_actor()
serialization.init_manager()
