def get_output(self, workflow_id: str) -> "ray.ObjectRef":
"""Get the output of a running workflow.
Args:
workflow_id: The ID of a workflow job.
Returns:
An object reference that can be used to retrieve the
workflow result.
"""
if workflow_id in self._workflow_outputs:
return self._workflow_outputs[workflow_id].output
wf_store = workflow_storage.WorkflowStorage(workflow_id, self._store)
meta = wf_store.load_workflow_meta()
if meta is None:
raise ValueError(f"No such workflow {workflow_id}")
if meta == common.WorkflowStatus.FAILED:
raise ValueError(
f"Workflow {workflow_id} failed, please resume it")
step_id = wf_store.get_entrypoint_step_id()
result = recovery.resume_workflow_step(workflow_id, step_id,
self._store.storage_url)
latest_output = LatestWorkflowOutput(result.persisted_output,
workflow_id, step_id)
self._workflow_outputs[workflow_id] = latest_output
wf_store = workflow_storage.WorkflowStorage(workflow_id, self._store)
wf_store.save_workflow_meta(
common.WorkflowMetaData(common.WorkflowStatus.RUNNING))
self._step_status.setdefault(workflow_id, {})
# "persisted_output" is the return value of a step or the state of
# a virtual actor.
return result.persisted_output
def _resume_workflow_step_executor(
workflow_id: str, step_id: "StepID", store_url: str,
current_output: [ray.ObjectRef
]) -> Tuple[ray.ObjectRef, ray.ObjectRef]:
# TODO (yic): We need better dependency management for virtual actor
# The current output will always be empty for normal workflow
# For virtual actor, if it's not empty, it means the previous job is
# running. This is a really bad one.
for ref in current_output:
try:
while isinstance(ref, ray.ObjectRef):
ref = ray.get(ref)
except Exception:
pass
try:
store = storage.create_storage(store_url)
wf_store = workflow_storage.WorkflowStorage(workflow_id, store)
r = _construct_resume_workflow_from_step(wf_store, step_id)
except Exception as e:
raise WorkflowNotResumableError(workflow_id) from e
if isinstance(r, Workflow):
with workflow_context.workflow_step_context(workflow_id,
store.storage_url):
from ray.experimental.workflow.step_executor import (
execute_workflow)
result = execute_workflow(r, last_step_of_workflow=True)
return result.persisted_output, result.volatile_output
assert isinstance(r, StepID)
return wf_store.load_step_output(r), None
def update_step_status(self, workflow_id: str, step_id: str,
status: common.WorkflowStatus):
# Note: For virtual actor, we could add more steps even if
# the workflow finishes.
self._step_status.setdefault(workflow_id, {})
if status == common.WorkflowStatus.SUCCESSFUL:
self._step_status[workflow_id].pop(step_id, None)
else:
self._step_status.setdefault(workflow_id, {})[step_id] = status
remaining = len(self._step_status[workflow_id])
if status != common.WorkflowStatus.RUNNING:
self._step_output_cache.pop((workflow_id, step_id), None)
if status != common.WorkflowStatus.FAILED and remaining != 0:
return
wf_store = workflow_storage.WorkflowStorage(workflow_id, self._store)
if status == common.WorkflowStatus.FAILED:
if workflow_id in self._workflow_outputs:
cancel_job(self._workflow_outputs.pop(workflow_id).output)
wf_store.save_workflow_meta(
common.WorkflowMetaData(common.WorkflowStatus.FAILED))
self._step_status.pop(workflow_id)
else:
# remaining = 0
wf_store.save_workflow_meta(
common.WorkflowMetaData(common.WorkflowStatus.SUCCESSFUL))
self._step_status.pop(workflow_id)
def _recover_workflow_step(input_object_refs: List[str],
input_workflows: List[Any],
instant_workflow_inputs: Dict[int, StepID]):
"""A workflow step that recovers the output of an unfinished step.
Args:
input_object_refs: The object refs in the argument of
the (original) step.
input_workflows: The workflows in the argument of the (original) step.
They are resolved into physical objects (i.e. the output of the
workflows) here. They come from other recover workflows we
construct recursively.
instant_workflow_inputs: Same as 'input_workflows', but they come
point to workflow steps that have output checkpoints. They override
corresponding workflows in 'input_workflows'.
Returns:
The output of the recovered step.
"""
reader = workflow_storage.WorkflowStorage()
for index, _step_id in instant_workflow_inputs.items():
# override input workflows with instant workflows
input_workflows[index] = reader.load_step_output(_step_id)
input_object_refs = [reader.load_object_ref(r) for r in input_object_refs]
step_id = workflow_context.get_current_step_id()
func: Callable = reader.load_step_func_body(step_id)
args, kwargs = reader.load_step_args(step_id, input_workflows,
input_object_refs)
return func(*args, **kwargs)
def resume_workflow_job(workflow_id: str,
store: storage.Storage) -> ray.ObjectRef:
"""Resume a workflow job.
Args:
workflow_id: The ID of the workflow job. The ID is used to identify
the workflow.
store: The storage to access the workflow.
Raises:
WorkflowNotResumableException: fail to resume the workflow.
Returns:
The execution result of the workflow, represented by Ray ObjectRef.
"""
reader = workflow_storage.WorkflowStorage(workflow_id, store)
try:
entrypoint_step_id: StepID = reader.get_entrypoint_step_id()
r = _construct_resume_workflow_from_step(reader, entrypoint_step_id)
except Exception as e:
raise WorkflowNotResumableError(workflow_id) from e
if isinstance(r, Workflow):
try:
workflow_context.init_workflow_step_context(
workflow_id, store.storage_url)
return execute_workflow(r)
finally:
workflow_context.set_workflow_step_context(None)
return ray.put(reader.load_step_output(r))
def run_or_resume(self, workflow_id: str, ignore_existing: bool = False
) -> "WorkflowExecutionResult":
"""Run or resume a workflow.
Args:
workflow_id: The ID of the workflow.
ignore_existing: Ignore we already have an existing output. When
set false, raise an exception if there has already been a workflow
running with this id
Returns:
Workflow execution result that contains the state and output.
"""
if workflow_id in self._workflow_outputs and not ignore_existing:
raise RuntimeError(f"The output of workflow[id={workflow_id}] "
"already exists.")
wf_store = workflow_storage.WorkflowStorage(workflow_id, self._store)
step_id = wf_store.get_entrypoint_step_id()
result = recovery.resume_workflow_step(workflow_id, step_id,
self._store.storage_url)
latest_output = LatestWorkflowOutput(result.persisted_output,
workflow_id, step_id)
self._workflow_outputs[workflow_id] = latest_output
self._step_output_cache[workflow_id, step_id] = latest_output
wf_store.save_workflow_meta(
common.WorkflowMetaData(common.WorkflowStatus.RUNNING))
if workflow_id not in self._step_status:
self._step_status[workflow_id] = {}
logger.info(f"Workflow job [id={workflow_id}] started.")
return result
def resume_workflow_job(workflow_id: str, store_url: str) -> ray.ObjectRef:
"""Resume a workflow job.
Args:
workflow_id: The ID of the workflow job. The ID is used to identify
the workflow.
store_url: The url of the storage to access the workflow.
Raises:
WorkflowNotResumableException: fail to resume the workflow.
Returns:
The execution result of the workflow, represented by Ray ObjectRef.
"""
try:
store = storage.create_storage(store_url)
wf_store = workflow_storage.WorkflowStorage(workflow_id, store)
entrypoint_step_id: StepID = wf_store.get_entrypoint_step_id()
r = _construct_resume_workflow_from_step(wf_store, entrypoint_step_id)
except Exception as e:
raise WorkflowNotResumableError(workflow_id) from e
if isinstance(r, Workflow):
with workflow_context.workflow_step_context(workflow_id,
store.storage_url):
from ray.experimental.workflow.step_executor import (
execute_workflow)
return execute_workflow(r)
return wf_store.load_step_output(r)
def get_latest_output(workflow_id: str, store: storage.Storage) -> Any:
"""Get the latest output of a workflow. This function is intended to be
used by readonly virtual actors. To resume a workflow,
`resume_workflow_job` should be used instead.
Args:
workflow_id: The ID of the workflow.
store: The storage of the workflow.
Returns:
The output of the workflow.
"""
reader = workflow_storage.WorkflowStorage(workflow_id, store)
try:
step_id: StepID = reader.get_entrypoint_step_id()
while True:
result: workflow_storage.StepInspectResult = reader.inspect_step(
step_id)
if result.output_object_valid:
# we already have the output
return reader.load_step_output(step_id)
if isinstance(result.output_step_id, str):
step_id = result.output_step_id
else:
raise ValueError(
"Workflow output does not exists or not valid.")
except Exception as e:
raise WorkflowNotResumableError(workflow_id) from e
def test_embedded_objectrefs(workflow_start_regular):
workflow_id = test_workflow_storage.__name__
class ObjectRefsWrapper:
def __init__(self, refs):
self.refs = refs
wf_storage = workflow_storage.WorkflowStorage(workflow_id,
storage.get_global_storage())
url = storage.get_global_storage().storage_url
wrapped = ObjectRefsWrapper([ray.put(1), ray.put(2)])
asyncio_run(wf_storage._put(["key"], wrapped))
# Be extremely explicit about shutting down. We want to make sure the
# `_get` call deserializes the full object and puts it in the object store.
# Shutting down the cluster should guarantee we don't accidently get the
# old object and pass the test.
ray.shutdown()
subprocess.check_output("ray stop --force", shell=True)
workflow.init(url)
storage2 = get_workflow_storage(workflow_id)
result = asyncio_run(storage2._get(["key"]))
assert ray.get(result.refs) == [1, 2]
def gen_step_id(self, workflow_id: str, step_name: str) -> int:
wf_store = workflow_storage.WorkflowStorage(workflow_id, self._store)
idx = wf_store.gen_step_id(step_name)
if idx == 0:
return step_name
else:
return f"{step_name}_{idx}"
def actor_ready(self, actor_id: str, storage_url: str) -> ray.ObjectRef:
"""Check if a workflow virtual actor is fully initialized.
Args:
actor_id: The ID of a workflow virtual actor.
storage_url: A string that represents the storage.
Returns:
A future object that represents the state of the actor.
"ray.get" the object successfully indicates the actor is
initialized successfully.
"""
store = storage.create_storage(storage_url)
ws = workflow_storage.WorkflowStorage(actor_id, store)
try:
step_id = ws.get_entrypoint_step_id()
output_exists = ws.inspect_step(step_id).output_object_valid
if output_exists:
return ray.put(None)
except Exception:
pass
if actor_id not in self._actor_initialized:
raise ValueError(f"Actor '{actor_id}' has not been created, or "
"it has failed before initialization.")
return self._actor_initialized[actor_id]
def test_shortcut():
ray.init()
output = workflow.run(recursive_chain.step(0), workflow_id="shortcut")
assert ray.get(output) == 100
# the shortcut points to the step with output checkpoint
store = workflow_storage.WorkflowStorage("shortcut")
step_id = store.get_entrypoint_step_id()
assert store.inspect_step(step_id).output_object_valid
ray.shutdown()
def get_output(self, workflow_id: str,
name: Optional[str]) -> "ray.ObjectRef":
"""Get the output of a running workflow.
Args:
workflow_id: The ID of a workflow job.
Returns:
An object reference that can be used to retrieve the
workflow result.
"""
if workflow_id in self._workflow_outputs and name is None:
return self._workflow_outputs[workflow_id].output
wf_store = workflow_storage.WorkflowStorage(workflow_id, self._store)
meta = wf_store.load_workflow_meta()
if meta is None:
raise ValueError(f"No such workflow {workflow_id}")
if meta == common.WorkflowStatus.CANCELED:
raise ValueError(f"Workflow {workflow_id} is canceled")
if name is None:
# For resumable workflow, the workflow result is not ready.
# It has to be resumed first.
if meta == common.WorkflowStatus.RESUMABLE:
raise ValueError(
f"Workflow {workflow_id} is in resumable status, "
"please resume it")
if name is None:
step_id = wf_store.get_entrypoint_step_id()
else:
step_id = name
output = self.get_cached_step_output(workflow_id, step_id)
if output is not None:
return ray.put(_SelfDereferenceObject(None, output))
@ray.remote
def load(wf_store, workflow_id, step_id):
result = wf_store.inspect_step(step_id)
if result.output_object_valid:
# we already have the output
return wf_store.load_step_output(step_id)
if isinstance(result.output_step_id, str):
actor = get_management_actor()
return actor.get_output.remote(workflow_id,
result.output_step_id)
raise ValueError(
f"No such step id {step_id} in workflow {workflow_id}")
return ray.put(
_SelfDereferenceObject(None,
load.remote(wf_store, workflow_id,
step_id)))
def commit_step(ret: Union[Workflow, Any],
outer_most_step_id: Optional[str] = None):
"""Checkpoint the step output.
Args:
The returned object of the workflow step.
outer_most_step_id: The ID of the outer most workflow. None if it
does not exists. See "step_executor.execute_workflow" for detailed
explanation.
"""
store = workflow_storage.WorkflowStorage()
if isinstance(ret, Workflow):
store.save_subworkflow(ret)
step_id = workflow_context.get_current_step_id()
store.save_step_output(step_id, ret, outer_most_step_id)
def _resume_workflow_step_executor(
workflow_id: str, step_id: "StepID",
store_url: str) -> Tuple[ray.ObjectRef, ray.ObjectRef]:
try:
store = storage.create_storage(store_url)
wf_store = workflow_storage.WorkflowStorage(workflow_id, store)
r = _construct_resume_workflow_from_step(wf_store, step_id)
except Exception as e:
raise WorkflowNotResumableError(workflow_id) from e
if isinstance(r, Workflow):
with workflow_context.workflow_step_context(workflow_id,
store.storage_url):
from ray.experimental.workflow.step_executor import (
execute_workflow)
result = execute_workflow(r, last_step_of_workflow=True)
return result.persisted_output, result.volatile_output
return wf_store.load_step_output(r), None
def run(entry_workflow: Workflow,
workflow_id: Optional[str] = None) -> ray.ObjectRef:
"""Run a workflow asynchronously. See "api.run()" for details."""
store = get_global_storage()
assert ray.is_initialized()
if workflow_id is None:
# Workflow ID format: {Entry workflow UUID}.{Unix time to nanoseconds}
workflow_id = f"{entry_workflow.id}.{time.time():.9f}"
logger.info(f"Workflow job created. [id=\"{workflow_id}\", storage_url="
f"\"{store.storage_url}\"].")
# checkpoint the workflow
ws = workflow_storage.WorkflowStorage(workflow_id, store)
commit_step(ws, "", entry_workflow)
workflow_manager = get_or_create_management_actor()
# NOTE: It is important to 'ray.get' the returned output. This
# ensures caller of 'run()' holds the reference to the workflow
# result. Otherwise if the actor removes the reference of the
# workflow output, the caller may fail to resolve the result.
output = ray.get(workflow_manager.run_or_resume.remote(workflow_id))
return flatten_workflow_output(workflow_id, output)
def run_or_resume(self, workflow_id: str) -> ray.ObjectRef:
"""Run or resume a workflow.
Args:
workflow_id: The ID of the workflow.
Returns:
An object reference that can be used to retrieve the
workflow result.
"""
if workflow_id in self._workflow_outputs:
raise ValueError(f"The output of workflow[id={workflow_id}] "
"already exists.")
output = recovery.resume_workflow_job.remote(workflow_id,
self._store.storage_url)
self._workflow_outputs[workflow_id] = output
wf_store = workflow_storage.WorkflowStorage(workflow_id, self._store)
wf_store.save_workflow_meta(
common.WorkflowMetaData(common.WorkflowStatus.RUNNING))
self._step_status[workflow_id] = {}
logger.info(f"Workflow job [id={workflow_id}] started.")
return output
def postprocess_workflow_step(ret: Union[Workflow, Any],
outer_most_step_id: Optional[StepID] = None):
"""Execute workflow and checkpoint outputs.
To fully explain what we are doing, we need to introduce some syntax first.
The syntax for dependencies between workflow steps
"A.step(B.step())" is "A - B"; the syntax for nested workflow steps
"def A(): return B.step()" is "A / B".
In a chain/DAG of step dependencies, the "output step" is the step of last
(topological) order. For example, in "A - B - C", C is the output step.
In a chain of nested workflow steps, the initial "output step" is
called the "outer most step" for other "output steps". For example, in
"A / B / C / D", "A" is the outer most step for "B", "C", "D";
in the hybrid workflow "((A - B) / C / D) - (E / (F - G) / H)",
"B" is the outer most step for "C", "D"; "E" is the outer most step
for "G", "H".
Args:
ret: The returned object of the workflow step.
outer_most_step_id: The ID of the outer most workflow. None if it
does not exists.
"""
store = workflow_storage.WorkflowStorage()
step_id = workflow_context.get_current_step_id()
store.commit_step(step_id, ret, outer_most_step_id)
if isinstance(ret, Workflow):
if outer_most_step_id is None:
# The current workflow step returns a nested workflow, and
# there is no outer step for the current step. So the current
# step is the outer most step for the inner nested workflow
# steps.
outer_most_step_id = step_id
# Passing down outer most step so inner nested steps would
# access the same outer most step.
return ret.execute(outer_most_step_id)
return ret
def update_step_status(self, workflow_id: str, step_id: str,
status: common.WorkflowStatus):
if status == common.WorkflowStatus.FINISHED:
self._step_status[workflow_id].pop(step_id, None)
else:
self._step_status.setdefault(workflow_id, {})[step_id] = status
remaining = len(self._step_status[workflow_id])
if status != common.WorkflowStatus.RESUMABLE and remaining != 0:
return
wf_store = workflow_storage.WorkflowStorage(workflow_id, self._store)
if status == common.WorkflowStatus.RESUMABLE:
if workflow_id in self._workflow_outputs:
cancel_job(self._workflow_outputs.pop(workflow_id))
wf_store.save_workflow_meta(
common.WorkflowMetaData(common.WorkflowStatus.RESUMABLE))
self._step_status.pop(workflow_id)
else:
# remaining = 0
wf_store.save_workflow_meta(
common.WorkflowMetaData(common.WorkflowStatus.FINISHED))
self._step_status.pop(workflow_id)
def test_workflow_storage(ray_start_regular, raw_storage):
workflow_id = test_workflow_storage.__name__
step_id = "some_step"
input_metadata = {
"name": "test_basic_workflows.append1",
"object_refs": ["abc"],
"workflows": ["def"]
}
output_metadata = {
"output_step_id": "a12423",
"dynamic_output_step_id": "b1234"
}
args = ([1, "2"], {"k": b"543"})
output = ["the_answer"]
object_resolved = 42
obj_ref = ray.put(object_resolved)
# test basics
asyncio.run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
asyncio.run(
raw_storage.save_step_func_body(workflow_id, step_id, some_func))
asyncio.run(raw_storage.save_step_args(workflow_id, step_id, args))
asyncio.run(raw_storage.save_object_ref(workflow_id, obj_ref))
asyncio.run(
raw_storage.save_step_output_metadata(workflow_id, step_id,
output_metadata))
asyncio.run(raw_storage.save_step_output(workflow_id, step_id, output))
wf_storage = workflow_storage.WorkflowStorage(workflow_id)
assert wf_storage.load_step_output(step_id) == output
assert wf_storage.load_step_args(step_id, [], []) == args
assert wf_storage.load_step_func_body(step_id)(33) == 34
assert ray.get(wf_storage.load_object_ref(
obj_ref.hex())) == object_resolved
# test "inspect_step"
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
output_object_valid=True)
assert inspect_result.is_recoverable()
step_id = "some_step2"
asyncio.run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
asyncio.run(
raw_storage.save_step_func_body(workflow_id, step_id, some_func))
asyncio.run(raw_storage.save_step_args(workflow_id, step_id, args))
asyncio.run(
raw_storage.save_step_output_metadata(workflow_id, step_id,
output_metadata))
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
output_step_id=output_metadata["dynamic_output_step_id"])
assert inspect_result.is_recoverable()
step_id = "some_step3"
asyncio.run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
asyncio.run(
raw_storage.save_step_func_body(workflow_id, step_id, some_func))
asyncio.run(raw_storage.save_step_args(workflow_id, step_id, args))
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
args_valid=True,
func_body_valid=True,
object_refs=input_metadata["object_refs"],
workflows=input_metadata["workflows"])
assert inspect_result.is_recoverable()
step_id = "some_step4"
asyncio.run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
asyncio.run(
raw_storage.save_step_func_body(workflow_id, step_id, some_func))
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
func_body_valid=True,
object_refs=input_metadata["object_refs"],
workflows=input_metadata["workflows"])
assert not inspect_result.is_recoverable()
step_id = "some_step5"
asyncio.run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
object_refs=input_metadata["object_refs"],
workflows=input_metadata["workflows"])
assert not inspect_result.is_recoverable()
step_id = "some_step6"
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult()
assert not inspect_result.is_recoverable()
def cancel_workflow(self, workflow_id: str) -> None:
self._step_status.pop(workflow_id)
cancel_job(self._workflow_outputs.pop(workflow_id).output)
wf_store = workflow_storage.WorkflowStorage(workflow_id, self._store)
wf_store.save_workflow_meta(
common.WorkflowMetaData(common.WorkflowStatus.CANCELED))
def test_workflow_storage(workflow_start_regular):
raw_storage = workflow_storage._StorageImpl(storage.get_global_storage())
workflow_id = test_workflow_storage.__name__
step_id = "some_step"
input_metadata = {
"name": "test_basic_workflows.append1",
"step_type": StepType.FUNCTION,
"object_refs": ["abc"],
"workflows": ["def"],
"workflow_refs": ["some_ref"],
"max_retries": 1,
"catch_exceptions": False,
"ray_options": {},
}
output_metadata = {
"output_step_id": "a12423",
"dynamic_output_step_id": "b1234"
}
flattened_args = [
signature.DUMMY_TYPE, 1, signature.DUMMY_TYPE, "2", "k", b"543"
]
args = signature.recover_args(flattened_args)
output = ["the_answer"]
object_resolved = 42
obj_ref = ray.put(object_resolved)
# test basics
asyncio_run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
asyncio_run(
raw_storage.save_step_func_body(workflow_id, step_id, some_func))
asyncio_run(
raw_storage.save_step_args(workflow_id, step_id, flattened_args))
asyncio_run(raw_storage.save_object_ref(workflow_id, obj_ref))
asyncio_run(
raw_storage.save_step_output_metadata(workflow_id, step_id,
output_metadata))
asyncio_run(raw_storage.save_step_output(workflow_id, step_id, output))
wf_storage = workflow_storage.WorkflowStorage(workflow_id,
storage.get_global_storage())
assert wf_storage.load_step_output(step_id) == output
assert wf_storage.load_step_args(step_id, [], [], []) == args
assert wf_storage.load_step_func_body(step_id)(33) == 34
assert ray.get(wf_storage.load_object_ref(
obj_ref.hex())) == object_resolved
# test "inspect_step"
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
output_object_valid=True)
assert inspect_result.is_recoverable()
step_id = "some_step2"
asyncio_run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
asyncio_run(
raw_storage.save_step_func_body(workflow_id, step_id, some_func))
asyncio_run(raw_storage.save_step_args(workflow_id, step_id, args))
asyncio_run(
raw_storage.save_step_output_metadata(workflow_id, step_id,
output_metadata))
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
output_step_id=output_metadata["dynamic_output_step_id"])
assert inspect_result.is_recoverable()
step_id = "some_step3"
asyncio_run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
asyncio_run(
raw_storage.save_step_func_body(workflow_id, step_id, some_func))
asyncio_run(raw_storage.save_step_args(workflow_id, step_id, args))
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
step_type=StepType.FUNCTION,
args_valid=True,
func_body_valid=True,
object_refs=input_metadata["object_refs"],
workflows=input_metadata["workflows"],
workflow_refs=input_metadata["workflow_refs"],
ray_options={})
assert inspect_result.is_recoverable()
step_id = "some_step4"
asyncio_run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
asyncio_run(
raw_storage.save_step_func_body(workflow_id, step_id, some_func))
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
step_type=StepType.FUNCTION,
func_body_valid=True,
object_refs=input_metadata["object_refs"],
workflows=input_metadata["workflows"],
workflow_refs=input_metadata["workflow_refs"],
ray_options={})
assert not inspect_result.is_recoverable()
step_id = "some_step5"
asyncio_run(
raw_storage.save_step_input_metadata(workflow_id, step_id,
input_metadata))
inspect_result = wf_storage.inspect_step(step_id)
assert inspect_result == workflow_storage.StepInspectResult(
step_type=StepType.FUNCTION,
object_refs=input_metadata["object_refs"],
workflows=input_metadata["workflows"],
workflow_refs=input_metadata["workflow_refs"],
ray_options={})
assert not inspect_result.is_recoverable()
step_id = "some_step6"
inspect_result = wf_storage.inspect_step(step_id)
print(inspect_result)
assert inspect_result == workflow_storage.StepInspectResult()
assert not inspect_result.is_recoverable()
def test_shortcut(ray_start_regular):
assert recursive_chain.step(0).run(workflow_id="shortcut") == 100
# the shortcut points to the step with output checkpoint
store = workflow_storage.WorkflowStorage("shortcut")
step_id = store.get_entrypoint_step_id()
assert store.inspect_step(step_id).output_object_valid
