def _ensure_workflow_initialized() -> None: # NOTE: Trying to get the actor has a side effect: it initializes Ray with # default arguments. This is different in "init()": it assigns a temporary # storage. This is why we need to check "ray.is_initialized()" first. if not ray.is_initialized(): init() else: try: workflow_access.get_management_actor() except ValueError: init()
def list_all( status_filter: Set[WorkflowStatus] ) -> List[Tuple[str, WorkflowStatus]]: try: workflow_manager = get_management_actor() except ValueError: workflow_manager = None if workflow_manager is None: runnings = [] else: runnings = ray.get(workflow_manager.list_running_workflow.remote()) if WorkflowStatus.RUNNING in status_filter and len(status_filter) == 1: return [(r, WorkflowStatus.RUNNING) for r in runnings] runnings = set(runnings) # Here we don't have workflow id, so use empty one instead store = workflow_storage.get_workflow_storage("") ret = [] for (k, s) in store.list_workflow(): if s == WorkflowStatus.RUNNING and k not in runnings: s = WorkflowStatus.RESUMABLE if s in status_filter: ret.append((k, s)) return ret
def cancel(workflow_id: str) -> None: """Cancel a workflow. Workflow checkpoints will still be saved in storage. To clean up saved checkpoints, see `workflow.delete()`. Args: workflow_id: The workflow to cancel. Examples: >>> from ray import workflow >>> some_job = ... # doctest: +SKIP >>> workflow_step = some_job.step() # doctest: +SKIP >>> output = workflow_step.run_async(workflow_id="some_job") # doctest: +SKIP >>> workflow.cancel(workflow_id="some_job") # doctest: +SKIP >>> assert [ # doctest: +SKIP ... ("some_job", workflow.CANCELED)] == workflow.list_all() Returns: None """ _ensure_workflow_initialized() if not isinstance(workflow_id, str): raise TypeError("workflow_id has to be a string type.") workflow_manager = workflow_access.get_management_actor() ray.get(workflow_manager.cancel_workflow.remote(workflow_id))
def cancel(workflow_id: str) -> None: try: workflow_manager = get_management_actor() ray.get(workflow_manager.cancel_workflow.remote(workflow_id)) except ValueError: wf_store = workflow_storage.get_workflow_storage(workflow_id) wf_store.save_workflow_meta(WorkflowMetaData(WorkflowStatus.CANCELED))
def resume_all(with_failed: bool) -> List[Tuple[str, ray.ObjectRef]]: filter_set = {WorkflowStatus.RESUMABLE} if with_failed: filter_set.add(WorkflowStatus.FAILED) all_failed = list_all(filter_set) try: workflow_manager = get_management_actor() except Exception as e: raise RuntimeError("Failed to get management actor") from e job_id = ray.get_runtime_context().job_id.hex() reconstructed_refs = [] reconstructed_workflows = [] for wid, _ in all_failed: context = workflow_context.WorkflowStepContext(workflow_id=wid) reconstructed_refs.append( (context, workflow_manager.reconstruct_workflow.remote(job_id, context))) for context, ref in reconstructed_refs: try: ray.get(ref) # make sure the workflow is already reconstructed reconstructed_workflows.append(( context.workflow_id, workflow_manager.execute_workflow.remote(job_id, context), )) except Exception: # TODO(suquark): Here some workflows got resumed successfully but some # failed and the user has no idea about this, which is very wired. # Maybe we should raise an exception here instead? logger.error(f"Failed to resume workflow {context.workflow_id}") return reconstructed_workflows
def get_output_async(workflow_id: str, *, name: Optional[str] = None) -> ray.ObjectRef: """Get the output of a running workflow asynchronously. Args: workflow_id: The workflow to get the output of. name: If set, fetch the specific step instead of the output of the workflow. Returns: An object reference that can be used to retrieve the workflow task result. """ _ensure_workflow_initialized() try: workflow_manager = workflow_access.get_management_actor() except ValueError as e: raise ValueError( "Failed to connect to the workflow management " "actor. The workflow could have already failed. You can use " "workflow.resume() to resume the workflow.") from e try: # check storage first wf_store = WorkflowStorage(workflow_id) tid = wf_store.inspect_output(name) if tid is not None: return workflow_access.load_step_output_from_storage.remote( workflow_id, name) except ValueError: pass return workflow_manager.get_output.remote(workflow_id, name)
def resume_all(with_failed: bool) -> List[Tuple[str, ray.ObjectRef]]: filter_set = {WorkflowStatus.RESUMABLE} if with_failed: filter_set.add(WorkflowStatus.FAILED) all_failed = list_all(filter_set) try: workflow_manager = get_management_actor() except Exception as e: raise RuntimeError("Failed to get management actor") from e async def _resume_one(wid: str) -> Tuple[str, Optional[ray.ObjectRef]]: try: result: "WorkflowExecutionResult" = ( await workflow_manager.run_or_resume.remote(wid) ) obj = flatten_workflow_output(wid, result.persisted_output) return wid, obj except Exception: logger.error(f"Failed to resume workflow {wid}") return (wid, None) ret = workflow_storage.asyncio_run( asyncio.gather(*[_resume_one(wid) for (wid, _) in all_failed]) ) return [(wid, obj) for (wid, obj) in ret if obj is not None]
def get_output(workflow_id: str, name: Optional[str]) -> ray.ObjectRef: """Get the output of a running workflow. See "api.get_output()" for details. """ from ray.workflow.api import _ensure_workflow_initialized _ensure_workflow_initialized() try: workflow_manager = get_management_actor() except ValueError as e: raise ValueError( "Failed to connect to the workflow management " "actor. The workflow could have already failed. You can use " "workflow.resume() to resume the workflow.") from e try: # check storage first wf_store = workflow_storage.WorkflowStorage(workflow_id) tid = wf_store.inspect_output(name) if tid is not None: return load_step_output_from_storage.remote(workflow_id, name) except ValueError: pass return workflow_manager.get_output.remote(workflow_id, name)
def _record_step_status(step_id: "StepID", status: "WorkflowStatus", outputs: List["ObjectRef"] = []) -> None: workflow_id = workflow_context.get_current_workflow_id() workflow_manager = get_management_actor() ray.get( workflow_manager.update_step_status.remote(workflow_id, step_id, status, outputs))
def cancel(workflow_id: str) -> None: try: workflow_manager = get_management_actor() except ValueError: wf_store = workflow_storage.get_workflow_storage(workflow_id) # TODO(suquark): Here we update workflow status "offline", so it is likely # thread-safe because there is no workflow management actor updating the # workflow concurrently. But we should be careful if we are going to # update more workflow status offline in the future. wf_store.update_workflow_status(WorkflowStatus.CANCELED) return ray.get(workflow_manager.cancel_workflow.remote(workflow_id))
def get_status(workflow_id: str) -> Optional[WorkflowStatus]: try: workflow_manager = get_management_actor() running = ray.get( workflow_manager.is_workflow_running.remote(workflow_id)) except Exception: running = False if running: return WorkflowStatus.RUNNING store = workflow_storage.get_workflow_storage(workflow_id) meta = store.load_workflow_meta() if meta is None: raise ValueError(f"No such workflow_id {workflow_id}") return meta.status
def get_output(workflow_id: str, name: Optional[str]) -> ray.ObjectRef: """Get the output of a running workflow. See "api.get_output()" for details. """ assert ray.is_initialized() try: workflow_manager = get_management_actor() except ValueError as e: raise ValueError( "Failed to connect to the workflow management " "actor. The workflow could have already failed. You can use " "workflow.resume() to resume the workflow.") from e output = ray.get(workflow_manager.get_output.remote(workflow_id, name)) return flatten_workflow_output(workflow_id, output)
def get_status(workflow_id: str) -> Optional[WorkflowStatus]: try: workflow_manager = get_management_actor() running = ray.get(workflow_manager.is_workflow_running.remote(workflow_id)) except Exception: running = False if running: return WorkflowStatus.RUNNING store = workflow_storage.get_workflow_storage(workflow_id) meta = store.load_workflow_meta() if meta is None: raise WorkflowNotFoundError(workflow_id) if meta.status == WorkflowStatus.RUNNING: return WorkflowStatus.RESUMABLE return meta.status
def list_all( status_filter: Set[WorkflowStatus] ) -> List[Tuple[str, WorkflowStatus]]: try: workflow_manager = get_management_actor() except ValueError: workflow_manager = None if workflow_manager is None: runnings = [] else: runnings = ray.get(workflow_manager.list_running_workflow.remote()) if WorkflowStatus.RUNNING in status_filter and len(status_filter) == 1: return [(r, WorkflowStatus.RUNNING) for r in runnings] runnings = set(runnings) # Here we don't have workflow id, so use empty one instead store = workflow_storage.get_workflow_storage("") # This is the tricky part: the status "RESUMABLE" neither come from # the workflow management actor nor the storage. It is the status where # the storage says it is running but the workflow management actor # is not running it. This usually happened when there was a sudden crash # of the whole Ray runtime or the workflow management actor # (due to cluster etc.). So we includes 'RUNNING' status in the storage # filter to get "RESUMABLE" candidates. storage_status_filter = status_filter.copy() storage_status_filter.add(WorkflowStatus.RUNNING) status_from_storage = store.list_workflow(storage_status_filter) ret = [] for (k, s) in status_from_storage: if s == WorkflowStatus.RUNNING: if k not in runnings: s = WorkflowStatus.RESUMABLE else: continue if s in status_filter: ret.append((k, s)) if WorkflowStatus.RUNNING in status_filter: ret.extend((k, WorkflowStatus.RUNNING) for k in runnings) return ret
def get_status(workflow_id: str) -> WorkflowStatus: """Get the status for a given workflow. Args: workflow_id: The workflow to query. Examples: >>> from ray import workflow >>> trip = ... # doctest: +SKIP >>> workflow_step = trip.step() # doctest: +SKIP >>> output = workflow_step.run(workflow_id="trip") # doctest: +SKIP >>> assert workflow.SUCCESSFUL == workflow.get_status("trip") # doctest: +SKIP Returns: The status of that workflow """ _ensure_workflow_initialized() if not isinstance(workflow_id, str): raise TypeError("workflow_id has to be a string type.") workflow_manager = workflow_access.get_management_actor() return ray.get(workflow_manager.get_workflow_status.remote(workflow_id))
def resume_async(workflow_id: str) -> ray.ObjectRef: """Resume a workflow asynchronously. Resume a workflow and retrieve its output. If the workflow was incomplete, it will be re-executed from its checkpointed outputs. If the workflow was complete, returns the result immediately. Examples: >>> from ray import workflow >>> start_trip = ... # doctest: +SKIP >>> trip = start_trip.step() # doctest: +SKIP >>> res1 = trip.run_async(workflow_id="trip1") # doctest: +SKIP >>> res2 = workflow.resume("trip1") # doctest: +SKIP >>> assert ray.get(res1) == ray.get(res2) # doctest: +SKIP Args: workflow_id: The id of the workflow to resume. Returns: An object reference that can be used to retrieve the workflow result. """ _ensure_workflow_initialized() logger.info(f'Resuming workflow [id="{workflow_id}"].') workflow_manager = workflow_access.get_management_actor() if ray.get( workflow_manager.is_workflow_non_terminating.remote(workflow_id)): raise RuntimeError( f"Workflow '{workflow_id}' is already running or pending.") # NOTE: It is important to 'ray.get' the returned output. This # ensures caller of 'run()' holds the reference to the workflow # result. Otherwise if the actor removes the reference of the # workflow output, the caller may fail to resolve the result. job_id = ray.get_runtime_context().job_id.hex() context = workflow_context.WorkflowStepContext(workflow_id=workflow_id) ray.get(workflow_manager.reconstruct_workflow.remote(job_id, context)) result = workflow_manager.execute_workflow.remote(job_id, context) logger.info(f"Workflow job {workflow_id} resumed.") return result
def list_all( status_filter: Set[WorkflowStatus] ) -> List[Tuple[str, WorkflowStatus]]: try: workflow_manager = get_management_actor() except ValueError: workflow_manager = None if workflow_manager is None: runnings = [] else: runnings = ray.get(workflow_manager.list_running_workflow.remote()) if WorkflowStatus.RUNNING in status_filter and len(status_filter) == 1: return [(r, WorkflowStatus.RUNNING) for r in runnings] runnings = set(runnings) # Here we don't have workflow id, so use empty one instead store = workflow_storage.get_workflow_storage("") exclude_running = False if (WorkflowStatus.RESUMABLE in status_filter and WorkflowStatus.RUNNING not in status_filter): # Here we have to add "RUNNING" to the status filter, because some "RESUMABLE" # workflows are converted from "RUNNING" workflows below. exclude_running = True status_filter.add(WorkflowStatus.RUNNING) status_from_storage = store.list_workflow(status_filter) ret = [] for (k, s) in status_from_storage: if s == WorkflowStatus.RUNNING: if k not in runnings: s = WorkflowStatus.RESUMABLE elif exclude_running: continue if s in status_filter: ret.append((k, s)) return ret
def run_async( dag: DAGNode, *args, workflow_id: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, **kwargs, ) -> ray.ObjectRef: """Run a workflow asynchronously. If the workflow with the given id already exists, it will be resumed. Args: workflow_id: A unique identifier that can be used to resume the workflow. If not specified, a random id will be generated. metadata: The metadata to add to the workflow. It has to be able to serialize to json. Returns: The running result as ray.ObjectRef. """ _ensure_workflow_initialized() if not isinstance(dag, DAGNode): raise TypeError("Input should be a DAG.") input_data = DAGInputData(*args, **kwargs) validate_user_metadata(metadata) metadata = metadata or {} if workflow_id is None: # Workflow ID format: {Entry workflow UUID}.{Unix time to nanoseconds} workflow_id = f"{str(uuid.uuid4())}.{time.time():.9f}" state = workflow_state_from_dag(dag, input_data, workflow_id) logger.info(f'Workflow job created. [id="{workflow_id}"].') context = workflow_context.WorkflowStepContext(workflow_id=workflow_id) with workflow_context.workflow_step_context(context): # checkpoint the workflow ws = WorkflowStorage(workflow_id) ws.save_workflow_user_metadata(metadata) job_id = ray.get_runtime_context().job_id.hex() try: ws.get_entrypoint_step_id() wf_exists = True except Exception: # The workflow does not exist. We must checkpoint entry workflow. ws.save_workflow_execution_state("", state) wf_exists = False workflow_manager = workflow_access.get_management_actor() if ray.get( workflow_manager.is_workflow_non_terminating.remote( workflow_id)): raise RuntimeError( f"Workflow '{workflow_id}' is already running or pending.") if wf_exists: return resume_async(workflow_id) ignore_existing = ws.load_workflow_status() == WorkflowStatus.NONE ray.get( workflow_manager.submit_workflow.remote( workflow_id, state, ignore_existing=ignore_existing)) return workflow_manager.execute_workflow.remote(job_id, context)
def workflow_state_from_dag(dag_node: DAGNode, input_context: Optional[DAGInputData], workflow_id: str): """ Transform a Ray DAG to a workflow. Map FunctionNode to workflow step with the workflow decorator. Args: dag_node: The DAG to be converted to a workflow. input_context: The input data that wraps varibles for the input node of the DAG. workflow_id: The ID of the workflow. """ if not isinstance(dag_node, FunctionNode): raise TypeError( "Currently workflow does not support classes as DAG inputs.") state = WorkflowExecutionState() # TODO(suquark): remove this cyclic importing later by changing the way of # task ID assignment. from ray.workflow.workflow_access import get_management_actor mgr = get_management_actor() context = workflow_context.get_workflow_step_context() def _node_visitor(node: Any) -> Any: if isinstance(node, FunctionNode): bound_options = node._bound_options.copy() num_returns = bound_options.get("num_returns", 1) if num_returns is None: # ray could use `None` as default value num_returns = 1 if num_returns > 1: raise ValueError("Workflow steps can only have one return.") workflow_options = bound_options.pop("_metadata", {}).get(WORKFLOW_OPTIONS, {}) # If checkpoint option is not specified, inherit checkpoint # options from context (i.e. checkpoint options of the outer # step). If it is still not specified, it's True by default. checkpoint = workflow_options.get("checkpoint", None) if checkpoint is None: checkpoint = context.checkpoint if context is not None else True # When it returns a nested workflow, catch_exception # should be passed recursively. catch_exceptions = workflow_options.get("catch_exceptions", None) if catch_exceptions is None: # TODO(suquark): should we also handle exceptions from a "leaf node" # in the continuation? For example, we have a workflow # > @ray.remote # > def A(): pass # > @ray.remote # > def B(x): return x # > @ray.remote # > def C(x): return workflow.continuation(B.bind(A.bind())) # > dag = C.options(**workflow.options(catch_exceptions=True)).bind() # Should C catches exceptions of A? if node.get_stable_uuid() == dag_node.get_stable_uuid(): # 'catch_exception' context should be passed down to # its direct continuation task. # In this case, the direct continuation is the output node. catch_exceptions = (context.catch_exceptions if context is not None else False) else: catch_exceptions = False max_retries = bound_options.get("max_retries", 3) if not isinstance(max_retries, int) or max_retries < -1: raise ValueError( "'max_retries' only accepts 0, -1 or a positive integer.") step_options = WorkflowStepRuntimeOptions( step_type=StepType.FUNCTION, catch_exceptions=catch_exceptions, max_retries=max_retries, allow_inplace=False, checkpoint=checkpoint, ray_options=bound_options, ) workflow_refs: List[WorkflowRef] = [] with serialization_context.workflow_args_serialization_context( workflow_refs): _func_signature = signature.extract_signature(node._body) flattened_args = signature.flatten_args( _func_signature, node._bound_args, node._bound_kwargs) # NOTE: When calling 'ray.put', we trigger python object # serialization. Under our serialization context, # Workflows are separated from the arguments, # leaving a placeholder object with all other python objects. # Then we put the placeholder object to object store, # so it won't be mutated later. This guarantees correct # semantics. See "tests/test_variable_mutable.py" as # an example. input_placeholder: ray.ObjectRef = ray.put(flattened_args) name = workflow_options.get("name") if name is None: name = f"{get_module(node._body)}.{slugify(get_qualname(node._body))}" task_id = ray.get(mgr.gen_step_id.remote(workflow_id, name)) state.add_dependencies(task_id, [s.task_id for s in workflow_refs]) state.task_input_args[task_id] = input_placeholder user_metadata = workflow_options.pop("metadata", {}) validate_user_metadata(user_metadata) state.tasks[task_id] = Task( name=name, options=step_options, user_metadata=user_metadata, func_body=node._body, ) return WorkflowRef(task_id) if isinstance(node, InputAttributeNode): return node._execute_impl() # get data from input node if isinstance(node, InputNode): return input_context # replace input node with input data if not isinstance(node, DAGNode): return node # return normal objects raise TypeError(f"Unsupported DAG node: {node}") output_workflow_ref = dag_node.apply_recursive(_node_visitor) state.output_task_id = output_workflow_ref.task_id return state
def list_all( status_filter: Optional[Union[Union[WorkflowStatus, str], Set[Union[WorkflowStatus, str]]]] = None ) -> List[Tuple[str, WorkflowStatus]]: """List all workflows matching a given status filter. When returning "RESUMEABLE" workflows, the workflows that was running ranks before the workflow that was pending in the result list. Args: status_filter: If given, only returns workflow with that status. This can be a single status or set of statuses. The string form of the status is also acceptable, i.e., "RUNNING"/"FAILED"/"SUCCESSFUL"/"CANCELED"/"RESUMABLE"/"PENDING". Examples: >>> from ray import workflow >>> long_running_job = ... # doctest: +SKIP >>> workflow_step = long_running_job.step() # doctest: +SKIP >>> wf = workflow_step.run_async( # doctest: +SKIP ... workflow_id="long_running_job") >>> jobs = workflow.list_all() # doctest: +SKIP >>> assert jobs == [ ("long_running_job", workflow.RUNNING) ] # doctest: +SKIP >>> ray.get(wf) # doctest: +SKIP >>> jobs = workflow.list_all({workflow.RUNNING}) # doctest: +SKIP >>> assert jobs == [] # doctest: +SKIP >>> jobs = workflow.list_all(workflow.SUCCESSFUL) # doctest: +SKIP >>> assert jobs == [ # doctest: +SKIP ... ("long_running_job", workflow.SUCCESSFUL)] Returns: A list of tuple with workflow id and workflow status """ _ensure_workflow_initialized() if isinstance(status_filter, str): status_filter = set({WorkflowStatus(status_filter)}) elif isinstance(status_filter, WorkflowStatus): status_filter = set({status_filter}) elif isinstance(status_filter, set): if all(isinstance(s, str) for s in status_filter): status_filter = {WorkflowStatus(s) for s in status_filter} elif not all(isinstance(s, WorkflowStatus) for s in status_filter): raise TypeError("status_filter contains element which is not" " a type of `WorkflowStatus or str`." f" {status_filter}") elif status_filter is None: status_filter = set(WorkflowStatus) status_filter.discard(WorkflowStatus.NONE) else: raise TypeError( "status_filter must be WorkflowStatus or a set of WorkflowStatus.") try: workflow_manager = workflow_access.get_management_actor() except ValueError: workflow_manager = None if workflow_manager is None: non_terminating_workflows = {} else: non_terminating_workflows = ray.get( workflow_manager.list_non_terminating_workflows.remote()) ret = [] if set(non_terminating_workflows.keys()).issuperset(status_filter): for status, workflows in non_terminating_workflows.items(): if status in status_filter: for w in workflows: ret.append((w, status)) return ret ret = [] # Here we don't have workflow id, so use empty one instead store = WorkflowStorage("") modified_status_filter = status_filter.copy() # Here we have to add non-terminating status to the status filter, because some # "RESUMABLE" workflows are converted from non-terminating workflows below. # This is the tricky part: the status "RESUMABLE" neither come from # the workflow management actor nor the storage. It is the status where # the storage says it is non-terminating but the workflow management actor # is not running it. This usually happened when there was a sudden crash # of the whole Ray runtime or the workflow management actor # (due to cluster etc.). So we includes non terminating status in the storage # filter to get "RESUMABLE" candidates. modified_status_filter.update(WorkflowStatus.non_terminating_status()) status_from_storage = store.list_workflow(modified_status_filter) non_terminating_workflows = { k: set(v) for k, v in non_terminating_workflows.items() } resume_running = [] resume_pending = [] for (k, s) in status_from_storage: if s in non_terminating_workflows and k not in non_terminating_workflows[ s]: if s == WorkflowStatus.RUNNING: resume_running.append(k) elif s == WorkflowStatus.PENDING: resume_pending.append(k) else: assert False, "This line of code should not be reachable." continue if s in status_filter: ret.append((k, s)) if WorkflowStatus.RESUMABLE in status_filter: # The running workflows ranks before the pending workflows. for w in resume_running: ret.append((w, WorkflowStatus.RESUMABLE)) for w in resume_pending: ret.append((w, WorkflowStatus.RESUMABLE)) return ret
def resume_all( include_failed: bool = False) -> List[Tuple[str, ray.ObjectRef]]: """Resume all resumable workflow jobs. This can be used after cluster restart to resume all tasks. Args: include_failed: Whether to resume FAILED workflows. Examples: >>> from ray import workflow >>> failed_job = ... # doctest: +SKIP >>> workflow_step = failed_job.step() # doctest: +SKIP >>> output = workflow_step.run_async(workflow_id="failed_job") # doctest: +SKIP >>> try: # doctest: +SKIP >>> ray.get(output) # doctest: +SKIP >>> except Exception: # doctest: +SKIP >>> print("JobFailed") # doctest: +SKIP >>> jobs = workflow.list_all() # doctest: +SKIP >>> assert jobs == [("failed_job", workflow.FAILED)] # doctest: +SKIP >>> assert workflow.resume_all( # doctest: +SKIP ... include_failed=True).get("failed_job") is not None # doctest: +SKIP Returns: A list of (workflow_id, returned_obj_ref) resumed. """ _ensure_workflow_initialized() filter_set = {WorkflowStatus.RESUMABLE} if include_failed: filter_set.add(WorkflowStatus.FAILED) all_failed = list_all(filter_set) try: workflow_manager = workflow_access.get_management_actor() except Exception as e: raise RuntimeError("Failed to get management actor") from e job_id = ray.get_runtime_context().job_id.hex() reconstructed_workflows = [] for wid, _ in all_failed: context = workflow_context.WorkflowStepContext(workflow_id=wid) # TODO(suquark): This is not very efficient, but it makes sure # running workflows has higher priority when getting reconstructed. try: ray.get( workflow_manager.reconstruct_workflow.remote(job_id, context)) except Exception as e: # TODO(suquark): Here some workflows got resumed successfully but some # failed and the user has no idea about this, which is very wired. # Maybe we should raise an exception here instead? logger.error(f"Failed to resume workflow {context.workflow_id}", exc_info=e) raise reconstructed_workflows.append(context) results = [] for context in reconstructed_workflows: results.append(( context.workflow_id, workflow_manager.execute_workflow.remote(job_id, context), )) return results