def _resume_workflow_step_executor(workflow_id: str, step_id: "StepID", store_url: str, current_output: [ ray.ObjectRef ]) -> Tuple[ray.ObjectRef, ray.ObjectRef]: # TODO (yic): We need better dependency management for virtual actor # The current output will always be empty for normal workflow # For virtual actor, if it's not empty, it means the previous job is # running. This is a really bad one. for ref in current_output: try: while isinstance(ref, ray.ObjectRef): ref = ray.get(ref) except Exception: pass try: store = storage.create_storage(store_url) wf_store = workflow_storage.WorkflowStorage(workflow_id, store) r = _construct_resume_workflow_from_step(wf_store, step_id) except Exception as e: raise WorkflowNotResumableError(workflow_id) from e if isinstance(r, Workflow): with workflow_context.workflow_step_context(workflow_id, store.storage_url): from ray.workflow.step_executor import (execute_workflow) result = execute_workflow(r, last_step_of_workflow=True) return result.persisted_output, result.volatile_output assert isinstance(r, StepID) return wf_store.load_step_output(r), None
def update_workflow_step_context(context: Optional[WorkflowStepContext], step_id: str): global _context _context = context _context.workflow_scope.append(step_id) # avoid cyclic import from ray.workflow import storage # TODO(suquark): [optimization] if the original storage has the same URL, # skip creating the new one storage.set_global_storage(storage.create_storage(context.storage_url))
def init(storage: "Optional[Union[str, Storage]]" = None) -> None: """Initialize workflow. Args: storage: The external storage URL or a custom storage class. If not specified, ``/tmp/ray/workflow_data`` will be used. """ if storage is None: storage = os.environ.get("RAY_WORKFLOW_STORAGE") if storage is None: # We should use get_temp_dir_path, but for ray client, we don't # have this one. We need a flag to tell whether it's a client # or a driver to use the right dir. # For now, just use /tmp/ray/workflow_data storage = "file:///tmp/ray/workflow_data" if isinstance(storage, str): logger.info(f"Using storage: {storage}") storage = storage_base.create_storage(storage) elif not isinstance(storage, Storage): raise TypeError("'storage' should be None, str, or Storage type.") try: _storage = storage_base.get_global_storage() except RuntimeError: pass else: # we have to use the 'else' branch because we would raise a # runtime error, but we do not want to be captured by 'except' if _storage.storage_url == storage.storage_url: logger.warning("Calling 'workflow.init()' again with the same " "storage.") else: raise RuntimeError("Calling 'workflow.init()' again with a " "different storage") storage_base.set_global_storage(storage) workflow_access.init_management_actor() serialization.init_manager()