def get_output(workflow_id: str, name: Optional[str]) -> ray.ObjectRef: """Get the output of a running workflow. See "api.get_output()" for details. """ from ray.workflow.api import _ensure_workflow_initialized _ensure_workflow_initialized() try: workflow_manager = get_management_actor() except ValueError as e: raise ValueError( "Failed to connect to the workflow management " "actor. The workflow could have already failed. You can use " "workflow.resume() to resume the workflow.") from e try: # check storage first wf_store = workflow_storage.WorkflowStorage(workflow_id) tid = wf_store.inspect_output(name) if tid is not None: return load_step_output_from_storage.remote(workflow_id, name) except ValueError: pass return workflow_manager.get_output.remote(workflow_id, name)
def __init__(self, workflow_id: str): from ray.workflow.api import _ensure_workflow_initialized _ensure_workflow_initialized() self._storage = storage.get_client( os.path.join(WORKFLOW_ROOT, workflow_id)) self._workflow_id = workflow_id
def run( entry_workflow: Workflow, workflow_id: Optional[str] = None, metadata: Optional[Dict] = None, ) -> ray.ObjectRef: """Run a workflow asynchronously.""" validate_user_metadata(metadata) metadata = metadata or {} from ray.workflow.api import _ensure_workflow_initialized _ensure_workflow_initialized() if workflow_id is None: # Workflow ID format: {Entry workflow UUID}.{Unix time to nanoseconds} workflow_id = f"{str(uuid.uuid4())}.{time.time():.9f}" step_type = entry_workflow.data.step_options.step_type logger.info( f'Workflow job created. [id="{workflow_id}"]. Type: {step_type}.') with workflow_context.workflow_step_context(workflow_id): # checkpoint the workflow ws = workflow_storage.get_workflow_storage(workflow_id) ws.save_workflow_user_metadata(metadata) wf_exists = True try: ws.get_entrypoint_step_id() except Exception: wf_exists = False # "Is growing" means we could adding steps to the (top-level) # workflow to grow the workflow dynamically at runtime. is_growing = step_type not in (StepType.FUNCTION, StepType.WAIT) # We only commit for # - virtual actor tasks: it's dynamic tasks, so we always add # - it's a new workflow # TODO (yic): follow up with force rerun if is_growing or not wf_exists: # We must checkpoint entry workflow. commit_step(ws, "", entry_workflow, exception=None) workflow_manager = get_or_create_management_actor() ignore_existing = is_growing # NOTE: It is important to 'ray.get' the returned output. This # ensures caller of 'run()' holds the reference to the workflow # result. Otherwise if the actor removes the reference of the # workflow output, the caller may fail to resolve the result. job_id = ray.get_runtime_context().job_id.hex() result: "WorkflowExecutionResult" = ray.get( workflow_manager.run_or_resume.remote(job_id, workflow_id, ignore_existing)) if not is_growing: return flatten_workflow_output(workflow_id, result.persisted_output) else: return flatten_workflow_output(workflow_id, result.volatile_output)
def run_async( self, workflow_id: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, ) -> ObjectRef: """Run a workflow asynchronously. If the workflow with the given id already exists, it will be resumed. Examples: >>> from ray import workflow >>> Flight, Reservation, Trip = ... # doctest: +SKIP >>> @workflow.step # doctest: +SKIP ... def book_flight(origin: str, dest: str) -> Flight: # doctest: +SKIP ... return Flight(...) # doctest: +SKIP >>> @workflow.step # doctest: +SKIP ... def book_hotel(location: str) -> Reservation: # doctest: +SKIP ... return Reservation(...) # doctest: +SKIP >>> @workflow.step # doctest: +SKIP ... def finalize_trip(bookings: List[Any]) -> Trip: # doctest: +SKIP ... return Trip(...) # doctest: +SKIP >>> flight1 = book_flight.step("OAK", "SAN") # doctest: +SKIP >>> flight2 = book_flight.step("SAN", "OAK") # doctest: +SKIP >>> hotel = book_hotel.step("SAN") # doctest: +SKIP >>> trip = finalize_trip.step([flight1, flight2, hotel]) # doctest: +SKIP >>> result = ray.get(trip.run_async()) # doctest: +SKIP Args: workflow_id: A unique identifier that can be used to resume the workflow. If not specified, a random id will be generated. metadata: The metadata to add to the workflow. It has to be able to serialize to json. Returns: The running result as ray.ObjectRef. """ # TODO(suquark): avoid cyclic importing from ray.workflow.execution import run from ray.workflow.api import _ensure_workflow_initialized _ensure_workflow_initialized() self._step_id = None return run(self, workflow_id, metadata)
def get_output(workflow_id: str, name: Optional[str]) -> ray.ObjectRef: """Get the output of a running workflow. See "api.get_output()" for details. """ from ray.workflow.api import _ensure_workflow_initialized _ensure_workflow_initialized() try: workflow_manager = get_management_actor() except ValueError as e: raise ValueError( "Failed to connect to the workflow management " "actor. The workflow could have already failed. You can use " "workflow.resume() to resume the workflow.") from e output = ray.get(workflow_manager.get_output.remote(workflow_id, name)) return flatten_workflow_output(workflow_id, output)
def run( dag: DAGNode, dag_inputs: DAGInputData, workflow_id: Optional[str] = None, metadata: Optional[Dict] = None, ) -> ray.ObjectRef: """Run a workflow asynchronously.""" validate_user_metadata(metadata) metadata = metadata or {} from ray.workflow.api import _ensure_workflow_initialized _ensure_workflow_initialized() if workflow_id is None: # Workflow ID format: {Entry workflow UUID}.{Unix time to nanoseconds} workflow_id = f"{str(uuid.uuid4())}.{time.time():.9f}" state = workflow_state_from_dag(dag, dag_inputs, workflow_id) logger.info(f'Workflow job created. [id="{workflow_id}"].') context = workflow_context.WorkflowStepContext(workflow_id=workflow_id) with workflow_context.workflow_step_context(context): # checkpoint the workflow ws = workflow_storage.get_workflow_storage(workflow_id) ws.save_workflow_user_metadata(metadata) job_id = ray.get_runtime_context().job_id.hex() try: ws.get_entrypoint_step_id() wf_exists = True except Exception: # The workflow does not exist. We must checkpoint entry workflow. ws.save_workflow_execution_state("", state) wf_exists = False workflow_manager = get_or_create_management_actor() if ray.get(workflow_manager.is_workflow_running.remote(workflow_id)): raise RuntimeError(f"Workflow '{workflow_id}' is already running.") if wf_exists: return resume(workflow_id) ignore_existing = ws.load_workflow_status() == WorkflowStatus.NONE ray.get( workflow_manager.submit_workflow.remote( workflow_id, state, ignore_existing=ignore_existing)) return workflow_manager.execute_workflow.remote(job_id, context)
def prepare_inputs(): from ray.workflow.api import _ensure_workflow_initialized _ensure_workflow_initialized() return serialization_context.make_workflow_inputs( flattened_args)