def wait_for_flow_run( flow_run_id: str, stream_states: bool = True, stream_logs: bool = False, raise_final_state: bool = False, max_duration: timedelta = timedelta(hours=12), ) -> "FlowRunView": """ Task to wait for a flow run to finish executing, streaming state and log information Args: - flow_run_id: The flow run id to wait for - stream_states: Stream information about the flow run state changes - stream_logs: Stream flow run logs; if `stream_state` is `False` this will be ignored - raise_final_state: If set, the state of this task will be set to the final state of the child flow run on completion. - max_duration: Duration to wait for flow run to complete. Defaults to 12 hours. Returns: FlowRunView: A view of the flow run after completion """ flow_run = FlowRunView.from_flow_run_id(flow_run_id) for log in watch_flow_run( flow_run_id, stream_states=stream_states, stream_logs=stream_logs, max_duration=max_duration, ): message = f"Flow {flow_run.name!r}: {log.message}" prefect.context.logger.log(log.level, message) # Get the final view of the flow run flow_run = flow_run.get_latest() if raise_final_state: state_signal = signal_from_state(flow_run.state)( message=f"{flow_run_id} finished in state {flow_run.state}", result=flow_run, ) raise state_signal else: return flow_run
def run( self, flow_name: str = None, project_name: str = None, parameters: dict = None, idempotency_key: str = None, new_flow_context: dict = None, run_name: str = None, scheduled_start_time: datetime.datetime = None, ) -> str: """ Run method for the task; responsible for scheduling the specified flow run. Args: - flow_name (str, optional): the name of the flow to schedule; if not provided, this method will use the flow name provided at initialization - project_name (str, optional): the Cloud project in which the flow is located; if not provided, this method will use the project provided at initialization. If running with Prefect Core's server as the backend, this should not be provided. - parameters (dict, optional): the parameters to pass to the flow run being scheduled; if not provided, this method will use the parameters provided at initialization - idempotency_key (str, optional): an optional idempotency key for scheduling the flow run; if provided, ensures that only one run is created if this task is retried or rerun with the same inputs. If not provided, the current flow run ID will be used. - new_flow_context (dict, optional): the optional run context for the new flow run - run_name (str, optional): name to be set for the flow run - scheduled_start_time (datetime, optional): the time to schedule the execution for; if not provided, defaults to now Returns: - str: the ID of the newly-scheduled flow run Raises: - ValueError: if flow was not provided, cannot be found, or if a project name was not provided while using Cloud as a backend Example: ```python from prefect.tasks.prefect.flow_run import StartFlowRun kickoff_task = StartFlowRun(project_name="Hello, World!", flow_name="My Cloud Flow") ``` """ # verify that flow and project names were passed where necessary if flow_name is None: raise ValueError("Must provide a flow name.") if project_name is None: raise ValueError("Must provide a project name.") where_clause = { "name": { "_eq": flow_name }, "archived": { "_eq": False }, "project": { "name": { "_eq": project_name } }, } # find the flow ID to schedule query = { "query": { with_args( "flow", { "where": where_clause, "order_by": { "version": EnumValue("desc") }, "limit": 1, }, ): {"id"} } } client = Client() flow = client.graphql(query).data.flow # verify that a flow has been returned if not flow: raise ValueError("Flow '{}' not found.".format(flow_name)) # grab the ID for the most recent version flow_id = flow[0].id idem_key = None if context.get("flow_run_id"): map_index = context.get("map_index") default = context.get("flow_run_id") + (f"-{map_index}" if map_index else "") idem_key = idempotency_key or default # providing an idempotency key ensures that retries for this task # will not create additional flow runs flow_run_id = client.create_flow_run( flow_id=flow_id, parameters=parameters, idempotency_key=idem_key or idempotency_key, context=new_flow_context, run_name=run_name, scheduled_start_time=scheduled_start_time, ) self.logger.debug(f"Flow Run {flow_run_id} created.") if not self.wait: return flow_run_id while True: time.sleep(10) flow_run_state = client.get_flow_run_info(flow_run_id).state if flow_run_state.is_finished(): exc = signal_from_state(flow_run_state)( f"{flow_run_id} finished in state {flow_run_state}") raise exc
def run( self, flow_name: str = None, project_name: str = None, parameters: dict = None, run_config: RunConfig = None, new_flow_context: dict = None, run_name: str = None, idempotency_key: str = None, scheduled_start_time: datetime.datetime = None, ) -> str: """ Run method for the task; responsible for scheduling the specified flow run. Args: - flow_name (str, optional): the name of the flow to schedule; if not provided, this method will use the flow name provided at initialization - project_name (str, optional): the Cloud project in which the flow is located; if not provided, this method will use the project provided at initialization. If running with Prefect Core's server as the backend, this should not be provided. - parameters (dict, optional): the parameters to pass to the flow run being scheduled; if not provided, this method will use the parameters provided at initialization - run_config (RunConfig, optional): a run-config to use for this flow run, overriding any existing flow settings. - new_flow_context (dict, optional): the optional run context for the new flow run - run_name (str, optional): name to be set for the flow run - idempotency_key (str, optional): a unique idempotency key for scheduling the flow run. Duplicate flow runs with the same idempotency key will only create a single flow run. This is useful for ensuring that only one run is created if this task is retried. If not provided, defaults to the active `task_run_id`. - scheduled_start_time (datetime, optional): the time to schedule the execution for; if not provided, defaults to now Returns: - str: the ID of the newly-scheduled flow run Raises: - ValueError: if flow was not provided, cannot be found, or if a project name was not provided while using Cloud as a backend Example: ```python from prefect.tasks.prefect.flow_run import StartFlowRun kickoff_task = StartFlowRun(project_name="Hello, World!", flow_name="My Cloud Flow") ``` """ # verify that flow and project names were passed where necessary if flow_name is None: raise ValueError("Must provide a flow name.") if project_name is None: raise ValueError("Must provide a project name.") where_clause = { "name": { "_eq": flow_name }, "archived": { "_eq": False }, "project": { "name": { "_eq": project_name } }, } # find the flow ID to schedule query = { "query": { with_args( "flow", { "where": where_clause, "order_by": { "version": EnumValue("desc") }, "limit": 1, }, ): {"id"} } } client = Client() flow = client.graphql(query).data.flow # verify that a flow has been returned if not flow: raise ValueError("Flow '{}' not found.".format(flow_name)) # grab the ID for the most recent version flow_id = flow[0].id if idempotency_key is None: idempotency_key = prefect.context.get("task_run_id", None) # providing an idempotency key ensures that retries for this task # will not create additional flow runs flow_run_id = client.create_flow_run( flow_id=flow_id, parameters=parameters, run_config=run_config, idempotency_key=idempotency_key, context=new_flow_context, run_name=run_name, scheduled_start_time=scheduled_start_time, ) self.logger.debug(f"Flow Run {flow_run_id} created.") self.logger.debug( f"Creating link artifact for Flow Run {flow_run_id}.") run_link = client.get_cloud_url("flow-run", flow_run_id, as_user=False) create_link(urlparse(run_link).path) self.logger.info(f"Flow Run: {run_link}") if not self.wait: return flow_run_id while True: time.sleep(self.poll_interval.total_seconds()) flow_run_state = client.get_flow_run_info(flow_run_id).state if flow_run_state.is_finished(): exc = signal_from_state(flow_run_state)( f"{flow_run_id} finished in state {flow_run_state}") raise exc
def test_signal_from_state_returns_correct_signal(signal, state): assert signal_from_state(state("Dummy message")) == signal
def run( self, flow_name: str = None, project_name: str = None, parameters: dict = None ) -> str: """ Run method for the task; responsible for scheduling the specified flow run. Args: - flow_name (str, optional): the name of the flow to schedule; if not provided, this method will use the flow name provided at initialization - project_name (str, optional): the Cloud project in which the flow is located; if not provided, this method will use the project provided at initialization. If running with Prefect Core's server as the backend, this should not be provided. - parameters (dict, optional): the parameters to pass to the flow run being scheduled; if not provided, this method will use the parameters provided at initialization Returns: - str: the ID of the newly-scheduled flow run Raises: - ValueError: if flow was not provided, cannot be found, or if a project name was not provided while using Cloud as a backend Example: ```python from prefect.tasks.prefect.flow_run import FlowRunTask kickoff_task = FlowRunTask(project_name="Hello, World!", flow_name="My Cloud Flow") ``` """ is_hosted_backend = "prefect.io" in urlparse(config.cloud.api).netloc # verify that flow and project names were passed where necessary if flow_name is None: raise ValueError("Must provide a flow name.") if project_name is None and is_hosted_backend: raise ValueError("Must provide a project name.") where_clause = { "name": {"_eq": flow_name}, "archived": {"_eq": False}, } if project_name: where_clause["project"] = {"name": {"_eq": project_name}} # find the flow ID to schedule query = { "query": { with_args( "flow", { "where": where_clause, "order_by": {"version": EnumValue("desc")}, "limit": 1, }, ): {"id"} } } client = Client() flow = client.graphql(query).data.flow # verify that a flow has been returned if not flow: raise ValueError("Flow '{}' not found.".format(flow_name)) # grab the ID for the most recent version flow_id = flow[0].id # providing an idempotency key ensures that retries for this task # will not create additional flow runs flow_run_id = client.create_flow_run( flow_id=flow_id, parameters=parameters, idempotency_key=context.get("flow_run_id"), ) self.logger.debug(f"Flow Run {flow_run_id} created.") if not self.wait: return flow_run_id while True: time.sleep(10) flow_run_state = client.get_flow_run_info(flow_run_id).state if flow_run_state.is_finished(): exc = signal_from_state(flow_run_state)( f"{flow_run_id} finished in state {flow_run_state}" ) raise exc
self.logger.debug(f"Flow Run {flow_run_id} created.") <<<<<<< HEAD self.logger.debug(f"Creating link artifact for Flow Run {flow_run_id}.") run_link = client.get_cloud_url("flow-run", flow_run_id, as_user=False) create_link(urlparse(run_link).path) ======= >>>>>>> prefect clone if not self.wait: return flow_run_id while True: time.sleep(10) flow_run_state = client.get_flow_run_info(flow_run_id).state if flow_run_state.is_finished(): exc = signal_from_state(flow_run_state)( f"{flow_run_id} finished in state {flow_run_state}" ) raise exc class FlowRunTask(StartFlowRun): def __new__(cls, *args, **kwargs): # type: ignore warnings.warn( "`FlowRunTask` has been renamed to `prefect.tasks.prefect.StartFlowRun`," "please update your code accordingly", stacklevel=2, ) return super().__new__(cls)