def run( self, flow_name: str = None, project_name: str = None, parameters: dict = None, run_config: RunConfig = None, new_flow_context: dict = None, run_name: str = None, idempotency_key: str = None, scheduled_start_time: datetime.datetime = None, ) -> str: """ Run method for the task; responsible for scheduling the specified flow run. Args: - flow_name (str, optional): the name of the flow to schedule; if not provided, this method will use the flow name provided at initialization - project_name (str, optional): the Cloud project in which the flow is located; if not provided, this method will use the project provided at initialization. If running with Prefect Core's server as the backend, this should not be provided. - parameters (dict, optional): the parameters to pass to the flow run being scheduled; if not provided, this method will use the parameters provided at initialization - run_config (RunConfig, optional): a run-config to use for this flow run, overriding any existing flow settings. - new_flow_context (dict, optional): the optional run context for the new flow run - run_name (str, optional): name to be set for the flow run - idempotency_key (str, optional): a unique idempotency key for scheduling the flow run. Duplicate flow runs with the same idempotency key will only create a single flow run. This is useful for ensuring that only one run is created if this task is retried. If not provided, defaults to the active `task_run_id`. - scheduled_start_time (datetime, optional): the time to schedule the execution for; if not provided, defaults to now Returns: - str: the ID of the newly-scheduled flow run Raises: - ValueError: if flow was not provided, cannot be found, or if a project name was not provided while using Cloud as a backend Example: ```python from prefect.tasks.prefect.flow_run import StartFlowRun kickoff_task = StartFlowRun(project_name="Hello, World!", flow_name="My Cloud Flow") ``` """ # verify that flow and project names were passed where necessary if flow_name is None: raise ValueError("Must provide a flow name.") if project_name is None: raise ValueError("Must provide a project name.") where_clause = { "name": { "_eq": flow_name }, "archived": { "_eq": False }, "project": { "name": { "_eq": project_name } }, } # find the flow ID to schedule query = { "query": { with_args( "flow", { "where": where_clause, "order_by": { "version": EnumValue("desc") }, "limit": 1, }, ): {"id"} } } client = Client() flow = client.graphql(query).data.flow # verify that a flow has been returned if not flow: raise ValueError("Flow '{}' not found.".format(flow_name)) # grab the ID for the most recent version flow_id = flow[0].id if idempotency_key is None: idempotency_key = prefect.context.get("task_run_id", None) # providing an idempotency key ensures that retries for this task # will not create additional flow runs flow_run_id = client.create_flow_run( flow_id=flow_id, parameters=parameters, run_config=run_config, idempotency_key=idempotency_key, context=new_flow_context, run_name=run_name, scheduled_start_time=scheduled_start_time, ) self.logger.debug(f"Flow Run {flow_run_id} created.") self.logger.debug( f"Creating link artifact for Flow Run {flow_run_id}.") run_link = client.get_cloud_url("flow-run", flow_run_id, as_user=False) create_link(urlparse(run_link).path) self.logger.info(f"Flow Run: {run_link}") if not self.wait: return flow_run_id while True: time.sleep(self.poll_interval.total_seconds()) flow_run_state = client.get_flow_run_info(flow_run_id).state if flow_run_state.is_finished(): exc = signal_from_state(flow_run_state)( f"{flow_run_id} finished in state {flow_run_state}") raise exc
def create_flow_run( flow_id: str = None, flow_name: str = None, project_name: str = "", parameters: dict = None, context: dict = None, labels: Iterable[str] = None, run_name: str = None, run_config: Optional[RunConfig] = None, scheduled_start_time: Optional[Union[pendulum.DateTime, datetime.datetime]] = None, ) -> str: """ Task to create a flow run in the Prefect backend. The flow to run must be registered and an agent must be available to deploy the flow run. Args: - flow_id: The flow or flow group uuid to lookup the flow to run - flow_name: The flow name to lookup the flow to run - project_name: The project name to lookup the flow to run. For use with `flow_name` if you have flows with the same name in multiple projects - parameters: An optional dictionary of parameters to pass to the flow run - context: An optional dictionary of context variables to pass to the flow run - labels: An optional iterable of labels to set on the flow run; if not provided, the default set of labels for the flow will be used - run_name: An optional name for the flow run; if not provided, the name will be generated as "{current_run_name}-{flow_name}" - run_config: An optional run config to use for the flow run; will override any existing run config settings - scheduled_start_time: An optional time in the future to schedule flow run execution for. If not provided, the flow run will be scheduled to start now Returns: str: The UUID of the created flow run """ if flow_id and flow_name: raise ValueError( "Received both `flow_id` and `flow_name`. Only one flow identifier " "can be passed.") if not flow_id and not flow_name: raise ValueError( "Both `flow_id` and `flow_name` are null. You must pass a flow " "identifier") logger = prefect.context.logger logger.debug("Looking up flow metadata...") if flow_id: flow = FlowView.from_id(flow_id) if flow_name: flow = FlowView.from_flow_name(flow_name, project_name=project_name) # Generate a 'sub-flow' run name if not run_name: current_run = prefect.context.get("flow_run_name") if current_run: run_name = f"{current_run}-{flow.name}" # A run name for logging display; robust to 'run_name' being empty run_name_dsp = run_name or "<generated-name>" logger.info( f"Creating flow run {run_name_dsp!r} for flow {flow.name!r}...") client = Client() flow_run_id = client.create_flow_run( flow_id=flow.flow_id, parameters=parameters, context=context, labels=labels, run_name=run_name, run_config=run_config, scheduled_start_time=scheduled_start_time, ) run_url = client.get_cloud_url("flow-run", flow_run_id) logger.info(f"Created flow run {run_name_dsp!r}: {run_url}") return flow_run_id
def create_flow_run( flow_id: str = None, flow_name: str = None, project_name: str = "", parameters: dict = None, context: dict = None, labels: Iterable[str] = None, run_name: str = None, run_config: Optional[RunConfig] = None, scheduled_start_time: Optional[Union[pendulum.DateTime, datetime.datetime, pendulum.Duration, datetime.timedelta, ]] = None, idempotency_key: str = None, ) -> str: """ Task to create a flow run in the Prefect backend. The flow to run must be registered and an agent must be available to deploy the flow run. Args: - flow_id: The flow or flow group uuid to lookup the flow to run - flow_name: The flow name to lookup the flow to run - project_name: The project name to lookup the flow to run. For use with `flow_name` if you have flows with the same name in multiple projects - parameters: An optional dictionary of parameters to pass to the flow run - context: An optional dictionary of context variables to pass to the flow run - labels: An optional iterable of labels to set on the flow run; if not provided, the default set of labels for the flow will be used - run_name: An optional name for the flow run; if not provided, the name will be generated as "{current_run_name}-{flow_name}" - run_config: An optional run config to use for the flow run; will override any existing run config settings - scheduled_start_time: An optional time in the future to schedule flow run execution for. If not provided, the flow run will be scheduled to start now - idempotency_key: a unique idempotency key for scheduling the flow run. Duplicate flow runs with the same idempotency key will only create a single flow run. This is useful for ensuring that only one run is created if this task is retried. If not provided, defaults to the active task run id and its map index. Returns: str: The UUID of the created flow run """ if flow_id and flow_name: raise ValueError( "Received both `flow_id` and `flow_name`. Only one flow identifier " "can be passed.") if not flow_id and not flow_name: raise ValueError( "Both `flow_id` and `flow_name` are null. You must pass a flow " "identifier") logger = prefect.context.logger logger.debug("Looking up flow metadata...") if flow_id: flow = FlowView.from_id(flow_id) if flow_name: flow = FlowView.from_flow_name(flow_name, project_name=project_name) # Generate a 'sub-flow' run name if not run_name: current_run = prefect.context.get("flow_run_name") if current_run: run_name = f"{current_run}-{flow.name}" # A run name for logging display; robust to 'run_name' being empty run_name_dsp = run_name or "<generated-name>" logger.info( f"Creating flow run {run_name_dsp!r} for flow {flow.name!r}...") if idempotency_key is None: # Generate a default key, if the context is missing this data just fall through # to `None` idempotency_key = prefect.context.get("task_run_id") map_index = prefect.context.get("map_index") if idempotency_key and map_index is not None: idempotency_key += f"-{map_index}" if isinstance(scheduled_start_time, (pendulum.Duration, datetime.timedelta)): scheduled_start_time = pendulum.now("utc") + scheduled_start_time client = Client() flow_run_id = client.create_flow_run( flow_id=flow.flow_id, parameters=parameters, context=context, labels=labels, run_name=run_name, run_config=run_config, scheduled_start_time=scheduled_start_time, idempotency_key=idempotency_key, ) run_url = client.get_cloud_url("flow-run", flow_run_id) logger.info(f"Created flow run {run_name_dsp!r}: {run_url}") return flow_run_id