async def _create_flow_run( flow_id: str = None, parameters: dict = None, context: dict = None, scheduled_start_time: datetime.datetime = None, flow_run_name: str = None, version_group_id: str = None, labels: List[str] = None, run_config: dict = None, ) -> Any: """ Creates a new flow run for an existing flow. Args: - flow_id (str): A string representing the current flow id - parameters (dict, optional): A dictionary of parameters that were specified for the flow - context (dict, optional): A dictionary of context values - scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`, defaults to right now. Must be UTC. - flow_run_name (str, optional): An optional string representing this flow run - version_group_id (str, optional): An optional version group ID; if provided, will run the most recent unarchived version of the group - labels (List[str], optional): a list of labels to apply to this individual flow run - run-config (dict, optional): A run-config override for this flow run. """ if flow_id is None and version_group_id is None: raise ValueError( "One of flow_id or version_group_id must be provided.") scheduled_start_time = scheduled_start_time or pendulum.now() if flow_id: where_clause = {"id": {"_eq": flow_id}} elif version_group_id: where_clause = { "version_group_id": { "_eq": version_group_id }, "archived": { "_eq": False }, } flow = await models.Flow.where(where=where_clause).first( { "id": True, "archived": True, "tenant_id": True, "environment": True, "run_config": True, "parameters": True, "flow_group_id": True, "flow_group": { "default_parameters": True, "labels": True, "run_config": True, }, }, order_by={"version": EnumValue("desc")}, ) # type: Any if not flow: msg = (f"Flow {flow_id} not found" if flow_id else f"Version group {version_group_id} has no unarchived flows.") raise exceptions.NotFound(msg) elif flow.archived: raise ValueError(f"Flow {flow.id} is archived.") # determine active labels if labels is not None: run_labels = labels elif run_config is not None: run_labels = run_config.get("labels") or [] elif flow.flow_group.labels is not None: run_labels = flow.flow_group.labels elif flow.flow_group.run_config is not None: run_labels = flow.flow_group.run_config.get("labels") or [] elif flow.run_config is not None: run_labels = flow.run_config.get("labels") or [] elif flow.environment is not None: run_labels = flow.environment.get("labels") or [] else: run_labels = [] run_labels.sort() # determine active run_config if run_config is None: if flow.flow_group.run_config is not None: run_config = flow.flow_group.run_config else: run_config = flow.run_config # check parameters run_parameters = flow.flow_group.default_parameters run_parameters.update((parameters or {})) required_parameters = [p["name"] for p in flow.parameters if p["required"]] missing = set(required_parameters).difference(run_parameters) if missing: raise ValueError(f"Required parameters were not supplied: {missing}") state = Scheduled(message="Flow run scheduled.", start_time=scheduled_start_time) run = models.FlowRun( tenant_id=flow.tenant_id, flow_id=flow_id or flow.id, labels=run_labels, parameters=run_parameters, run_config=run_config, context=context or {}, scheduled_start_time=scheduled_start_time, name=flow_run_name or names.generate_slug(2), states=[ models.FlowRunState( tenant_id=flow.tenant_id, **models.FlowRunState.fields_from_state( Pending(message="Flow run created")), ) ], ) flow_run_id = await run.insert() # apply the flow run's initial state via `set_flow_run_state` await api.states.set_flow_run_state(flow_run_id=flow_run_id, state=state) return flow_run_id
async def test_cloud_hook( cloud_hook_id: str, flow_run_id: str = None, state: prefect.engine.state.State = None, ): """ Sends a test payload to a hook Args: - cloud_hook_id (str): the hook id - flow_run_id (str, optional): a flow run ID to test with. If provided, the event will pull event details from the flow run - state (State, optional): a sample state """ if state is None: state = prefect.engine.state.Success(message="Test success state") serialized_state = state.serialize() if cloud_hook_id is None: raise ValueError("Invalid ID") # verify that the hook actually exists hook = await models.CloudHook.where(id=cloud_hook_id).first( {"type", "config", "tenant_id"}) if not hook: raise ValueError("Invalid ID") # if a flow run has been provided, construct an event mirroring the actual event # we'd expect to see if flow_run_id: flow_run = await models.FlowRun.where(id=flow_run_id).first( {"id", "tenant_id", "flow_id", "name", "version"}) new_flow_run = Box( id=flow_run.id, is_test_event=True, tenant_id=flow_run.tenant_id, flow_id=flow_run.flow_id, name=flow_run.name, version=flow_run.version + 1, state=type(state).__name__, serialized_state=state.serialize(), ) flow = await models.Flow.where(id=flow_run.flow_id).first( selection_set={ "id": True, "tenant_id": True, "name": True, "environment": True, "version_group_id": True, }) tenant = await models.Tenant.where(id=flow_run.tenant_id ).first({"id", "slug"}) test_event = events.FlowRunStateChange( flow_run=new_flow_run.to_dict(), flow=flow.dict(), tenant=tenant.dict(), state=dict( version=1, state=serialized_state["type"], serialized_state=serialized_state, ), ) # otherwise populate the event with dummy data else: test_event = events.FlowRunStateChange( is_test_event=True, flow_run=dict(id=str(uuid.uuid4()), name=names.generate_slug(2)), tenant=dict(id=hook.tenant_id, slug="test-slug"), flow=dict(id=str(uuid.uuid4()), name="Cloud Hook Test Flow"), state=dict( version=1, state=serialized_state["type"], serialized_state=serialized_state, ), ) if hook.type == "WEBHOOK": await _call_webhook(url=hook.config["url"], event=test_event) elif hook.type == "SLACK_WEBHOOK": await _call_slack_webhook(url=hook.config["url"], event=test_event) elif hook.type == "PREFECT_MESSAGE": await _call_prefect_message(event=test_event) elif hook.type == "TWILIO": await _call_twilio( account_sid=hook.config["account_sid"], auth_token=hook.config["auth_token"], messaging_service_sid=hook.config["messaging_service_sid"], to=hook.config["to"], event=test_event, ) elif hook.type == "PAGERDUTY": await _call_pagerduty( api_token=hook.config["api_token"], routing_key=hook.config["routing_key"], severity=hook.config["severity"], event=test_event, ) else: raise ValueError("Invalid type")
async def create_flow_run( flow_id: str = None, parameters: dict = None, context: dict = None, scheduled_start_time: datetime.datetime = None, flow_run_name: str = None, version_group_id: str = None, labels: List[str] = None, run_config: dict = None, idempotency_key: str = None, ) -> Any: """ Creates a new flow run for an existing flow. Args: - flow_id (str): A string representing the current flow id - parameters (dict, optional): A dictionary of parameters that were specified for the flow - context (dict, optional): A dictionary of context values - scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`, defaults to right now. Must be UTC. - flow_run_name (str, optional): An optional string representing this flow run - version_group_id (str, optional): An optional version group ID; if provided, will run the most recent unarchived version of the group - labels (List[str], optional): a list of labels to apply to this individual flow run - run-config (dict, optional): A run-config override for this flow run. - idempotency_key (str, optional): An optional idempotency key to prevent duplicate run creation. idempotency keys are unique to each flow, but can be repeated across different flows. """ if flow_id is None and version_group_id is None: raise ValueError("One of flow_id or version_group_id must be provided.") scheduled_start_time = scheduled_start_time or pendulum.now() if flow_id: where_clause = {"id": {"_eq": flow_id}} elif version_group_id: where_clause = { "version_group_id": {"_eq": version_group_id}, "archived": {"_eq": False}, } flow = await models.Flow.where(where=where_clause).first( { "id": True, "archived": True, "tenant_id": True, "environment": True, "run_config": True, "parameters": True, "flow_group_id": True, "flow_group": { "default_parameters": True, "labels": True, "run_config": True, }, }, order_by={"version": EnumValue("desc")}, ) # type: Any if not flow: msg = ( f"Flow {flow_id} not found" if flow_id else f"Version group {version_group_id} has no unarchived flows." ) raise exceptions.NotFound(msg) elif flow.archived: raise ValueError(f"Flow {flow.id} is archived.") # Determine active labels based on the following hierarchy: # - Provided labels # - Provided run config # - Flow group labels # - Flow group run config # - Flow run config # - Flow environment if labels is not None: run_labels = labels elif run_config is not None: run_labels = run_config.get("labels") or [] elif flow.flow_group.labels is not None: run_labels = flow.flow_group.labels elif flow.flow_group.run_config is not None: run_labels = flow.flow_group.run_config.get("labels") or [] elif flow.run_config is not None: run_labels = flow.run_config.get("labels") or [] elif flow.environment is not None: run_labels = flow.environment.get("labels") or [] else: run_labels = [] run_labels.sort() # determine active run_config if run_config is None: if flow.flow_group.run_config is not None: run_config = flow.flow_group.run_config else: run_config = flow.run_config # check parameters run_parameters = flow.flow_group.default_parameters run_parameters.update((parameters or {})) required_parameters = [p["name"] for p in flow.parameters if p["required"]] missing = set(required_parameters).difference(run_parameters) if missing: raise ValueError(f"Required parameters were not supplied: {missing}") run = models.FlowRun( id=str(uuid.uuid4()), tenant_id=flow.tenant_id, flow_id=flow_id or flow.id, labels=run_labels, parameters=run_parameters, run_config=run_config, context=context or {}, scheduled_start_time=scheduled_start_time, name=flow_run_name or names.generate_slug(2), idempotency_key=idempotency_key, ) # upsert the flow run against the idempotency key, returning the id of the # new or existing run insert_result = await run.insert( on_conflict=dict( constraint="ix_flow_run_idempotency_key_unique", update_columns=["idempotency_key"], ), selection_set={"returning": {"id"}}, ) # if the run id matches the returned id, the insert succeeded and the # initial state should be set otherwise, the idempotency key was matched and # we take no action if run.id == insert_result.returning.id: # apply the flow run's initial state via `set_flow_run_state` await api.states.set_flow_run_state( flow_run_id=run.id, state=Scheduled( message="Flow run scheduled.", start_time=scheduled_start_time ), ) logger.debug( f"Flow run {insert_result.returning.id} of flow {flow_id or flow.id} " f"scheduled for {scheduled_start_time}" ) # return the database id return insert_result.returning.id
async def create_flow_run( flow_id: str = None, parameters: dict = None, context: dict = None, scheduled_start_time: datetime.datetime = None, flow_run_name: str = None, version_group_id: str = None, ) -> Any: """ Creates a new flow run for an existing flow. Args: - flow_id (str): A string representing the current flow id - parameters (dict, optional): A dictionary of parameters that were specified for the flow - context (dict, optional): A dictionary of context values - scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`, defaults to right now. Must be UTC. - flow_run_name (str, optional): An optional string representing this flow run - version_group_id (str, optional): An optional version group ID; if provided, will run the most recent unarchived version of the group """ scheduled_start_time = scheduled_start_time or pendulum.now() if flow_id: where_clause = {"id": {"_eq": flow_id}} elif version_group_id: where_clause = { "version_group_id": { "_eq": version_group_id }, "archived": { "_eq": False }, } else: raise ValueError( "One of flow_id or version_group_id must be provided.") flow = await models.Flow.where(where=where_clause).first( { "id": True, "archived": True, "parameters": True }, order_by={"version": EnumValue("desc")}, ) # type: Any if not flow: msg = (f"Flow {flow_id} not found" if flow_id else f"Version group {version_group_id} has no unarchived flows.") raise exceptions.NotFound(msg) elif flow.archived: raise ValueError(f"Flow {flow.id} is archived.") # check parameters parameters = parameters or {} required_parameters = [p["name"] for p in flow.parameters if p["required"]] missing = set(required_parameters).difference(parameters) if missing: raise ValueError(f"Required parameters were not supplied: {missing}") extra = set(parameters).difference([p["name"] for p in flow.parameters]) if extra: raise ValueError(f"Extra parameters were supplied: {extra}") state = Scheduled(message="Flow run scheduled.", start_time=scheduled_start_time) run = models.FlowRun( flow_id=flow_id or flow.id, parameters=parameters, context=context or {}, scheduled_start_time=scheduled_start_time, name=flow_run_name or names.generate_slug(2), states=[ models.FlowRunState(**models.FlowRunState.fields_from_state( Pending(message="Flow run created"))) ], ) flow_run_id = await run.insert() # apply the flow run's initial state via `set_flow_run_state` await api.states.set_flow_run_state(flow_run_id=flow_run_id, state=state) return flow_run_id