async def _create_flow_run(
flow_id: str = None,
parameters: dict = None,
context: dict = None,
scheduled_start_time: datetime.datetime = None,
flow_run_name: str = None,
version_group_id: str = None,
labels: List[str] = None,
run_config: dict = None,
) -> Any:
"""
Creates a new flow run for an existing flow.
Args:
- flow_id (str): A string representing the current flow id
- parameters (dict, optional): A dictionary of parameters that were specified for the flow
- context (dict, optional): A dictionary of context values
- scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`,
defaults to right now. Must be UTC.
- flow_run_name (str, optional): An optional string representing this flow run
- version_group_id (str, optional): An optional version group ID; if provided, will run the most
recent unarchived version of the group
- labels (List[str], optional): a list of labels to apply to this individual flow run
- run-config (dict, optional): A run-config override for this flow run.
"""
if flow_id is None and version_group_id is None:
raise ValueError(
"One of flow_id or version_group_id must be provided.")
scheduled_start_time = scheduled_start_time or pendulum.now()
if flow_id:
where_clause = {"id": {"_eq": flow_id}}
elif version_group_id:
where_clause = {
"version_group_id": {
"_eq": version_group_id
},
"archived": {
"_eq": False
},
}
flow = await models.Flow.where(where=where_clause).first(
{
"id": True,
"archived": True,
"tenant_id": True,
"environment": True,
"run_config": True,
"parameters": True,
"flow_group_id": True,
"flow_group": {
"default_parameters": True,
"labels": True,
"run_config": True,
},
},
order_by={"version": EnumValue("desc")},
) # type: Any
if not flow:
msg = (f"Flow {flow_id} not found" if flow_id else
f"Version group {version_group_id} has no unarchived flows.")
raise exceptions.NotFound(msg)
elif flow.archived:
raise ValueError(f"Flow {flow.id} is archived.")
# determine active labels
if labels is not None:
run_labels = labels
elif run_config is not None:
run_labels = run_config.get("labels") or []
elif flow.flow_group.labels is not None:
run_labels = flow.flow_group.labels
elif flow.flow_group.run_config is not None:
run_labels = flow.flow_group.run_config.get("labels") or []
elif flow.run_config is not None:
run_labels = flow.run_config.get("labels") or []
elif flow.environment is not None:
run_labels = flow.environment.get("labels") or []
else:
run_labels = []
run_labels.sort()
# determine active run_config
if run_config is None:
if flow.flow_group.run_config is not None:
run_config = flow.flow_group.run_config
else:
run_config = flow.run_config
# check parameters
run_parameters = flow.flow_group.default_parameters
run_parameters.update((parameters or {}))
required_parameters = [p["name"] for p in flow.parameters if p["required"]]
missing = set(required_parameters).difference(run_parameters)
if missing:
raise ValueError(f"Required parameters were not supplied: {missing}")
state = Scheduled(message="Flow run scheduled.",
start_time=scheduled_start_time)
run = models.FlowRun(
tenant_id=flow.tenant_id,
flow_id=flow_id or flow.id,
labels=run_labels,
parameters=run_parameters,
run_config=run_config,
context=context or {},
scheduled_start_time=scheduled_start_time,
name=flow_run_name or names.generate_slug(2),
states=[
models.FlowRunState(
tenant_id=flow.tenant_id,
**models.FlowRunState.fields_from_state(
Pending(message="Flow run created")),
)
],
)
flow_run_id = await run.insert()
# apply the flow run's initial state via `set_flow_run_state`
await api.states.set_flow_run_state(flow_run_id=flow_run_id, state=state)
return flow_run_id
async def test_cloud_hook(
cloud_hook_id: str,
flow_run_id: str = None,
state: prefect.engine.state.State = None,
):
"""
Sends a test payload to a hook
Args:
- cloud_hook_id (str): the hook id
- flow_run_id (str, optional): a flow run ID to test with. If provided, the event will pull event details
from the flow run
- state (State, optional): a sample state
"""
if state is None:
state = prefect.engine.state.Success(message="Test success state")
serialized_state = state.serialize()
if cloud_hook_id is None:
raise ValueError("Invalid ID")
# verify that the hook actually exists
hook = await models.CloudHook.where(id=cloud_hook_id).first(
{"type", "config", "tenant_id"})
if not hook:
raise ValueError("Invalid ID")
# if a flow run has been provided, construct an event mirroring the actual event
# we'd expect to see
if flow_run_id:
flow_run = await models.FlowRun.where(id=flow_run_id).first(
{"id", "tenant_id", "flow_id", "name", "version"})
new_flow_run = Box(
id=flow_run.id,
is_test_event=True,
tenant_id=flow_run.tenant_id,
flow_id=flow_run.flow_id,
name=flow_run.name,
version=flow_run.version + 1,
state=type(state).__name__,
serialized_state=state.serialize(),
)
flow = await models.Flow.where(id=flow_run.flow_id).first(
selection_set={
"id": True,
"tenant_id": True,
"name": True,
"environment": True,
"version_group_id": True,
})
tenant = await models.Tenant.where(id=flow_run.tenant_id
).first({"id", "slug"})
test_event = events.FlowRunStateChange(
flow_run=new_flow_run.to_dict(),
flow=flow.dict(),
tenant=tenant.dict(),
state=dict(
version=1,
state=serialized_state["type"],
serialized_state=serialized_state,
),
)
# otherwise populate the event with dummy data
else:
test_event = events.FlowRunStateChange(
is_test_event=True,
flow_run=dict(id=str(uuid.uuid4()), name=names.generate_slug(2)),
tenant=dict(id=hook.tenant_id, slug="test-slug"),
flow=dict(id=str(uuid.uuid4()), name="Cloud Hook Test Flow"),
state=dict(
version=1,
state=serialized_state["type"],
serialized_state=serialized_state,
),
)
if hook.type == "WEBHOOK":
await _call_webhook(url=hook.config["url"], event=test_event)
elif hook.type == "SLACK_WEBHOOK":
await _call_slack_webhook(url=hook.config["url"], event=test_event)
elif hook.type == "PREFECT_MESSAGE":
await _call_prefect_message(event=test_event)
elif hook.type == "TWILIO":
await _call_twilio(
account_sid=hook.config["account_sid"],
auth_token=hook.config["auth_token"],
messaging_service_sid=hook.config["messaging_service_sid"],
to=hook.config["to"],
event=test_event,
)
elif hook.type == "PAGERDUTY":
await _call_pagerduty(
api_token=hook.config["api_token"],
routing_key=hook.config["routing_key"],
severity=hook.config["severity"],
event=test_event,
)
else:
raise ValueError("Invalid type")
async def create_flow_run(
flow_id: str = None,
parameters: dict = None,
context: dict = None,
scheduled_start_time: datetime.datetime = None,
flow_run_name: str = None,
version_group_id: str = None,
labels: List[str] = None,
run_config: dict = None,
idempotency_key: str = None,
) -> Any:
"""
Creates a new flow run for an existing flow.
Args:
- flow_id (str): A string representing the current flow id
- parameters (dict, optional): A dictionary of parameters that were specified for the flow
- context (dict, optional): A dictionary of context values
- scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`,
defaults to right now. Must be UTC.
- flow_run_name (str, optional): An optional string representing this flow run
- version_group_id (str, optional): An optional version group ID; if provided, will run the most
recent unarchived version of the group
- labels (List[str], optional): a list of labels to apply to this individual flow run
- run-config (dict, optional): A run-config override for this flow run.
- idempotency_key (str, optional): An optional idempotency key to prevent duplicate run creation.
idempotency keys are unique to each flow, but can be repeated across different flows.
"""
if flow_id is None and version_group_id is None:
raise ValueError("One of flow_id or version_group_id must be provided.")
scheduled_start_time = scheduled_start_time or pendulum.now()
if flow_id:
where_clause = {"id": {"_eq": flow_id}}
elif version_group_id:
where_clause = {
"version_group_id": {"_eq": version_group_id},
"archived": {"_eq": False},
}
flow = await models.Flow.where(where=where_clause).first(
{
"id": True,
"archived": True,
"tenant_id": True,
"environment": True,
"run_config": True,
"parameters": True,
"flow_group_id": True,
"flow_group": {
"default_parameters": True,
"labels": True,
"run_config": True,
},
},
order_by={"version": EnumValue("desc")},
) # type: Any
if not flow:
msg = (
f"Flow {flow_id} not found"
if flow_id
else f"Version group {version_group_id} has no unarchived flows."
)
raise exceptions.NotFound(msg)
elif flow.archived:
raise ValueError(f"Flow {flow.id} is archived.")
# Determine active labels based on the following hierarchy:
# - Provided labels
# - Provided run config
# - Flow group labels
# - Flow group run config
# - Flow run config
# - Flow environment
if labels is not None:
run_labels = labels
elif run_config is not None:
run_labels = run_config.get("labels") or []
elif flow.flow_group.labels is not None:
run_labels = flow.flow_group.labels
elif flow.flow_group.run_config is not None:
run_labels = flow.flow_group.run_config.get("labels") or []
elif flow.run_config is not None:
run_labels = flow.run_config.get("labels") or []
elif flow.environment is not None:
run_labels = flow.environment.get("labels") or []
else:
run_labels = []
run_labels.sort()
# determine active run_config
if run_config is None:
if flow.flow_group.run_config is not None:
run_config = flow.flow_group.run_config
else:
run_config = flow.run_config
# check parameters
run_parameters = flow.flow_group.default_parameters
run_parameters.update((parameters or {}))
required_parameters = [p["name"] for p in flow.parameters if p["required"]]
missing = set(required_parameters).difference(run_parameters)
if missing:
raise ValueError(f"Required parameters were not supplied: {missing}")
run = models.FlowRun(
id=str(uuid.uuid4()),
tenant_id=flow.tenant_id,
flow_id=flow_id or flow.id,
labels=run_labels,
parameters=run_parameters,
run_config=run_config,
context=context or {},
scheduled_start_time=scheduled_start_time,
name=flow_run_name or names.generate_slug(2),
idempotency_key=idempotency_key,
)
# upsert the flow run against the idempotency key, returning the id of the
# new or existing run
insert_result = await run.insert(
on_conflict=dict(
constraint="ix_flow_run_idempotency_key_unique",
update_columns=["idempotency_key"],
),
selection_set={"returning": {"id"}},
)
# if the run id matches the returned id, the insert succeeded and the
# initial state should be set otherwise, the idempotency key was matched and
# we take no action
if run.id == insert_result.returning.id:
# apply the flow run's initial state via `set_flow_run_state`
await api.states.set_flow_run_state(
flow_run_id=run.id,
state=Scheduled(
message="Flow run scheduled.", start_time=scheduled_start_time
),
)
logger.debug(
f"Flow run {insert_result.returning.id} of flow {flow_id or flow.id} "
f"scheduled for {scheduled_start_time}"
)
# return the database id
return insert_result.returning.id
async def create_flow_run(
flow_id: str = None,
parameters: dict = None,
context: dict = None,
scheduled_start_time: datetime.datetime = None,
flow_run_name: str = None,
version_group_id: str = None,
) -> Any:
"""
Creates a new flow run for an existing flow.
Args:
- flow_id (str): A string representing the current flow id
- parameters (dict, optional): A dictionary of parameters that were specified for the flow
- context (dict, optional): A dictionary of context values
- scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`,
defaults to right now. Must be UTC.
- flow_run_name (str, optional): An optional string representing this flow run
- version_group_id (str, optional): An optional version group ID; if provided, will run the most
recent unarchived version of the group
"""
scheduled_start_time = scheduled_start_time or pendulum.now()
if flow_id:
where_clause = {"id": {"_eq": flow_id}}
elif version_group_id:
where_clause = {
"version_group_id": {
"_eq": version_group_id
},
"archived": {
"_eq": False
},
}
else:
raise ValueError(
"One of flow_id or version_group_id must be provided.")
flow = await models.Flow.where(where=where_clause).first(
{
"id": True,
"archived": True,
"parameters": True
},
order_by={"version": EnumValue("desc")},
) # type: Any
if not flow:
msg = (f"Flow {flow_id} not found" if flow_id else
f"Version group {version_group_id} has no unarchived flows.")
raise exceptions.NotFound(msg)
elif flow.archived:
raise ValueError(f"Flow {flow.id} is archived.")
# check parameters
parameters = parameters or {}
required_parameters = [p["name"] for p in flow.parameters if p["required"]]
missing = set(required_parameters).difference(parameters)
if missing:
raise ValueError(f"Required parameters were not supplied: {missing}")
extra = set(parameters).difference([p["name"] for p in flow.parameters])
if extra:
raise ValueError(f"Extra parameters were supplied: {extra}")
state = Scheduled(message="Flow run scheduled.",
start_time=scheduled_start_time)
run = models.FlowRun(
flow_id=flow_id or flow.id,
parameters=parameters,
context=context or {},
scheduled_start_time=scheduled_start_time,
name=flow_run_name or names.generate_slug(2),
states=[
models.FlowRunState(**models.FlowRunState.fields_from_state(
Pending(message="Flow run created")))
],
)
flow_run_id = await run.insert()
# apply the flow run's initial state via `set_flow_run_state`
await api.states.set_flow_run_state(flow_run_id=flow_run_id, state=state)
return flow_run_id