async def create_flow_run(
flow_id: str = None,
parameters: dict = None,
context: dict = None,
scheduled_start_time: datetime.datetime = None,
flow_run_name: str = None,
version_group_id: str = None,
labels: List[str] = None,
run_config: dict = None,
idempotency_key: str = None,
) -> Any:
"""
Creates a new flow run for an existing flow.
Args:
- flow_id (str): A string representing the current flow id
- parameters (dict, optional): A dictionary of parameters that were specified for the flow
- context (dict, optional): A dictionary of context values
- scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`,
defaults to right now. Must be UTC.
- flow_run_name (str, optional): An optional string representing this flow run
- version_group_id (str, optional): An optional version group ID; if provided, will run the most
recent unarchived version of the group
- labels (List[str], optional): a list of labels to apply to this individual flow run
- run-config (dict, optional): A run-config override for this flow run.
- idempotency_key (str, optional): An optional idempotency key to prevent duplicate run creation.
idempotency keys are unique to each flow, but can be repeated across different flows.
"""
if flow_id is None and version_group_id is None:
raise ValueError("One of flow_id or version_group_id must be provided.")
scheduled_start_time = scheduled_start_time or pendulum.now()
if flow_id:
where_clause = {"id": {"_eq": flow_id}}
elif version_group_id:
where_clause = {
"version_group_id": {"_eq": version_group_id},
"archived": {"_eq": False},
}
flow = await models.Flow.where(where=where_clause).first(
{
"id": True,
"archived": True,
"tenant_id": True,
"environment": True,
"run_config": True,
"parameters": True,
"flow_group_id": True,
"flow_group": {
"default_parameters": True,
"labels": True,
"run_config": True,
},
},
order_by={"version": EnumValue("desc")},
) # type: Any
if not flow:
msg = (
f"Flow {flow_id} not found"
if flow_id
else f"Version group {version_group_id} has no unarchived flows."
)
raise exceptions.NotFound(msg)
elif flow.archived:
raise ValueError(f"Flow {flow.id} is archived.")
# Determine active labels based on the following hierarchy:
# - Provided labels
# - Provided run config
# - Flow group labels
# - Flow group run config
# - Flow run config
# - Flow environment
if labels is not None:
run_labels = labels
elif run_config is not None:
run_labels = run_config.get("labels") or []
elif flow.flow_group.labels is not None:
run_labels = flow.flow_group.labels
elif flow.flow_group.run_config is not None:
run_labels = flow.flow_group.run_config.get("labels") or []
elif flow.run_config is not None:
run_labels = flow.run_config.get("labels") or []
elif flow.environment is not None:
run_labels = flow.environment.get("labels") or []
else:
run_labels = []
run_labels.sort()
# determine active run_config
if run_config is None:
if flow.flow_group.run_config is not None:
run_config = flow.flow_group.run_config
else:
run_config = flow.run_config
# check parameters
run_parameters = flow.flow_group.default_parameters
run_parameters.update((parameters or {}))
required_parameters = [p["name"] for p in flow.parameters if p["required"]]
missing = set(required_parameters).difference(run_parameters)
if missing:
raise ValueError(f"Required parameters were not supplied: {missing}")
run = models.FlowRun(
id=str(uuid.uuid4()),
tenant_id=flow.tenant_id,
flow_id=flow_id or flow.id,
labels=run_labels,
parameters=run_parameters,
run_config=run_config,
context=context or {},
scheduled_start_time=scheduled_start_time,
name=flow_run_name or names.generate_slug(2),
idempotency_key=idempotency_key,
)
# upsert the flow run against the idempotency key, returning the id of the
# new or existing run
insert_result = await run.insert(
on_conflict=dict(
constraint="ix_flow_run_idempotency_key_unique",
update_columns=["idempotency_key"],
),
selection_set={"returning": {"id"}},
)
# if the run id matches the returned id, the insert succeeded and the
# initial state should be set otherwise, the idempotency key was matched and
# we take no action
if run.id == insert_result.returning.id:
# apply the flow run's initial state via `set_flow_run_state`
await api.states.set_flow_run_state(
flow_run_id=run.id,
state=Scheduled(
message="Flow run scheduled.", start_time=scheduled_start_time
),
)
logger.debug(
f"Flow run {insert_result.returning.id} of flow {flow_id or flow.id} "
f"scheduled for {scheduled_start_time}"
)
# return the database id
return insert_result.returning.id
async def _create_flow_run(
flow_id: str = None,
parameters: dict = None,
context: dict = None,
scheduled_start_time: datetime.datetime = None,
flow_run_name: str = None,
version_group_id: str = None,
labels: List[str] = None,
run_config: dict = None,
) -> Any:
"""
Creates a new flow run for an existing flow.
Args:
- flow_id (str): A string representing the current flow id
- parameters (dict, optional): A dictionary of parameters that were specified for the flow
- context (dict, optional): A dictionary of context values
- scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`,
defaults to right now. Must be UTC.
- flow_run_name (str, optional): An optional string representing this flow run
- version_group_id (str, optional): An optional version group ID; if provided, will run the most
recent unarchived version of the group
- labels (List[str], optional): a list of labels to apply to this individual flow run
- run-config (dict, optional): A run-config override for this flow run.
"""
if flow_id is None and version_group_id is None:
raise ValueError(
"One of flow_id or version_group_id must be provided.")
scheduled_start_time = scheduled_start_time or pendulum.now()
if flow_id:
where_clause = {"id": {"_eq": flow_id}}
elif version_group_id:
where_clause = {
"version_group_id": {
"_eq": version_group_id
},
"archived": {
"_eq": False
},
}
flow = await models.Flow.where(where=where_clause).first(
{
"id": True,
"archived": True,
"tenant_id": True,
"environment": True,
"run_config": True,
"parameters": True,
"flow_group_id": True,
"flow_group": {
"default_parameters": True,
"labels": True,
"run_config": True,
},
},
order_by={"version": EnumValue("desc")},
) # type: Any
if not flow:
msg = (f"Flow {flow_id} not found" if flow_id else
f"Version group {version_group_id} has no unarchived flows.")
raise exceptions.NotFound(msg)
elif flow.archived:
raise ValueError(f"Flow {flow.id} is archived.")
# determine active labels
if labels is not None:
run_labels = labels
elif run_config is not None:
run_labels = run_config.get("labels") or []
elif flow.flow_group.labels is not None:
run_labels = flow.flow_group.labels
elif flow.flow_group.run_config is not None:
run_labels = flow.flow_group.run_config.get("labels") or []
elif flow.run_config is not None:
run_labels = flow.run_config.get("labels") or []
elif flow.environment is not None:
run_labels = flow.environment.get("labels") or []
else:
run_labels = []
run_labels.sort()
# determine active run_config
if run_config is None:
if flow.flow_group.run_config is not None:
run_config = flow.flow_group.run_config
else:
run_config = flow.run_config
# check parameters
run_parameters = flow.flow_group.default_parameters
run_parameters.update((parameters or {}))
required_parameters = [p["name"] for p in flow.parameters if p["required"]]
missing = set(required_parameters).difference(run_parameters)
if missing:
raise ValueError(f"Required parameters were not supplied: {missing}")
state = Scheduled(message="Flow run scheduled.",
start_time=scheduled_start_time)
run = models.FlowRun(
tenant_id=flow.tenant_id,
flow_id=flow_id or flow.id,
labels=run_labels,
parameters=run_parameters,
run_config=run_config,
context=context or {},
scheduled_start_time=scheduled_start_time,
name=flow_run_name or names.generate_slug(2),
states=[
models.FlowRunState(
tenant_id=flow.tenant_id,
**models.FlowRunState.fields_from_state(
Pending(message="Flow run created")),
)
],
)
flow_run_id = await run.insert()
# apply the flow run's initial state via `set_flow_run_state`
await api.states.set_flow_run_state(flow_run_id=flow_run_id, state=state)
return flow_run_id
async def create_flow_run(
flow_id: str = None,
parameters: dict = None,
context: dict = None,
scheduled_start_time: datetime.datetime = None,
flow_run_name: str = None,
version_group_id: str = None,
) -> Any:
"""
Creates a new flow run for an existing flow.
Args:
- flow_id (str): A string representing the current flow id
- parameters (dict, optional): A dictionary of parameters that were specified for the flow
- context (dict, optional): A dictionary of context values
- scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`,
defaults to right now. Must be UTC.
- flow_run_name (str, optional): An optional string representing this flow run
- version_group_id (str, optional): An optional version group ID; if provided, will run the most
recent unarchived version of the group
"""
scheduled_start_time = scheduled_start_time or pendulum.now()
if flow_id:
where_clause = {"id": {"_eq": flow_id}}
elif version_group_id:
where_clause = {
"version_group_id": {
"_eq": version_group_id
},
"archived": {
"_eq": False
},
}
else:
raise ValueError(
"One of flow_id or version_group_id must be provided.")
flow = await models.Flow.where(where=where_clause).first(
{
"id": True,
"archived": True,
"parameters": True
},
order_by={"version": EnumValue("desc")},
) # type: Any
if not flow:
msg = (f"Flow {flow_id} not found" if flow_id else
f"Version group {version_group_id} has no unarchived flows.")
raise exceptions.NotFound(msg)
elif flow.archived:
raise ValueError(f"Flow {flow.id} is archived.")
# check parameters
parameters = parameters or {}
required_parameters = [p["name"] for p in flow.parameters if p["required"]]
missing = set(required_parameters).difference(parameters)
if missing:
raise ValueError(f"Required parameters were not supplied: {missing}")
extra = set(parameters).difference([p["name"] for p in flow.parameters])
if extra:
raise ValueError(f"Extra parameters were supplied: {extra}")
state = Scheduled(message="Flow run scheduled.",
start_time=scheduled_start_time)
run = models.FlowRun(
flow_id=flow_id or flow.id,
parameters=parameters,
context=context or {},
scheduled_start_time=scheduled_start_time,
name=flow_run_name or names.generate_slug(2),
states=[
models.FlowRunState(**models.FlowRunState.fields_from_state(
Pending(message="Flow run created")))
],
)
flow_run_id = await run.insert()
# apply the flow run's initial state via `set_flow_run_state`
await api.states.set_flow_run_state(flow_run_id=flow_run_id, state=state)
return flow_run_id
