async def _create_flow_run( flow_id: str = None, parameters: dict = None, context: dict = None, scheduled_start_time: datetime.datetime = None, flow_run_name: str = None, version_group_id: str = None, labels: List[str] = None, run_config: dict = None, ) -> Any: """ Creates a new flow run for an existing flow. Args: - flow_id (str): A string representing the current flow id - parameters (dict, optional): A dictionary of parameters that were specified for the flow - context (dict, optional): A dictionary of context values - scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`, defaults to right now. Must be UTC. - flow_run_name (str, optional): An optional string representing this flow run - version_group_id (str, optional): An optional version group ID; if provided, will run the most recent unarchived version of the group - labels (List[str], optional): a list of labels to apply to this individual flow run - run-config (dict, optional): A run-config override for this flow run. """ if flow_id is None and version_group_id is None: raise ValueError( "One of flow_id or version_group_id must be provided.") scheduled_start_time = scheduled_start_time or pendulum.now() if flow_id: where_clause = {"id": {"_eq": flow_id}} elif version_group_id: where_clause = { "version_group_id": { "_eq": version_group_id }, "archived": { "_eq": False }, } flow = await models.Flow.where(where=where_clause).first( { "id": True, "archived": True, "tenant_id": True, "environment": True, "run_config": True, "parameters": True, "flow_group_id": True, "flow_group": { "default_parameters": True, "labels": True, "run_config": True, }, }, order_by={"version": EnumValue("desc")}, ) # type: Any if not flow: msg = (f"Flow {flow_id} not found" if flow_id else f"Version group {version_group_id} has no unarchived flows.") raise exceptions.NotFound(msg) elif flow.archived: raise ValueError(f"Flow {flow.id} is archived.") # determine active labels if labels is not None: run_labels = labels elif run_config is not None: run_labels = run_config.get("labels") or [] elif flow.flow_group.labels is not None: run_labels = flow.flow_group.labels elif flow.flow_group.run_config is not None: run_labels = flow.flow_group.run_config.get("labels") or [] elif flow.run_config is not None: run_labels = flow.run_config.get("labels") or [] elif flow.environment is not None: run_labels = flow.environment.get("labels") or [] else: run_labels = [] run_labels.sort() # determine active run_config if run_config is None: if flow.flow_group.run_config is not None: run_config = flow.flow_group.run_config else: run_config = flow.run_config # check parameters run_parameters = flow.flow_group.default_parameters run_parameters.update((parameters or {})) required_parameters = [p["name"] for p in flow.parameters if p["required"]] missing = set(required_parameters).difference(run_parameters) if missing: raise ValueError(f"Required parameters were not supplied: {missing}") state = Scheduled(message="Flow run scheduled.", start_time=scheduled_start_time) run = models.FlowRun( tenant_id=flow.tenant_id, flow_id=flow_id or flow.id, labels=run_labels, parameters=run_parameters, run_config=run_config, context=context or {}, scheduled_start_time=scheduled_start_time, name=flow_run_name or names.generate_slug(2), states=[ models.FlowRunState( tenant_id=flow.tenant_id, **models.FlowRunState.fields_from_state( Pending(message="Flow run created")), ) ], ) flow_run_id = await run.insert() # apply the flow run's initial state via `set_flow_run_state` await api.states.set_flow_run_state(flow_run_id=flow_run_id, state=state) return flow_run_id
async def create_flow_run( flow_id: str = None, parameters: dict = None, context: dict = None, scheduled_start_time: datetime.datetime = None, flow_run_name: str = None, version_group_id: str = None, labels: List[str] = None, run_config: dict = None, idempotency_key: str = None, ) -> Any: """ Creates a new flow run for an existing flow. Args: - flow_id (str): A string representing the current flow id - parameters (dict, optional): A dictionary of parameters that were specified for the flow - context (dict, optional): A dictionary of context values - scheduled_start_time (datetime.datetime): When the flow_run should be scheduled to run. If `None`, defaults to right now. Must be UTC. - flow_run_name (str, optional): An optional string representing this flow run - version_group_id (str, optional): An optional version group ID; if provided, will run the most recent unarchived version of the group - labels (List[str], optional): a list of labels to apply to this individual flow run - run-config (dict, optional): A run-config override for this flow run. - idempotency_key (str, optional): An optional idempotency key to prevent duplicate run creation. idempotency keys are unique to each flow, but can be repeated across different flows. """ if flow_id is None and version_group_id is None: raise ValueError("One of flow_id or version_group_id must be provided.") scheduled_start_time = scheduled_start_time or pendulum.now() if flow_id: where_clause = {"id": {"_eq": flow_id}} elif version_group_id: where_clause = { "version_group_id": {"_eq": version_group_id}, "archived": {"_eq": False}, } flow = await models.Flow.where(where=where_clause).first( { "id": True, "archived": True, "tenant_id": True, "environment": True, "run_config": True, "parameters": True, "flow_group_id": True, "flow_group": { "default_parameters": True, "labels": True, "run_config": True, }, }, order_by={"version": EnumValue("desc")}, ) # type: Any if not flow: msg = ( f"Flow {flow_id} not found" if flow_id else f"Version group {version_group_id} has no unarchived flows." ) raise exceptions.NotFound(msg) elif flow.archived: raise ValueError(f"Flow {flow.id} is archived.") # Determine active labels based on the following hierarchy: # - Provided labels # - Provided run config # - Flow group labels # - Flow group run config # - Flow run config # - Flow environment if labels is not None: run_labels = labels elif run_config is not None: run_labels = run_config.get("labels") or [] elif flow.flow_group.labels is not None: run_labels = flow.flow_group.labels elif flow.flow_group.run_config is not None: run_labels = flow.flow_group.run_config.get("labels") or [] elif flow.run_config is not None: run_labels = flow.run_config.get("labels") or [] elif flow.environment is not None: run_labels = flow.environment.get("labels") or [] else: run_labels = [] run_labels.sort() # determine active run_config if run_config is None: if flow.flow_group.run_config is not None: run_config = flow.flow_group.run_config else: run_config = flow.run_config # check parameters run_parameters = flow.flow_group.default_parameters run_parameters.update((parameters or {})) required_parameters = [p["name"] for p in flow.parameters if p["required"]] missing = set(required_parameters).difference(run_parameters) if missing: raise ValueError(f"Required parameters were not supplied: {missing}") run = models.FlowRun( id=str(uuid.uuid4()), tenant_id=flow.tenant_id, flow_id=flow_id or flow.id, labels=run_labels, parameters=run_parameters, run_config=run_config, context=context or {}, scheduled_start_time=scheduled_start_time, name=flow_run_name or names.generate_slug(2), idempotency_key=idempotency_key, ) # upsert the flow run against the idempotency key, returning the id of the # new or existing run insert_result = await run.insert( on_conflict=dict( constraint="ix_flow_run_idempotency_key_unique", update_columns=["idempotency_key"], ), selection_set={"returning": {"id"}}, ) # if the run id matches the returned id, the insert succeeded and the # initial state should be set otherwise, the idempotency key was matched and # we take no action if run.id == insert_result.returning.id: # apply the flow run's initial state via `set_flow_run_state` await api.states.set_flow_run_state( flow_run_id=run.id, state=Scheduled( message="Flow run scheduled.", start_time=scheduled_start_time ), ) logger.debug( f"Flow run {insert_result.returning.id} of flow {flow_id or flow.id} " f"scheduled for {scheduled_start_time}" ) # return the database id return insert_result.returning.id