def wait_for_flow_run(
flow_run_id: str,
stream_states: bool = True,
stream_logs: bool = False,
raise_final_state: bool = False,
max_duration: timedelta = timedelta(hours=12),
) -> "FlowRunView":
"""
Task to wait for a flow run to finish executing, streaming state and log information
Args:
- flow_run_id: The flow run id to wait for
- stream_states: Stream information about the flow run state changes
- stream_logs: Stream flow run logs; if `stream_state` is `False` this will be
ignored
- raise_final_state: If set, the state of this task will be set to the final
state of the child flow run on completion.
- max_duration: Duration to wait for flow run to complete. Defaults to 12 hours.
Returns:
FlowRunView: A view of the flow run after completion
"""
flow_run = FlowRunView.from_flow_run_id(flow_run_id)
for log in watch_flow_run(
flow_run_id,
stream_states=stream_states,
stream_logs=stream_logs,
max_duration=max_duration,
):
message = f"Flow {flow_run.name!r}: {log.message}"
prefect.context.logger.log(log.level, message)
# Get the final view of the flow run
flow_run = flow_run.get_latest()
if raise_final_state:
state_signal = signal_from_state(flow_run.state)(
message=f"{flow_run_id} finished in state {flow_run.state}",
result=flow_run,
)
raise state_signal
else:
return flow_run
def run(
self,
flow_name: str = None,
project_name: str = None,
parameters: dict = None,
idempotency_key: str = None,
new_flow_context: dict = None,
run_name: str = None,
scheduled_start_time: datetime.datetime = None,
) -> str:
"""
Run method for the task; responsible for scheduling the specified flow run.
Args:
- flow_name (str, optional): the name of the flow to schedule; if not provided,
this method will use the flow name provided at initialization
- project_name (str, optional): the Cloud project in which the flow is located; if
not provided, this method will use the project provided at initialization. If
running with Prefect Core's server as the backend, this should not be provided.
- parameters (dict, optional): the parameters to pass to the flow run being
scheduled; if not provided, this method will use the parameters provided at
initialization
- idempotency_key (str, optional): an optional idempotency key for scheduling the
flow run; if provided, ensures that only one run is created if this task is retried
or rerun with the same inputs. If not provided, the current flow run ID will be used.
- new_flow_context (dict, optional): the optional run context for the new flow run
- run_name (str, optional): name to be set for the flow run
- scheduled_start_time (datetime, optional): the time to schedule the execution
for; if not provided, defaults to now
Returns:
- str: the ID of the newly-scheduled flow run
Raises:
- ValueError: if flow was not provided, cannot be found, or if a project name was
not provided while using Cloud as a backend
Example:
```python
from prefect.tasks.prefect.flow_run import StartFlowRun
kickoff_task = StartFlowRun(project_name="Hello, World!", flow_name="My Cloud Flow")
```
"""
# verify that flow and project names were passed where necessary
if flow_name is None:
raise ValueError("Must provide a flow name.")
if project_name is None:
raise ValueError("Must provide a project name.")
where_clause = {
"name": {
"_eq": flow_name
},
"archived": {
"_eq": False
},
"project": {
"name": {
"_eq": project_name
}
},
}
# find the flow ID to schedule
query = {
"query": {
with_args(
"flow",
{
"where": where_clause,
"order_by": {
"version": EnumValue("desc")
},
"limit": 1,
},
): {"id"}
}
}
client = Client()
flow = client.graphql(query).data.flow
# verify that a flow has been returned
if not flow:
raise ValueError("Flow '{}' not found.".format(flow_name))
# grab the ID for the most recent version
flow_id = flow[0].id
idem_key = None
if context.get("flow_run_id"):
map_index = context.get("map_index")
default = context.get("flow_run_id") + (f"-{map_index}"
if map_index else "")
idem_key = idempotency_key or default
# providing an idempotency key ensures that retries for this task
# will not create additional flow runs
flow_run_id = client.create_flow_run(
flow_id=flow_id,
parameters=parameters,
idempotency_key=idem_key or idempotency_key,
context=new_flow_context,
run_name=run_name,
scheduled_start_time=scheduled_start_time,
)
self.logger.debug(f"Flow Run {flow_run_id} created.")
if not self.wait:
return flow_run_id
while True:
time.sleep(10)
flow_run_state = client.get_flow_run_info(flow_run_id).state
if flow_run_state.is_finished():
exc = signal_from_state(flow_run_state)(
f"{flow_run_id} finished in state {flow_run_state}")
raise exc
def run(
self,
flow_name: str = None,
project_name: str = None,
parameters: dict = None,
run_config: RunConfig = None,
new_flow_context: dict = None,
run_name: str = None,
idempotency_key: str = None,
scheduled_start_time: datetime.datetime = None,
) -> str:
"""
Run method for the task; responsible for scheduling the specified flow run.
Args:
- flow_name (str, optional): the name of the flow to schedule; if not provided,
this method will use the flow name provided at initialization
- project_name (str, optional): the Cloud project in which the flow is located; if
not provided, this method will use the project provided at initialization. If
running with Prefect Core's server as the backend, this should not be provided.
- parameters (dict, optional): the parameters to pass to the flow run being
scheduled; if not provided, this method will use the parameters provided at
initialization
- run_config (RunConfig, optional): a run-config to use for this flow
run, overriding any existing flow settings.
- new_flow_context (dict, optional): the optional run context for the new flow run
- run_name (str, optional): name to be set for the flow run
- idempotency_key (str, optional): a unique idempotency key for scheduling the
flow run. Duplicate flow runs with the same idempotency key will only create
a single flow run. This is useful for ensuring that only one run is created
if this task is retried. If not provided, defaults to the active `task_run_id`.
- scheduled_start_time (datetime, optional): the time to schedule the execution
for; if not provided, defaults to now
Returns:
- str: the ID of the newly-scheduled flow run
Raises:
- ValueError: if flow was not provided, cannot be found, or if a project name was
not provided while using Cloud as a backend
Example:
```python
from prefect.tasks.prefect.flow_run import StartFlowRun
kickoff_task = StartFlowRun(project_name="Hello, World!", flow_name="My Cloud Flow")
```
"""
# verify that flow and project names were passed where necessary
if flow_name is None:
raise ValueError("Must provide a flow name.")
if project_name is None:
raise ValueError("Must provide a project name.")
where_clause = {
"name": {
"_eq": flow_name
},
"archived": {
"_eq": False
},
"project": {
"name": {
"_eq": project_name
}
},
}
# find the flow ID to schedule
query = {
"query": {
with_args(
"flow",
{
"where": where_clause,
"order_by": {
"version": EnumValue("desc")
},
"limit": 1,
},
): {"id"}
}
}
client = Client()
flow = client.graphql(query).data.flow
# verify that a flow has been returned
if not flow:
raise ValueError("Flow '{}' not found.".format(flow_name))
# grab the ID for the most recent version
flow_id = flow[0].id
if idempotency_key is None:
idempotency_key = prefect.context.get("task_run_id", None)
# providing an idempotency key ensures that retries for this task
# will not create additional flow runs
flow_run_id = client.create_flow_run(
flow_id=flow_id,
parameters=parameters,
run_config=run_config,
idempotency_key=idempotency_key,
context=new_flow_context,
run_name=run_name,
scheduled_start_time=scheduled_start_time,
)
self.logger.debug(f"Flow Run {flow_run_id} created.")
self.logger.debug(
f"Creating link artifact for Flow Run {flow_run_id}.")
run_link = client.get_cloud_url("flow-run", flow_run_id, as_user=False)
create_link(urlparse(run_link).path)
self.logger.info(f"Flow Run: {run_link}")
if not self.wait:
return flow_run_id
while True:
time.sleep(self.poll_interval.total_seconds())
flow_run_state = client.get_flow_run_info(flow_run_id).state
if flow_run_state.is_finished():
exc = signal_from_state(flow_run_state)(
f"{flow_run_id} finished in state {flow_run_state}")
raise exc
def test_signal_from_state_returns_correct_signal(signal, state):
assert signal_from_state(state("Dummy message")) == signal
def run(
self, flow_name: str = None, project_name: str = None, parameters: dict = None
) -> str:
"""
Run method for the task; responsible for scheduling the specified flow run.
Args:
- flow_name (str, optional): the name of the flow to schedule; if not provided,
this method will use the flow name provided at initialization
- project_name (str, optional): the Cloud project in which the flow is located; if
not provided, this method will use the project provided at initialization. If
running with Prefect Core's server as the backend, this should not be provided.
- parameters (dict, optional): the parameters to pass to the flow run being
scheduled; if not provided, this method will use the parameters provided at
initialization
Returns:
- str: the ID of the newly-scheduled flow run
Raises:
- ValueError: if flow was not provided, cannot be found, or if a project name was
not provided while using Cloud as a backend
Example:
```python
from prefect.tasks.prefect.flow_run import FlowRunTask
kickoff_task = FlowRunTask(project_name="Hello, World!", flow_name="My Cloud Flow")
```
"""
is_hosted_backend = "prefect.io" in urlparse(config.cloud.api).netloc
# verify that flow and project names were passed where necessary
if flow_name is None:
raise ValueError("Must provide a flow name.")
if project_name is None and is_hosted_backend:
raise ValueError("Must provide a project name.")
where_clause = {
"name": {"_eq": flow_name},
"archived": {"_eq": False},
}
if project_name:
where_clause["project"] = {"name": {"_eq": project_name}}
# find the flow ID to schedule
query = {
"query": {
with_args(
"flow",
{
"where": where_clause,
"order_by": {"version": EnumValue("desc")},
"limit": 1,
},
): {"id"}
}
}
client = Client()
flow = client.graphql(query).data.flow
# verify that a flow has been returned
if not flow:
raise ValueError("Flow '{}' not found.".format(flow_name))
# grab the ID for the most recent version
flow_id = flow[0].id
# providing an idempotency key ensures that retries for this task
# will not create additional flow runs
flow_run_id = client.create_flow_run(
flow_id=flow_id,
parameters=parameters,
idempotency_key=context.get("flow_run_id"),
)
self.logger.debug(f"Flow Run {flow_run_id} created.")
if not self.wait:
return flow_run_id
while True:
time.sleep(10)
flow_run_state = client.get_flow_run_info(flow_run_id).state
if flow_run_state.is_finished():
exc = signal_from_state(flow_run_state)(
f"{flow_run_id} finished in state {flow_run_state}"
)
raise exc
self.logger.debug(f"Flow Run {flow_run_id} created.")
<<<<<<< HEAD
self.logger.debug(f"Creating link artifact for Flow Run {flow_run_id}.")
run_link = client.get_cloud_url("flow-run", flow_run_id, as_user=False)
create_link(urlparse(run_link).path)
=======
>>>>>>> prefect clone
if not self.wait:
return flow_run_id
while True:
time.sleep(10)
flow_run_state = client.get_flow_run_info(flow_run_id).state
if flow_run_state.is_finished():
exc = signal_from_state(flow_run_state)(
f"{flow_run_id} finished in state {flow_run_state}"
)
raise exc
class FlowRunTask(StartFlowRun):
def __new__(cls, *args, **kwargs): # type: ignore
warnings.warn(
"`FlowRunTask` has been renamed to `prefect.tasks.prefect.StartFlowRun`,"
"please update your code accordingly",
stacklevel=2,
)
return super().__new__(cls)
