def wait_for_flow_run(flow_run_id: str, stream_states: bool = True, stream_logs: bool = False) -> "FlowRunView": """ Task to wait for a flow run to finish executing, streaming state and log information Args: - flow_run_id: The flow run id to wait for - stream_states: Stream information about the flow run state changes - stream_logs: Stream flow run logs; if `stream_state` is `False` this will be ignored Returns: FlowRunView: A view of the flow run after completion """ flow_run = FlowRunView.from_flow_run_id(flow_run_id) for log in watch_flow_run(flow_run_id, stream_states=stream_states, stream_logs=stream_logs): message = f"Flow {flow_run.name!r}: {log.message}" prefect.context.logger.log(log.level, message) # Return the final view of the flow run return flow_run.get_latest()
def test_watch_flow_run_already_finished(patch_post): data = FLOW_RUN_DATA_1.copy() # Change the updated timestamp for the "ago" message data["updated"] = pendulum.now().subtract(minutes=5).isoformat() patch_post({"data": {"flow_run": [data]}}) logs = [log for log in watch_flow_run("id")] assert len(logs) == 1 log = logs[0] assert log.message == "Your flow run finished 5 minutes ago" assert log.level == logging.INFO
def wait_for_flow_run( flow_run_id: str, stream_states: bool = True, stream_logs: bool = False, raise_final_state: bool = False, max_duration: timedelta = timedelta(hours=12), ) -> "FlowRunView": """ Task to wait for a flow run to finish executing, streaming state and log information Args: - flow_run_id: The flow run id to wait for - stream_states: Stream information about the flow run state changes - stream_logs: Stream flow run logs; if `stream_state` is `False` this will be ignored - raise_final_state: If set, the state of this task will be set to the final state of the child flow run on completion. - max_duration: Duration to wait for flow run to complete. Defaults to 12 hours. Returns: FlowRunView: A view of the flow run after completion """ flow_run = FlowRunView.from_flow_run_id(flow_run_id) for log in watch_flow_run( flow_run_id, stream_states=stream_states, stream_logs=stream_logs, max_duration=max_duration, ): message = f"Flow {flow_run.name!r}: {log.message}" prefect.context.logger.log(log.level, message) # Get the final view of the flow run flow_run = flow_run.get_latest() if raise_final_state: state_signal = signal_from_state(flow_run.state)( message=f"{flow_run_id} finished in state {flow_run.state}", result=flow_run, ) raise state_signal else: return flow_run
def test_watch_flow_run_timeout(monkeypatch): flow_run = FlowRunView._from_flow_run_data(FLOW_RUN_DATA_1) flow_run.state = Running() # Not finished flow_run.get_latest = MagicMock(return_value=flow_run) flow_run.get_logs = MagicMock() MockView = MagicMock() MockView.from_flow_run_id.return_value = flow_run monkeypatch.setattr("prefect.backend.flow_run.FlowRunView", MockView) # Mock sleep so that we do not have a slow test monkeypatch.setattr("prefect.backend.flow_run.time.sleep", MagicMock()) with pytest.raises(RuntimeError, match="timed out after 12 hours of waiting"): for log in watch_flow_run("id"): pass
def test_watch_flow_run_default_timeout(monkeypatch): # Test the default behavior, which sets the timeout to 12 hours # when the `max_duration` kwarg is not provided flow_run = FlowRunView._from_flow_run_data(FLOW_RUN_DATA_1) flow_run.state = Running() # Not finished flow_run.get_latest = MagicMock(return_value=flow_run) flow_run.get_logs = MagicMock() MockView = MagicMock() MockView.from_flow_run_id.return_value = flow_run monkeypatch.setattr("prefect.backend.flow_run.FlowRunView", MockView) # Mock sleep so that we do not have a slow test monkeypatch.setattr("prefect.backend.flow_run.time.sleep", MagicMock()) with pytest.raises(RuntimeError, match="timed out after 12.0 hours of waiting"): for log in watch_flow_run("id"): pass
def run( ctx, flow_or_group_id, project, path, module, name, labels, context_vars, params, execute, idempotency_key, schedule, log_level, param_file, run_name, quiet, no_logs, watch, ): """Run a flow""" # Since the old command was a subcommand of this, we have to do some # mucking to smoothly deprecate it. Can be removed with `prefect run flow` # is removed. if ctx.invoked_subcommand is not None: if any([params, no_logs, quiet, flow_or_group_id]): # These options are not supported by `prefect run flow` raise ClickException("Got unexpected extra argument (%s)" % ctx.invoked_subcommand) return # Define a simple function so we don't have to have a lot of `if not quiet` logic quiet_echo = ((lambda *_, **__: None) if quiet else lambda *args, **kwargs: click.secho(*args, **kwargs)) # Cast labels to a list instead of a tuple so we can extend it labels = list(labels) # Ensure that the user has not passed conflicting options given_lookup_options = { key for key, option in { "--id": flow_or_group_id, "--project": project, "--path": path, "--module": module, }.items() if option is not None } # Since `name` can be passed in conjunction with several options and also alone # it requires a special case here if not given_lookup_options and not name: raise ClickException("Received no options to look up the flow." + FLOW_LOOKUP_MSG) if "--id" in given_lookup_options and name: raise ClickException("Received too many options to look up the flow; " "cannot specifiy both `--name` and `--id`" + FLOW_LOOKUP_MSG) if len(given_lookup_options) > 1: raise ClickException("Received too many options to look up the flow: " f"{', '.join(given_lookup_options)}" + FLOW_LOOKUP_MSG) # Load parameters and context ------------------------------------------------------ context_dict = load_json_key_values(context_vars, "context") file_params = {} if param_file: try: with open(param_file) as fp: file_params = json.load(fp) except FileNotFoundError: raise TerminalError( f"Parameter file does not exist: {os.path.abspath(param_file)!r}" ) except ValueError as exc: raise TerminalError( f"Failed to parse JSON at {os.path.abspath(param_file)!r}: {exc}" ) cli_params = load_json_key_values(params, "parameter") conflicting_keys = set(cli_params.keys()).intersection(file_params.keys()) if conflicting_keys: quiet_echo( "The following parameters were specified by file and CLI, the CLI value " f"will be used: {conflicting_keys}") params_dict = {**file_params, **cli_params} # Local flow run ------------------------------------------------------------------- if path or module: # We can load a flow for local execution immediately if given a path or module, # otherwise, we'll lookup the flow then pull from storage for a local run with try_error_done("Retrieving local flow...", quiet_echo, traceback=True): flow = get_flow_from_path_or_module(path=path, module=module, name=name) # Set the desired log level if no_logs: log_level = 100 # CRITICAL is 50 so this should do it run_info = "" if params_dict: run_info += f"└── Parameters: {params_dict}\n" if context_dict: run_info += f"└── Context: {context_dict}\n" if run_info: quiet_echo("Configured local flow run") quiet_echo(run_info, nl=False) quiet_echo("Running flow locally...") with temporary_logger_config( level=log_level, stream_fmt="└── %(asctime)s | %(levelname)-7s | %(message)s", stream_datefmt="%H:%M:%S", ): with prefect.context(**context_dict): try: result_state = flow.run(parameters=params_dict, run_on_schedule=schedule) except Exception as exc: quiet_echo("Flow runner encountered an exception!") log_exception(exc, indent=2) raise TerminalError("Flow run failed!") if result_state.is_failed(): quiet_echo("Flow run failed!", fg="red") sys.exit(1) else: quiet_echo("Flow run succeeded!", fg="green") return # Backend flow run ----------------------------------------------------------------- if schedule: raise ClickException( "`--schedule` can only be specified for local flow runs") client = Client() # Validate the flow look up options we've been given and get the flow from the # backend with try_error_done("Looking up flow metadata...", quiet_echo): flow_view = get_flow_view( flow_or_group_id=flow_or_group_id, project=project, name=name, ) if log_level: run_config = flow_view.run_config if not run_config.env: run_config.env = {} run_config.env["PREFECT__LOGGING__LEVEL"] = log_level else: run_config = None if execute: # Add a random label to prevent an agent from picking up this run labels.append(f"agentless-run-{str(uuid.uuid4())[:8]}") try: # Handle keyboard interrupts during creation flow_run_id = None # Create a flow run in the backend with try_error_done( f"Creating run for flow {flow_view.name!r}...", quiet_echo, traceback=True, # Display 'Done' manually after querying for data to display so there is not # a lag skip_done=True, ): flow_run_id = client.create_flow_run( flow_id=flow_view.flow_id, parameters=params_dict, context=context_dict, # If labels is an empty list pass `None` to get defaults # https://github.com/PrefectHQ/server/blob/77c301ce0c8deda4f8771f7e9991b25e7911224a/src/prefect_server/api/runs.py#L136 labels=labels or None, run_name=run_name, # We only use the run config for setting logging levels right now run_config=run_config, idempotency_key=idempotency_key, ) if quiet: # Just display the flow run id in quiet mode click.echo(flow_run_id) flow_run = None else: # Grab information about the flow run (if quiet we can skip this query) flow_run = FlowRunView.from_flow_run_id(flow_run_id) run_url = client.get_cloud_url("flow-run", flow_run_id) # Display "Done" for creating flow run after pulling the info so there # isn't a weird lag quiet_echo(" Done", fg="green") quiet_echo( textwrap.dedent(f""" └── Name: {flow_run.name} └── UUID: {flow_run.flow_run_id} └── Labels: {flow_run.labels} └── Parameters: {flow_run.parameters} └── Context: {flow_run.context} └── URL: {run_url} """).strip()) except KeyboardInterrupt: # If the user interrupts here, they will expect the flow run to be cancelled quiet_echo("\nKeyboard interrupt detected! Aborting...", fg="yellow") if flow_run_id: client.cancel_flow_run(flow_run_id=flow_run_id) quiet_echo("Cancelled flow run.") else: # The flow run was not created so we can just exit quiet_echo("Aborted.") return # Handle agentless execution if execute: quiet_echo("Executing flow run...") try: with temporary_logger_config( level=(100 if no_logs or quiet else log_level), # Disable logging if asked stream_fmt= "└── %(asctime)s | %(levelname)-7s | %(message)s", stream_datefmt="%H:%M:%S", ): execute_flow_run_in_subprocess(flow_run_id) except KeyboardInterrupt: quiet_echo("Keyboard interrupt detected! Aborting...", fg="yellow") pass elif watch: try: quiet_echo("Watching flow run execution...") for log in watch_flow_run( flow_run_id=flow_run_id, stream_logs=not no_logs, ): level_name = logging.getLevelName(log.level) timestamp = log.timestamp.in_tz(tz="local") echo_with_log_color( log.level, f"└── {timestamp:%H:%M:%S} | {level_name:<7} | {log.message}", ) except KeyboardInterrupt: quiet_echo("Keyboard interrupt detected!", fg="yellow") try: cancel = click.confirm( "On exit, we can leave your flow run executing or cancel it.\n" "Do you want to cancel this flow run?", default=True, ) except click.Abort: # A second keyboard interrupt will exit without cancellation pass else: if cancel: client.cancel_flow_run(flow_run_id=flow_run_id) quiet_echo("Cancelled flow run.", fg="green") return quiet_echo("Exiting without cancelling flow run!", fg="yellow") raise # Re-raise the interrupt else: # If not watching or executing, exit without checking state return # Get the final flow run state flow_run = FlowRunView.from_flow_run_id(flow_run_id) # Wait for the flow run to be done up to 3 seconds elapsed_time = 0 while not flow_run.state.is_finished() and elapsed_time < 3: time.sleep(1) elapsed_time += 1 flow_run = flow_run.get_latest() # Display the final state if flow_run.state.is_failed(): quiet_echo("Flow run failed!", fg="red") sys.exit(1) elif flow_run.state.is_successful(): quiet_echo("Flow run succeeded!", fg="green") else: quiet_echo(f"Flow run is in unexpected state: {flow_run.state}", fg="yellow") sys.exit(1)
def test_watch_flow_run(monkeypatch): flow_run = FlowRunView._from_flow_run_data(FLOW_RUN_DATA_1) flow_run.state = Scheduled() # Not running flow_run.states = [] flow_run.get_latest = MagicMock(return_value=flow_run) flow_run.get_logs = MagicMock() MockView = MagicMock() MockView.from_flow_run_id.return_value = flow_run monkeypatch.setattr("prefect.backend.flow_run.FlowRunView", MockView) monkeypatch.setattr( "prefect.backend.flow_run.check_for_compatible_agents", MagicMock(return_value="Helpful agent message."), ) # Mock sleep so that we do not have a slow test monkeypatch.setattr("prefect.backend.flow_run.time.sleep", MagicMock()) for i, log in enumerate(watch_flow_run("id")): # Assert that we get the agent warning a couple times then update the state if i == 0: assert log.message == ( "It has been 15 seconds and your flow run has not been submitted by an agent. " "Helpful agent message.") assert log.level == logging.WARNING elif i == 1: assert log.message == ( "It has been 50 seconds and your flow run has not been submitted by an agent. " "Helpful agent message.") # Mark the flow run as finished and give it a few past states to log # If this test times out, we did not reach this log flow_run.state = Success() scheduled = Scheduled("My message") scheduled.timestamp = pendulum.now() running = Running("Another message") running.timestamp = pendulum.now().add(seconds=10) # Given intentionally out of order states to prove sorting flow_run.states = [running, scheduled] # Add a log between the states and a log at the end flow_run.get_logs = MagicMock(return_value=[ FlowRunLog( timestamp=pendulum.now().add(seconds=5), message="Foo", level=logging.DEBUG, ), FlowRunLog( timestamp=pendulum.now().add(seconds=15), message="Bar", level=logging.ERROR, ), ]) elif i == 2: assert log.message == "Entered state <Scheduled>: My message" assert log.level == logging.INFO elif i == 3: assert log.message == "Foo" assert log.level == logging.DEBUG elif i == 4: assert log.message == "Entered state <Running>: Another message" assert log.level == logging.INFO elif i == 5: assert log.message == "Bar" assert log.level == logging.ERROR assert i == 5 # Assert we saw all of the expected logs