def run_flow(self) -> None:
"""
Run the flow from specified flow_file_path location using the default executor
"""
# Call on_start callback if specified
if self.on_start:
self.on_start()
try:
from prefect.engine import (
get_default_flow_runner_class,
get_default_executor_class,
)
# Load serialized flow from file and run it with the executor
with open(
prefect.context.get("flow_file_path",
"/root/.prefect/flow_env.prefect"),
"rb",
) as f:
flow = cloudpickle.load(f)
runner_cls = get_default_flow_runner_class()
executor_cls = get_default_executor_class()
runner_cls(flow=flow).run(executor=executor_cls)
except Exception as exc:
self.logger.exception(
"Unexpected error raised during flow run: {}".format(exc))
raise exc
finally:
# Call on_exit callback if specified
if self.on_exit:
self.on_exit()
def execute( # type: ignore
self, storage: "Storage", flow_location: str,
**kwargs: Any) -> None:
"""
Run a flow from the `flow_location` here using the specified executor and
executor kwargs.
Args:
- storage (Storage): the storage object that contains information relating
to where and how the flow is stored
- flow_location (str): the location of the Flow to execute
- **kwargs (Any): additional keyword arguments to pass to the runner
"""
try:
from prefect.engine import (
get_default_executor_class,
get_default_flow_runner_class,
)
# Load serialized flow from file and run it with a DaskExecutor
with open(flow_location, "rb") as f:
flow = cloudpickle.load(f)
with set_temporary_config(
{"engine.executor.default_class": self.executor}):
executor = get_default_executor_class()
executor = executor(**self.executor_kwargs)
runner_cls = get_default_flow_runner_class()
runner_cls(flow=flow).run(executor=executor)
except Exception as exc:
self.logger.exception(
"Unexpected error raised during flow run: {}".format(exc))
raise exc
def test_default_executor_responds_to_config():
with utilities.configuration.set_temporary_config({
"engine.executor.default_class":
"prefect.engine.executors.LocalDaskExecutor"
}):
assert engine.get_default_executor_class(
) is engine.executors.LocalDaskExecutor
def execute(self, flow: "Flow", **kwargs: Any) -> None: # type: ignore
"""
Run the provided flow here using the specified executor and executor kwargs.
Args:
- flow (Flow): the Flow object
- **kwargs (Any): additional keyword arguments to pass to the runner
"""
# Call on_start callback if specified
if self.on_start:
self.on_start()
try:
from prefect.engine import (
get_default_executor_class,
get_default_flow_runner_class,
)
# Run flow with default executor class
with set_temporary_config(
{"engine.executor.default_class": self.executor}):
executor = get_default_executor_class()
executor = executor(**self.executor_kwargs)
runner_cls = get_default_flow_runner_class()
runner_cls(flow=flow).run(executor=executor)
except Exception as exc:
self.logger.exception(
"Unexpected error raised during flow run: {}".format(exc))
raise exc
finally:
# Call on_exit callback if specified
if self.on_exit:
self.on_exit()
def run_flow(self) -> None:
"""
Run the flow from specified flow_file_path location using the default executor
"""
# Call on_start callback if specified
if self.on_start:
self.on_start()
try:
from prefect.engine import (
get_default_flow_runner_class,
get_default_executor_class,
)
# Load serialized flow from file and run it with the executor
with open(
prefect.context.get("flow_file_path",
"/root/.prefect/flow_env.prefect"),
"rb",
) as f:
flow = cloudpickle.load(f)
## populate global secrets
secrets = prefect.context.get("secrets", {})
for secret in flow.storage.secrets:
secrets[secret] = prefect.tasks.secrets.PrefectSecret(
name=secret).run()
with prefect.context(secrets=secrets):
runner_cls = get_default_flow_runner_class()
executor_cls = get_default_executor_class()(
**self.executor_kwargs)
runner_cls(flow=flow).run(executor=executor_cls)
except Exception as exc:
self.logger.exception(
"Unexpected error raised during flow run: {}".format(exc))
raise exc
finally:
# Call on_exit callback if specified
if self.on_exit:
self.on_exit()
def run_flow(self) -> None:
"""
Run the flow using the default executor
Raises:
- ValueError: if no `flow_run_id` is found in context
"""
# Call on_start callback if specified
if self.on_start:
self.on_start()
try:
from prefect.engine import (
get_default_flow_runner_class,
get_default_executor_class,
)
flow_run_id = prefect.context.get("flow_run_id")
if not flow_run_id:
raise ValueError("No flow run ID found in context.")
query = {
"query": {
with_args("flow_run", {
"where": {
"id": {
"_eq": flow_run_id
}
}
}): {
"flow": {
"name": True,
"storage": True,
},
}
}
}
client = Client()
result = client.graphql(query)
flow_run = result.data.flow_run[0]
flow_data = flow_run.flow
storage_schema = prefect.serialization.storage.StorageSchema()
storage = storage_schema.load(flow_data.storage)
## populate global secrets
secrets = prefect.context.get("secrets", {})
for secret in storage.secrets:
secrets[secret] = prefect.tasks.secrets.PrefectSecret(
name=secret).run()
with prefect.context(secrets=secrets):
flow = storage.get_flow(storage.flows[flow_data.name])
runner_cls = get_default_flow_runner_class()
if hasattr(self, "executor_kwargs"):
executor_cls = get_default_executor_class()(
**self.executor_kwargs) # type: ignore
else:
executor_cls = get_default_executor_class()
runner_cls(flow=flow).run(executor=executor_cls)
except Exception as exc:
self.logger.exception(
"Unexpected error raised during flow run: {}".format(exc))
raise exc
finally:
# Call on_exit callback if specified
if self.on_exit:
self.on_exit()
def test_default_executor():
assert engine.get_default_executor_class() is engine.executors.LocalExecutor
def test_default_executor_with_bad_config():
with utilities.configuration.set_temporary_config(
{"engine.executor.default_class": "prefect.engine.bad import path"}
):
with pytest.warns(UserWarning):
assert engine.get_default_executor_class() is engine.executors.LocalExecutor
def test_default_executor():
assert engine.get_default_executor_class(
) is engine.executors.SynchronousExecutor
def test_default_executor_responds_to_config_object():
with utilities.configuration.set_temporary_config(
{"engine.executor.default_class": engine.executors.LocalExecutor}):
assert engine.get_default_executor_class(
) is engine.executors.LocalExecutor
