def test_import_object():
rand = import_object("random")
randint = import_object("random.randint")
import random
assert rand is random
assert randint is random.randint
def load_flows_from_module(name: str) -> "List[prefect.Flow]":
"""
Given a module name (or full import path to a flow), load all flows found in the
module
"""
# TODO: This is copied and slightly modified from `prefect.cli.build_register`
# we should probably abstract this in the future
try:
with prefect.context({"loading_flow": True}):
mod_or_obj = import_object(name)
except Exception as exc:
# If the requested module isn't found, log without a traceback
# otherwise log a general message with the traceback.
if isinstance(exc, ModuleNotFoundError) and (
name == exc.name or
(name.startswith(exc.name) and name[len(exc.name)] == ".")):
raise TerminalError(str(exc).capitalize())
elif isinstance(exc, AttributeError):
raise TerminalError(str(exc).capitalize())
else:
raise
if isinstance(mod_or_obj, ModuleType):
flows = [
f for f in vars(mod_or_obj).values()
if isinstance(f, prefect.Flow)
]
elif isinstance(mod_or_obj, prefect.Flow):
flows = [mod_or_obj]
else:
raise TerminalError(
f"Invalid object of type {type(mod_or_obj).__name__!r} found at {name!r}. "
f"Expected Module or Flow.")
return flows
def load_flows_from_module(name: str) -> "List[prefect.Flow]":
"""
Given a module name (or full import path to a flow), load all flows found in the
module
"""
try:
with prefect.context({"loading_flow": True}):
mod_or_obj = import_object(name)
except Exception as exc:
# If the requested module (or any parent module) isn't found, log
# without a traceback, otherwise log a general message with the
# traceback.
if isinstance(exc, ModuleNotFoundError) and (
name == exc.name or
(name.startswith(exc.name) and name[len(exc.name)] == ".")):
raise TerminalError(str(exc))
elif isinstance(exc, AttributeError):
raise TerminalError(str(exc))
else:
click.secho(f"Error loading {name!r}:", fg="red")
log_exception(exc, 2)
raise TerminalError
if isinstance(mod_or_obj, ModuleType):
flows = [
f for f in vars(mod_or_obj).values()
if isinstance(f, prefect.Flow)
]
elif isinstance(mod_or_obj, prefect.Flow):
flows = [mod_or_obj]
# Get a valid module name for f.storage
name, _ = name.rsplit(".", 1)
else:
click.secho(
f"Invalid object of type {type(mod_or_obj).__name__!r} found at {name!r}. "
f"Expected Module or Flow.")
raise TerminalError
if flows:
for f in flows:
if f.storage is None:
f.storage = Module(name)
return flows
def __init__(
self,
address: str = None,
cluster_class: Union[str, Callable] = None,
cluster_kwargs: dict = None,
adapt_kwargs: dict = None,
client_kwargs: dict = None,
debug: bool = None,
**kwargs: Any,
):
if address is None:
address = context.config.engine.executor.dask.address or None
# XXX: deprecated
if address == "local":
warnings.warn(
"`address='local'` is deprecated. To use a local cluster, leave the "
"`address` field empty.")
address = None
# XXX: deprecated
local_processes = kwargs.pop("local_processes", None)
if local_processes is None:
local_processes = context.config.engine.executor.dask.get(
"local_processes", None)
if local_processes is not None:
warnings.warn(
"`local_processes` is deprecated, please use "
"`cluster_kwargs={'processes': local_processes}`. The default is "
"now `local_processes=True`.")
if address is not None:
if cluster_class is not None or cluster_kwargs is not None:
raise ValueError(
"Cannot specify `address` and `cluster_class`/`cluster_kwargs`"
)
else:
if cluster_class is None:
cluster_class = context.config.engine.executor.dask.cluster_class
if isinstance(cluster_class, str):
cluster_class = import_object(cluster_class)
if cluster_kwargs is None:
cluster_kwargs = {}
else:
cluster_kwargs = cluster_kwargs.copy()
from distributed.deploy.local import LocalCluster
if cluster_class == LocalCluster:
if debug is None:
debug = context.config.debug
cluster_kwargs.setdefault(
"silence_logs",
logging.CRITICAL if not debug else logging.WARNING)
if local_processes is not None:
cluster_kwargs.setdefault("processes", local_processes)
for_cluster = set(kwargs).difference(_valid_client_kwargs)
if for_cluster:
warnings.warn(
"Forwarding executor kwargs to `LocalCluster` is now handled by the "
"`cluster_kwargs` parameter, please update accordingly"
)
for k in for_cluster:
cluster_kwargs[k] = kwargs.pop(k)
if adapt_kwargs is None:
adapt_kwargs = {}
if client_kwargs is None:
client_kwargs = {}
else:
client_kwargs = client_kwargs.copy()
if kwargs:
warnings.warn(
"Forwarding executor kwargs to `Client` is now handled by the "
"`client_kwargs` parameter, please update accordingly")
client_kwargs.update(kwargs)
client_kwargs.setdefault("set_as_default", False)
self.address = address
self.cluster_class = cluster_class
self.cluster_kwargs = cluster_kwargs
self.adapt_kwargs = adapt_kwargs
self.client_kwargs = client_kwargs
# Runtime attributes
self.client = None
# These are coupled - they're either both None, or both non-None.
# They're used in the case we can't forcibly kill all the dask workers,
# and need to wait for all the dask tasks to cleanup before exiting.
self._futures = None # type: Optional[weakref.WeakSet[Future]]
self._should_run_var = None # type: Optional[Variable]
super().__init__()
def test_import_object_attribute_does_not_exist():
with pytest.raises(AttributeError):
import_object("random.random_attribute_that_does_not_exist")
def test_import_object_module_does_not_exist():
with pytest.raises(ImportError):
import_object("random_module_name_that_does_not_exist")
def test_import_object_submodule_not_an_attribute():
# `hello_world` is not in the `prefect` top-level
hello_world = import_object("prefect.hello_world")
import prefect.hello_world
assert hello_world is prefect.hello_world
def __init__(
self,
address: str = None,
cluster_class: Union[str, Callable] = None,
cluster_kwargs: dict = None,
adapt_kwargs: dict = None,
client_kwargs: dict = None,
debug: bool = None,
**kwargs: Any,
):
if address is None:
address = context.config.engine.executor.dask.address or None
# XXX: deprecated
if address == "local":
warnings.warn(
"`address='local'` is deprecated. To use a local cluster, leave the "
"`address` field empty.")
address = None
# XXX: deprecated
local_processes = kwargs.pop("local_processes", None)
if local_processes is None:
local_processes = context.config.engine.executor.dask.get(
"local_processes", None)
if local_processes is not None:
warnings.warn(
"`local_processes` is deprecated, please use "
"`cluster_kwargs={'processes': local_processes}`. The default is "
"now `local_processes=True`.")
if address is not None:
if cluster_class is not None or cluster_kwargs is not None:
raise ValueError(
"Cannot specify `address` and `cluster_class`/`cluster_kwargs`"
)
else:
if cluster_class is None:
cluster_class = context.config.engine.executor.dask.cluster_class
if isinstance(cluster_class, str):
cluster_class = import_object(cluster_class)
if cluster_kwargs is None:
cluster_kwargs = {}
else:
cluster_kwargs = cluster_kwargs.copy()
from distributed.deploy.local import LocalCluster
if cluster_class == LocalCluster:
if debug is None:
debug = context.config.debug
cluster_kwargs.setdefault(
"silence_logs",
logging.CRITICAL if not debug else logging.WARNING)
if local_processes is not None:
cluster_kwargs.setdefault("processes", local_processes)
for_cluster = set(kwargs).difference(_valid_client_kwargs)
if for_cluster:
warnings.warn(
"Forwarding executor kwargs to `LocalCluster` is now handled by the "
"`cluster_kwargs` parameter, please update accordingly"
)
for k in for_cluster:
cluster_kwargs[k] = kwargs.pop(k)
if adapt_kwargs is None:
adapt_kwargs = {}
if client_kwargs is None:
client_kwargs = {}
if kwargs:
warnings.warn(
"Forwarding executor kwargs to `Client` is now handled by the "
"`client_kwargs` parameter, please update accordingly")
client_kwargs.update(kwargs)
self.address = address
self.cluster_class = cluster_class
self.cluster_kwargs = cluster_kwargs
self.adapt_kwargs = adapt_kwargs
self.client_kwargs = client_kwargs
self.client = None
super().__init__()
def __init__(
self,
address: str = None,
cluster_class: Union[str, Callable] = None,
cluster_kwargs: dict = None,
adapt_kwargs: dict = None,
client_kwargs: dict = None,
debug: bool = None,
performance_report_path: str = None,
disable_cancellation_event: bool = False,
):
if address is None:
address = context.config.engine.executor.dask.address or None
if address is not None:
if cluster_class is not None or cluster_kwargs is not None:
raise ValueError(
"Cannot specify `address` and `cluster_class`/`cluster_kwargs`"
)
else:
if cluster_class is None:
cluster_class = context.config.engine.executor.dask.cluster_class
if isinstance(cluster_class, str):
cluster_class = import_object(cluster_class)
if cluster_kwargs is None:
cluster_kwargs = {}
else:
cluster_kwargs = cluster_kwargs.copy()
from distributed.deploy.local import LocalCluster
if cluster_class == LocalCluster:
if debug is None:
debug = context.config.debug
cluster_kwargs.setdefault(
"silence_logs",
logging.CRITICAL if not debug else logging.WARNING)
if adapt_kwargs is None:
adapt_kwargs = {}
if client_kwargs is None:
client_kwargs = {}
else:
client_kwargs = client_kwargs.copy()
client_kwargs.setdefault("set_as_default", False)
self.address = address
self.cluster_class = cluster_class
self.cluster_kwargs = cluster_kwargs
self.adapt_kwargs = adapt_kwargs
self.client_kwargs = client_kwargs
self.disable_cancellation_event = disable_cancellation_event
# Runtime attributes
self.client = None
# These are coupled - they're either both None, or both non-None.
# They're used in the case we can't forcibly kill all the dask workers,
# and need to wait for all the dask tasks to cleanup before exiting.
self._futures = None # type: Optional[weakref.WeakSet[Future]]
self._should_run_event = None # type: Optional[Event]
# A ref to a background task subscribing to dask cluster events
self._watch_dask_events_task = None # type: Optional[concurrent.futures.Future]
self.performance_report_path = performance_report_path
super().__init__()
