def test_import_object(): rand = import_object("random") randint = import_object("random.randint") import random assert rand is random assert randint is random.randint
def load_flows_from_module(name: str) -> "List[prefect.Flow]": """ Given a module name (or full import path to a flow), load all flows found in the module """ # TODO: This is copied and slightly modified from `prefect.cli.build_register` # we should probably abstract this in the future try: with prefect.context({"loading_flow": True}): mod_or_obj = import_object(name) except Exception as exc: # If the requested module isn't found, log without a traceback # otherwise log a general message with the traceback. if isinstance(exc, ModuleNotFoundError) and ( name == exc.name or (name.startswith(exc.name) and name[len(exc.name)] == ".")): raise TerminalError(str(exc).capitalize()) elif isinstance(exc, AttributeError): raise TerminalError(str(exc).capitalize()) else: raise if isinstance(mod_or_obj, ModuleType): flows = [ f for f in vars(mod_or_obj).values() if isinstance(f, prefect.Flow) ] elif isinstance(mod_or_obj, prefect.Flow): flows = [mod_or_obj] else: raise TerminalError( f"Invalid object of type {type(mod_or_obj).__name__!r} found at {name!r}. " f"Expected Module or Flow.") return flows
def load_flows_from_module(name: str) -> "List[prefect.Flow]": """ Given a module name (or full import path to a flow), load all flows found in the module """ try: with prefect.context({"loading_flow": True}): mod_or_obj = import_object(name) except Exception as exc: # If the requested module (or any parent module) isn't found, log # without a traceback, otherwise log a general message with the # traceback. if isinstance(exc, ModuleNotFoundError) and ( name == exc.name or (name.startswith(exc.name) and name[len(exc.name)] == ".")): raise TerminalError(str(exc)) elif isinstance(exc, AttributeError): raise TerminalError(str(exc)) else: click.secho(f"Error loading {name!r}:", fg="red") log_exception(exc, 2) raise TerminalError if isinstance(mod_or_obj, ModuleType): flows = [ f for f in vars(mod_or_obj).values() if isinstance(f, prefect.Flow) ] elif isinstance(mod_or_obj, prefect.Flow): flows = [mod_or_obj] # Get a valid module name for f.storage name, _ = name.rsplit(".", 1) else: click.secho( f"Invalid object of type {type(mod_or_obj).__name__!r} found at {name!r}. " f"Expected Module or Flow.") raise TerminalError if flows: for f in flows: if f.storage is None: f.storage = Module(name) return flows
def __init__( self, address: str = None, cluster_class: Union[str, Callable] = None, cluster_kwargs: dict = None, adapt_kwargs: dict = None, client_kwargs: dict = None, debug: bool = None, **kwargs: Any, ): if address is None: address = context.config.engine.executor.dask.address or None # XXX: deprecated if address == "local": warnings.warn( "`address='local'` is deprecated. To use a local cluster, leave the " "`address` field empty.") address = None # XXX: deprecated local_processes = kwargs.pop("local_processes", None) if local_processes is None: local_processes = context.config.engine.executor.dask.get( "local_processes", None) if local_processes is not None: warnings.warn( "`local_processes` is deprecated, please use " "`cluster_kwargs={'processes': local_processes}`. The default is " "now `local_processes=True`.") if address is not None: if cluster_class is not None or cluster_kwargs is not None: raise ValueError( "Cannot specify `address` and `cluster_class`/`cluster_kwargs`" ) else: if cluster_class is None: cluster_class = context.config.engine.executor.dask.cluster_class if isinstance(cluster_class, str): cluster_class = import_object(cluster_class) if cluster_kwargs is None: cluster_kwargs = {} else: cluster_kwargs = cluster_kwargs.copy() from distributed.deploy.local import LocalCluster if cluster_class == LocalCluster: if debug is None: debug = context.config.debug cluster_kwargs.setdefault( "silence_logs", logging.CRITICAL if not debug else logging.WARNING) if local_processes is not None: cluster_kwargs.setdefault("processes", local_processes) for_cluster = set(kwargs).difference(_valid_client_kwargs) if for_cluster: warnings.warn( "Forwarding executor kwargs to `LocalCluster` is now handled by the " "`cluster_kwargs` parameter, please update accordingly" ) for k in for_cluster: cluster_kwargs[k] = kwargs.pop(k) if adapt_kwargs is None: adapt_kwargs = {} if client_kwargs is None: client_kwargs = {} else: client_kwargs = client_kwargs.copy() if kwargs: warnings.warn( "Forwarding executor kwargs to `Client` is now handled by the " "`client_kwargs` parameter, please update accordingly") client_kwargs.update(kwargs) client_kwargs.setdefault("set_as_default", False) self.address = address self.cluster_class = cluster_class self.cluster_kwargs = cluster_kwargs self.adapt_kwargs = adapt_kwargs self.client_kwargs = client_kwargs # Runtime attributes self.client = None # These are coupled - they're either both None, or both non-None. # They're used in the case we can't forcibly kill all the dask workers, # and need to wait for all the dask tasks to cleanup before exiting. self._futures = None # type: Optional[weakref.WeakSet[Future]] self._should_run_var = None # type: Optional[Variable] super().__init__()
def test_import_object_attribute_does_not_exist(): with pytest.raises(AttributeError): import_object("random.random_attribute_that_does_not_exist")
def test_import_object_module_does_not_exist(): with pytest.raises(ImportError): import_object("random_module_name_that_does_not_exist")
def test_import_object_submodule_not_an_attribute(): # `hello_world` is not in the `prefect` top-level hello_world = import_object("prefect.hello_world") import prefect.hello_world assert hello_world is prefect.hello_world
def __init__( self, address: str = None, cluster_class: Union[str, Callable] = None, cluster_kwargs: dict = None, adapt_kwargs: dict = None, client_kwargs: dict = None, debug: bool = None, **kwargs: Any, ): if address is None: address = context.config.engine.executor.dask.address or None # XXX: deprecated if address == "local": warnings.warn( "`address='local'` is deprecated. To use a local cluster, leave the " "`address` field empty.") address = None # XXX: deprecated local_processes = kwargs.pop("local_processes", None) if local_processes is None: local_processes = context.config.engine.executor.dask.get( "local_processes", None) if local_processes is not None: warnings.warn( "`local_processes` is deprecated, please use " "`cluster_kwargs={'processes': local_processes}`. The default is " "now `local_processes=True`.") if address is not None: if cluster_class is not None or cluster_kwargs is not None: raise ValueError( "Cannot specify `address` and `cluster_class`/`cluster_kwargs`" ) else: if cluster_class is None: cluster_class = context.config.engine.executor.dask.cluster_class if isinstance(cluster_class, str): cluster_class = import_object(cluster_class) if cluster_kwargs is None: cluster_kwargs = {} else: cluster_kwargs = cluster_kwargs.copy() from distributed.deploy.local import LocalCluster if cluster_class == LocalCluster: if debug is None: debug = context.config.debug cluster_kwargs.setdefault( "silence_logs", logging.CRITICAL if not debug else logging.WARNING) if local_processes is not None: cluster_kwargs.setdefault("processes", local_processes) for_cluster = set(kwargs).difference(_valid_client_kwargs) if for_cluster: warnings.warn( "Forwarding executor kwargs to `LocalCluster` is now handled by the " "`cluster_kwargs` parameter, please update accordingly" ) for k in for_cluster: cluster_kwargs[k] = kwargs.pop(k) if adapt_kwargs is None: adapt_kwargs = {} if client_kwargs is None: client_kwargs = {} if kwargs: warnings.warn( "Forwarding executor kwargs to `Client` is now handled by the " "`client_kwargs` parameter, please update accordingly") client_kwargs.update(kwargs) self.address = address self.cluster_class = cluster_class self.cluster_kwargs = cluster_kwargs self.adapt_kwargs = adapt_kwargs self.client_kwargs = client_kwargs self.client = None super().__init__()
def __init__( self, address: str = None, cluster_class: Union[str, Callable] = None, cluster_kwargs: dict = None, adapt_kwargs: dict = None, client_kwargs: dict = None, debug: bool = None, performance_report_path: str = None, disable_cancellation_event: bool = False, ): if address is None: address = context.config.engine.executor.dask.address or None if address is not None: if cluster_class is not None or cluster_kwargs is not None: raise ValueError( "Cannot specify `address` and `cluster_class`/`cluster_kwargs`" ) else: if cluster_class is None: cluster_class = context.config.engine.executor.dask.cluster_class if isinstance(cluster_class, str): cluster_class = import_object(cluster_class) if cluster_kwargs is None: cluster_kwargs = {} else: cluster_kwargs = cluster_kwargs.copy() from distributed.deploy.local import LocalCluster if cluster_class == LocalCluster: if debug is None: debug = context.config.debug cluster_kwargs.setdefault( "silence_logs", logging.CRITICAL if not debug else logging.WARNING) if adapt_kwargs is None: adapt_kwargs = {} if client_kwargs is None: client_kwargs = {} else: client_kwargs = client_kwargs.copy() client_kwargs.setdefault("set_as_default", False) self.address = address self.cluster_class = cluster_class self.cluster_kwargs = cluster_kwargs self.adapt_kwargs = adapt_kwargs self.client_kwargs = client_kwargs self.disable_cancellation_event = disable_cancellation_event # Runtime attributes self.client = None # These are coupled - they're either both None, or both non-None. # They're used in the case we can't forcibly kill all the dask workers, # and need to wait for all the dask tasks to cleanup before exiting. self._futures = None # type: Optional[weakref.WeakSet[Future]] self._should_run_event = None # type: Optional[Event] # A ref to a background task subscribing to dask cluster events self._watch_dask_events_task = None # type: Optional[concurrent.futures.Future] self.performance_report_path = performance_report_path super().__init__()