def dump_plugins(args):
"""Dump plugins information"""
plugins_manager.log.setLevel(logging.DEBUG)
plugins_manager.ensure_plugins_loaded()
plugins_manager.integrate_dag_plugins()
plugins_manager.integrate_executor_plugins()
plugins_manager.initialize_extra_operators_links_plugins()
plugins_manager.initialize_web_ui_plugins()
_header("PLUGINS MANGER:", "#")
for attr_name in PLUGINS_MANAGER_ATTRIBUTES_TO_DUMP:
attr_value = getattr(plugins_manager, attr_name)
print(f"{attr_name} = ", end='')
pprint(attr_value)
print()
_header("PLUGINS:", "#")
if not plugins_manager.plugins:
print("No plugins loaded")
else:
print(f"Loaded {len(plugins_manager.plugins)} plugins")
for plugin_no, plugin in enumerate(plugins_manager.plugins, 1):
_header(f"{plugin_no}. {plugin.name}", "=")
for attr_name in PLUGINS_ATTRIBUTES_TO_DUMP:
attr_value = getattr(plugin, attr_name)
print(f"{attr_name} = ", end='')
pprint(attr_value)
print()
def import_executor_cls(
cls, executor_name: str
) -> Tuple[Type["BaseExecutor"], ConnectorSource]:
"""
Imports the executor class.
Supports the same formats as ExecutorLoader.load_executor.
:return: executor class via executor_name and executor import source
"""
if executor_name in cls.executors:
return import_string(
cls.executors[executor_name]), ConnectorSource.CORE
if executor_name.count(".") == 1:
log.debug(
"The executor name looks like the plugin path (executor_name=%s). Trying to import a "
"executor from a plugin",
executor_name,
)
with suppress(ImportError, AttributeError):
# Load plugins here for executors as at that time the plugins might not have been
# initialized yet
from airflow import plugins_manager
plugins_manager.integrate_executor_plugins()
return import_string(f"airflow.executors.{executor_name}"
), ConnectorSource.PLUGIN
return import_string(executor_name), ConnectorSource.CUSTOM_PATH
def load_executor(cls, executor_name: str) -> Type[BaseExecutor]:
"""
Loads the executor.
This supports the following formats:
* by executor name for core executor
* by ``{plugin_name}.{class_name}`` for executor from plugins
* by import path.
"""
if executor_name in cls.executors:
log.debug("Loading core executor: %s", executor_name)
return import_string(cls.executors[executor_name])
# If the executor name looks like "plugin executor path" then try to load plugins.
if executor_name.count(".") == 1:
log.debug(
"The executor name looks like the plugin path (executor_name=%s). Trying to load a "
"executor from a plugin", executor_name)
with suppress(ImportError), suppress(AttributeError):
# Load plugins here for executors as at that time the plugins might not have been
# initialized yet
from airflow import plugins_manager
plugins_manager.integrate_executor_plugins()
return import_string(f"airflow.executors.{executor_name}")
log.debug("Loading executor from custom path: %s", executor_name)
try:
executor = import_string(executor_name)
except ImportError as e:
log.error(e)
raise AirflowConfigException(
f'The module/attribute could not be loaded. Please check "executor" key in "core" section. '
f'Current value: "{executor_name}".')
log.info("Loaded executor: %s", executor_name)
return executor
def _get_executor(executor_name: str) -> BaseExecutor:
"""
Creates a new instance of the named executor.
In case the executor name is unknown in airflow,
look for it in the plugins
"""
if executor_name == ExecutorLoader.LOCAL_EXECUTOR:
from airflow.executors.local_executor import LocalExecutor
return LocalExecutor()
elif executor_name == ExecutorLoader.SEQUENTIAL_EXECUTOR:
from airflow.executors.sequential_executor import SequentialExecutor
return SequentialExecutor()
elif executor_name == ExecutorLoader.CELERY_EXECUTOR:
from airflow.executors.celery_executor import CeleryExecutor
return CeleryExecutor()
elif executor_name == ExecutorLoader.DASK_EXECUTOR:
from airflow.executors.dask_executor import DaskExecutor
return DaskExecutor()
elif executor_name == ExecutorLoader.KUBERNETES_EXECUTOR:
from airflow.executors.kubernetes_executor import KubernetesExecutor
return KubernetesExecutor()
else:
# Load plugins here for executors as at that time the plugins might not have been initialized yet
# TODO: verify the above and remove two lines below in case plugins are always initialized first
from airflow import plugins_manager
plugins_manager.integrate_executor_plugins()
executor_path = executor_name.split('.')
assert len(executor_path) == 2, f"Executor {executor_name} not supported: " \
f"please specify in format plugin_module.executor"
assert executor_path[0] in globals(
), f"Executor {executor_name} not supported"
return globals()[executor_path[0]].__dict__[executor_path[1]]()
def dump_plugins(args):
"""Dump plugins information"""
plugins_manager.ensure_plugins_loaded()
plugins_manager.integrate_macros_plugins()
plugins_manager.integrate_executor_plugins()
plugins_manager.initialize_extra_operators_links_plugins()
plugins_manager.initialize_web_ui_plugins()
if not plugins_manager.plugins:
print("No plugins loaded")
return
plugins_info: List[Dict[str, str]] = []
for plugin in plugins_manager.plugins:
info = {"name": plugin.name}
info.update(
{n: getattr(plugin, n)
for n in PLUGINS_ATTRIBUTES_TO_DUMP})
plugins_info.append(info)
# Remove empty info
if args.output == "table": # pylint: disable=too-many-nested-blocks
# We can do plugins_info[0] as the element it will exist as there's
# at least one plugin at this point
for col in list(plugins_info[0]):
if all(not bool(p[col]) for p in plugins_info):
for plugin in plugins_info:
del plugin[col]
AirflowConsole().print_as(plugins_info, output=args.output)
def _load_executor(cls, executor_name: str) -> BaseExecutor:
"""
Loads the executor.
This supports the following following formats:
* by executor name for core executor
* by ``{plugin_name}.{class_name}`` for executor from plugins
* by import path.
"""
if executor_name in cls.executors:
log.debug("Loading core executor: %s", executor_name)
return import_string(cls.executors[executor_name])()
# If the executor name looks like "plugin executor path" then try to load plugins.
if executor_name.count(".") == 1:
log.debug(
"The executor name looks like the plugin path (executor_name=%s). Trying to load a "
"executor from a plugin", executor_name)
with suppress(ImportError), suppress(AttributeError):
# Load plugins here for executors as at that time the plugins might not have been
# initialized yet
from airflow import plugins_manager
plugins_manager.integrate_executor_plugins()
return import_string(f"airflow.executors.{executor_name}")()
log.debug("Loading executor from custom path: %s", executor_name)
executor = import_string(executor_name)()
log.info("Loaded executor: %s", executor_name)
return executor
def _get_executor(cls, executor_name: str) -> BaseExecutor:
"""
Creates a new instance of the named executor.
In case the executor name is unknown in airflow,
look for it in the plugins
"""
if executor_name in cls.executors:
executor_module = importlib.import_module(cls.executors[executor_name])
executor = getattr(executor_module, executor_name)
return executor()
else:
# Load plugins here for executors as at that time the plugins might not have been initialized yet
from airflow import plugins_manager
plugins_manager.integrate_executor_plugins()
executor_path = executor_name.split('.')
if len(executor_path) != 2:
raise ValueError(f"Executor {executor_name} not supported: "
f"please specify in format plugin_module.executor")
if executor_path[0] not in globals():
raise ValueError(f"Executor {executor_name} not supported")
return globals()[executor_path[0]].__dict__[executor_path[1]]()
def dump_plugins(args):
"""Dump plugins information"""
plugins_manager.ensure_plugins_loaded()
plugins_manager.integrate_macros_plugins()
plugins_manager.integrate_executor_plugins()
plugins_manager.initialize_extra_operators_links_plugins()
plugins_manager.initialize_web_ui_plugins()
if not plugins_manager.plugins:
print("No plugins loaded")
return
console = Console()
console.print("[bold yellow]SUMMARY:[/bold yellow]")
console.print(
f"[bold green]Plugins directory[/bold green]: {conf.get('core', 'plugins_folder')}\n",
highlight=False)
console.print(
f"[bold green]Loaded plugins[/bold green]: {len(plugins_manager.plugins)}\n",
highlight=False)
for attr_name in PLUGINS_MANAGER_ATTRIBUTES_TO_DUMP:
attr_value: Optional[List[Any]] = getattr(plugins_manager, attr_name)
if not attr_value:
continue
table = SimpleTable(title=attr_name.capitalize().replace("_", " "))
table.add_column(width=100)
for item in attr_value: # pylint: disable=not-an-iterable
table.add_row(f"- {_get_name(item)}", )
console.print(table)
console.print("[bold yellow]DETAILED INFO:[/bold yellow]")
for plugin in plugins_manager.plugins:
table = SimpleTable(title=plugin.name)
for attr_name in PLUGINS_ATTRIBUTES_TO_DUMP:
value = getattr(plugin, attr_name)
if not value:
continue
table.add_row(attr_name.capitalize().replace("_", " "),
_join_plugins_names(value))
console.print(table)
