def dump_plugins(args):
"""Dump plugins information"""
plugins_manager.ensure_plugins_loaded()
plugins_manager.integrate_macros_plugins()
plugins_manager.integrate_executor_plugins()
plugins_manager.initialize_extra_operators_links_plugins()
plugins_manager.initialize_web_ui_plugins()
if not plugins_manager.plugins:
print("No plugins loaded")
return
console = Console()
console.print("[bold yellow]SUMMARY:[/bold yellow]")
console.print(
f"[bold green]Plugins directory[/bold green]: {conf.get('core', 'plugins_folder')}\n",
highlight=False)
console.print(
f"[bold green]Loaded plugins[/bold green]: {len(plugins_manager.plugins)}\n",
highlight=False)
for attr_name in PLUGINS_MANAGER_ATTRIBUTES_TO_DUMP:
attr_value: Optional[List[Any]] = getattr(plugins_manager, attr_name)
if not attr_value:
continue
table = SimpleTable(title=attr_name.capitalize().replace("_", " "))
table.add_column(width=100)
for item in attr_value: # pylint: disable=not-an-iterable
table.add_row(f"- {_get_name(item)}", )
console.print(table)
console.print("[bold yellow]DETAILED INFO:[/bold yellow]")
for plugin in plugins_manager.plugins:
table = SimpleTable(title=plugin.name)
for attr_name in PLUGINS_ATTRIBUTES_TO_DUMP:
value = getattr(plugin, attr_name)
if not value:
continue
table.add_row(attr_name.capitalize().replace("_", " "),
_join_plugins_names(value))
console.print(table)
def test_should_load_plugins_from_property(self, caplog):
class AirflowTestPropertyPlugin(AirflowPlugin):
name = "test_property_plugin"
@property
def hooks(self):
class TestPropertyHook(BaseHook):
pass
return [TestPropertyHook]
with mock_plugin_manager(plugins=[AirflowTestPropertyPlugin()]):
from airflow import plugins_manager
plugins_manager.ensure_plugins_loaded()
assert 'AirflowTestPropertyPlugin' in str(plugins_manager.plugins)
assert 'TestPropertyHook' in str(plugins_manager.registered_hooks)
assert caplog.records[0].levelname == 'INFO'
assert caplog.records[
0].msg == 'Loading %d plugin(s) took %.2f seconds'
def __getattr__(name):
# PEP-562: Lazy loaded attributes on python modules
if name == "DAG":
from airflow.models.dag import DAG # pylint: disable=redefined-outer-name
return DAG
if name == "AirflowException":
from airflow.exceptions import AirflowException # pylint: disable=redefined-outer-name
return AirflowException
raise AttributeError(f"module {__name__} has no attribute {name}")
if not settings.LAZY_LOAD_PLUGINS:
from airflow import plugins_manager
plugins_manager.ensure_plugins_loaded()
# This is never executed, but tricks static analyzers (PyDev, PyCharm,
# pylint, etc.) into knowing the types of these symbols, and what
# they contain.
STATICA_HACK = True
globals()['kcah_acitats'[::-1].upper()] = False
if STATICA_HACK: # pragma: no cover
from airflow.models.dag import DAG
from airflow.exceptions import AirflowException
if not PY37:
from pep562 import Pep562
Pep562(__name__)
def _deserialize_operator_extra_links(
cls, encoded_op_links: list) -> Dict[str, BaseOperatorLink]:
"""
Deserialize Operator Links if the Classes are registered in Airflow Plugins.
Error is raised if the OperatorLink is not found in Plugins too.
:param encoded_op_links: Serialized Operator Link
:return: De-Serialized Operator Link
"""
from airflow import plugins_manager
plugins_manager.ensure_plugins_loaded()
op_predefined_extra_links = {}
for _operator_links_source in encoded_op_links:
# Get the key, value pair as Tuple where key is OperatorLink ClassName
# and value is the dictionary containing the arguments passed to the OperatorLink
#
# Example of a single iteration:
#
# _operator_links_source =
# {
# 'airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink': {
# 'index': 0
# }
# },
#
# list(_operator_links_source.items()) =
# [
# (
# 'airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink',
# {'index': 0}
# )
# ]
#
# list(_operator_links_source.items())[0] =
# (
# 'airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink',
# {
# 'index': 0
# }
# )
_operator_link_class_path, data = list(
_operator_links_source.items())[0]
if _operator_link_class_path in BUILTIN_OPERATOR_EXTRA_LINKS:
single_op_link_class = import_string(_operator_link_class_path)
elif _operator_link_class_path in plugins_manager.registered_operator_link_classes:
single_op_link_class = plugins_manager.registered_operator_link_classes[
_operator_link_class_path]
else:
raise KeyError("Operator Link class %r not registered" %
_operator_link_class_path)
op_predefined_extra_link: BaseOperatorLink = cattr.structure(
data, single_op_link_class)
op_predefined_extra_links.update(
{op_predefined_extra_link.name: op_predefined_extra_link})
return op_predefined_extra_links
def deserialize_operator(cls, encoded_op: Dict[str, Any]) -> BaseOperator:
"""Deserializes an operator from a JSON object.
"""
from airflow import plugins_manager
plugins_manager.ensure_plugins_loaded()
op = SerializedBaseOperator(task_id=encoded_op['task_id'])
# Extra Operator Links defined in Plugins
op_extra_links_from_plugin = {}
for ope in plugins_manager.operator_extra_links:
for operator in ope.operators:
if operator.__name__ == encoded_op["_task_type"] and \
operator.__module__ == encoded_op["_task_module"]:
op_extra_links_from_plugin.update({ope.name: ope})
# If OperatorLinks are defined in Plugins but not in the Operator that is being Serialized
# set the Operator links attribute
# The case for "If OperatorLinks are defined in the operator that is being Serialized"
# is handled in the deserialization loop where it matches k == "_operator_extra_links"
if op_extra_links_from_plugin and "_operator_extra_links" not in encoded_op:
setattr(op, "operator_extra_links",
list(op_extra_links_from_plugin.values()))
for k, v in encoded_op.items():
if k == "_downstream_task_ids":
v = set(v)
elif k == "subdag":
v = SerializedDAG.deserialize_dag(v)
elif k in {"retry_delay", "execution_timeout"}:
v = cls._deserialize_timedelta(v)
elif k in encoded_op["template_fields"]:
pass
elif k.endswith("_date"):
v = cls._deserialize_datetime(v)
elif k == "_operator_extra_links":
op_predefined_extra_links = cls._deserialize_operator_extra_links(
v)
# If OperatorLinks with the same name exists, Links via Plugin have higher precedence
op_predefined_extra_links.update(op_extra_links_from_plugin)
v = list(op_predefined_extra_links.values())
k = "operator_extra_links"
elif k in cls._decorated_fields or k not in op.get_serialized_fields(
):
v = cls._deserialize(v)
# else use v as it is
setattr(op, k, v)
for k in op.get_serialized_fields() - encoded_op.keys(
) - cls._CONSTRUCTOR_PARAMS.keys():
setattr(op, k, None)
# Set all the template_field to None that were not present in Serialized JSON
for field in op.template_fields:
if not hasattr(op, field):
setattr(op, field, None)
return op
