def dump_plugins(args): """Dump plugins information""" plugins_manager.ensure_plugins_loaded() plugins_manager.integrate_macros_plugins() plugins_manager.integrate_executor_plugins() plugins_manager.initialize_extra_operators_links_plugins() plugins_manager.initialize_web_ui_plugins() if not plugins_manager.plugins: print("No plugins loaded") return console = Console() console.print("[bold yellow]SUMMARY:[/bold yellow]") console.print( f"[bold green]Plugins directory[/bold green]: {conf.get('core', 'plugins_folder')}\n", highlight=False) console.print( f"[bold green]Loaded plugins[/bold green]: {len(plugins_manager.plugins)}\n", highlight=False) for attr_name in PLUGINS_MANAGER_ATTRIBUTES_TO_DUMP: attr_value: Optional[List[Any]] = getattr(plugins_manager, attr_name) if not attr_value: continue table = SimpleTable(title=attr_name.capitalize().replace("_", " ")) table.add_column(width=100) for item in attr_value: # pylint: disable=not-an-iterable table.add_row(f"- {_get_name(item)}", ) console.print(table) console.print("[bold yellow]DETAILED INFO:[/bold yellow]") for plugin in plugins_manager.plugins: table = SimpleTable(title=plugin.name) for attr_name in PLUGINS_ATTRIBUTES_TO_DUMP: value = getattr(plugin, attr_name) if not value: continue table.add_row(attr_name.capitalize().replace("_", " "), _join_plugins_names(value)) console.print(table)
def test_should_load_plugins_from_property(self, caplog): class AirflowTestPropertyPlugin(AirflowPlugin): name = "test_property_plugin" @property def hooks(self): class TestPropertyHook(BaseHook): pass return [TestPropertyHook] with mock_plugin_manager(plugins=[AirflowTestPropertyPlugin()]): from airflow import plugins_manager plugins_manager.ensure_plugins_loaded() assert 'AirflowTestPropertyPlugin' in str(plugins_manager.plugins) assert 'TestPropertyHook' in str(plugins_manager.registered_hooks) assert caplog.records[0].levelname == 'INFO' assert caplog.records[ 0].msg == 'Loading %d plugin(s) took %.2f seconds'
def __getattr__(name): # PEP-562: Lazy loaded attributes on python modules if name == "DAG": from airflow.models.dag import DAG # pylint: disable=redefined-outer-name return DAG if name == "AirflowException": from airflow.exceptions import AirflowException # pylint: disable=redefined-outer-name return AirflowException raise AttributeError(f"module {__name__} has no attribute {name}") if not settings.LAZY_LOAD_PLUGINS: from airflow import plugins_manager plugins_manager.ensure_plugins_loaded() # This is never executed, but tricks static analyzers (PyDev, PyCharm, # pylint, etc.) into knowing the types of these symbols, and what # they contain. STATICA_HACK = True globals()['kcah_acitats'[::-1].upper()] = False if STATICA_HACK: # pragma: no cover from airflow.models.dag import DAG from airflow.exceptions import AirflowException if not PY37: from pep562 import Pep562 Pep562(__name__)
def _deserialize_operator_extra_links( cls, encoded_op_links: list) -> Dict[str, BaseOperatorLink]: """ Deserialize Operator Links if the Classes are registered in Airflow Plugins. Error is raised if the OperatorLink is not found in Plugins too. :param encoded_op_links: Serialized Operator Link :return: De-Serialized Operator Link """ from airflow import plugins_manager plugins_manager.ensure_plugins_loaded() op_predefined_extra_links = {} for _operator_links_source in encoded_op_links: # Get the key, value pair as Tuple where key is OperatorLink ClassName # and value is the dictionary containing the arguments passed to the OperatorLink # # Example of a single iteration: # # _operator_links_source = # { # 'airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink': { # 'index': 0 # } # }, # # list(_operator_links_source.items()) = # [ # ( # 'airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink', # {'index': 0} # ) # ] # # list(_operator_links_source.items())[0] = # ( # 'airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleIndexableLink', # { # 'index': 0 # } # ) _operator_link_class_path, data = list( _operator_links_source.items())[0] if _operator_link_class_path in BUILTIN_OPERATOR_EXTRA_LINKS: single_op_link_class = import_string(_operator_link_class_path) elif _operator_link_class_path in plugins_manager.registered_operator_link_classes: single_op_link_class = plugins_manager.registered_operator_link_classes[ _operator_link_class_path] else: raise KeyError("Operator Link class %r not registered" % _operator_link_class_path) op_predefined_extra_link: BaseOperatorLink = cattr.structure( data, single_op_link_class) op_predefined_extra_links.update( {op_predefined_extra_link.name: op_predefined_extra_link}) return op_predefined_extra_links
def deserialize_operator(cls, encoded_op: Dict[str, Any]) -> BaseOperator: """Deserializes an operator from a JSON object. """ from airflow import plugins_manager plugins_manager.ensure_plugins_loaded() op = SerializedBaseOperator(task_id=encoded_op['task_id']) # Extra Operator Links defined in Plugins op_extra_links_from_plugin = {} for ope in plugins_manager.operator_extra_links: for operator in ope.operators: if operator.__name__ == encoded_op["_task_type"] and \ operator.__module__ == encoded_op["_task_module"]: op_extra_links_from_plugin.update({ope.name: ope}) # If OperatorLinks are defined in Plugins but not in the Operator that is being Serialized # set the Operator links attribute # The case for "If OperatorLinks are defined in the operator that is being Serialized" # is handled in the deserialization loop where it matches k == "_operator_extra_links" if op_extra_links_from_plugin and "_operator_extra_links" not in encoded_op: setattr(op, "operator_extra_links", list(op_extra_links_from_plugin.values())) for k, v in encoded_op.items(): if k == "_downstream_task_ids": v = set(v) elif k == "subdag": v = SerializedDAG.deserialize_dag(v) elif k in {"retry_delay", "execution_timeout"}: v = cls._deserialize_timedelta(v) elif k in encoded_op["template_fields"]: pass elif k.endswith("_date"): v = cls._deserialize_datetime(v) elif k == "_operator_extra_links": op_predefined_extra_links = cls._deserialize_operator_extra_links( v) # If OperatorLinks with the same name exists, Links via Plugin have higher precedence op_predefined_extra_links.update(op_extra_links_from_plugin) v = list(op_predefined_extra_links.values()) k = "operator_extra_links" elif k in cls._decorated_fields or k not in op.get_serialized_fields( ): v = cls._deserialize(v) # else use v as it is setattr(op, k, v) for k in op.get_serialized_fields() - encoded_op.keys( ) - cls._CONSTRUCTOR_PARAMS.keys(): setattr(op, k, None) # Set all the template_field to None that were not present in Serialized JSON for field in op.template_fields: if not hasattr(op, field): setattr(op, field, None) return op