def load_plugins_from_plugin_directory(): """Load and register Airflow Plugins from plugins directory""" global import_errors log.debug("Loading plugins from directory: %s", settings.PLUGINS_FOLDER) for file_path in find_path_from_directory(settings.PLUGINS_FOLDER, ".airflowignore"): if not os.path.isfile(file_path): continue mod_name, file_ext = os.path.splitext(os.path.split(file_path)[-1]) if file_ext != '.py': continue try: loader = importlib.machinery.SourceFileLoader(mod_name, file_path) spec = importlib.util.spec_from_loader(mod_name, loader) mod = importlib.util.module_from_spec(spec) sys.modules[spec.name] = mod loader.exec_module(mod) log.debug('Importing plugin module %s', file_path) for mod_attr_value in (m for m in mod.__dict__.values() if is_valid_plugin(m)): plugin_instance = mod_attr_value() plugin_instance.source = PluginsDirectorySource(file_path) register_plugin(plugin_instance) except Exception as e: log.exception('Failed to import plugin %s', file_path) import_errors[file_path] = str(e)
def test_find_not_should_ignore_path(self): """ Test that the .airflowignore work and whether the file is properly ignored. """ detected_files = set() should_ignore_files = { 'test_notload.py', 'test_notload_sub.py', 'test_noneload_sub1.py', 'test_shouldignore.py', } should_not_ignore_files = { 'test_load.py', 'test_load_sub1.py', } ignore_list_file = ".airflowignore" for file_path in find_path_from_directory(self.plugin_folder_path, ignore_list_file): if not os.path.isfile(file_path): continue _, file_ext = os.path.splitext(os.path.split(file_path)[-1]) if file_ext != '.py': continue detected_files.add(os.path.basename(file_path)) assert detected_files == should_not_ignore_files assert detected_files & should_ignore_files == set()