def collect_data(self, **kwargs): self.logger.info("Begin collect_data") simplePluginManager = PluginManager() logging.getLogger('yapsy').setLevel(logging.DEBUG) simplePluginManager.setCategoriesFilter({ "DataCollector": data_collector_plugin }) # Tell it the default place(s) where to find plugins self.logger.debug("Plugin directories: %s" % (kwargs['data_collector_plugin_directories'])) simplePluginManager.setPluginPlaces(kwargs['data_collector_plugin_directories']) simplePluginManager.collectPlugins() plugin_cnt = 0 plugin_start_time = time.time() for plugin in simplePluginManager.getAllPlugins(): self.logger.info("Starting plugin: %s" % (plugin.name)) if plugin.plugin_object.initialize_plugin(details=plugin.details): plugin.plugin_object.start() else: self.logger.error("Failed to initialize plugin: %s" % (plugin.name)) plugin_cnt += 1 #Wait for the plugings to finish up. self.logger.info("Waiting for %d plugins to complete." % (plugin_cnt)) for plugin in simplePluginManager.getAllPlugins(): plugin.plugin_object.join() self.logger.info("%d Plugins completed in %f seconds" % (plugin_cnt, time.time() - plugin_start_time))
def loadPlugins(root_folder, plugin_name, categories_filter): from yapsy.PluginManager import PluginManager plugin_dirs = [] plugin_dirs.append(configuration.fabfile_basedir + '/.fabalicious/plugins') plugin_dirs.append(expanduser("~") + '/.fabalicious/plugins') plugin_dirs.append(root_folder + '/plugins') log.debug("Looking for %s-plugins in %s" % (plugin_name, ", ".join(plugin_dirs))) manager = PluginManager() manager.setPluginPlaces(plugin_dirs) manager.setCategoriesFilter(categories_filter) manager.collectPlugins() # Activate all loaded plugins for pluginInfo in manager.getAllPlugins(): manager.activatePluginByName(pluginInfo.name) result = {} for plugin in manager.getAllPlugins(): if hasattr(plugin.plugin_object, 'aliases') and isinstance(plugin.plugin_object.aliases, list): for alias in plugin.plugin_object.aliases: result[alias] = plugin.plugin_object elif hasattr(plugin.plugin_object, 'alias'): result[plugin.plugin_object.alias] = plugin.plugin_object else: result[plugin.name] = plugin.plugin_object return result
def run_output_plugins(**kwargs): logger = logging.getLogger(__name__) logger.info("Begin run_output_plugins") simplePluginManager = PluginManager() logging.getLogger('yapsy').setLevel(logging.DEBUG) simplePluginManager.setCategoriesFilter({"OutputResults": output_plugin}) # Tell it the default place(s) where to find plugins if logger: logger.debug("Plugin directories: %s" % (kwargs['output_plugin_directories'])) simplePluginManager.setPluginPlaces(kwargs['output_plugin_directories']) simplePluginManager.collectPlugins() plugin_cnt = 0 plugin_start_time = time.time() for plugin in simplePluginManager.getAllPlugins(): if logger: logger.info("Starting plugin: %s" % (plugin.name)) if plugin.plugin_object.initialize_plugin(details=plugin.details): plugin.plugin_object.emit( prediction_date=kwargs['prediction_date'].astimezone( timezone("US/Eastern")).strftime("%Y-%m-%d %H:%M:%S"), execution_date=kwargs['prediction_run_date'].strftime( "%Y-%m-%d %H:%M:%S"), ensemble_tests=kwargs['site_model_ensemble']) plugin_cnt += 1 else: logger.error("Failed to initialize plugin: %s" % (plugin.name)) logger.debug("%d output plugins run in %f seconds" % (plugin_cnt, time.time() - plugin_start_time)) logger.info("Finished run_output_plugins")
def run_output_plugins(**kwargs): logger = logging.getLogger(__name__) logger.info("Begin run_output_plugins") simplePluginManager = PluginManager() logging.getLogger('yapsy').setLevel(logging.DEBUG) simplePluginManager.setCategoriesFilter({ "OutputResults": output_plugin }) # Tell it the default place(s) where to find plugins if logger: logger.debug("Plugin directories: %s" % (kwargs['output_plugin_directories'])) simplePluginManager.setPluginPlaces(kwargs['output_plugin_directories']) simplePluginManager.collectPlugins() plugin_cnt = 0 plugin_start_time = time.time() for plugin in simplePluginManager.getAllPlugins(): if logger: logger.info("Starting plugin: %s" % (plugin.name)) if plugin.plugin_object.initialize_plugin(details=plugin.details): plugin.plugin_object.emit(prediction_date=kwargs['prediction_date'].astimezone(timezone("US/Eastern")).strftime("%Y-%m-%d %H:%M:%S"), execution_date=kwargs['prediction_run_date'].strftime("%Y-%m-%d %H:%M:%S"), ensemble_tests=kwargs['site_model_ensemble']) plugin_cnt += 1 else: logger.error("Failed to initialize plugin: %s" % (plugin.name)) logger.debug("%d output plugins run in %f seconds" % (plugin_cnt, time.time() - plugin_start_time)) logger.info("Finished run_output_plugins")
def loadPlugins(root_folder, plugin_name, categories_filter): from yapsy.PluginManager import PluginManager plugin_dirs = [] plugin_dirs.append(configuration.fabfile_basedir + '/.fabalicious/plugins') plugin_dirs.append(expanduser("~") + '/.fabalicious/plugins') plugin_dirs.append(root_folder + '/plugins') log.debug("Looking for %s-plugins in %s" % (plugin_name, ", ".join(plugin_dirs))) manager = PluginManager() manager.setPluginPlaces(plugin_dirs) manager.setCategoriesFilter(categories_filter) manager.collectPlugins() # Activate all loaded plugins for pluginInfo in manager.getAllPlugins(): manager.activatePluginByName(pluginInfo.name) result = {} for plugin in manager.getAllPlugins(): if hasattr(plugin.plugin_object, 'aliases') and isinstance( plugin.plugin_object.aliases, list): for alias in plugin.plugin_object.aliases: result[alias] = plugin.plugin_object elif hasattr(plugin.plugin_object, 'alias'): result[plugin.plugin_object.alias] = plugin.plugin_object else: result[plugin.name] = plugin.plugin_object return result
def test_plugins_found(self): pm = PluginManager() libpath = '%s/OpenMesher/plugins' % (get_python_lib()) pm.setPluginPlaces([ '/usr/share/openmesher/plugins', '~/.openmesher/plugins', './OpenMesher/plugins', './plugins', libpath, ]) pm.setPluginInfoExtension('plugin') pm.setCategoriesFilter({ 'config': IOpenMesherConfigPlugin, 'package': IOpenMesherPackagePlugin, 'deploy': IOpenMesherDeployPlugin, }) pm.collectPlugins() for plugin in pm.getAllPlugins(): print('Author: %s' % (plugin.author)) print('Categories: %s' % (plugin.categories)) print('Copyright: %s' % (plugin.copyright)) print('Descr: %s' % (plugin.description)) print('Error: %s' % (plugin.error)) print('Name: %s' % (plugin.name)) print('Path: %s' % (plugin.path)) print('Version: %s' % (plugin.version)) print('Website: %s' % (plugin.website)) print('') return True
def collect_data(self, **kwargs): self.logger.info("Begin collect_data") try: simplePluginManager = PluginManager() logging.getLogger('yapsy').setLevel(logging.DEBUG) simplePluginManager.setCategoriesFilter( {"DataCollector": data_collector_plugin}) # Tell it the default place(s) where to find plugins self.logger.debug("Plugin directories: %s" % (kwargs['data_collector_plugin_directories'])) yapsy_logger = logging.getLogger('yapsy') yapsy_logger.setLevel(logging.DEBUG) #yapsy_logger.parent.level = logging.DEBUG yapsy_logger.disabled = False simplePluginManager.setPluginPlaces( kwargs['data_collector_plugin_directories']) simplePluginManager.collectPlugins() output_queue = Queue() plugin_cnt = 0 plugin_start_time = time.time() for plugin in simplePluginManager.getAllPlugins(): plugin_start_time = time.time() self.logger.info("Starting plugin: %s" % (plugin.name)) if plugin.plugin_object.initialize_plugin( details=plugin.details, queue=output_queue, begin_date=kwargs['begin_date']): plugin.plugin_object.start() self.logger.info("Waiting for %s plugin to complete." % (plugin.name)) plugin.plugin_object.join() self.logger.info( "%s plugin to completed in %f seconds." % (plugin.name, time.time() - plugin_start_time)) else: self.logger.error("Failed to initialize plugin: %s" % (plugin.name)) plugin_cnt += 1 #Wait for the plugings to finish up. self.logger.info("Waiting for %d plugins to complete." % (plugin_cnt)) for plugin in simplePluginManager.getAllPlugins(): plugin.plugin_object.join() while not output_queue.empty(): results = output_queue.get() if results[0] == data_result_types.SAMPLING_DATA_TYPE: self.bacteria_sample_data = results[1] self.logger.info("%d Plugins completed in %f seconds" % (plugin_cnt, time.time() - plugin_start_time)) except Exception as e: self.logger.exception(e)
class Plugins(): ''' Plugins ''' def __init__(self): ''' Constructor ''' super(Plugins, self).__init__() self.write_tab_dock_plugin_dict = {} self.write_panel_dock_plugin_dict = {} # Build the manager self._plugin_manager = PluginManager() # List all sub-directories of "plugins" plugin_path = os.path.sep.join([os.getcwd(), "plugins"]) plugin_places = [plugin_path] for dirname, dirnames, filenames in os.walk(plugin_path): # print path to all subdirectories first. for subdirname in dirnames: plugin_places.append(os.path.join(dirname, subdirname)) # Tell it the default place(s) where to find plugins self._plugin_manager.setPluginPlaces(plugin_places) sys.path.append(plugin_path) # Define the various categories corresponding to the different # kinds of plugins you have defined self._plugin_manager.setCategoriesFilter({ "CoreWriteTabDockPlugin": CoreWriteTabDockPlugin, "CoreWritePanelDockPlugin": CoreWritePanelDockPlugin }) self._plugin_manager.collectPlugins() self.load_plugins( ["CoreWriteTabDockPlugin", "CoreWritePanelDockPlugin"]) def load_plugins(self, categories): ''' function:: load_plugins(categories) :param categories: list ''' for category in categories: for pluginInfo in self._plugin_manager.getPluginsOfCategory( category): setattr(self, pluginInfo.plugin_object.core_class().__name__, pluginInfo.plugin_object.core_class()) if category is "CoreWriteTabDockPlugin": self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.core_class().dock_name] \ = pluginInfo.plugin_object.core_class() if category is "CoreWritePanelDockPlugin": self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.core_class().dock_name] \ = pluginInfo.plugin_object.core_class()
class Plugins(): ''' Plugins ''' def __init__(self): ''' Constructor ''' super(Plugins, self).__init__() self.write_tab_dock_plugin_dict = {} self.write_panel_dock_plugin_dict = {} # Build the manager self._plugin_manager = PluginManager() # List all sub-directories of "plugins" plugin_path = os.path.sep.join([os.getcwd(), "plugins"]) plugin_places = [plugin_path] for dirname, dirnames, filenames in os.walk(plugin_path): # print path to all subdirectories first. for subdirname in dirnames: plugin_places.append(os.path.join(dirname, subdirname)) # Tell it the default place(s) where to find plugins self._plugin_manager.setPluginPlaces(plugin_places) sys.path.append(plugin_path) # Define the various categories corresponding to the different # kinds of plugins you have defined self._plugin_manager.setCategoriesFilter({ "GuiWriteTabDockPlugin": GuiWriteTabDockPlugin, "GuiWritePanelDockPlugin": GuiWritePanelDockPlugin }) self._plugin_manager.collectPlugins() self.load_plugins( ["GuiWriteTabDockPlugin", "GuiWritePanelDockPlugin"]) def load_plugins(self, categories): ''' function:: load_plugins(categories) :param categories: list ''' for category in categories: for pluginInfo in self._plugin_manager.getPluginsOfCategory(category): setattr(self, pluginInfo.plugin_object.gui_class( ).__name__, pluginInfo.plugin_object.gui_class()) if category is "GuiWriteTabDockPlugin": self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \ = pluginInfo.plugin_object.gui_class() if category is "GuiWritePanelDockPlugin": self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \ = pluginInfo.plugin_object.gui_class()
class PluginHandler: def __init__(self): self._plugin_manager = PluginManager() self._category_active = {"Modifier": False, "Analyzer": False, "Comparator": False} self._plugin_from_category = {"Modifier": [], "Analyzer": [], "Comparator": []} self._modifier_plugins = [] self._analyzer_plugins = [] self._collect_all_plugins() def _collect_all_plugins(self): self._plugin_manager.setPluginPlaces( ["hugin/analyze/modifier", "hugin/analyze/analyzer", "hugin/analyze/comparator"] ) # setting filter categories for pluginmanager self._plugin_manager.setCategoriesFilter( { # movie metadata provider "Modifier": IModifier, # movie metadata provider "Comparator": IComparator, # sub metadata provider "Analyzer": IAnalyzer, } ) self._plugin_manager.collectPlugins() def activate_plugins_by_category(self, category): self._toggle_activate_plugins_by_category(category) def deactivate_plugins_by_category(self, category): self._toggle_activate_plugins_by_category(category) def _toggle_activate_plugins_by_category(self, category): plugins = self._plugin_manager.getPluginsOfCategory(category) is_active = self._category_active[category] for pluginInfo in plugins: if is_active: self._plugin_manager.deactivatePluginByName(name=pluginInfo.name, category=category) self._plugin_from_category[category].remove(pluginInfo) else: self._plugin_manager.activatePluginByName(name=pluginInfo.name, category=category) self._plugin_from_category[category].append(pluginInfo) self._category_active[category] = not is_active def get_plugins_from_category(self, category): plugins = [] for plugin in self._plugin_from_category[category]: plugin.plugin_object.name = plugin.name plugin.plugin_object.description = plugin.description plugins.append(plugin.plugin_object) return plugins def is_activated(self, category): """ True if category is activated. """ return self._category_active[category]
def initialize_plugins(self): self.__log.info("Collecting and loading plugins") try: plugin_manager = PluginManager() # TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT) plugin_manager.setCategoriesFilter({ CARTRIDGE_AGENT_PLUGIN: ICartridgeAgentPlugin, ARTIFACT_MGT_PLUGIN: IArtifactManagementPlugin }) plugin_manager.setPluginPlaces( [self.__config.read_property(constants.PLUGINS_DIR)]) plugin_manager.collectPlugins() # activate cartridge agent plugins plugins = plugin_manager.getPluginsOfCategory( CARTRIDGE_AGENT_PLUGIN) grouped_plugins = {} for plugin_info in plugins: self.__log.debug("Found plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) plugin_manager.activatePluginByName(plugin_info.name) self.__log.info("Activated plugin [%s]" % plugin_info.name) mapped_events = plugin_info.description.split(",") for mapped_event in mapped_events: if mapped_event.strip() != "": if grouped_plugins.get(mapped_event) is None: grouped_plugins[mapped_event] = [] grouped_plugins[mapped_event].append(plugin_info) # activate artifact management plugins artifact_mgt_plugins = plugin_manager.getPluginsOfCategory( ARTIFACT_MGT_PLUGIN) for plugin_info in artifact_mgt_plugins: self.__log.debug( "Found artifact management plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) plugin_manager.activatePluginByName(plugin_info.name) self.__log.info("Activated artifact management plugin [%s]" % plugin_info.name) return plugin_manager, grouped_plugins, artifact_mgt_plugins except ParameterNotFoundException as e: self.__log.exception( "Could not load plugins. Plugins directory not set: %s" % e) return None, None, None except Exception as e: self.__log.exception("Error while loading plugin: %s" % e) return None, None, None
def ConfigurePluginManager(categories=None, pluginLocation=None): if categories is None: categories = dict(Test=ITestPlugin, DAQ=IDAQPlugin, App=IAppPlugin, Decoder=IDecoderPlugin) if pluginLocation is None: pluginLocation = ["plugins"] manager = PluginManager() # todo: plugin directory and categories should be set some other way (command line arg parsing?) manager.setPluginPlaces(pluginLocation) manager.setCategoriesFilter(categories) manager.collectPlugins() return manager
def loadPlugins(): # Find all plugin directories (xxx.plugin) dirs = [f for f in os.listdir('.') if f.endswith('.plugin')] # Load the plugins from the plugin directory. manager = PluginManager() manager.setPluginPlaces(dirs) manager.setCategoriesFilter({"plugin" : IInfoBoxPlugin}) manager.collectPlugins() return manager
class MUCBot(sleekxmpp.ClientXMPP): def __init__(self, jid, password, room, nick): sleekxmpp.ClientXMPP.__init__(self, jid, password) self.room = room self.nick = nick # Load plugins self.plugin_manager = PluginManager() self.plugin_manager.setPluginPlaces(["plugins_enabled"]) self.plugin_manager.setCategoriesFilter({ "Message" : MessagePlugin, "Presence": PresencePlugin }) self.plugin_manager.collectPlugins() # Activate all loaded plugins for pluginInfo in self.plugin_manager.getAllPlugins(): self.plugin_manager.activatePluginByName(pluginInfo.name) self.add_event_handler("session_start", self.start) self.add_event_handler("message", self.message) self.add_event_handler("groupchat_message", self.muc_message) self.add_event_handler("muc::%s::got_online" % self.room, self.muc_online) self.add_event_handler("muc::%s::got_offline" % self.room, self.muc_offline) def start(self, event): self.get_roster() self.send_presence() if self.room and len(self.room) > 0: self.plugin['xep_0045'].joinMUC(self.room, self.nick, wait=True) def message(self, msg): if msg['type'] in ('chat', 'normal'): for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"): pluginInfo.plugin_object.message_received(msg) def muc_message(self, msg): if msg['mucnick'] != self.nick: for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"): pluginInfo.plugin_object.message_received(msg, nick=self.nick) def muc_online(self, presence): if presence['muc']['nick'] != self.nick: for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"): pluginInfo.plugin_object.got_online(self, presence) def muc_offline(self, presence): if presence['muc']['nick'] != self.nick: for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"): pluginInfo.plugin_object.got_offline(self, presence)
def get_plugins(category_filter={}, plugin_places=['plugins']): pm = PluginManager(plugin_info_ext='plugin') # Normalize the paths to the location of this file. # XXX-ricarkol: there has to be a better way to do this. plugin_places = [misc.execution_path(x) for x in plugin_places] pm.setPluginPlaces(plugin_places) pm.setCategoriesFilter(category_filter) pm.collectPlugins() return pm.getAllPlugins()
def output_results(self, **kwargs): self.logger.info("Begin run_output_plugins") simplePluginManager = PluginManager() logging.getLogger('yapsy').setLevel(logging.DEBUG) simplePluginManager.setCategoriesFilter( {"OutputResults": data_collector_plugin}) # Tell it the default place(s) where to find plugins self.logger.debug("Plugin directories: %s" % (kwargs['output_plugin_directories'])) simplePluginManager.setPluginPlaces( kwargs['output_plugin_directories']) yapsy_logger = logging.getLogger('yapsy') yapsy_logger.setLevel(logging.DEBUG) # yapsy_logger.parent.level = logging.DEBUG yapsy_logger.disabled = False simplePluginManager.collectPlugins() plugin_cnt = 0 plugin_start_time = time.time() for plugin in simplePluginManager.getAllPlugins(): try: self.logger.info("Starting plugin: %s" % (plugin.name)) if plugin.plugin_object.initialize_plugin( details=plugin.details, prediction_date=kwargs['prediction_date'].astimezone( timezone("US/Eastern")).strftime( "%Y-%m-%d %H:%M:%S"), execution_date=kwargs['prediction_run_date'].strftime( "%Y-%m-%d %H:%M:%S"), ensemble_tests=kwargs['site_model_ensemble']): plugin.plugin_object.start() plugin_cnt += 1 else: self.logger.error("Failed to initialize plugin: %s" % (plugin.details)) except Exception as e: self.logger.exception(e) #Wait for the plugings to finish up. self.logger.info("Waiting for %d plugins to complete." % (plugin_cnt)) for plugin in simplePluginManager.getAllPlugins(): plugin.plugin_object.join() plugin.plugin_object.finalize() self.logger.debug("%d output plugins run in %f seconds" % (plugin_cnt, time.time() - plugin_start_time)) self.logger.info("Finished output_results")
def modelBehaviorImplementations(paths): """ Return an iterable of plugin-info for every locatable implementation of this interface on a given path. PyDev for Eclipse reports a compilation error here on a line that is actually legal Python due to the schedule upon which module resolution and imports happen. """ manager = PluginManager() manager.setPluginPlaces(paths) from fakeDataGenerator.model import IModelBehavior as foreignModelBehavior # hey PyDev: the previous line is weird, but is actually legal manager.setCategoriesFilter({"ModelBehavior": foreignModelBehavior}) manager.collectPlugins() return manager.getPluginsOfCategory("ModelBehavior")
def create_plugin_manager(category_filter, plugin_place): """ Creates a PluginManager object from the given folder according to the given filter :param category_filter: :param plugin_place: :return: :rtype: PluginManager """ plugin_manager = PluginManager() plugin_manager.setCategoriesFilter(category_filter) plugin_manager.setPluginPlaces([plugin_place]) plugin_manager.collectPlugins() return plugin_manager
def main(): # Load the plugins from the plugin directory. manager = PluginManager() manager.setPluginPlaces(["plugins\\plugins1", "plugins\\plugins2"]) manager.setCategoriesFilter({ "PluginOne": category.PluginOne, "PluginTwo": category.PluginTwo, }) print { "PluginOne": category.PluginOne, "PluginTwo": category.PluginTwo } manager.collectPlugins() print manager.getPluginsOfCategory("PluginTwo") for plugin in manager.getPluginsOfCategory("PluginTwo"): plugin.plugin_object.print_name()
def get_manager(): categories = { catg.ApkChecker.category: catg.ApkChecker, catg.Unpacker.category: catg.Unpacker, catg.ProtectChecker.category: catg.ProtectChecker, catg.Auditor.category: catg.Auditor, } manager = PluginManager() plugin_paths = map(lambda e: os.path.join(settings.PLUGINS_DIR, e), categories.keys()) manager.setPluginPlaces(plugin_paths) manager.setCategoriesFilter(categories) manager.collectPlugins() return manager
def modelBehaviorImplementations(paths): """ Return an iterable of plugin-info for every locatable implementation of this interface on a given path. PyDev for Eclipse reports a compilation error here on a line that is actually legal Python due to the schedule upon which module resolution and imports happen. """ manager = PluginManager() manager.setPluginPlaces(paths) from fakeDataGenerator.model import IModelBehavior as foreignModelBehavior #hey PyDev: the previous line is weird, but is actually legal manager.setCategoriesFilter({ "ModelBehavior" : foreignModelBehavior, }) manager.collectPlugins() return manager.getPluginsOfCategory("ModelBehavior")
def get_plugin_manager(config): plugin_manager = PluginManager() plugin_manager.setPluginPlaces( [config.get("GENERAL", "PLUGIN_DIRECTORY")] ) plugin_manager.setCategoriesFilter({ PluginTypeName.SENSOR: ISensorPlugin, PluginTypeName.REPOSITORY: IRepositoryPlugin, PluginTypeName.DISPLAY: IDisplayPlugin }) plugin_manager.collectPlugins() return plugin_manager
def test_json_reporting_plugin_found(): """Test that the plugin manager finds the JSON plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')]) manager.setCategoriesFilter({ "Reporting": ReportingPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "upload_risk_assessment" assert any(plugin_info.plugin_object.get_name() == 'upload_risk_assessment' for plugin_info in manager.getPluginsOfCategory("Reporting")) # While we're at it, verify that a plugin is named JSON Risk Assessment Reporting Plugin assert any(plugin_info.name == 'JSON Risk Assessment Reporting Plugin' for plugin_info in manager.getPluginsOfCategory("Reporting"))
def test_console_reporting_plugin_found(): """Test that the plugin manager finds the console reporting plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')]) manager.setCategoriesFilter({ "Reporting": ReportingPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "c" assert any(plugin_info.plugin_object.get_name() == 'print_to_console' for plugin_info in manager.getPluginsOfCategory("Reporting")) # While we're at it, verify that a plugin is named C Discovery Plugin assert any(plugin_info.name == 'Print To Console Reporting Plugin' for plugin_info in manager.getPluginsOfCategory("Reporting"))
def test_maven_discovery_plugin_found(): """Test that the plugin manager finds the Maven discovery plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]) manager.setCategoriesFilter({ "Discovery": DiscoveryPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "java" assert any(plugin_info.plugin_object.get_name() == "maven" for plugin_info in manager.getPluginsOfCategory("Discovery")) # While we're at it, verify that a plugin is named Maven Discovery Plugin assert any(plugin_info.name == "Maven Discovery Plugin" for plugin_info in manager.getPluginsOfCategory("Discovery"))
def test_yaml_discovery_plugin_found(): """Test that the YAML discovery plugin is detected by the plugin system.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')]) manager.setCategoriesFilter({ "Discovery": DiscoveryPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "yaml" assert any(plugin_info.plugin_object.get_name() == 'yaml' for plugin_info in manager.getPluginsOfCategory("Discovery")) # While we're at it, verify that a plugin is named YAML Discovery Plugin assert any(plugin_info.name == 'YAML Discovery Plugin' for plugin_info in manager.getPluginsOfCategory("Discovery"))
def test_cccc_tool_plugin_found(): """Test that the CCCC tool plugin is detected by the plugin system.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]) manager.setCategoriesFilter({ "Tool": ToolPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "cccc" assert any(plugin_info.plugin_object.get_name() == "cccc" for plugin_info in manager.getPluginsOfCategory("Tool")) # While we're at it, verify that a plugin is named CCCC Tool Plugin assert any(plugin_info.name == "CCCC Tool Plugin" for plugin_info in manager.getPluginsOfCategory("Tool"))
def test_clang_tidy_tool_plugin_found(): """Test that the plugin manager can find the clang-tidy plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')]) manager.setCategoriesFilter({ "Tool": ToolPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "clang_tidy" assert any(plugin_info.plugin_object.get_name() == 'clang-tidy' for plugin_info in manager.getPluginsOfCategory("Tool")) # While we're at it, verify that a plugin is named ClangTidy Tool Plugin assert any(plugin_info.name == 'clang-tidy Tool Plugin' for plugin_info in manager.getPluginsOfCategory("Tool"))
def initialize_plugins(self): self.__log.info("Collecting and loading plugins") try: plugin_manager = PluginManager() # TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT) plugin_manager.setCategoriesFilter({ CARTRIDGE_AGENT_PLUGIN: ICartridgeAgentPlugin, ARTIFACT_MGT_PLUGIN: IArtifactManagementPlugin }) plugin_manager.setPluginPlaces([self.__config.read_property(constants.PLUGINS_DIR)]) plugin_manager.collectPlugins() # activate cartridge agent plugins plugins = plugin_manager.getPluginsOfCategory(CARTRIDGE_AGENT_PLUGIN) grouped_plugins = {} for plugin_info in plugins: self.__log.debug("Found plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) plugin_manager.activatePluginByName(plugin_info.name) self.__log.info("Activated plugin [%s]" % plugin_info.name) mapped_events = plugin_info.description.split(",") for mapped_event in mapped_events: if mapped_event.strip() != "": if grouped_plugins.get(mapped_event) is None: grouped_plugins[mapped_event] = [] grouped_plugins[mapped_event].append(plugin_info) # activate artifact management plugins artifact_mgt_plugins = plugin_manager.getPluginsOfCategory(ARTIFACT_MGT_PLUGIN) for plugin_info in artifact_mgt_plugins: self.__log.debug("Found artifact management plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) plugin_manager.activatePluginByName(plugin_info.name) self.__log.info("Activated artifact management plugin [%s]" % plugin_info.name) return plugin_manager, grouped_plugins, artifact_mgt_plugins except ParameterNotFoundException as e: self.__log.exception("Could not load plugins. Plugins directory not set: %s" % e) return None, None, None except Exception as e: self.__log.exception("Error while loading plugin: %s" % e) return None, None, None
def test_write_jenkins_warnings_ng_reporting_plugin_found(): """Test that the plugin manager finds the file writing plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]) manager.setCategoriesFilter({ "Reporting": ReportingPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "c" assert any( plugin_info.plugin_object.get_name() == "write_jenkins_warnings_ng" for plugin_info in manager.getPluginsOfCategory("Reporting")) # While we're at it, verify that a plugin is named C Discovery Plugin assert any(plugin_info.name == "Write Jenkins Warnings NG Reporting Plugin" for plugin_info in manager.getPluginsOfCategory("Reporting"))
def init_plugin_manager(): global tom_plugin_manager from engine import BuiltinEngine, AnsibleEngine, Engine if not holder.plugin_manager: tom_plugin_manager = PluginManager() tom_plugin_manager.setPluginPlaces(config.plugin_dirs) tom_plugin_manager.setPluginInfoExtension('plug') # 3 types of plugins, Built-in for plugin come with Tom, # Ansible for ansible simplerunner or playbook running on tom # User for user-defined plugin tom_plugin_manager.setCategoriesFilter({ 'Built-in': BuiltinEngine, 'Ansible': AnsibleEngine, 'User': Engine }) holder.plugin_manager = tom_plugin_manager else: tom_plugin_manager = holder.plugin_manager
def load_env_plugin(plugin_places=[misc.execution_path('plugins')], environment='cloudsight'): pm = PluginManager(plugin_info_ext='plugin') # Normalize the paths to the location of this file. # XXX-ricarkol: there has to be a better way to do this. plugin_places = [misc.execution_path(x) for x in plugin_places] pm.setPluginPlaces(plugin_places) pm.setCategoriesFilter({"RuntimeEnvironment" : IRuntimeEnvironment}) pm.collectPlugins() for env_plugin in pm.getAllPlugins(): # There should be only one plugin of the given category and type; # but in case there are more, pick the first one. if env_plugin.plugin_object.get_environment_name() == environment: return env_plugin.plugin_object raise RuntimeEnvironmentPluginNotFound('Could not find a valid runtime ' 'environment plugin at %s' % plugin_places)
def __init__(self): self.ver = "0.0.1" wx.Frame.__init__(self, None, -1, "xshock v" + self.ver, size = (1000, 700)) plugin_mger = PluginManager(categories_filter={"toolbar":IToolBarPlugin}) plugin_mger.setPluginPlaces(["plugins"]) plugin_mger.setCategoriesFilter({ "all":IPlugin, "toolbar":IToolBarPlugin, "Win":IWinPlugin, }) plugin_mger.collectPlugins() tbs = [] nbs = [] #tb_plugin_objs = plugin_mger.getPluginsOfCategory("all") tb_plugin_objs = plugin_mger.getAllPlugins() for obj in tb_plugin_objs: if isinstance (obj.plugin_object,IToolBarPlugin): tbs.append(obj) elif isinstance (obj.plugin_object,IWinPlugin): nbs.append(obj) self.tm = ECG_Timer() vbox = wx.BoxSizer(wx.VERTICAL) self.tb = self.createToolBar(tbs) vbox.Add(self.tb, 0 , wx.EXPAND) self.splitter = wx.SplitterWindow(self, -1, style = wx.SP_LIVE_UPDATE) self.ann_tree = LogTree(self.splitter) self.ecg_notebook = Notebook_div(self.splitter,nbs) self.splitter.SplitVertically(self.ann_tree, self.ecg_notebook, 200) vbox.Add(self.splitter, 2, wx.EXPAND) self.SetSizer(vbox)
def test_isort_tool_plugin_found(): """Test that the plugin manager can find the Isort plugin.""" if sys.version_info.major == 3 and sys.version_info.minor < 6: pytest.skip("isort is only available for Python 3.6+, unable to test") manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]) manager.setCategoriesFilter({ "Tool": ToolPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "isort" assert any(plugin_info.plugin_object.get_name() == "isort" for plugin_info in manager.getPluginsOfCategory("Tool")) # While we're at it, verify that a plugin is named Isort Tool Plugin assert any(plugin_info.name == "Isort Tool Plugin" for plugin_info in manager.getPluginsOfCategory("Tool"))
def testChangingCategoriesFilter(self): """ Test the effect of setting a new category filer. """ spm = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # load the plugins that may be found spm.collectPlugins() newCategory = "Mouf" # Pre-requisite for the test previousCategories = spm.getCategories() self.assertTrue(len(previousCategories) >= 1) self.assertTrue(newCategory not in previousCategories) # change the category and see what's happening spm.setCategoriesFilter({newCategory: IPlugin}) spm.collectPlugins() for categoryName in previousCategories: self.assertRaises(KeyError, spm.getPluginsOfCategory, categoryName) self.assertTrue(len(spm.getPluginsOfCategory(newCategory)) >= 1)
def load_env_plugin(plugin_places=[misc.execution_path('plugins')], environment='cloudsight'): pm = PluginManager(plugin_info_ext='plugin') # Normalize the paths to the location of this file. # XXX-ricarkol: there has to be a better way to do this. plugin_places = [misc.execution_path(x) for x in plugin_places] pm.setPluginPlaces(plugin_places) pm.setCategoriesFilter({"RuntimeEnvironment": IRuntimeEnvironment}) pm.collectPlugins() for env_plugin in pm.getAllPlugins(): # There should be only one plugin of the given category and type; # but in case there are more, pick the first one. if env_plugin.plugin_object.get_environment_name() == environment: return env_plugin.plugin_object raise RuntimeEnvironmentPluginNotFound( 'Could not find a valid "%s" runtime ' 'environment plugin at %s' % (environment, plugin_places))
class ExtensionLoader(object): def __init__(self, core_config): default_dir = os.path.join(os.path.expanduser("~"), "epops") self.plugin_path = core_config.get('plugin_dir', [default_dir]) self.simple_plugin_manager = PluginManager() self.simple_plugin_manager.setPluginPlaces(self.plugin_path) self.simple_plugin_manager.setPluginInfoExtension('plugin-meta') self.simple_plugin_manager.setCategoriesFilter({"Extension": IExtension}) self.pluginManager = AutoInstallPluginManager(self.plugin_path[0], self.simple_plugin_manager) def ext_init(self): self.simple_plugin_manager.collectPlugins() return self.simple_plugin_manager.getAllPlugins() def ext_load_new(self, filename): zip_path = os.path.join(self.plugin_path[0], filename) self.pluginManager.installFromZIP(zip_path) self.simple_plugin_manager.collectPlugins() return self.simple_plugin_manager.getAllPlugins()
def testChangingCategoriesFilter(self): """ Test the effect of setting a new category filer. """ spm = PluginManager(directories_list=[ os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins") ]) # load the plugins that may be found spm.collectPlugins() newCategory = "Mouf" # Pre-requisite for the test previousCategories = spm.getCategories() self.assertTrue(len(previousCategories) >= 1) self.assertTrue(newCategory not in previousCategories) # change the category and see what's happening spm.setCategoriesFilter({newCategory: IPlugin}) spm.collectPlugins() for categoryName in previousCategories: self.assertRaises(KeyError, spm.getPluginsOfCategory, categoryName) self.assertTrue(len(spm.getPluginsOfCategory(newCategory)) >= 1)
def _load_plugins(plugin_places=[misc.execution_path('plugins')], category_filter={}, filter_func=lambda *arg: True): pm = PluginManager(plugin_info_ext='plugin') # Normalize the paths to the location of this file. # XXX-ricarkol: there has to be a better way to do this. plugin_places = [misc.execution_path(x) for x in plugin_places] pm.setPluginPlaces(plugin_places) pm.setCategoriesFilter(category_filter) pm.collectPlugins() config = config_parser.get_config() enabled_plugins = [p for p in config['crawlers']] for plugin in pm.getAllPlugins(): if filter_func(plugin.plugin_object, plugin.name, enabled_plugins): plugin_args = {} if plugin.name in config['crawlers']: plugin_args = config['crawlers'][plugin.name] yield (plugin.plugin_object, plugin_args)
def test_print_json_reporting_plugin_found(): """Test that the plugin manager finds the print json reporting plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")] ) manager.setCategoriesFilter( { "Reporting": ReportingPlugin, } ) manager.collectPlugins() assert any( plugin_info.plugin_object.get_name() == "print_json" for plugin_info in manager.getPluginsOfCategory("Reporting") ) assert any( plugin_info.name == "Print JSON Reporting Plugin" for plugin_info in manager.getPluginsOfCategory("Reporting") )
def load_plugins(): """ Load plugin for environment. See lib/base.py """ # Create plugin manager manager = PluginManager() # Tell it the default place(s) where to find plugins manager.setPluginPlaces(["pylons_yapsy_plugin/plugins/"]) # Define the various categories corresponding to the different # kinds of plugins you have defined manager.setCategoriesFilter({ "Menu" : menu.Menu, "Inline" : inline.Inline, }) manager.locatePlugins() # Deactivate plugin # Список деактивированных плагинов из БД deactivatedPlugins = [plugin.name for plugin in\ s.query(DeactivatedPlugins).all()] # Список кандидатов на загрузку candidates = manager.getPluginCandidates() # Список деактивированных плагинов в формате yapsy deactivatedList = [] for candidate in candidates: if candidate[2].name in deactivatedPlugins: deactivatedList.append(candidate) # Исключаем плагины которые указанны в БД for plugin in deactivatedList: manager.removePluginCandidate(plugin) # Load plugins manager.loadPlugins() return manager, [plugin[2] for plugin in deactivatedList]
def test_groovylint_tool_plugin_found(): """Test that the plugin manager can find the groovylint plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")] ) manager.setCategoriesFilter( { "Tool": ToolPlugin, } ) manager.collectPlugins() # Verify that a plugin's get_name() function returns "groovylint" assert any( plugin_info.plugin_object.get_name() == "groovylint" for plugin_info in manager.getPluginsOfCategory("Tool") ) # While we're at it, verify that a plugin is named NPM Groovy Lint Tool Plugin assert any( plugin_info.name == "NPM Groovy Lint Tool Plugin" for plugin_info in manager.getPluginsOfCategory("Tool") )
def _load_plugins( category_filter={}, filter_func=lambda *arg: True, features=['os', 'cpu'], plugin_places=['plugins'], options={}): pm = PluginManager(plugin_info_ext='plugin') # Normalize the paths to the location of this file. # XXX-ricarkol: there has to be a better way to do this. plugin_places = [misc.execution_path(x) for x in plugin_places] pm.setPluginPlaces(plugin_places) pm.setCategoriesFilter(category_filter) pm.collectPlugins() config = config_parser.get_config() enabled_plugins = [] if 'enabled_plugins' in config['general']: enabled_plugins = config['general']['enabled_plugins'] if 'ALL' in enabled_plugins: enabled_plugins = [p for p in config['crawlers']] # Reading from 'crawlers' section inside crawler.conf # Alternatively, 'ALL' can be made to signify # all crawlers in plugins/* for plugin in pm.getAllPlugins(): if filter_func( plugin.plugin_object, plugin.name, enabled_plugins, features): plugin_args = get_plugin_args(plugin, config, options) yield (plugin.plugin_object, plugin_args)
class GiantCommands(object): _client_category = 'Client' _server_category = 'Server' def __init__(self): self.client_plugins = None self.server_plugins = None self._plugin_manager = PluginManager(plugin_info_ext='giant') self._plugin_manager.setCategoriesFilter({ GiantCommands._client_category : BaseGiantClient, GiantCommands._server_category: BaseGiantServer, }) path = os.path.dirname(os.path.realpath(__file__)) self._plugin_manager.setPluginPlaces([path + '/plugins']) self._plugin_manager.collectPlugins() self.plugin_infos = self._plugin_manager.getAllPlugins() def new_plugin(self, args): import getpass username = getpass.getuser() safe_language = (args.language .replace('#', 'Sharp') .replace('+', 'Plus') .replace(' ', '') .replace('-', '_') .replace('!', 'Exclamation') .replace('/', 'Slash') .replace('*', 'Star')) repo_name = safe_language.lower()+'_'+args.framework.lower() cookiecutter('gh:lixar/giant-plugin', extra_context={ 'full_name': username, 'project_language': args.language, 'project_framework': args.framework, 'project_type': args.type, 'repo_name': repo_name, 'command': repo_name.replace('_', '-') }) def install_plugin(self, args): import glob input_path = args.input_plugin plugin_file = glob.glob(input_path + '/*.giant') with open(plugin_file[0], 'r') as plugin: for line in plugin.readlines(): if line.startswith('Name'): plugin_name = line.split(' = ')[1].strip() if line.startswith('Type'): plugin_type = line.split(' = ')[1].strip() path = os.path.dirname(os.path.abspath(__file__)) plugins_dir = os.path.join(path, 'plugins') plugins_dir = os.path.join(plugins_dir, plugin_type.lower()) try: os.makedirs(plugins_dir) except: pass plugins_dir = os.path.join(plugins_dir, plugin_name) if args.symlink: try: os.symlink(os.path.abspath(args.input_plugin), plugins_dir) except: logging.error('Plugin with this name already exists.') else: import shutil shutil.copytree(os.path.abspath(args.input_plugin), plugins_dir) # TODO Update giant-1.0.0.dist-info/RECORD to have new files added. def uninstall_plugin(self, args): import shutil plugin = self._get_plugin(args.command_name) path = plugin.path while not path.endswith('client') and not path.endswith('server'): path, plugin_dirname = os.path.split(path) shutil.rmtree(os.path.join(path, plugin_dirname)) def get_client_plugins(self): if self.client_plugins == None: self.client_plugins = self._plugin_manager.getPluginsOfCategory(GiantCommands._client_category) return self.client_plugins def get_server_plugins(self): if self.server_plugins == None: self.server_plugins = self._plugin_manager.getPluginsOfCategory(GiantCommands._server_category) return self.server_plugins def get_all_plugins(self): return self.get_client_plugins() + self.get_server_plugins() def _get_plugin(self, command_name): return next(plugin for plugin in self.get_all_plugins() if plugin.details.get('Details', 'Command') == command_name) def generate_client(self, args, name): plugin_info = next(plugin_info for plugin_info in self.client_plugins if plugin_info.details.get('Details', 'Command') == name) self._plugin_manager.activatePluginByName(plugin_info.name) self._generate_project(args, plugin_info.plugin_object) def generate_server(self, args, name): plugin_info = next(plugin_info for plugin_info in self.server_plugins if plugin_info.details.get('Details', 'Command') == name) plugin = self._plugin_manager.activatePluginByName(plugin_info.name) self._generate_project(args, plugin_info.plugin_object) def _generate_project(self, args, plugin): if args.swagger_files == None: import requests response = requests.get(args.swagger_url) try: swagger = response.json() except: import yaml try: swagger = yaml.load(response.text) except: logging.critical("Failed to parse server response to JSON or YAML.") exit(1) else: swagger = {} for f in args.swagger_files: if os.path.isdir(f): files = os.listdir(f) else: files = [f] for swagger_file_path in files: with open(swagger_file_path, 'r') as swagger_file: try: api = json.load(swagger_file) except: swagger_file.seek(0) import yaml api = yaml.load(swagger_file) paths = {} for path_name, path in api['paths'].iteritems(): if len(args.swagger_files) > 1: paths[api['host'] + path_name] = path else: paths[path_name] = path api['paths'] = paths tag = f.split('.')[0] self.mergedicts(swagger, api, conflicts=['paths', 'definitions'], conflict_tag=tag) plugin.setup(swagger, args.output_dir, args.force_overwrite) print('Generating Project...') plugin.generate() def mergedicts(self, a, b, path=None, conflicts=[], conflict_tag=None): "merges b into a" if path is None: path = [] for key in b: if key in a: if isinstance(a[key], dict) and isinstance(b[key], dict): self.mergedicts(a[key], b[key], path + [str(key)], conflicts=conflicts, conflict_tag=conflict_tag) elif a[key] == b[key]: pass # same leaf value else: conflict_path = '.'.join(path + [str(key)]) if any(conflict in conflicts for conflict in conflict_path.split('.')): if conflict_tag is not None and key + conflict_tag not in a.keys(): a[key + conflict_tag] = b[key] continue # resolve conflict using tag. raise Exception('Conflict at ' + '.'.join(path + [str(key)])) else: print('Ignoring conflict ' + conflict_path) else: a[key] = b[key] return a
def main(): parser = optparse.OptionParser() parser.add_option("-l", "--LogConfigFile", dest="log_config_file", help="Logging Configuration file.") parser.add_option( "-p", "--PluginDirList", dest="plugin_directories", help="Comma separated directory list to search for plugins.") parser.add_option( "-s", "--StartDateTime", dest="start_date_time", help= "A date to re-run the predictions for, if not provided, the default is the current day. Format is YYYY-MM-DD HH:MM:SS." ) (options, args) = parser.parse_args() if (options.plugin_directories is None): parser.print_help() sys.exit(-1) if options.log_config_file: logging.config.fileConfig(options.log_config_file) logging.getLogger('yapsy').setLevel(logging.DEBUG) """ log_stop_event = Event() log_queue = Queue() log_config_plugin = { 'version': 1, 'disable_existing_loggers': True, 'handlers': { 'queue': { 'class': 'logutils.queue.QueueHandler', 'queue': log_queue, }, }, 'root': { 'level': 'NOTSET', 'handlers': ['queue'] } } lp = Process(target=listener_process, name='listener', args=(log_queue, log_stop_event, options.log_config_file)) lp.start() """ #logging.config.fileConfig(logfile) #logging.config.dictConfig(log_config_plugin) logger = logging.getLogger("WQ-Engine") logger.info("Log file opened.") plugin_dirs = options.plugin_directories.split(',') #plugin_dirs = (["plugins"]) # Build the manager simplePluginManager = PluginManager() # Tell it the default place(s) where to find plugins if logger: logger.debug("Plugin directories: %s" % (options.plugin_directories)) simplePluginManager.setCategoriesFilter( {"PredictionEngine": wq_prediction_engine_plugin}) simplePluginManager.setPluginPlaces(plugin_dirs) # Load all plugins if logger: logger.info("Begin loading plugins") simplePluginManager.collectPlugins() plugin_proc_start = time.time() cnt = 0 try: for plugin in simplePluginManager.getAllPlugins(): if logger: logger.info("Starting plugin: %s" % (plugin.name)) if plugin.plugin_object.initialize_plugin(ini=plugin.details.get( "Core", "Ini"), name=plugin.name): plugin.plugin_object.start() else: logger.error("Failed to initialize plugin: %s" % (plugin.name)) cnt += 1 #Wait for the plugings to finish up. if logger: logger.info("Waiting for %d plugins to complete." % (cnt)) for plugin in simplePluginManager.getAllPlugins(): plugin.plugin_object.join() if logger: logger.info("Plugins completed in %f seconds" % (time.time() - plugin_proc_start)) logger.debug("Shutting down logger.") """ log_stop_event.set() lp.join() """ except Exception as e: logger.exception(e) if logger: logger.info("Log file closed.")
try: from compatible.urlresolver.curlresolver import getPlugins as getURPlugins except: ur_plugins = False else: ur_plugins = True from yapsy.PluginManager import PluginManager from yapsy.PluginFileLocator import PluginFileLocator, PluginFileAnalyzerMathingRegex import re import os plugins = PluginManager(plugin_locator=PluginFileLocator([PluginFileAnalyzerMathingRegex("all", "^[a-zA-Z][a-zA-Z_]*\.py$")])) plugins.setCategoriesFilter({ "Page" : Page, }) def loadExtension(): # folder from this project path = os.path.dirname(os.path.abspath(__file__)) pathes = [] pathes.append(os.path.join(path, 'pages')) # folder from config # for con in ('page_extension_dir'): # c_path = config.get(con, "") # if len(c_path) > 1: # pathes.append(c_path) plugins.setPluginPlaces(pathes) plugins.collectPlugins()
class TestDef(object): def __init__(self): # set aside storage for options and cmd line args self.options = {} self.args = [] # record if we have loaded the plugins or # not - this is just a bozo check to ensure # someone doesn't tell us to do it twice self.loaded = False # set aside a spot for a logger object, and # note that it hasn't yet been defined self.logger = None self.modcmd = None self.execmd = None self.harasser = None self.config = None self.stages = None self.tools = None self.utilities = None self.defaults = None self.log = {} self.watchdog = None self.plugin_trans_sem = Semaphore() def setOptions(self, args): self.options = vars(args) self.args = args # private function to convert values def __convert_value(self, opt, inval): if opt is None or type(opt) is str: return 0, inval elif type(opt) is bool: if type(inval) is bool: return 0, inval elif type(inval) is str: if inval.lower() in ['true', '1', 't', 'y', 'yes']: return 0, True else: return 0, False elif type(inval) is int: if 0 == inval: return 0, False else: return 0, True elif is_py2 and type(inval) is unicode: return 0, int(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None elif type(opt) is int: if type(inval) is int: return 0, inval elif type(inval) is str: return 0, int(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None elif type(opt) is float: if type(inval) is float: return 0, inval elif type(inval) is str or type(inval) is int: return 0, float(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None else: return 1, None # scan the key-value pairs obtained from the configuration # parser and compare them with the options defined for a # given plugin. Generate an output dictionary that contains # the updated set of option values, the default value for # any option that wasn't included in the configuration file, # and return an error status plus output identifying any # keys in the configuration file that are not supported # by the list of options # # @log [INPUT] # - a dictionary that will return the status plus # stderr containing strings identifying any # provided keyvals that don't have a corresponding # supported option # @options [INPUT] # - a dictionary of tuples, each consisting of three # entries: # (a) the default value # (b) data type # (c) a help-like description # @keyvals [INPUT] # - a dictionary of key-value pairs obtained from # the configuration parser # @target [OUTPUT] # - the resulting dictionary of key-value pairs def parseOptions(self, log, options, keyvals, target): # parse the incoming keyvals dictionary against the source # options. If a source option isn't provided, then # copy it across to the target. opts = list(options.keys()) kvkeys = list(keyvals.keys()) for opt in opts: found = False for kvkey in kvkeys: if kvkey == opt: # they provided us with an update, so # pass this value into the target - expand # any provided lists if keyvals[kvkey] is None: continue st, outval = self.__convert_value(options[opt][0], keyvals[kvkey]) if 0 == st: target[opt] = outval else: if len(keyvals[kvkey]) == 0: # this indicates they do not want this option found = True break if keyvals[kvkey][0][0] == "[": # they provided a list - remove the brackets val = keyvals[kvkey].replace('[','') val = val.replace(']','') # split the input to pickup sets of options newvals = list(val) # convert the values to specified type i=0 for val in newvals: st, newvals[i] = self.__convert_value(opt[0], val) i = i + 1 target[opt] = newvals else: st, target[opt] = self.__convert_value(opt[0], keyvals[kvkey]) found = True break if not found: # they didn't provide this one, so # transfer only the value across target[opt] = options[opt][0] # add in any default settings that have not # been overridden - anything set by this input # stage will override the default if self.defaults is not None: keys = self.defaults.options.keys() for key in keys: if key not in target: target[key] = self.defaults.options[key][0] # now go thru in the reverse direction to see # if any keyvals they provided aren't supported # as this would be an error unsupported_options = [] for kvkey in kvkeys: # ignore some standard keys if kvkey in ['section', 'plugin']: continue try: if target[kvkey] is not None: pass except KeyError: # some always need to be passed if kvkey in ['parent', 'asis']: target[kvkey] = keyvals[kvkey] else: unsupported_options.append(kvkey) if unsupported_options: sys.exit("ERROR: Unsupported options for section [%s]: %s" % (log['section'], ",".join(unsupported_options))) log['status'] = 0 log['options'] = target return def loadPlugins(self, basedir, topdir): if self.loaded: print("Cannot load plugins multiple times") sys.exit(1) self.loaded = True # find the loader utility so we can bootstrap ourselves try: m = imp.load_source("LoadClasses", os.path.join(basedir, "LoadClasses.py")); except ImportError: print("ERROR: unable to load LoadClasses that must contain the class loader object") sys.exit(1) cls = getattr(m, "LoadClasses") a = cls() # setup the loader object self.loader = a.__class__(); # Setup the array of directories we will search for plugins # Note that we always look at the topdir location by default plugindirs = [] plugindirs.append(topdir) if self.options['plugindir']: # could be a comma-delimited list, so split on commas x = self.options['plugindir'].split(',') for y in x: # prepend so we always look at the given # location first in case the user wants # to "overload/replace" a default MTT # class definition plugindirs.insert(0, y) # Load plugins from each of the specified plugin dirs for dirPath in plugindirs: if not Path(dirPath).exists(): print("Attempted to load plugins from non-existent path:", dirPath) continue try: self.loader.load(dirPath) except Exception as e: print("Exception caught while loading plugins:") print(e) sys.exit(1) # Build plugin managers, # class yapsy.PluginManager.PluginManager(categories_filter=None, # directories_list=None, plugin_info_ext=None, plugin_locator=None) # Build the stages plugin manager self.stages = PluginManager(None, plugindirs, None, None) # Get a list of all the categories - this corresponds to # the MTT stages that have been defined. Note that we # don't need to formally define the stages here - anyone # can add a new stage, or delete an old one, by simply # adding or removing a plugin directory. self.stages.setCategoriesFilter(self.loader.stages) # Load all plugins we find there self.stages.collectPlugins() # Build the tools plugin manager - tools differ from sections # in that they are plugins we will use to execute the various # sections. For example, the TestRun section clearly needs the # ability to launch jobs. There are many ways to launch jobs # depending on the environment, and sometimes several ways to # start jobs even within one environment (e.g., mpirun vs # direct launch). self.tools = PluginManager(None, plugindirs, None, None) # Get the list of tools - not every tool will be capable # of executing. For example, a tool that supports direct launch # against a specific resource manager cannot be used on a # system being managed by a different RM. self.tools.setCategoriesFilter(self.loader.tools) # Load all the tool plugins self.tools.collectPlugins() # Tool plugins are required to provide a function we can # probe to determine if they are capable of operating - check # those now and prune those tools that cannot support this # environment # Build the utilities plugins self.utilities = PluginManager(None, plugindirs, None, None) # Get the list of available utilities. self.utilities.setCategoriesFilter(self.loader.utilities) # Load all the utility plugins self.utilities.collectPlugins() # since we use these all over the place, find the # ExecuteCmd and ModuleCmd plugins and record them availUtil = list(self.loader.utilities.keys()) for util in availUtil: for pluginInfo in self.utilities.getPluginsOfCategory(util): if "ExecuteCmd" == pluginInfo.plugin_object.print_name(): self.execmd = pluginInfo.plugin_object elif "ModuleCmd" == pluginInfo.plugin_object.print_name(): self.modcmd = pluginInfo.plugin_object # initialize this module self.modcmd.setCommand(self.options) elif "Watchdog" == pluginInfo.plugin_object.print_name(): self.watchdog = pluginInfo.plugin_object if self.execmd is not None and self.modcmd is not None and self.watchdog is not None: break if self.execmd is None: print("ExecuteCmd plugin was not found") print("This is a basic capability required") print("for MTT operations - cannot continue") sys.exit(1) # Configure harasser plugin for pluginInfo in self.tools.getPluginsOfCategory("Harasser"): if "Harasser" == pluginInfo.plugin_object.print_name(): self.harasser = pluginInfo.plugin_object break if self.harasser is None: print("Harasser plugin was not found") print("This is required for all TestRun plugins") print("cannot continue") sys.exit(1) # similarly, capture the highest priority defaults stage here pri = -1 for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"): if pri < pluginInfo.plugin_object.priority(): self.defaults = pluginInfo.plugin_object pri = pluginInfo.plugin_object.priority() return def printInfo(self): # Print the available MTT sections out, if requested if self.options['listsections']: print("Supported MTT stages:") # print them in the default order of execution for stage in self.loader.stageOrder: print(" " + stage) sys.exit(0) # Print the detected plugins for a given stage if self.options['listplugins']: # if the list is '*', print the plugins for every stage if self.options['listplugins'] == "*": sections = self.loader.stageOrder else: sections = self.options['listplugins'].split(',') print() for section in sections: print(section + ":") try: for pluginInfo in self.stages.getPluginsOfCategory(section): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid stage name " + section) print() sys.exit(1) # Print the options for a given plugin if self.options['liststageoptions']: # if the list is '*', print the options for every stage/plugin if self.options['liststageoptions'] == "*": sections = self.loader.stageOrder else: sections = self.options['liststageoptions'].split(',') print() for section in sections: print(section + ":") try: for pluginInfo in self.stages.getPluginsOfCategory(section): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options(self, " ") except KeyError: print(" Invalid stage name " + section) print() sys.exit(1) # Print the available MTT tools out, if requested if self.options['listtools']: print("Available MTT tools:") availTools = list(self.loader.tools.keys()) for tool in availTools: print(" " + tool) sys.exit(0) # Print the detected tool plugins for a given tool type if self.options['listtoolmodules']: # if the list is '*', print the plugins for every type if self.options['listtoolmodules'] == "*": print() availTools = list(self.loader.tools.keys()) else: availTools = self.options['listtoolmodules'].split(',') print() for tool in availTools: print(tool + ":") try: for pluginInfo in self.tools.getPluginsOfCategory(tool): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid tool type name",tool) print() sys.exit(1) # Print the options for a given plugin if self.options['listtooloptions']: # if the list is '*', print the options for every stage/plugin if self.options['listtooloptions'] == "*": availTools = list(self.loader.tools.keys()) else: availTools = self.options['listtooloptions'].split(',') print() for tool in availTools: print(tool + ":") try: for pluginInfo in self.tools.getPluginsOfCategory(tool): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options(self, " ") except KeyError: print(" Invalid tool type name " + tool) print() sys.exit(1) # Print the available MTT utilities out, if requested if self.options['listutils']: print("Available MTT utilities:") availUtils = list(self.loader.utilities.keys()) for util in availUtils: print(" " + util) sys.exit(0) # Print the detected utility plugins for a given tool type if self.options['listutilmodules']: # if the list is '*', print the plugins for every type if self.options['listutilmodules'] == "*": print() availUtils = list(self.loader.utilities.keys()) else: availUtils = self.options['listutilitymodules'].split(',') print() for util in availUtils: print(util + ":") try: for pluginInfo in self.utilities.getPluginsOfCategory(util): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid utility type name") print() sys.exit(1) # Print the options for a given plugin if self.options['listutiloptions']: # if the list is '*', print the options for every stage/plugin if self.options['listutiloptions'] == "*": availUtils = list(self.loader.utilities.keys()) else: availUtils = self.options['listutiloptions'].split(',') print() for util in availUtils: print(util + ":") try: for pluginInfo in self.utilities.getPluginsOfCategory(util): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options(self, " ") except KeyError: print(" Invalid utility type name " + util) print() sys.exit(1) # if they asked for the version info, print it and exit if self.options['version']: for pluginInfo in self.tools.getPluginsOfCategory("Version"): print("MTT Base: " + pluginInfo.plugin_object.getVersion()) print("MTT Client: " + pluginInfo.plugin_object.getClientVersion()) sys.exit(0) def openLogger(self): # there must be a logger utility or we can't do # anything useful if not self.utilities.activatePluginByName("Logger", "Base"): print("Required Logger plugin not found or could not be activated") sys.exit(1) # execute the provided test description self.logger = self.utilities.getPluginByName("Logger", "Base").plugin_object self.logger.open(self) return def fill_log_interpolation(self, basestr, sublog): if isinstance(sublog, str): self.config.set("LOG", basestr, sublog.replace("$","$$")) elif isinstance(sublog, dict): for k,v in sublog.items(): self.fill_log_interpolation("%s.%s" % (basestr, k), v) elif isinstance(sublog, list): if sum([((isinstance(t, list) or isinstance(t, tuple)) and len(t) == 2) for t in sublog]) == len(sublog): self.fill_log_interpolation(basestr, {k:v for k,v in sublog}) else: for i,v in enumerate(sublog): self.fill_log_interpolation("%s.%d" % (basestr, i), v) else: self.fill_log_interpolation(basestr, str(sublog)) def expandWildCards(self, sections): expsec = [] cpsections = list(sections) for sec in cpsections: if '*' in sec: modsec = sec.split('*') startswith = modsec[0] endswith = modsec[-1] findsec = modsec[1:-1] allsections = self.config.sections() for s in allsections: if not s.startswith(startswith): continue if not s.endswith(endswith): continue found = True s_tmp = s for f in findsec: if not f in s_tmp: found = False break s_tmp = f.join(s_tmp.split(f)[1:]) if not found: continue expsec.append(s) sections.remove(sec) return sections + expsec def fill_env_hidden_section(self): """fill ENV section with environment variables """ try: self.config.add_section('ENV') except configparser.DuplicateSectionError: pass for k,v in os.environ.items(): self.config.set('ENV', k, v.replace("$","$$")) def fill_log_hidden_section(self): """Add LOG section filled with log results of stages """ try: self.config.add_section('LOG') except configparser.DuplicateSectionError: pass thefulllog = self.logger.getLog(None) for e in thefulllog: self.fill_log_interpolation(e['section'].replace(":","_"), e) def check_for_nondefined_env_variables(self): # Check for ENV input required_env = [] all_file_contents = [] for testFile in self.log['inifiles']: file_contents = open(testFile, "r").read() file_contents = "\n".join(["%s %d: %s" % (testFile.split("/")[-1],i,l) for i,l in enumerate(file_contents.split("\n")) if not l.lstrip().startswith("#")]) all_file_contents.append(file_contents) if "${ENV:" in file_contents: required_env.extend([s.split("}")[0] for s in file_contents.split("${ENV:")[1:]]) env_not_found = set([e for e in required_env if e not in os.environ.keys()]) lines_with_env_not_found = [] for file_contents in all_file_contents: lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \ for l in file_contents.split("\n") \ if sum(["${ENV:%s}"%e in l for e in env_not_found])]) if lines_with_env_not_found: print("ERROR: Not all required environment variables are defined.") print("ERROR: Still need:") for l in lines_with_env_not_found: print("ERROR: %s"%l) sys.exit(1) def configTest(self): # setup the configuration parser self.config = configparser.SafeConfigParser(interpolation=configparser.ExtendedInterpolation()) # Set the config parser to make option names case sensitive. self.config.optionxform = str # fill ENV section with environemt variables self.fill_env_hidden_section() # log the list of files - note that the argument parser # puts the input files in a list, with the first member # being the list of input files self.log['inifiles'] = self.args.ini_files[0] # initialize the list of active sections self.actives = [] # if they specified a list to execute, then use it sections = [] if self.args.section: sections = self.args.section.split(",") skip = False elif self.args.skipsections: sections = self.args.skipsections.split(",") skip = True else: sections = None # cycle thru the input files for testFile in self.log['inifiles']: if not os.path.isfile(testFile): print("Test description file",testFile,"not found!") sys.exit(1) self.config.read(self.log['inifiles']) # Check for ENV input self.check_for_nondefined_env_variables() # find all the sections that match the wild card and expand them # this is simple wild carding, ie *text, text*, *text* and * # should all work if sections is not None: sections = self.expandWildCards(sections) #if sections is not None: # expsec = [] # cpsections = list(sections) # for sec in cpsections: # if '*' in sec: # modsec = sec.replace('*','') # for s in self.config.sections(): # if modsec in s: # expsec.append(s) # sections.remove(sec) # sections = sections + expsec for section in self.config.sections(): if section.startswith("SKIP") or section.startswith("skip"): # users often want to temporarily ignore a section # of their test definition file, but don't want to # remove it lest they forget what it did. So let # them just mark the section as "skip" to be ignored continue # if we are to filter the sections, then do so takeus = True if sections is not None: found = False for sec in sections: if sec == section: found = True sections.remove(sec) if skip: takeus = False break if not found and not skip: takeus = False if takeus: self.actives.append(section) if sections is not None and 0 != len(sections) and not skip: print("ERROR: sections were specified for execution and not found:",sections) sys.exit(1) # set Defaults -command line args supercede .ini args try: if not self.options['scratchdir']: self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratchdir') except: try: self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratch') except: self.options['scratchdir'] = os.path.abspath('./mttscratch') self.options['scratchdir'] = os.path.abspath(self.options['scratchdir']) try: if not self.options['executor']: self.options['executor'] = self.config.get('MTTDefaults', 'executor') except: self.options['executor'] = 'sequential' # if they want us to clear the scratch, then do so if self.options['clean'] and os.path.isdir(self.options['scratchdir']) : shutil.rmtree(self.options['scratchdir']) # setup the scratch directory _mkdir_recursive(self.options['scratchdir']) return # Used with combinatorial executor, loads next .ini file to be run with the # sequential executor def configNewTest(self, file): # clear the configuration parser for section in self.config.sections(): self.config.remove_section(section) # read in the file self.config.read(file) for section in self.config.sections(): if section.startswith("SKIP") or section.startswith("skip"): # users often want to temporarily ignore a section # of their test definition file, but don't want to # remove it lest they forget what it did. So let # them just mark the section as "skip" to be ignored continue if self.logger is not None: self.logger.verbose_print("SECTION: " + section) self.logger.verbose_print(self.config.items(section)) return def executeTest(self, executor="sequential"): self.logger.print_cmdline_args(self) if not self.loaded: print("Plugins have not been loaded - cannot execute test") sys.exit(1) if self.config is None: print("No test definition file was parsed - cannot execute test") sys.exit(1) if not self.tools.getPluginByName(executor, "Executor"): print("Specified executor %s not found" % executor) sys.exit(1) # activate the specified plugin self.tools.activatePluginByName(executor, "Executor") # execute the provided test description executor = self.tools.getPluginByName(executor, "Executor") status = executor.plugin_object.execute(self) if status == 0 and self.options['clean_after'] and os.path.isdir(self.options['scratchdir']): self.logger.verbose_print("Cleaning up scratchdir after successful run") shutil.rmtree(self.options['scratchdir']) return status def printOptions(self, options): # if the options are empty, report that if not options: lines = ["None"] return lines # create the list of options opts = [] vals = list(options.keys()) for val in vals: opts.append(val) if options[val][0] is None: opts.append("None") elif isinstance(options[val][0], bool): if options[val][0]: opts.append("True") else: opts.append("False") elif isinstance(options[val][0], list): opts.append(" ".join(options[val][0])) elif isinstance(options[val][0], int): opts.append(str(options[val][0])) else: opts.append(options[val][0]) opts.append(options[val][1]) # print the options, their default value, and # the help description in 3 column format max1 = 0 max2 = 0 for i in range(0,len(opts),3): # we want all the columns to line up # and left-justify, so first find out # the max len of each of the first two # column entries if len(opts[i]) > max1: max1 = len(opts[i]) if type(opts[i+1]) is not str: optout = str(opts[i+1]) else: optout = opts[i+1] if len(optout) > max2: max2 = len(optout) # provide some spacing max1 = max1 + 4 max2 = max2 + 4 # cycle thru again, padding each entry to # align the columns lines = [] sp = " " for i in range(0,len(opts),3): line = opts[i] + (max1-len(opts[i]))*sp if type(opts[i+1]) is not str: optout = str(opts[i+1]) else: optout = opts[i+1] line = line + optout + (max2-len(optout))*sp # to make this more readable, we will wrap the line at # 130 characters. First, see if the line is going to be # too long if 130 < (len(line) + len(opts[i+2])): # split the remaining column into individual words words = opts[i+2].split() first = True for word in words: if (len(line) + len(word)) < 130: if first: line = line + word first = False else: line = line + " " + word else: lines.append(line) line = (max1 + max2)*sp + word if 0 < len(line): lines.append(line) else: # the line is fine - so just add the last piece line = line + opts[i+2] # append the result lines.append(line) # add one blank line lines.append("") return lines def selectPlugin(self, name, category): if category == "stage": try: availStages = list(self.loader.stages.keys()) for stage in availStages: for pluginInfo in self.stages.getPluginsOfCategory(stage): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None elif category == "tool": try: availTools = list(self.loader.tools.keys()) for tool in availTools: for pluginInfo in self.tools.getPluginsOfCategory(tool): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None elif category == "utility": try: availUtils = list(self.loader.utilities.keys()) for util in availUtils: for pluginInfo in self.utilities.getPluginsOfCategory(util): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None else: print("Unrecognized category:",category) return None
if category == 'Default features': plugin = pluginManager.getPluginByName("Standard Object Features", "ObjectFeatures") else: plugin = pluginManager.getPluginByName(category, "ObjectFeatures") if plugin: plugin_feature_names = {name: {}} all_props = plugin.plugin_object.fill_properties(plugin_feature_names) # fill in display name and such if all_props: long_name = all_props[name]["displaytext"] else: long_name = name return long_name ############### # the manager # ############### pluginManager = PluginManager() pluginManager.setPluginPlaces(plugin_paths) pluginManager.setCategoriesFilter({ "ObjectFeatures" : ObjectFeaturesPlugin, "TrackingExportFormats": TrackingExportFormatPlugin }) pluginManager.collectPlugins() for pluginInfo in pluginManager.getAllPlugins(): pluginManager.activatePluginByName(pluginInfo.name)
class PluginSystem: def __init__(self, config=None, extra_plugin_paths=[]): 'Initialize a new PluginSystem. Needs no arguments.' # Build the manager self._mngr = PluginManager(plugin_info_ext='plugin') # Tell it the default place(s) where to find plugins plugin_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'plugins') self._mngr.setPluginPlaces([plugin_path] + extra_plugin_paths) # find the categories specified in moosecat/plugins/__init__.py self._categories = _get_interfaces_from_module() LOGGER.debug('Discovered following plugins categories: ' + ', '.join(self.list_categories())) # tell yapsy about our extra categories self._mngr.setCategoriesFilter(self._categories) try: self._mngr.collectPlugins() except SystemError as e: LOGGER.exception('Some plugin could not be loaded.') # Get a list of plugin names to load load_these = [pluginInfo.name for pluginInfo in self._mngr.getAllPlugins()] if config is not None: config.add_defaults({'plugins_to_load': load_these}) load_these = config.get('plugins_to_load') # Actually load them for name in load_these: self._mngr.activatePluginByName(name) def list_plugin_info(self): ''' Get a list of PluginInfo objects. These contain additionnal metadata found in the .plugin files. See here for more information: http://yapsy.sourceforge.net/PluginInfo.html Useful if you want to realize a list of Plugins. :returns: A list of PluginInfo objects ''' return self._mngr.getAllPlugins() def list_plugin_info_by_category(self, name): return self._mngr.getPluginsOfCategory(name) def list_categories(self): 'Get a string list of categories.' return self._categories.keys() def by_name(self, plugin_name): 'Access a Plugin by its actual name - discouraged' return self._mngr.getPluginByName(plugin_name) @functools.lru_cache(maxsize=25) def category(self, name): ''' Get a list of all Plugins of a Category. :name: The name of a category. :returns: A list of instances of a certain Plugin Classes. ''' cat = [info.plugin_object for info in self.list_plugin_info_by_category(name)] cat.sort(key=lambda pobj: pobj.priority(), reverse=True) return cat def first(self, name): ''' Shortcut for ``psys.categories(name)[0]`` Will return None if no plugins for this category. :name: A Category Name, Same as in ``category()`` :returns: An Instance of a certain Plugin Class. ''' cat = self.category(name) if len(cat) > 0: return cat[0]
import os import codecs from yapsy.PluginManager import PluginManager from pendium.plugins import IRenderPlugin, ISearchPlugin from pendium import app import logging logger = logging.getLogger(__name__) # Populate plugins manager = PluginManager() manager.setPluginPlaces(["pendium/plugins"]) manager.setCategoriesFilter({"Search": ISearchPlugin, "Render": IRenderPlugin}) manager.collectPlugins() class PathExists(Exception): pass class PathNotFound(Exception): pass class CannotRender(Exception): pass class NoSearchPluginAvailable(Exception):
"""Helper for features that only take one channel. :param fn: function that computes features """ results = [] slc = [slice(None)] * 4 for channel in range(image.shape[axes.c]): slc[axes.c] = channel #a dictionary for the channel result = fn(image[slc], axes=axes, **kwargs) results.append(result) return self.combine_dicts_with_numpy(results) ############### # the manager # ############### pluginManager = PluginManager() pluginManager.setPluginPlaces(plugin_paths) pluginManager.setCategoriesFilter({ "ObjectFeatures": ObjectFeaturesPlugin, }) pluginManager.collectPlugins() for pluginInfo in pluginManager.getAllPlugins(): pluginManager.activatePluginByName(pluginInfo.name)
if not os.path.isdir(app.config['UPLOAD_FOLDER']): os.makedirs(app.config['UPLOAD_FOLDER'], 0o770) app.jinja_env.filters['fixify'] = utils.printable from flaskext.markdown import Markdown from markdown.extensions.toc import TocExtension Markdown(app, extensions=['extra', 'codehilite', QuotedExtension(), TocExtension(baselevel=2, marker='')], lazy_ol=False) logging.basicConfig(level=logging.INFO) logger = logging.getLogger("quoted-forsooth") logging.getLogger("werkzeug").setLevel(logging.WARNING) manager = PluginManager() manager.setPluginPlaces([os.path.join(os.path.dirname(__file__), 'plugins')]) manager.setCategoriesFilter({ 'Report' : report_plugin.ReportPlugin }) manager.collectPlugins() with app.app_context(): for plugin in manager.getAllPlugins(): manager.activatePluginByName(plugin.name, plugin.category) @app.route('/files/<path:path>') def send_file(path): return send_from_directory(app.config['UPLOAD_FOLDER'], path) @app.after_request def after_request(response): if db.config['file_store']['type'] == 'xenodermus': response.headers.add('Accept-Ranges', 'bytes') return response
def main(): parser = optparse.OptionParser() parser.add_option("-l", "--LogConfigFile", dest="log_config_file", help="Logging Configuration file." ) parser.add_option("-p", "--PluginDirList", dest="plugin_directories", help="Comma separated directory list to search for plugins.") parser.add_option("-s", "--StartDateTime", dest="start_date_time", help="A date to re-run the predictions for, if not provided, the default is the current day. Format is YYYY-MM-DD HH:MM:SS." ) (options, args) = parser.parse_args() if(options.plugin_directories is None): parser.print_help() sys.exit(-1) if options.log_config_file: logging.config.fileConfig(options.log_config_file) logging.getLogger('yapsy').setLevel(logging.DEBUG) """ log_stop_event = Event() log_queue = Queue() log_config_plugin = { 'version': 1, 'disable_existing_loggers': True, 'handlers': { 'queue': { 'class': 'logutils.queue.QueueHandler', 'queue': log_queue, }, }, 'root': { 'level': 'NOTSET', 'handlers': ['queue'] } } lp = Process(target=listener_process, name='listener', args=(log_queue, log_stop_event, options.log_config_file)) lp.start() """ #logging.config.fileConfig(logfile) #logging.config.dictConfig(log_config_plugin) logger = logging.getLogger("WQ-Engine") logger.info("Log file opened.") plugin_dirs = options.plugin_directories.split(',') #plugin_dirs = (["plugins"]) # Build the manager simplePluginManager = PluginManager() # Tell it the default place(s) where to find plugins if logger: logger.debug("Plugin directories: %s" % (options.plugin_directories)) simplePluginManager.setCategoriesFilter({ "PredictionEngine" : wq_prediction_engine_plugin }) simplePluginManager.setPluginPlaces(plugin_dirs) # Load all plugins if logger: logger.info("Begin loading plugins") simplePluginManager.collectPlugins() plugin_proc_start = time.time() cnt = 0 try: for plugin in simplePluginManager.getAllPlugins(): if logger: logger.info("Starting plugin: %s" % (plugin.name)) if plugin.plugin_object.initialize_plugin(ini=plugin.details.get("Core", "Ini"), name=plugin.name): plugin.plugin_object.start() else: logger.error("Failed to initialize plugin: %s" % (plugin.name)) cnt += 1 #Wait for the plugings to finish up. if logger: logger.info("Waiting for %d plugins to complete." % (cnt)) for plugin in simplePluginManager.getAllPlugins(): plugin.plugin_object.join() if logger: logger.info("Plugins completed in %f seconds" % (time.time() - plugin_proc_start)) logger.debug("Shutting down logger.") """ log_stop_event.set() lp.join() """ except Exception as e: logger.exception(e) if logger: logger.info("Log file closed.")
m = getattr(m.storage, storage_package) STORAGE = getattr(m, storage_class)(CONFIG['storage config']) STORAGE.load_data() # plugin system from yapsy.PluginManager import PluginManager # Build the manager pluginManager = PluginManager() # Tell it the default place(s) where to find plugins myPath = os.path.dirname(__file__) modulePath = os.path.join(myPath, "plugins") pluginManager.setPluginPlaces([modulePath]) # Tell it about our categories pluginManager.setCategoriesFilter({ 'WebPlugin': telescope.plugins.abstract.ITelescopeWebPlugin, 'AnnouncePlugin': telescope.plugins.abstract.ITelescopeAnnouncePlugin }) # Load all plugins pluginManager.collectPlugins() # Activate all loaded plugins for pluginInfo in pluginManager.getAllPlugins(): pluginManager.activatePluginByName(pluginInfo.name) PLUGIN_MANAGER = pluginManager def go(): import bottle import telescope.handler
class WeaponSystem(rpyc.Service): ''' RPC Services: This is the code that does the actual password cracking and returns the results to orbital control. Currently only supports cracking using rainbow tables (RCrackPy) ''' is_initialized = False mutex = Lock() is_busy = False job_id = None def initialize(self): ''' Initializes variables, this should only be called once ''' logging.info("Weapon system initializing ...") self.plugin_manager = PluginManager() self.plugin_manager.setPluginPlaces(["plugins/"]) self.plugin_manager.setCategoriesFilter(FILTERS) self.plugin_manager.collectPlugins() self.plugins = {} logging.info( "Loaded %d plugin(s)" % len(self.plugin_manager.getAllPlugins()) ) self.__cpu__() logging.info("Weapon system online, good hunting.") @atomic def on_connect(self): ''' Called when successfully connected ''' if not self.is_initialized: self.initialize() self.is_initialized = True logging.info("Uplink to orbital control active") def on_disconnect(self): ''' Called if the connection is lost/disconnected ''' logging.info("Disconnected from orbital command server.") def __cpu__(self): ''' Detects the number of CPU cores on a system (including virtual cores) ''' if cpu_count is not None: try: self.cpu_cores = cpu_count() logging.info("Detected %d CPU core(s)" % self.cpu_cores) except NotImplementedError: logging.error("Could not detect number of processors; assuming 1") self.cpu_cores = 1 else: try: self.cpu_cores = int(sysconf("SC_NPROCESSORS_CONF")) logging.info("Detected %d CPU core(s)" % self.cpu_cores) except ValueError: logging.error("Could not detect number of processors; assuming 1") self.cpu_cores = 1 ############################ [ EXPOSED METHODS ] ############################ @atomic def exposed_crack(self, plugin_name, job_id, hashes, **kwargs): ''' Exposes plugins calls ''' assert plugin_name in self.plugins self.is_busy = True self.job_id = job_id self.plugin_manager.activatePluginByName(plugin_name) plugin = self.plugin_manager.getPluginByName(plugin_name) results = plugin.execute(hashes, **kwargs) self.plugin_manager.deactivatePluginByName(plugin_name) self.job_id = None self.is_busy = False return results def exposed_get_plugin_names(self): ''' Returns what algorithms can be cracked ''' logging.info("Method called: exposed_get_capabilities") plugins = self.plugin_manager.getAllPlugins() return [plugin.name for plugin in plugins] def exposed_get_categories(self): ''' Return categories for which we have plugins ''' categories = [] for category in self.plugin_manager.getCategories(): if 0 < len(self.plugin_manager.getPluginsOfCategory(category)): categories.append(category) return categories def exposed_get_category_plugins(self, category): ''' Get plugin names for a category ''' plugins = self.plugin_manager.getPluginsOfCategory(category) return [plugin.name for plugin in plugins] def exposed_get_plugin_details(self, category, plugin_name): ''' Get plugin based on name details ''' plugin = self.plugin_manager.getPluginByName(plugin_name, category) info = {'name': plugin.name} info['author'] = plugin.details.get('Documentation', 'author') info['website'] = plugin.details.get('Documentation', 'website') info['version'] = plugin.details.get('Documentation', 'version') info['description'] = plugin.details.get('Documentation', 'description') info['copyright'] = plugin.details.get('Documentation', 'copyright') info['precomputation'] = plugin.details.getboolean('Core', 'precomputation') return info def exposed_ping(self): ''' Returns a pong message ''' return "PONG" def exposed_is_busy(self): ''' Returns True/False if the current system is busy (thread safe) ''' return self.is_busy def exposed_current_job_id(self): ''' Returns the current job id (thread safe) ''' return self.job_id def exposed_cpu_count(self): ''' Returns the number of detected cpu cores ''' return self.cpu_cores
class PluginHandler: """ Handles management of provider, postprocessor and converter plugins. .. autosummary:: activate_plugins_by_category deactivate_plugins_by_category get_plugins_from_category is_activated Categories are Provider, Converter and Postprocessor. """ def __init__(self): self._plugin_manager = PluginManager() self._category_active = { 'Provider': False, 'Converter': False, 'Postprocessor': False } self._plugin_from_category = { 'Provider': [], 'Converter': [], 'Postprocessor': [] } self._provider_plugins = [] self._converter_plugins = [] self._postprocessor_plugins = [] self._collect_all_plugins() def _collect_all_plugins(self): """ Collect all provider, converter and postprocessor plugins. """ self._plugin_manager.setPluginPlaces([ 'hugin/harvest/provider', 'hugin/harvest/converter', 'hugin/harvest/postprocessor' ]) # setting filter categories for pluginmanager self._plugin_manager.setCategoriesFilter({ # movie metadata provider 'Provider': IProvider, # sub metadata provider 'Movie': IMovieProvider, 'Person': IPersonProvider, # output converter 'Converter': IConverter, # postprocessor filter 'Postprocessor': IPostprocessor }) self._plugin_manager.collectPlugins() def activate_plugins_by_category(self, category): """ Activate plugins from given category. """ self._toggle_activate_plugins_by_category(category) def deactivate_plugins_by_category(self, category): """ Deactivate plugins from given category. """ self._toggle_activate_plugins_by_category(category) def _toggle_activate_plugins_by_category(self, category): plugins = self._plugin_manager.getPluginsOfCategory(category) is_active = self._category_active[category] for pluginInfo in plugins: if is_active: self._plugin_manager.deactivatePluginByName( name=pluginInfo.name, category=category ) self._plugin_from_category[category].remove(pluginInfo) else: self._plugin_manager.activatePluginByName( name=pluginInfo.name, category=category ) self._plugin_from_category[category].append(pluginInfo) self._category_active[category] = not is_active def get_plugins_from_category(self, category): """ Retrun plugins from the given categrory. Gets plugins from given categrory. Plugin name is set according to name given in yapsy plugin file. :param categrory: The category plugins to load from. :returns: A list with plugins. """ plugins = [] for plugin in self._plugin_from_category[category]: plugin.plugin_object.name = plugin.name plugin.plugin_object.description = plugin.description plugins.append(plugin.plugin_object) return plugins def is_activated(self, category): """ True if category is activated. """ return self._category_active[category]
"""Helper for features that only take one channel. :param fn: function that computes features """ results = [] slc = [slice(None)] * 4 for channel in range(image.shape[axes.c]): slc[axes.c] = channel #a dictionary for the channel result = fn(image[slc], axes=axes, **kwargs) results.append(result) return self.combine_dicts_with_numpy(results) ############### # the manager # ############### pluginManager = PluginManager() pluginManager.setPluginPlaces(plugin_paths) pluginManager.setCategoriesFilter({ "ObjectFeatures" : ObjectFeaturesPlugin, }) pluginManager.collectPlugins() for pluginInfo in pluginManager.getAllPlugins(): pluginManager.activatePluginByName(pluginInfo.name)