def testTwoStepsLoad(self): """ Test loading the plugins in two steps in order to collect more deltailed informations. """ spm = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # trigger the first step to look up for plugins spm.locatePlugins() # make full use of the "feedback" the loadPlugins can give # - set-up the callback function that will be called *before* # loading each plugin callback_infos = [] def preload_cbk(plugin_info): callback_infos.append(plugin_info) # - gather infos about the processed plugins (loaded or not) loadedPlugins = spm.loadPlugins(callback=preload_cbk) self.assertEqual(len(loadedPlugins),1) self.assertEqual(len(callback_infos),1) self.assertEqual(loadedPlugins[0].error,None) self.assertEqual(loadedPlugins[0],callback_infos[0]) # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1) plugin_info = spm.getPluginsOfCategory(sole_category)[0] # try to remove it and check that is worked spm.removePluginFromCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),0) # now re-add this plugin the to same category spm.appendPluginToCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1)
def initialize_plugins(self): self.__log.info("Collecting and loading plugins") try: plugin_manager = PluginManager() # TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT) plugin_manager.setCategoriesFilter({ CARTRIDGE_AGENT_PLUGIN: ICartridgeAgentPlugin, ARTIFACT_MGT_PLUGIN: IArtifactManagementPlugin }) plugin_manager.setPluginPlaces( [self.__config.read_property(constants.PLUGINS_DIR)]) plugin_manager.collectPlugins() # activate cartridge agent plugins plugins = plugin_manager.getPluginsOfCategory( CARTRIDGE_AGENT_PLUGIN) grouped_plugins = {} for plugin_info in plugins: self.__log.debug("Found plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) plugin_manager.activatePluginByName(plugin_info.name) self.__log.info("Activated plugin [%s]" % plugin_info.name) mapped_events = plugin_info.description.split(",") for mapped_event in mapped_events: if mapped_event.strip() != "": if grouped_plugins.get(mapped_event) is None: grouped_plugins[mapped_event] = [] grouped_plugins[mapped_event].append(plugin_info) # activate artifact management plugins artifact_mgt_plugins = plugin_manager.getPluginsOfCategory( ARTIFACT_MGT_PLUGIN) for plugin_info in artifact_mgt_plugins: self.__log.debug( "Found artifact management plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) plugin_manager.activatePluginByName(plugin_info.name) self.__log.info("Activated artifact management plugin [%s]" % plugin_info.name) return plugin_manager, grouped_plugins, artifact_mgt_plugins except ParameterNotFoundException as e: self.__log.exception( "Could not load plugins. Plugins directory not set: %s" % e) return None, None, None except Exception as e: self.__log.exception("Error while loading plugin: %s" % e) return None, None, None
class MUCBot(sleekxmpp.ClientXMPP): def __init__(self, jid, password, room, nick): sleekxmpp.ClientXMPP.__init__(self, jid, password) self.room = room self.nick = nick # Load plugins self.plugin_manager = PluginManager() self.plugin_manager.setPluginPlaces(["plugins_enabled"]) self.plugin_manager.setCategoriesFilter({ "Message" : MessagePlugin, "Presence": PresencePlugin }) self.plugin_manager.collectPlugins() # Activate all loaded plugins for pluginInfo in self.plugin_manager.getAllPlugins(): self.plugin_manager.activatePluginByName(pluginInfo.name) self.add_event_handler("session_start", self.start) self.add_event_handler("message", self.message) self.add_event_handler("groupchat_message", self.muc_message) self.add_event_handler("muc::%s::got_online" % self.room, self.muc_online) self.add_event_handler("muc::%s::got_offline" % self.room, self.muc_offline) def start(self, event): self.get_roster() self.send_presence() if self.room and len(self.room) > 0: self.plugin['xep_0045'].joinMUC(self.room, self.nick, wait=True) def message(self, msg): if msg['type'] in ('chat', 'normal'): for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"): pluginInfo.plugin_object.message_received(msg) def muc_message(self, msg): if msg['mucnick'] != self.nick: for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"): pluginInfo.plugin_object.message_received(msg, nick=self.nick) def muc_online(self, presence): if presence['muc']['nick'] != self.nick: for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"): pluginInfo.plugin_object.got_online(self, presence) def muc_offline(self, presence): if presence['muc']['nick'] != self.nick: for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"): pluginInfo.plugin_object.got_offline(self, presence)
class ExtInfoTest(unittest.TestCase): def setUp(self): # create the plugin manager - use the base plugin manager to # remove any unecessary dependencies self.simplePluginManager = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"Plugins") ], plugin_info_ext="mm-plugin", ) #Set mode to test info class self.simplePluginManager.setPluginInfoClass(ExtensionInfo) # load the plugins that may be found self.simplePluginManager.collectPlugins() # Will be used later self.plugin_info = None def loading_check(self): """ Test the plugins load. """ if self.plugin_info is None: # check nb of categories self.assertEqual(len(self.simplePluginManager.getCategories()),1) sole_category = self.simplePluginManager.getCategories()[0] # check the number of plugins self.assertEqual(len(self.simplePluginManager.getPluginsOfCategory(sole_category)),2) self.plugin_info = [ None , None ] self.plugin_info[0] = self.simplePluginManager.getPluginsOfCategory(sole_category)[0] self.plugin_info[1] = self.simplePluginManager.getPluginsOfCategory(sole_category)[1] # test that the name of the plugin has been correctly defined self.assertTrue("FirstPlugin" in [x.name for x in self.plugin_info]) self.assertEqual(sole_category,self.plugin_info[0].category) else: self.assert_(True) def testBasic(self): self.loading_check() def testKnownHash(self): """ Test hash can be fetched thru ExtensionInfo """ self.loading_check() hash1=ExtensionSecureID.fromPathName(self.plugin_info[0].path+".py") self.assertEquals(hash1,self.plugin_info[0].getSecureID()) self.assertNotEquals(self.plugin_info[0].getSecureID(),self.plugin_info[1].getSecureID())
def main(): # Load the plugins from the plugin directory. manager = PluginManager() manager.setPluginPlaces(["plugins\\plugins1", "plugins\\plugins2"]) manager.setCategoriesFilter({ "PluginOne": category.PluginOne, "PluginTwo": category.PluginTwo, }) print { "PluginOne": category.PluginOne, "PluginTwo": category.PluginTwo } manager.collectPlugins() print manager.getPluginsOfCategory("PluginTwo") for plugin in manager.getPluginsOfCategory("PluginTwo"): plugin.plugin_object.print_name()
class DownloadManager(object): """DownloadManager will be used to download any kind of resource from a \ source system. DownloadManager will locate all plugins of\ type :class:`IDownloader` and will provide it back to the\ requesting object """ __single_download_manager = None def __new__(cls, *args, **kwargs): """Singleton class constructor """ if cls != type(cls.__single_download_manager): cls.__single_download_manager = object.__new__(cls, *args, **kwargs) return cls.__single_download_manager def __init__(self): """Constructor for DownloadManager """ self.plugins_path = constants.DOWNLOADER_PLUGINS_PATH self.save_downloads_to = constants.SAVE_DOWNLOADS_TO_PATH self.plugin_manager = PluginManager(categories_filter={constants.DOWNLOAD_PLUGIN_FILTER: iplugins.IDownloader}) self.plugin_manager.setPluginPlaces([self.plugins_path]) self.plugin_manager.locatePlugins() self.plugin_manager.loadPlugins() def downloaders_list(self): """Provides a list of all downloader plugins available """ return self.plugin_manager.getPluginsOfCategory(constants.DOWNLOAD_PLUGIN_FILTER) def get_downloader(self, downloader_name=None): """Returns a downloader plugin instance by finding through name """ return self.plugin_manager.getPluginByName(downloader_name, constants.DOWNLOAD_PLUGIN_FILTER)
class MarinheiroCmdApp(cmd2.Cmd): def __init__(self): super().__init__() self.manager = PluginManager( plugin_info_ext="marinheiro-test", directories_list=[os.path.join(os.path.dirname(__file__), "tests")], categories_filter={ "Basic" : marinheiro.tests.BasicTest, "Web" : marinheiro.tests.WebTest, }) self.manager.collectPlugins() @cmd2.options([cmd2.make_option("-c", "--category", action="store", help="Specify test category.")]) def do_list(self, arg, opts=None): if opts.category: categories = self.manager.getPluginsOfCategory(opts.category) else: categories = self.manager.getAllPlugins() for plugin in categories: print("{}: {p.name}".format(self.colorize("Name", "bold"), p=plugin)) print("{}: {p.description}".format(self.colorize("Description", "bold"), p=plugin)) print() @cmd2.options([cmd2.make_option("-c", "--category", action="store", help="Specify test category.")]) def do_run(self, arg, opts=None): if opts.category: categories = self.manager.getPluginsOfCategory(opts.category) else: categories = self.manager.getAllPlugins() for plugin in categories: print("{}: {p.name}".format(self.colorize("Name", "bold"), p=plugin)) print("{}: {p.description}".format(self.colorize("Description", "bold"), p=plugin)) try: output = plugin.plugin_object.run() except marinheiro.exceptions.FailedTest as err: status = self.colorize("FAILED", "red") output = err.msg else: status = self.colorize("PASSED", "green") finally: print("{}: {}".format(self.colorize("Status", "bold"), status)) print("{}: {}".format(self.colorize("Output", "bold"), output)) print()
def test_json_reporting_plugin_found(): """Test that the plugin manager finds the JSON plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')]) manager.setCategoriesFilter({ "Reporting": ReportingPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "upload_risk_assessment" assert any(plugin_info.plugin_object.get_name() == 'upload_risk_assessment' for plugin_info in manager.getPluginsOfCategory("Reporting")) # While we're at it, verify that a plugin is named JSON Risk Assessment Reporting Plugin assert any(plugin_info.name == 'JSON Risk Assessment Reporting Plugin' for plugin_info in manager.getPluginsOfCategory("Reporting"))
def test_maven_discovery_plugin_found(): """Test that the plugin manager finds the Maven discovery plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]) manager.setCategoriesFilter({ "Discovery": DiscoveryPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "java" assert any(plugin_info.plugin_object.get_name() == "maven" for plugin_info in manager.getPluginsOfCategory("Discovery")) # While we're at it, verify that a plugin is named Maven Discovery Plugin assert any(plugin_info.name == "Maven Discovery Plugin" for plugin_info in manager.getPluginsOfCategory("Discovery"))
def test_clang_tidy_tool_plugin_found(): """Test that the plugin manager can find the clang-tidy plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')]) manager.setCategoriesFilter({ "Tool": ToolPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "clang_tidy" assert any(plugin_info.plugin_object.get_name() == 'clang-tidy' for plugin_info in manager.getPluginsOfCategory("Tool")) # While we're at it, verify that a plugin is named ClangTidy Tool Plugin assert any(plugin_info.name == 'clang-tidy Tool Plugin' for plugin_info in manager.getPluginsOfCategory("Tool"))
def test_cccc_tool_plugin_found(): """Test that the CCCC tool plugin is detected by the plugin system.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]) manager.setCategoriesFilter({ "Tool": ToolPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "cccc" assert any(plugin_info.plugin_object.get_name() == "cccc" for plugin_info in manager.getPluginsOfCategory("Tool")) # While we're at it, verify that a plugin is named CCCC Tool Plugin assert any(plugin_info.name == "CCCC Tool Plugin" for plugin_info in manager.getPluginsOfCategory("Tool"))
def test_console_reporting_plugin_found(): """Test that the plugin manager finds the console reporting plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')]) manager.setCategoriesFilter({ "Reporting": ReportingPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "c" assert any(plugin_info.plugin_object.get_name() == 'print_to_console' for plugin_info in manager.getPluginsOfCategory("Reporting")) # While we're at it, verify that a plugin is named C Discovery Plugin assert any(plugin_info.name == 'Print To Console Reporting Plugin' for plugin_info in manager.getPluginsOfCategory("Reporting"))
def test_yaml_discovery_plugin_found(): """Test that the YAML discovery plugin is detected by the plugin system.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')]) manager.setCategoriesFilter({ "Discovery": DiscoveryPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "yaml" assert any(plugin_info.plugin_object.get_name() == 'yaml' for plugin_info in manager.getPluginsOfCategory("Discovery")) # While we're at it, verify that a plugin is named YAML Discovery Plugin assert any(plugin_info.name == 'YAML Discovery Plugin' for plugin_info in manager.getPluginsOfCategory("Discovery"))
class Plugins(): ''' Plugins ''' def __init__(self): ''' Constructor ''' super(Plugins, self).__init__() self.write_tab_dock_plugin_dict = {} self.write_panel_dock_plugin_dict = {} # Build the manager self._plugin_manager = PluginManager() # List all sub-directories of "plugins" plugin_path = os.path.sep.join([os.getcwd(), "plugins"]) plugin_places = [plugin_path] for dirname, dirnames, filenames in os.walk(plugin_path): # print path to all subdirectories first. for subdirname in dirnames: plugin_places.append(os.path.join(dirname, subdirname)) # Tell it the default place(s) where to find plugins self._plugin_manager.setPluginPlaces(plugin_places) sys.path.append(plugin_path) # Define the various categories corresponding to the different # kinds of plugins you have defined self._plugin_manager.setCategoriesFilter({ "CoreWriteTabDockPlugin": CoreWriteTabDockPlugin, "CoreWritePanelDockPlugin": CoreWritePanelDockPlugin }) self._plugin_manager.collectPlugins() self.load_plugins( ["CoreWriteTabDockPlugin", "CoreWritePanelDockPlugin"]) def load_plugins(self, categories): ''' function:: load_plugins(categories) :param categories: list ''' for category in categories: for pluginInfo in self._plugin_manager.getPluginsOfCategory( category): setattr(self, pluginInfo.plugin_object.core_class().__name__, pluginInfo.plugin_object.core_class()) if category is "CoreWriteTabDockPlugin": self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.core_class().dock_name] \ = pluginInfo.plugin_object.core_class() if category is "CoreWritePanelDockPlugin": self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.core_class().dock_name] \ = pluginInfo.plugin_object.core_class()
def initialize_plugins(self): self.__log.info("Collecting and loading plugins") try: plugin_manager = PluginManager() # TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT) plugin_manager.setCategoriesFilter({ CARTRIDGE_AGENT_PLUGIN: ICartridgeAgentPlugin, ARTIFACT_MGT_PLUGIN: IArtifactManagementPlugin }) plugin_manager.setPluginPlaces([self.__config.read_property(constants.PLUGINS_DIR)]) plugin_manager.collectPlugins() # activate cartridge agent plugins plugins = plugin_manager.getPluginsOfCategory(CARTRIDGE_AGENT_PLUGIN) grouped_plugins = {} for plugin_info in plugins: self.__log.debug("Found plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) plugin_manager.activatePluginByName(plugin_info.name) self.__log.info("Activated plugin [%s]" % plugin_info.name) mapped_events = plugin_info.description.split(",") for mapped_event in mapped_events: if mapped_event.strip() != "": if grouped_plugins.get(mapped_event) is None: grouped_plugins[mapped_event] = [] grouped_plugins[mapped_event].append(plugin_info) # activate artifact management plugins artifact_mgt_plugins = plugin_manager.getPluginsOfCategory(ARTIFACT_MGT_PLUGIN) for plugin_info in artifact_mgt_plugins: self.__log.debug("Found artifact management plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path)) plugin_manager.activatePluginByName(plugin_info.name) self.__log.info("Activated artifact management plugin [%s]" % plugin_info.name) return plugin_manager, grouped_plugins, artifact_mgt_plugins except ParameterNotFoundException as e: self.__log.exception("Could not load plugins. Plugins directory not set: %s" % e) return None, None, None except Exception as e: self.__log.exception("Error while loading plugin: %s" % e) return None, None, None
class Plugins(): ''' Plugins ''' def __init__(self): ''' Constructor ''' super(Plugins, self).__init__() self.write_tab_dock_plugin_dict = {} self.write_panel_dock_plugin_dict = {} # Build the manager self._plugin_manager = PluginManager() # List all sub-directories of "plugins" plugin_path = os.path.sep.join([os.getcwd(), "plugins"]) plugin_places = [plugin_path] for dirname, dirnames, filenames in os.walk(plugin_path): # print path to all subdirectories first. for subdirname in dirnames: plugin_places.append(os.path.join(dirname, subdirname)) # Tell it the default place(s) where to find plugins self._plugin_manager.setPluginPlaces(plugin_places) sys.path.append(plugin_path) # Define the various categories corresponding to the different # kinds of plugins you have defined self._plugin_manager.setCategoriesFilter({ "GuiWriteTabDockPlugin": GuiWriteTabDockPlugin, "GuiWritePanelDockPlugin": GuiWritePanelDockPlugin }) self._plugin_manager.collectPlugins() self.load_plugins( ["GuiWriteTabDockPlugin", "GuiWritePanelDockPlugin"]) def load_plugins(self, categories): ''' function:: load_plugins(categories) :param categories: list ''' for category in categories: for pluginInfo in self._plugin_manager.getPluginsOfCategory(category): setattr(self, pluginInfo.plugin_object.gui_class( ).__name__, pluginInfo.plugin_object.gui_class()) if category is "GuiWriteTabDockPlugin": self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \ = pluginInfo.plugin_object.gui_class() if category is "GuiWritePanelDockPlugin": self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \ = pluginInfo.plugin_object.gui_class()
def test_write_jenkins_warnings_ng_reporting_plugin_found(): """Test that the plugin manager finds the file writing plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]) manager.setCategoriesFilter({ "Reporting": ReportingPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "c" assert any( plugin_info.plugin_object.get_name() == "write_jenkins_warnings_ng" for plugin_info in manager.getPluginsOfCategory("Reporting")) # While we're at it, verify that a plugin is named C Discovery Plugin assert any(plugin_info.name == "Write Jenkins Warnings NG Reporting Plugin" for plugin_info in manager.getPluginsOfCategory("Reporting"))
def testTwoStepsLoad(self): """ Test loading the plugins in two steps in order to collect more deltailed informations. """ spm = PluginManager(directories_list=[ os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins") ]) # trigger the first step to look up for plugins spm.locatePlugins() # make full use of the "feedback" the loadPlugins can give # - set-up the callback function that will be called *before* # loading each plugin callback_infos = [] def preload_cbk(plugin_info): callback_infos.append(plugin_info) callback_after_infos = [] def postload_cbk(plugin_info): callback_after_infos.append(plugin_info) # - gather infos about the processed plugins (loaded or not) loadedPlugins = spm.loadPlugins(callback=preload_cbk, callback_after=postload_cbk) self.assertEqual(len(loadedPlugins), 1) self.assertEqual(len(callback_infos), 1) self.assertEqual(loadedPlugins[0].error, None) self.assertEqual(loadedPlugins[0], callback_infos[0]) self.assertEqual(len(callback_after_infos), 1) self.assertEqual(loadedPlugins[0], callback_infos[0]) # check that the getCategories works self.assertEqual(len(spm.getCategories()), 1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1) plugin_info = spm.getPluginsOfCategory(sole_category)[0] # try to remove it and check that is worked spm.removePluginFromCategory(plugin_info, sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0) # now re-add this plugin the to same category spm.appendPluginToCategory(plugin_info, sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
class PluginHandler: def __init__(self): self._plugin_manager = PluginManager() self._category_active = {"Modifier": False, "Analyzer": False, "Comparator": False} self._plugin_from_category = {"Modifier": [], "Analyzer": [], "Comparator": []} self._modifier_plugins = [] self._analyzer_plugins = [] self._collect_all_plugins() def _collect_all_plugins(self): self._plugin_manager.setPluginPlaces( ["hugin/analyze/modifier", "hugin/analyze/analyzer", "hugin/analyze/comparator"] ) # setting filter categories for pluginmanager self._plugin_manager.setCategoriesFilter( { # movie metadata provider "Modifier": IModifier, # movie metadata provider "Comparator": IComparator, # sub metadata provider "Analyzer": IAnalyzer, } ) self._plugin_manager.collectPlugins() def activate_plugins_by_category(self, category): self._toggle_activate_plugins_by_category(category) def deactivate_plugins_by_category(self, category): self._toggle_activate_plugins_by_category(category) def _toggle_activate_plugins_by_category(self, category): plugins = self._plugin_manager.getPluginsOfCategory(category) is_active = self._category_active[category] for pluginInfo in plugins: if is_active: self._plugin_manager.deactivatePluginByName(name=pluginInfo.name, category=category) self._plugin_from_category[category].remove(pluginInfo) else: self._plugin_manager.activatePluginByName(name=pluginInfo.name, category=category) self._plugin_from_category[category].append(pluginInfo) self._category_active[category] = not is_active def get_plugins_from_category(self, category): plugins = [] for plugin in self._plugin_from_category[category]: plugin.plugin_object.name = plugin.name plugin.plugin_object.description = plugin.description plugins.append(plugin.plugin_object) return plugins def is_activated(self, category): """ True if category is activated. """ return self._category_active[category]
def test_isort_tool_plugin_found(): """Test that the plugin manager can find the Isort plugin.""" if sys.version_info.major == 3 and sys.version_info.minor < 6: pytest.skip("isort is only available for Python 3.6+, unable to test") manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]) manager.setCategoriesFilter({ "Tool": ToolPlugin, }) manager.collectPlugins() # Verify that a plugin's get_name() function returns "isort" assert any(plugin_info.plugin_object.get_name() == "isort" for plugin_info in manager.getPluginsOfCategory("Tool")) # While we're at it, verify that a plugin is named Isort Tool Plugin assert any(plugin_info.name == "Isort Tool Plugin" for plugin_info in manager.getPluginsOfCategory("Tool"))
def __init__(self, manager=None, name=None): """ Create the plugin object :param manager: Manager instance :type manager: :class:`biomajmanager.manager.Manager` :param name: Name of the plugin to load. [DEFAULT: load all plugins] :type name: String :raises SystemExit: If 'manager' arg is not given :raises SystemExit: If 'PLUGINS' section not found in :py:data:`manager.properties` :raises SystemExit: If 'plugins.dir' not set in :py:data:`manager.properties` :raises SystemExit: If 'plugins.list' not set in :py:data:`manager.properties` :raises SystemExit: If 'plugins.dir' does not exist """ self.pm = None self.name = None self.config = None self.manager = None if not manager: Utils.error("'manager' is required") self.manager = manager self.config = self.manager.config if not self.config.has_section('PLUGINS'): Utils.error("Can't load plugins, no section found!") if not self.config.has_option('MANAGER', 'plugins.dir'): Utils.error("plugins.dir not defined!") if not self.config.has_option('PLUGINS', 'plugins.list'): Utils.error("plugins.list is not defined!") if not os.path.isdir(self.config.get('MANAGER', 'plugins.dir')): Utils.error("Can't find plugins.dir") plugin_manager = PluginManager(directories_list=[self.config.get('MANAGER', 'plugins.dir')], categories_filter={Plugins.CATEGORY: BMPlugin}) plugin_manager.collectPlugins() self.pm = plugin_manager self.name = name user_plugins = [] # Load user wanted plugin(s) for plugin in self.config.get('PLUGINS', 'plugins.list').split(','): plugin.strip() # We need to lower the plugin name user_plugins.append(plugin) # This means that all plugins must inherits from BMPlugin for pluginInfo in plugin_manager.getPluginsOfCategory(Plugins.CATEGORY): Utils.verbose("[manager] plugin name => %s" % pluginInfo.name) if pluginInfo.name in user_plugins: if not pluginInfo.is_activated: Utils.verbose("[manager] plugin %s activated" % pluginInfo.name) plugin_manager.activatePluginByName(pluginInfo.name) setattr(self, pluginInfo.name, pluginInfo.plugin_object) pluginInfo.plugin_object.set_config(self.config) pluginInfo.plugin_object.set_manager(self.manager)
def testTwoStepsLoadWithError(self): """ Test loading the plugins in two steps in order to collect more deltailed informations and take care of an erroneous plugin. """ spm = PluginManager(directories_list=[ os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins") ], plugin_info_ext="yapsy-error-plugin") # trigger the first step to look up for plugins spm.locatePlugins() # make full use of the "feedback" the loadPlugins can give # - set-up the callback function that will be called *before* # loading each plugin callback_infos = [] def preload_cbk(i_plugin_info): callback_infos.append(i_plugin_info) callback_after_infos = [] def postload_cbk(i_plugin_info): callback_after_infos.append(i_plugin_info) # - gather infos about the processed plugins (loaded or not) # and for the test, monkey patch the logger originalLogLevel = log.getEffectiveLevel() log.setLevel(logging.ERROR) errorLogCallFlag = [False] def errorMock(*args, **kwargs): errorLogCallFlag[0] = True originalErrorMethod = log.error log.error = errorMock try: loadedPlugins = spm.loadPlugins(callback=preload_cbk, callback_after=postload_cbk) finally: log.setLevel(originalLogLevel) log.error = originalErrorMethod self.assertTrue(errorLogCallFlag[0]) self.assertEqual(len(loadedPlugins), 1) self.assertEqual(len(callback_infos), 1) self.assertTrue(isinstance(callback_infos[0].error, tuple)) self.assertEqual(loadedPlugins[0], callback_infos[0]) self.assertTrue(issubclass(callback_infos[0].error[0], ImportError)) self.assertEqual(len(callback_after_infos), 0) # check that the getCategories works self.assertEqual(len(spm.getCategories()), 1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
def testCategoryManipulation(self): """ Test querying, removing and adding plugins from/to a category. """ spm = PluginManager(directories_list=[ os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins") ]) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()), 1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1) plugin_info = spm.getPluginsOfCategory(sole_category)[0] # try to remove it and check that is worked spm.removePluginFromCategory(plugin_info, sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0) # now re-add this plugin the to same category spm.appendPluginToCategory(plugin_info, sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
def testCategoryManipulation(self): """ Test querying, removing and adding plugins from/to a category. """ spm = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1) plugin_info = spm.getPluginsOfCategory(sole_category)[0] # try to remove it and check that is worked spm.removePluginFromCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),0) # now re-add this plugin the to same category spm.appendPluginToCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1)
def testMultipleCategoriesForASamePlugin(self): """ Test that associating a plugin to multiple categories works as expected. """ class AnotherPluginIfce(object): def __init__(self): pass def activate(self): pass def deactivate(self): pass spm = PluginManager(categories_filter={ "Default": IPlugin, "IP": IPlugin, "Other": AnotherPluginIfce, }, directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)), "plugins") ]) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()), 3) categories = spm.getCategories() self.assertTrue("Default" in categories) # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory("Default")), 1) plugin_info = spm.getPluginsOfCategory("Default")[0] self.assertTrue("Default" in plugin_info.categories) self.assertTrue("IP" in plugin_info.categories) self.assertTrue("IP" in categories) # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1) self.assertTrue("Other" in categories) # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory("Other")), 0) # try to remove the plugin from one category and check the # other category spm.removePluginFromCategory(plugin_info, "Default") self.assertEqual(len(spm.getPluginsOfCategory("Default")), 0) self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1) # now re-add this plugin the to same category spm.appendPluginToCategory(plugin_info, "Default") self.assertEqual(len(spm.getPluginsOfCategory("Default")), 1) self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1)
def test_print_json_reporting_plugin_found(): """Test that the plugin manager finds the print json reporting plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")] ) manager.setCategoriesFilter( { "Reporting": ReportingPlugin, } ) manager.collectPlugins() assert any( plugin_info.plugin_object.get_name() == "print_json" for plugin_info in manager.getPluginsOfCategory("Reporting") ) assert any( plugin_info.name == "Print JSON Reporting Plugin" for plugin_info in manager.getPluginsOfCategory("Reporting") )
def modelBehaviorImplementations(paths): """ Return an iterable of plugin-info for every locatable implementation of this interface on a given path. PyDev for Eclipse reports a compilation error here on a line that is actually legal Python due to the schedule upon which module resolution and imports happen. """ manager = PluginManager() manager.setPluginPlaces(paths) from fakeDataGenerator.model import IModelBehavior as foreignModelBehavior # hey PyDev: the previous line is weird, but is actually legal manager.setCategoriesFilter({"ModelBehavior": foreignModelBehavior}) manager.collectPlugins() return manager.getPluginsOfCategory("ModelBehavior")
def testMultipleCategoriesForASamePlugin(self): """ Test that associating a plugin to multiple categories works as expected. """ class AnotherPluginIfce(object): def __init__(self): pass def activate(self): pass def deactivate(self): pass spm = PluginManager( categories_filter = { "Default": IPlugin, "IP": IPlugin, "Other": AnotherPluginIfce, }, directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),3) categories = spm.getCategories() self.assertTrue("Default" in categories) # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory("Default")), 1) plugin_info = spm.getPluginsOfCategory("Default")[0] self.assertTrue("Default" in plugin_info.categories) self.assertTrue("IP" in plugin_info.categories) self.assertTrue("IP" in categories) # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory("IP")),1) self.assertTrue("Other" in categories) # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory("Other")),0) # try to remove the plugin from one category and check the # other category spm.removePluginFromCategory(plugin_info, "Default") self.assertEqual(len(spm.getPluginsOfCategory("Default")), 0) self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1) # now re-add this plugin the to same category spm.appendPluginToCategory(plugin_info, "Default") self.assertEqual(len(spm.getPluginsOfCategory("Default")),1) self.assertEqual(len(spm.getPluginsOfCategory("IP")),1)
def testRecursivePluginlocation(self): """ Test detection of plugins which by default must be recusrive. Here we give the test directory as a plugin place whereas we expect the plugins to be in test/plugins. """ spm = PluginManager( directories_list=[os.path.dirname(os.path.abspath(__file__))]) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()), 1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
def test_groovylint_tool_plugin_found(): """Test that the plugin manager can find the groovylint plugin.""" manager = PluginManager() # Get the path to statick_tool/__init__.py, get the directory part, and # add 'plugins' to that to get the standard plugins dir manager.setPluginPlaces( [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")] ) manager.setCategoriesFilter( { "Tool": ToolPlugin, } ) manager.collectPlugins() # Verify that a plugin's get_name() function returns "groovylint" assert any( plugin_info.plugin_object.get_name() == "groovylint" for plugin_info in manager.getPluginsOfCategory("Tool") ) # While we're at it, verify that a plugin is named NPM Groovy Lint Tool Plugin assert any( plugin_info.name == "NPM Groovy Lint Tool Plugin" for plugin_info in manager.getPluginsOfCategory("Tool") )
def testRecursivePluginlocation(self): """ Test detection of plugins which by default must be recusrive. Here we give the test directory as a plugin place whereas we expect the plugins to be in test/plugins. """ spm = PluginManager(directories_list=[ os.path.dirname(os.path.abspath(__file__))]) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1)
def modelBehaviorImplementations(paths): """ Return an iterable of plugin-info for every locatable implementation of this interface on a given path. PyDev for Eclipse reports a compilation error here on a line that is actually legal Python due to the schedule upon which module resolution and imports happen. """ manager = PluginManager() manager.setPluginPlaces(paths) from fakeDataGenerator.model import IModelBehavior as foreignModelBehavior #hey PyDev: the previous line is weird, but is actually legal manager.setCategoriesFilter({ "ModelBehavior" : foreignModelBehavior, }) manager.collectPlugins() return manager.getPluginsOfCategory("ModelBehavior")
def testTwoStepsLoadWithError(self): """ Test loading the plugins in two steps in order to collect more deltailed informations and take care of an erroneous plugin. """ spm = PluginManager( directories_list=[os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")], plugin_info_ext="yapsy-error-plugin", ) # trigger the first step to look up for plugins spm.locatePlugins() # make full use of the "feedback" the loadPlugins can give # - set-up the callback function that will be called *before* # loading each plugin callback_infos = [] def preload_cbk(i_plugin_info): callback_infos.append(i_plugin_info) # - gather infos about the processed plugins (loaded or not) # and for the test, monkey patch the logger originalLogLevel = log.getEffectiveLevel() log.setLevel(logging.ERROR) errorLogCallFlag = [False] def errorMock(*args, **kwargs): errorLogCallFlag[0] = True originalErrorMethod = log.error log.error = errorMock try: loadedPlugins = spm.loadPlugins(callback=preload_cbk) finally: log.setLevel(originalLogLevel) log.error = originalErrorMethod self.assertTrue(errorLogCallFlag[0]) self.assertEqual(len(loadedPlugins), 1) self.assertEqual(len(callback_infos), 1) self.assertTrue(isinstance(callback_infos[0].error, tuple)) self.assertEqual(loadedPlugins[0], callback_infos[0]) self.assertEqual(callback_infos[0].error[0], ImportError) # check that the getCategories works self.assertEqual(len(spm.getCategories()), 1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
def testNonRecursivePluginlocationNotFound(self): """ Test detection of plugins when the detection is non recursive. Here we test that it cannot look into subdirectories of the test directory. """ pluginLocator = PluginFileLocator() pluginLocator.setPluginPlaces([ os.path.dirname(os.path.abspath(__file__))]) pluginLocator.disableRecursiveScan() spm = PluginManager() spm.setPluginLocator(pluginLocator) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),0)
class FakeNikola(object): def __init__(self): self.config = {'DISABLED_PLUGINS': []} self.debug = False self.loghandlers = [] self.timeline = [] self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "RestExtension": RestExtension, "MarkdownExtension": MarkdownExtension, "SignalHandler": SignalHandler, }) self.plugin_manager.setPluginInfoExtension('plugin') extra_plugins_dirs = '' places = [ resource_filename('nikola', utils.sys_encode('plugins')), os.path.join(os.getcwd(), utils.sys_encode('plugins')), os.path.expanduser('~/.nikola/plugins'), ] + [utils.sys_encode(path) for path in extra_plugins_dirs if path] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() #self.pug = None #for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"): # if plugin_info.name == 'rest': # self.plugin_manager.activatePluginByName(plugin_info.name) # plugin_info.plugin_object.set_site(self) # self.pug = plugin_info # Now we have our pug for plugin_info in self.plugin_manager.getPluginsOfCategory( "RestExtension"): self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugin_info.plugin_object.short_help = plugin_info.description
def testNonRecursivePluginlocationNotFound(self): """ Test detection of plugins when the detection is non recursive. Here we test that it cannot look into subdirectories of the test directory. """ pluginLocator = PluginFileLocator() pluginLocator.setPluginPlaces( [os.path.dirname(os.path.abspath(__file__))]) pluginLocator.disableRecursiveScan() spm = PluginManager() spm.setPluginLocator(pluginLocator) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()), 1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
def testDisablingRecursivePluginLocationAllowsFindingTopLevelPlugins(self): """ Test detection of plugins when the detection is non recursive. Here we test that if we give test/plugin as the directory to scan it can find the plugin. """ pluginLocator = PluginFileLocator() pluginLocator.setPluginPlaces([ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) pluginLocator.disableRecursiveScan() spm = PluginManager() spm.setPluginLocator(pluginLocator) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1)
def testDisablingRecursivePluginLocationAllowsFindingTopLevelPlugins(self): """ Test detection of plugins when the detection is non recursive. Here we test that if we give test/plugin as the directory to scan it can find the plugin. """ pluginLocator = PluginFileLocator() pluginLocator.setPluginPlaces([ os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins") ]) pluginLocator.disableRecursiveScan() spm = PluginManager() spm.setPluginLocator(pluginLocator) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()), 1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
def testChangingCategoriesFilter(self): """ Test the effect of setting a new category filer. """ spm = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # load the plugins that may be found spm.collectPlugins() newCategory = "Mouf" # Pre-requisite for the test previousCategories = spm.getCategories() self.assertTrue(len(previousCategories) >= 1) self.assertTrue(newCategory not in previousCategories) # change the category and see what's happening spm.setCategoriesFilter({newCategory: IPlugin}) spm.collectPlugins() for categoryName in previousCategories: self.assertRaises(KeyError, spm.getPluginsOfCategory, categoryName) self.assertTrue(len(spm.getPluginsOfCategory(newCategory)) >= 1)
def testChangingCategoriesFilter(self): """ Test the effect of setting a new category filer. """ spm = PluginManager(directories_list=[ os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins") ]) # load the plugins that may be found spm.collectPlugins() newCategory = "Mouf" # Pre-requisite for the test previousCategories = spm.getCategories() self.assertTrue(len(previousCategories) >= 1) self.assertTrue(newCategory not in previousCategories) # change the category and see what's happening spm.setCategoriesFilter({newCategory: IPlugin}) spm.collectPlugins() for categoryName in previousCategories: self.assertRaises(KeyError, spm.getPluginsOfCategory, categoryName) self.assertTrue(len(spm.getPluginsOfCategory(newCategory)) >= 1)
class FakeNikola(object): def __init__(self): self.config = {'DISABLED_PLUGINS': []} self.debug = False self.loghandlers = [] self.timeline = [] self.plugin_manager = PluginManager(categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "RestExtension": RestExtension, "MarkdownExtension": MarkdownExtension, "SignalHandler": SignalHandler, }) self.plugin_manager.setPluginInfoExtension('plugin') extra_plugins_dirs = '' places = [ resource_filename('nikola', utils.sys_encode('plugins')), os.path.join(os.getcwd(), utils.sys_encode('plugins')), os.path.expanduser('~/.nikola/plugins'), ] + [utils.sys_encode(path) for path in extra_plugins_dirs if path] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() #self.pug = None #for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"): # if plugin_info.name == 'rest': # self.plugin_manager.activatePluginByName(plugin_info.name) # plugin_info.plugin_object.set_site(self) # self.pug = plugin_info # Now we have our pug for plugin_info in self.plugin_manager.getPluginsOfCategory("RestExtension"): self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugin_info.plugin_object.short_help = plugin_info.description
def main(): configpath = ".animedb.config" config = Config(configpath) if config['error']: print "No Config file found - using default config" print "please edit %s to include your anidb username and pass" % config['path'] config=DefaultConfig(configpath) config.save() raise IOError manager = PluginManager(categories_filter={"Parsers": Parser}, plugin_info_ext="parser") manager.setPluginPlaces(["plugins"]) manager.collectPlugins() parsers = [] for plugin in manager.getPluginsOfCategory("Parsers"): parsers.append(plugin.plugin_object) parsers.append(MiscParser()) for parser in parsers: parser.setdb("db.sqlite", "sqlite") parser.setconfig(config) for root,dirs,files in os.walk(os.path.abspath(config['ScanFolder'])): for filename in files: name = os.path.join(root,filename) print "Parsing: %s" % name for parser in parsers: catalogue = [] Info = parser.ScanFile(name) if Info: location = parser.SortFile(Info, name) catalogue.append((location, Info)) if catalogue: parser.Catalogue(catalogue) break for parser in parsers: parser.commit()
class SimpleTestsCase(unittest.TestCase): """ Test the correct loading of a simple plugin as well as basic commands. """ def setUp(self): """ init """ # create the plugin manager self.simplePluginManager = PluginManager(directories_list=[ os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins") ]) # load the plugins that may be found self.simplePluginManager.collectPlugins() # Will be used later self.plugin_info = None def plugin_loading_check(self): """ Test if the correct plugin has been loaded. """ if self.plugin_info is None: # check nb of categories self.assertEqual(len(self.simplePluginManager.getCategories()), 1) sole_category = self.simplePluginManager.getCategories()[0] # check the number of plugins self.assertEqual( len( self.simplePluginManager.getPluginsOfCategory( sole_category)), 1) self.plugin_info = self.simplePluginManager.getPluginsOfCategory( sole_category)[0] # test that the name of the plugin has been correctly defined self.assertEqual(self.plugin_info.name, "Simple Plugin") self.assertEqual(sole_category, self.plugin_info.category) else: self.assert_(True) def testLoaded(self): """ Test if the correct plugin has been loaded. """ self.plugin_loading_check() def testGetAll(self): """ Test if the correct plugin has been loaded. """ self.plugin_loading_check() self.assertEqual(len(self.simplePluginManager.getAllPlugins()), 1) self.assertEqual(self.simplePluginManager.getAllPlugins()[0], self.plugin_info) def testActivationAndDeactivation(self): """ Test if the activation procedure works. """ self.plugin_loading_check() self.assert_(not self.plugin_info.plugin_object.is_activated) self.simplePluginManager.activatePluginByName( self.plugin_info.name, self.plugin_info.category) self.assert_(self.plugin_info.plugin_object.is_activated) self.simplePluginManager.deactivatePluginByName( self.plugin_info.name, self.plugin_info.category) self.assert_(not self.plugin_info.plugin_object.is_activated)
class Nikola(object): """Class that handles site generation. Takes a site config as argument on creation. """ EXTRA_PLUGINS = ["planetoid", "ipynb", "local_search", "render_mustache"] def __init__(self, **config): """Setup proper environment for running tasks.""" self.global_data = {} self.posts_per_year = defaultdict(list) self.posts_per_month = defaultdict(list) self.posts_per_tag = defaultdict(list) self.timeline = [] self.pages = [] self._scanned = False if not config: self.configured = False else: self.configured = True # This is the default config self.config = { "ADD_THIS_BUTTONS": True, "ANALYTICS": "", "ARCHIVE_PATH": "", "ARCHIVE_FILENAME": "archive.html", "CACHE_FOLDER": "cache", "CODE_COLOR_SCHEME": "default", "COMMENTS_IN_GALLERIES": False, "COMMENTS_IN_STORIES": False, "CONTENT_FOOTER": "", "CREATE_MONTHLY_ARCHIVE": False, "DATE_FORMAT": "%Y-%m-%d %H:%M", "DEFAULT_LANG": "en", "DEPLOY_COMMANDS": [], "DISABLED_PLUGINS": (), "DISQUS_FORUM": "nikolademo", "ENABLED_EXTRAS": (), "EXTRA_HEAD_DATA": "", "FAVICONS": {}, "FILE_METADATA_REGEXP": None, "FILES_FOLDERS": {"files": ""}, "FILTERS": {}, "GALLERY_PATH": "galleries", "GZIP_FILES": False, "GZIP_EXTENSIONS": (".txt", ".htm", ".html", ".css", ".js", ".json"), "HIDE_UNTRANSLATED_POSTS": False, "INDEX_DISPLAY_POST_COUNT": 10, "INDEX_TEASERS": False, "INDEXES_TITLE": "", "INDEXES_PAGES": "", "INDEX_PATH": "", "LICENSE": "", "LISTINGS_FOLDER": "listings", "MAX_IMAGE_SIZE": 1280, "MATHJAX_CONFIG": "", "OLD_THEME_SUPPORT": True, "OUTPUT_FOLDER": "output", "post_compilers": { "rest": (".txt", ".rst"), "markdown": (".md", ".mdown", ".markdown"), "textile": (".textile",), "txt2tags": (".t2t",), "bbcode": (".bb",), "wiki": (".wiki",), "ipynb": (".ipynb",), "html": (".html", ".htm"), }, "POST_PAGES": ( ("posts/*.txt", "posts", "post.tmpl", True), ("stories/*.txt", "stories", "story.tmpl", False), ), "REDIRECTIONS": [], "RSS_LINK": None, "RSS_PATH": "", "RSS_TEASERS": True, "SEARCH_FORM": "", "SLUG_TAG_PATH": True, "STORY_INDEX": False, "STRIP_INDEX_HTML": False, "TAG_PATH": "categories", "TAG_PAGES_ARE_INDEXES": False, "THEME": "site", "THEME_REVEAL_CONGIF_SUBTHEME": "sky", "THEME_REVEAL_CONGIF_TRANSITION": "cube", "THUMBNAIL_SIZE": 180, "USE_BUNDLES": True, "USE_CDN": False, "USE_FILENAME_AS_TITLE": True, "TIMEZONE": None, } self.config.update(config) self.config["TRANSLATIONS"] = self.config.get("TRANSLATIONS", {self.config["DEFAULT_" "LANG"]: ""}) self.THEMES = utils.get_theme_chain(self.config["THEME"]) self.MESSAGES = utils.load_messages(self.THEMES, self.config["TRANSLATIONS"], self.config["DEFAULT_LANG"]) # SITE_URL is required, but if the deprecated BLOG_URL # is available, use it and warn if "SITE_URL" not in self.config: if "BLOG_URL" in self.config: print("WARNING: You should configure SITE_URL instead of BLOG_URL") self.config["SITE_URL"] = self.config["BLOG_URL"] self.default_lang = self.config["DEFAULT_LANG"] self.translations = self.config["TRANSLATIONS"] # BASE_URL defaults to SITE_URL if "BASE_URL" not in self.config: self.config["BASE_URL"] = self.config.get("SITE_URL") self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, } ) self.plugin_manager.setPluginInfoExtension("plugin") self.plugin_manager.setPluginPlaces( [str(os.path.join(os.path.dirname(__file__), "plugins")), str(os.path.join(os.getcwd(), "plugins"))] ) self.plugin_manager.collectPlugins() self.commands = {} # Activate all command plugins for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"): if plugin_info.name in self.config["DISABLED_PLUGINS"] or ( plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"] ): self.plugin_manager.removePluginFromCategory(plugin_info, "Command") continue self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugin_info.plugin_object.short_help = plugin_info.description self.commands[plugin_info.name] = plugin_info.plugin_object # Activate all task plugins for task_type in ["Task", "LateTask"]: for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type): if plugin_info.name in self.config["DISABLED_PLUGINS"] or ( plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"] ): self.plugin_manager.removePluginFromCategory(plugin_info, task_type) continue self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) # set global_context for template rendering self.GLOBAL_CONTEXT = {} self.GLOBAL_CONTEXT["messages"] = self.MESSAGES self.GLOBAL_CONTEXT["_link"] = self.link self.GLOBAL_CONTEXT["set_locale"] = s_l self.GLOBAL_CONTEXT["rel_link"] = self.rel_link self.GLOBAL_CONTEXT["abs_link"] = self.abs_link self.GLOBAL_CONTEXT["exists"] = self.file_exists self.GLOBAL_CONTEXT["SLUG_TAG_PATH"] = self.config["SLUG_TAG_PATH"] self.GLOBAL_CONTEXT["add_this_buttons"] = self.config["ADD_THIS_BUTTONS"] self.GLOBAL_CONTEXT["index_display_post_count"] = self.config["INDEX_DISPLAY_POST_COUNT"] self.GLOBAL_CONTEXT["use_bundles"] = self.config["USE_BUNDLES"] self.GLOBAL_CONTEXT["use_cdn"] = self.config.get("USE_CDN") self.GLOBAL_CONTEXT["favicons"] = self.config["FAVICONS"] self.GLOBAL_CONTEXT["date_format"] = self.config.get("DATE_FORMAT", "%Y-%m-%d %H:%M") self.GLOBAL_CONTEXT["blog_author"] = self.config.get("BLOG_AUTHOR") self.GLOBAL_CONTEXT["blog_title"] = self.config.get("BLOG_TITLE") self.GLOBAL_CONTEXT["blog_url"] = self.config.get("SITE_URL", self.config.get("BLOG_URL")) self.GLOBAL_CONTEXT["blog_desc"] = self.config.get("BLOG_DESCRIPTION") self.GLOBAL_CONTEXT["analytics"] = self.config.get("ANALYTICS") self.GLOBAL_CONTEXT["translations"] = self.config.get("TRANSLATIONS") self.GLOBAL_CONTEXT["license"] = self.config.get("LICENSE") self.GLOBAL_CONTEXT["search_form"] = self.config.get("SEARCH_FORM") self.GLOBAL_CONTEXT["disqus_forum"] = self.config.get("DISQUS_FORUM") self.GLOBAL_CONTEXT["mathjax_config"] = self.config.get("MATHJAX_CONFIG") self.GLOBAL_CONTEXT["subtheme"] = self.config.get("THEME_REVEAL_CONGIF_SUBTHEME") self.GLOBAL_CONTEXT["transition"] = self.config.get("THEME_REVEAL_CONGIF_TRANSITION") self.GLOBAL_CONTEXT["content_footer"] = self.config.get("CONTENT_FOOTER") self.GLOBAL_CONTEXT["rss_path"] = self.config.get("RSS_PATH") self.GLOBAL_CONTEXT["rss_link"] = self.config.get("RSS_LINK") self.GLOBAL_CONTEXT["sidebar_links"] = utils.Functionary(list, self.config["DEFAULT_LANG"]) for k, v in self.config.get("SIDEBAR_LINKS", {}).items(): self.GLOBAL_CONTEXT["sidebar_links"][k] = v self.GLOBAL_CONTEXT["twitter_card"] = self.config.get("TWITTER_CARD", {}) self.GLOBAL_CONTEXT["extra_head_data"] = self.config.get("EXTRA_HEAD_DATA") self.GLOBAL_CONTEXT.update(self.config.get("GLOBAL_CONTEXT", {})) # check if custom css exist and is not empty for files_path in list(self.config["FILES_FOLDERS"].keys()): custom_css_path = os.path.join(files_path, "assets/css/custom.css") if self.file_exists(custom_css_path, not_empty=True): self.GLOBAL_CONTEXT["has_custom_css"] = True break else: self.GLOBAL_CONTEXT["has_custom_css"] = False # Load template plugin template_sys_name = utils.get_template_engine(self.THEMES) pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem") if pi is None: sys.stderr.write("Error loading {0} template system " "plugin\n".format(template_sys_name)) sys.exit(1) self.template_system = pi.plugin_object lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES] self.template_system.set_directories(lookup_dirs, self.config["CACHE_FOLDER"]) # Check consistency of USE_CDN and the current THEME (Issue #386) if self.config["USE_CDN"]: bootstrap_path = utils.get_asset_path(os.path.join("assets", "css", "bootstrap.min.css"), self.THEMES) if bootstrap_path.split(os.sep)[-4] != "site": warnings.warn( "The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap." ) # Load compiler plugins self.compilers = {} self.inverse_compilers = {} for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"): self.compilers[plugin_info.name] = plugin_info.plugin_object.compile_html def get_compiler(self, source_name): """Get the correct compiler for a post from `conf.post_compilers` To make things easier for users, the mapping in conf.py is compiler->[extensions], although this is less convenient for us. The majority of this function is reversing that dictionary and error checking. """ ext = os.path.splitext(source_name)[1] try: compile_html = self.inverse_compilers[ext] except KeyError: # Find the correct compiler for this files extension langs = [lang for lang, exts in list(self.config["post_compilers"].items()) if ext in exts] if len(langs) != 1: if len(set(langs)) > 1: exit( "Your file extension->compiler definition is" "ambiguous.\nPlease remove one of the file extensions" "from 'post_compilers' in conf.py\n(The error is in" "one of {0})".format(", ".join(langs)) ) elif len(langs) > 1: langs = langs[:1] else: exit("post_compilers in conf.py does not tell me how to " "handle '{0}' extensions.".format(ext)) lang = langs[0] compile_html = self.compilers[lang] self.inverse_compilers[ext] = compile_html return compile_html def render_template(self, template_name, output_name, context): local_context = {} local_context["template_name"] = template_name local_context.update(self.GLOBAL_CONTEXT) local_context.update(context) data = self.template_system.render_template(template_name, None, local_context) assert output_name.startswith(self.config["OUTPUT_FOLDER"]) url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1 :] # Treat our site as if output/ is "/" and then make all URLs relative, # making the site "relocatable" src = os.sep + url_part src = os.path.normpath(src) # The os.sep is because normpath will change "/" to "\" on windows src = "/".join(src.split(os.sep)) parsed_src = urlsplit(src) src_elems = parsed_src.path.split("/")[1:] def replacer(dst): # Refuse to replace links that are full URLs. dst_url = urlparse(dst) if dst_url.netloc: if dst_url.scheme == "link": # Magic link dst = self.link(dst_url.netloc, dst_url.path.lstrip("/"), context["lang"]) else: return dst # Normalize dst = urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_dst = urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute dst_elems = parsed_dst.path.split("/")[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break # Now i is the longest common prefix result = "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:]) if not result: result = "." # Don't forget the fragment (anchor) part of the link if parsed_dst.fragment: result += "#" + parsed_dst.fragment assert result, (src, dst, i, src_elems, dst_elems) return result try: os.makedirs(os.path.dirname(output_name)) except: pass doc = lxml.html.document_fromstring(data) doc.rewrite_links(replacer) data = b"<!DOCTYPE html>" + lxml.html.tostring(doc, encoding="utf8") with open(output_name, "wb+") as post_file: post_file.write(data) def current_lang(self): # FIXME: this is duplicated, turn into a mixin """Return the currently set locale, if it's one of the available translations, or default_lang.""" lang = utils.LocaleBorg().current_lang if lang: if lang in self.translations: return lang lang = lang.split("_")[0] if lang in self.translations: return lang # whatever return self.default_lang def path(self, kind, name, lang=None, is_link=False): """Build the path to a certain kind of page. kind is one of: * tag_index (name is ignored) * tag (and name is the tag name) * tag_rss (name is the tag name) * archive (and name is the year, or None for the main archive index) * index (name is the number in index-number) * rss (name is ignored) * gallery (name is the gallery name) * listing (name is the source code file name) * post_path (name is 1st element in a post_pages tuple) The returned value is always a path relative to output, like "categories/whatever.html" If is_link is True, the path is absolute and uses "/" as separator (ex: "/archive/index.html"). If is_link is False, the path is relative to output and uses the platform's separator. (ex: "archive\\index.html") """ if lang is None: lang = self.current_lang() path = [] if kind == "tag_index": path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], "index.html"] if _f] elif kind == "tag": if self.config["SLUG_TAG_PATH"]: name = utils.slugify(name) path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".html"] if _f] elif kind == "tag_rss": if self.config["SLUG_TAG_PATH"]: name = utils.slugify(name) path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".xml"] if _f] elif kind == "index": if name not in [None, 0]: path = [ _f for _f in [ self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index-{0}.html".format(name), ] if _f ] else: path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index.html"] if _f] elif kind == "post_path": path = [_f for _f in [self.config["TRANSLATIONS"][lang], os.path.dirname(name), "index.html"] if _f] elif kind == "rss": path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["RSS_PATH"], "rss.xml"] if _f] elif kind == "archive": if name: path = [ _f for _f in [self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], name, "index.html"] if _f ] else: path = [ _f for _f in [ self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], self.config["ARCHIVE_FILENAME"], ] if _f ] elif kind == "gallery": path = [_f for _f in [self.config["GALLERY_PATH"], name, "index.html"] if _f] elif kind == "listing": path = [_f for _f in [self.config["LISTINGS_FOLDER"], name + ".html"] if _f] if is_link: link = "/" + ("/".join(path)) if self.config["STRIP_INDEX_HTML"] and link.endswith("/index.html"): return link[:-10] else: return link else: return os.path.join(*path) def link(self, *args): return self.path(*args, is_link=True) def abs_link(self, dst): # Normalize dst = urljoin(self.config["BASE_URL"], dst) return urlparse(dst).path def rel_link(self, src, dst): # Normalize src = urljoin(self.config["BASE_URL"], src) dst = urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_src = urlsplit(src) parsed_dst = urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute src_elems = parsed_src.path.split("/")[1:] dst_elems = parsed_dst.path.split("/")[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break else: i += 1 # Now i is the longest common prefix return "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:]) def file_exists(self, path, not_empty=False): """Returns True if the file exists. If not_empty is True, it also has to be not empty.""" exists = os.path.exists(path) if exists and not_empty: exists = os.stat(path).st_size > 0 return exists def gen_tasks(self): def create_gzipped_copy(in_path, out_path): with gzip.GzipFile(out_path, "wb+") as outf: with open(in_path, "rb") as inf: outf.write(inf.read()) def flatten(task): if isinstance(task, dict): yield task else: for t in task: for ft in flatten(t): yield ft def add_gzipped_copies(task): if not self.config["GZIP_FILES"]: return None if task.get("name") is None: return None gzip_task = { "file_dep": [], "targets": [], "actions": [], "basename": "gzip", "name": task.get("name") + ".gz", "clean": True, } targets = task.get("targets", []) flag = False for target in targets: ext = os.path.splitext(target)[1] if ext.lower() in self.config["GZIP_EXTENSIONS"] and target.startswith(self.config["OUTPUT_FOLDER"]): flag = True gzipped = target + ".gz" gzip_task["file_dep"].append(target) gzip_task["targets"].append(gzipped) gzip_task["actions"].append((create_gzipped_copy, (target, gzipped))) if not flag: return None return gzip_task if self.config["GZIP_FILES"]: task_dep = ["gzip"] else: task_dep = [] for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"): for task in flatten(pluginInfo.plugin_object.gen_tasks()): gztask = add_gzipped_copies(task) if gztask: yield gztask yield task if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"): for task in pluginInfo.plugin_object.gen_tasks(): gztask = add_gzipped_copies(task) if gztask: yield gztask yield task if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) yield {"name": b"all", "actions": None, "clean": True, "task_dep": task_dep} def scan_posts(self): """Scan all the posts.""" if self._scanned: return print("Scanning posts", end="") tzinfo = None if self.config["TIMEZONE"] is not None: tzinfo = pytz.timezone(self.config["TIMEZONE"]) targets = set([]) for wildcard, destination, template_name, use_in_feeds in self.config["post_pages"]: print(".", end="") dirname = os.path.dirname(wildcard) for dirpath, _, _ in os.walk(dirname): dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) dest_dir = os.path.normpath(os.path.join(destination, os.path.relpath(dirpath, dirname))) full_list = glob.glob(dir_glob) # Now let's look for things that are not in default_lang for lang in self.config["TRANSLATIONS"].keys(): lang_glob = dir_glob + "." + lang translated_list = glob.glob(lang_glob) for fname in translated_list: orig_name = os.path.splitext(fname)[0] if orig_name in full_list: continue full_list.append(orig_name) for base_path in full_list: post = Post( base_path, self.config["CACHE_FOLDER"], dest_dir, use_in_feeds, self.config["TRANSLATIONS"], self.config["DEFAULT_LANG"], self.config["BASE_URL"], self.MESSAGES, template_name, self.config["FILE_METADATA_REGEXP"], self.config["STRIP_INDEX_HTML"], tzinfo, self.config["HIDE_UNTRANSLATED_POSTS"], ) for lang, langpath in list(self.config["TRANSLATIONS"].items()): dest = (destination, langpath, dir_glob, post.meta[lang]["slug"]) if dest in targets: raise Exception( "Duplicated output path {0!r} " "in post {1!r}".format(post.meta[lang]["slug"], base_path) ) targets.add(dest) self.global_data[post.post_name] = post if post.use_in_feeds: self.posts_per_year[str(post.date.year)].append(post.post_name) self.posts_per_month["{0}/{1:02d}".format(post.date.year, post.date.month)].append( post.post_name ) for tag in post.tags: self.posts_per_tag[tag].append(post.post_name) else: self.pages.append(post) if self.config["OLD_THEME_SUPPORT"]: post._add_old_metadata() for name, post in list(self.global_data.items()): self.timeline.append(post) self.timeline.sort(key=lambda p: p.date) self.timeline.reverse() post_timeline = [p for p in self.timeline if p.use_in_feeds] for i, p in enumerate(post_timeline[1:]): p.next_post = post_timeline[i] for i, p in enumerate(post_timeline[:-1]): p.prev_post = post_timeline[i + 1] self._scanned = True print("done!") def generic_page_renderer(self, lang, post, filters): """Render post fragments to final HTML pages.""" context = {} deps = post.deps(lang) + self.template_system.template_deps(post.template_name) context["post"] = post context["lang"] = lang context["title"] = post.title(lang) context["description"] = post.description(lang) context["permalink"] = post.permalink(lang) context["page_list"] = self.pages if post.use_in_feeds: context["enable_comments"] = True else: context["enable_comments"] = self.config["COMMENTS_IN_STORIES"] output_name = os.path.join(self.config["OUTPUT_FOLDER"], post.destination_path(lang)) deps_dict = copy(context) deps_dict.pop("post") if post.prev_post: deps_dict["PREV_LINK"] = [post.prev_post.permalink(lang)] if post.next_post: deps_dict["NEXT_LINK"] = [post.next_post.permalink(lang)] deps_dict["OUTPUT_FOLDER"] = self.config["OUTPUT_FOLDER"] deps_dict["TRANSLATIONS"] = self.config["TRANSLATIONS"] deps_dict["global"] = self.GLOBAL_CONTEXT deps_dict["comments"] = context["enable_comments"] task = { "name": os.path.normpath(output_name), "file_dep": deps, "targets": [output_name], "actions": [(self.render_template, [post.template_name, output_name, context])], "clean": True, "uptodate": [config_changed(deps_dict)], } yield utils.apply_filters(task, filters) def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context): """Renders pages with lists of posts.""" deps = self.template_system.template_deps(template_name) for post in posts: deps += post.deps(lang) context = {} context["posts"] = posts context["title"] = self.config["BLOG_TITLE"] context["description"] = self.config["BLOG_DESCRIPTION"] context["lang"] = lang context["prevlink"] = None context["nextlink"] = None context.update(extra_context) deps_context = copy(context) deps_context["posts"] = [(p.meta[lang]["title"], p.permalink(lang)) for p in posts] deps_context["global"] = self.GLOBAL_CONTEXT task = { "name": os.path.normpath(output_name), "targets": [output_name], "file_dep": deps, "actions": [(self.render_template, [template_name, output_name, context])], "clean": True, "uptodate": [config_changed(deps_context)], } return utils.apply_filters(task, filters)
class Nikola(object): """Class that handles site generation. Takes a site config as argument on creation. """ EXTRA_PLUGINS = [ 'planetoid', 'ipynb', 'local_search', 'render_mustache', ] def __init__(self, **config): """Setup proper environment for running tasks.""" # Register our own path handlers self.path_handlers = { 'slug': self.slug_path, 'post_path': self.post_path, } self.strict = False self.global_data = {} self.posts = [] self.posts_per_year = defaultdict(list) self.posts_per_month = defaultdict(list) self.posts_per_tag = defaultdict(list) self.posts_per_category = defaultdict(list) self.post_per_file = {} self.timeline = [] self.pages = [] self._scanned = False self._template_system = None self._THEMES = None self.debug = DEBUG self.loghandlers = [] if not config: self.configured = False else: self.configured = True # This is the default config self.config = { 'ADD_THIS_BUTTONS': True, 'ANNOTATIONS': False, 'ARCHIVE_PATH': "", 'ARCHIVE_FILENAME': "archive.html", 'BLOG_TITLE': 'Default Title', 'BLOG_DESCRIPTION': 'Default Description', 'BODY_END': "", 'CACHE_FOLDER': 'cache', 'CODE_COLOR_SCHEME': 'default', 'COMMENT_SYSTEM': 'disqus', 'COMMENTS_IN_GALLERIES': False, 'COMMENTS_IN_STORIES': False, 'COMPILERS': { "rest": ('.txt', '.rst'), "markdown": ('.md', '.mdown', '.markdown'), "textile": ('.textile',), "txt2tags": ('.t2t',), "bbcode": ('.bb',), "wiki": ('.wiki',), "ipynb": ('.ipynb',), "html": ('.html', '.htm') }, 'CONTENT_FOOTER': '', 'COPY_SOURCES': True, 'CREATE_MONTHLY_ARCHIVE': False, 'CREATE_SINGLE_ARCHIVE': False, 'DATE_FORMAT': '%Y-%m-%d %H:%M', 'DEFAULT_LANG': "en", 'DEPLOY_COMMANDS': [], 'DISABLED_PLUGINS': (), 'EXTRA_PLUGINS_DIRS': [], 'COMMENT_SYSTEM_ID': 'nikolademo', 'ENABLED_EXTRAS': (), 'EXTRA_HEAD_DATA': '', 'FAVICONS': {}, 'FEED_LENGTH': 10, 'FILE_METADATA_REGEXP': None, 'ADDITIONAL_METADATA': {}, 'FILES_FOLDERS': {'files': ''}, 'FILTERS': {}, 'GALLERY_PATH': 'galleries', 'GALLERY_SORT_BY_DATE': True, 'GZIP_COMMAND': None, 'GZIP_FILES': False, 'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json', '.xml'), 'HIDE_SOURCELINK': False, 'HIDE_UNTRANSLATED_POSTS': False, 'HYPHENATE': False, 'INDEX_DISPLAY_POST_COUNT': 10, 'INDEX_FILE': 'index.html', 'INDEX_TEASERS': False, 'INDEXES_TITLE': "", 'INDEXES_PAGES': "", 'INDEXES_PAGES_MAIN': False, 'INDEX_PATH': '', 'IPYNB_CONFIG': {}, 'LESS_COMPILER': 'lessc', 'LICENSE': '', 'LINK_CHECK_WHITELIST': [], 'LISTINGS_FOLDER': 'listings', 'NAVIGATION_LINKS': None, 'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'], 'MAX_IMAGE_SIZE': 1280, 'MATHJAX_CONFIG': '', 'OLD_THEME_SUPPORT': True, 'OUTPUT_FOLDER': 'output', 'POSTS': (("posts/*.txt", "posts", "post.tmpl"),), 'PAGES': (("stories/*.txt", "stories", "story.tmpl"),), 'PRETTY_URLS': False, 'FUTURE_IS_NOW': False, 'READ_MORE_LINK': '<p class="more"><a href="{link}">{read_more}…</a></p>', 'REDIRECTIONS': [], 'RSS_LINK': None, 'RSS_PATH': '', 'RSS_TEASERS': True, 'SASS_COMPILER': 'sass', 'SEARCH_FORM': '', 'SLUG_TAG_PATH': True, 'SOCIAL_BUTTONS_CODE': SOCIAL_BUTTONS_CODE, 'SITE_URL': 'http://getnikola.com/', 'STORY_INDEX': False, 'STRIP_INDEXES': False, 'SITEMAP_INCLUDE_FILELESS_DIRS': True, 'TAG_PATH': 'categories', 'TAG_PAGES_ARE_INDEXES': False, 'THEME': 'bootstrap', 'THEME_REVEAL_CONFIG_SUBTHEME': 'sky', 'THEME_REVEAL_CONFIG_TRANSITION': 'cube', 'THUMBNAIL_SIZE': 180, 'URL_TYPE': 'rel_path', 'USE_BUNDLES': True, 'USE_CDN': False, 'USE_FILENAME_AS_TITLE': True, 'TIMEZONE': 'UTC', 'DEPLOY_DRAFTS': True, 'DEPLOY_FUTURE': False, 'SCHEDULE_ALL': False, 'SCHEDULE_RULE': '', 'SCHEDULE_FORCE_TODAY': False, 'LOGGING_HANDLERS': {'stderr': {'loglevel': 'WARNING', 'bubble': True}}, 'DEMOTE_HEADERS': 1, } self.config.update(config) # Make sure we have pyphen installed if we are using it if self.config.get('HYPHENATE') and pyphen is None: utils.LOGGER.warn('To use the hyphenation, you have to install ' 'the "pyphen" package.') utils.LOGGER.warn('Setting HYPHENATE to False.') self.config['HYPHENATE'] = False # Deprecating post_compilers # TODO: remove on v7 if 'post_compilers' in config: utils.LOGGER.warn('The post_compilers option is deprecated, use COMPILERS instead.') if 'COMPILERS' in config: utils.LOGGER.warn('COMPILERS conflicts with post_compilers, ignoring post_compilers.') else: self.config['COMPILERS'] = config['post_compilers'] # Deprecating post_pages # TODO: remove on v7 if 'post_pages' in config: utils.LOGGER.warn('The post_pages option is deprecated, use POSTS and PAGES instead.') if 'POSTS' in config or 'PAGES' in config: utils.LOGGER.warn('POSTS and PAGES conflict with post_pages, ignoring post_pages.') else: self.config['POSTS'] = [item[:3] for item in config['post_pages'] if item[-1]] self.config['PAGES'] = [item[:3] for item in config['post_pages'] if not item[-1]] # FIXME: Internally, we still use post_pages because it's a pain to change it self.config['post_pages'] = [] for i1, i2, i3 in self.config['POSTS']: self.config['post_pages'].append([i1, i2, i3, True]) for i1, i2, i3 in self.config['PAGES']: self.config['post_pages'].append([i1, i2, i3, False]) # Deprecating DISQUS_FORUM # TODO: remove on v7 if 'DISQUS_FORUM' in config: utils.LOGGER.warn('The DISQUS_FORUM option is deprecated, use COMMENT_SYSTEM_ID instead.') if 'COMMENT_SYSTEM_ID' in config: utils.LOGGER.warn('DISQUS_FORUM conflicts with COMMENT_SYSTEM_ID, ignoring DISQUS_FORUM.') else: self.config['COMMENT_SYSTEM_ID'] = config['DISQUS_FORUM'] # Deprecating the ANALYTICS option # TODO: remove on v7 if 'ANALYTICS' in config: utils.LOGGER.warn('The ANALYTICS option is deprecated, use BODY_END instead.') if 'BODY_END' in config: utils.LOGGER.warn('ANALYTICS conflicts with BODY_END, ignoring ANALYTICS.') else: self.config['BODY_END'] = config['ANALYTICS'] # Deprecating the SIDEBAR_LINKS option # TODO: remove on v7 if 'SIDEBAR_LINKS' in config: utils.LOGGER.warn('The SIDEBAR_LINKS option is deprecated, use NAVIGATION_LINKS instead.') if 'NAVIGATION_LINKS' in config: utils.LOGGER.warn('The SIDEBAR_LINKS conflicts with NAVIGATION_LINKS, ignoring SIDEBAR_LINKS.') else: self.config['NAVIGATION_LINKS'] = config['SIDEBAR_LINKS'] # Compatibility alias self.config['SIDEBAR_LINKS'] = self.config['NAVIGATION_LINKS'] if self.config['NAVIGATION_LINKS'] in (None, {}): self.config['NAVIGATION_LINKS'] = {self.config['DEFAULT_LANG']: ()} # Deprecating the ADD_THIS_BUTTONS option # TODO: remove on v7 if 'ADD_THIS_BUTTONS' in config: utils.LOGGER.warn('The ADD_THIS_BUTTONS option is deprecated, use SOCIAL_BUTTONS_CODE instead.') if not config['ADD_THIS_BUTTONS']: utils.LOGGER.warn('Setting SOCIAL_BUTTONS_CODE to empty because ADD_THIS_BUTTONS is False.') self.config['SOCIAL_BUTTONS_CODE'] = '' # STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES # Port it if only the oldef form is there # TODO: remove on v7 if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config: utils.LOGGER.warn('You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML') self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML'] # PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config: self.config['STRIP_INDEXES'] = True if config.get('COPY_SOURCES') and not self.config['HIDE_SOURCELINK']: self.config['HIDE_SOURCELINK'] = True self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS', {self.config['DEFAULT_LANG']: ''}) # SITE_URL is required, but if the deprecated BLOG_URL # is available, use it and warn # TODO: remove on v7 if 'SITE_URL' not in self.config: if 'BLOG_URL' in self.config: utils.LOGGER.warn('You should configure SITE_URL instead of BLOG_URL') self.config['SITE_URL'] = self.config['BLOG_URL'] self.default_lang = self.config['DEFAULT_LANG'] self.translations = self.config['TRANSLATIONS'] locale_fallback, locale_default, locales = sanitized_locales( self.config.get('LOCALE_FALLBACK', None), self.config.get('LOCALE_DEFAULT', None), self.config.get('LOCALES', {}), self.translations) # NOQA utils.LocaleBorg.initialize(locales, self.default_lang) # BASE_URL defaults to SITE_URL if 'BASE_URL' not in self.config: self.config['BASE_URL'] = self.config.get('SITE_URL') # BASE_URL should *always* end in / if self.config['BASE_URL'] and self.config['BASE_URL'][-1] != '/': utils.LOGGER.warn("Your BASE_URL doesn't end in / -- adding it.") self.plugin_manager = PluginManager(categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "RestExtension": RestExtension, "SignalHandler": SignalHandler, }) self.plugin_manager.setPluginInfoExtension('plugin') extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS'] if sys.version_info[0] == 3: places = [ os.path.join(os.path.dirname(__file__), 'plugins'), os.path.join(os.getcwd(), 'plugins'), ] + [path for path in extra_plugins_dirs if path] else: places = [ os.path.join(os.path.dirname(__file__), utils.sys_encode('plugins')), os.path.join(os.getcwd(), utils.sys_encode('plugins')), ] + [utils.sys_encode(path) for path in extra_plugins_dirs if path] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() # Activate all required SignalHandler plugins for plugin_info in self.plugin_manager.getPluginsOfCategory("SignalHandler"): if plugin_info.name in self.config.get('DISABLED_PLUGINS'): self.plugin_manager.removePluginFromCategory(plugin_info, "SignalHandler") else: self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) # Emit signal for SignalHandlers which need to start running immediately. signal('sighandlers_loaded').send(self) self.commands = {} # Activate all command plugins for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"): if (plugin_info.name in self.config['DISABLED_PLUGINS'] or (plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config['ENABLED_EXTRAS'])): self.plugin_manager.removePluginFromCategory(plugin_info, "Command") continue self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugin_info.plugin_object.short_help = plugin_info.description self.commands[plugin_info.name] = plugin_info.plugin_object # Activate all task plugins for task_type in ["Task", "LateTask"]: for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type): if (plugin_info.name in self.config['DISABLED_PLUGINS'] or (plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config['ENABLED_EXTRAS'])): self.plugin_manager.removePluginFromCategory(plugin_info, task_type) continue self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) # Activate all multiplier plugins for plugin_info in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"): if (plugin_info.name in self.config['DISABLED_PLUGINS'] or (plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config['ENABLED_EXTRAS'])): self.plugin_manager.removePluginFromCategory(plugin_info, task_type) continue self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) # Activate all required compiler plugins for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"): if plugin_info.name in self.config["COMPILERS"].keys(): self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) # set global_context for template rendering self._GLOBAL_CONTEXT = {} self._GLOBAL_CONTEXT['_link'] = self.link self._GLOBAL_CONTEXT['set_locale'] = utils.LocaleBorg().set_locale self._GLOBAL_CONTEXT['rel_link'] = self.rel_link self._GLOBAL_CONTEXT['abs_link'] = self.abs_link self._GLOBAL_CONTEXT['exists'] = self.file_exists self._GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config['SLUG_TAG_PATH'] self._GLOBAL_CONTEXT['annotations'] = self.config['ANNOTATIONS'] self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[ 'INDEX_DISPLAY_POST_COUNT'] self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES'] self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN") self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS'] self._GLOBAL_CONTEXT['date_format'] = self.config.get( 'DATE_FORMAT', '%Y-%m-%d %H:%M') self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR') self._GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE') # TODO: remove fallback in v7 self._GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL', self.config.get('BLOG_URL')) self._GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION') self._GLOBAL_CONTEXT['body_end'] = self.config.get('BODY_END') # TODO: remove in v7 self._GLOBAL_CONTEXT['analytics'] = self.config.get('BODY_END') # TODO: remove in v7 self._GLOBAL_CONTEXT['add_this_buttons'] = self.config.get('SOCIAL_BUTTONS_CODE') self._GLOBAL_CONTEXT['social_buttons_code'] = self.config.get('SOCIAL_BUTTONS_CODE') self._GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS') self._GLOBAL_CONTEXT['license'] = self.config.get('LICENSE') self._GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM') self._GLOBAL_CONTEXT['comment_system'] = self.config.get('COMMENT_SYSTEM') self._GLOBAL_CONTEXT['comment_system_id'] = self.config.get('COMMENT_SYSTEM_ID') # TODO: remove in v7 self._GLOBAL_CONTEXT['disqus_forum'] = self.config.get('COMMENT_SYSTEM_ID') self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get( 'MATHJAX_CONFIG') self._GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONFIG_SUBTHEME') self._GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONFIG_TRANSITION') self._GLOBAL_CONTEXT['content_footer'] = self.config.get( 'CONTENT_FOOTER') self._GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH') self._GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK') self._GLOBAL_CONTEXT['navigation_links'] = utils.Functionary(list, self.config['DEFAULT_LANG']) for k, v in self.config.get('NAVIGATION_LINKS', {}).items(): self._GLOBAL_CONTEXT['navigation_links'][k] = v # TODO: remove on v7 # Compatibility alias self._GLOBAL_CONTEXT['sidebar_links'] = self._GLOBAL_CONTEXT['navigation_links'] self._GLOBAL_CONTEXT['twitter_card'] = self.config.get( 'TWITTER_CARD', {}) self._GLOBAL_CONTEXT['hide_sourcelink'] = self.config.get( 'HIDE_SOURCELINK') self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA') self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {})) # Load compiler plugins self.compilers = {} self.inverse_compilers = {} for plugin_info in self.plugin_manager.getPluginsOfCategory( "PageCompiler"): self.compilers[plugin_info.name] = \ plugin_info.plugin_object signal('configured').send(self) def _get_themes(self): if self._THEMES is None: # Check for old theme names (Issue #650) TODO: remove in v7 theme_replacements = { 'site': 'bootstrap', 'orphan': 'base', 'default': 'oldfashioned', } if self.config['THEME'] in theme_replacements: utils.LOGGER.warn('You are using the old theme "{0}", using "{1}" instead.'.format( self.config['THEME'], theme_replacements[self.config['THEME']])) self.config['THEME'] = theme_replacements[self.config['THEME']] if self.config['THEME'] == 'oldfashioned': utils.LOGGER.warn('''You may need to install the "oldfashioned" theme ''' '''from themes.nikola.ralsina.com.ar because it's not ''' '''shipped by default anymore.''') utils.LOGGER.warn('Please change your THEME setting.') try: self._THEMES = utils.get_theme_chain(self.config['THEME']) except Exception: utils.LOGGER.warn('''Can't load theme "{0}", using 'bootstrap' instead.'''.format(self.config['THEME'])) self.config['THEME'] = 'bootstrap' return self._get_themes() # Check consistency of USE_CDN and the current THEME (Issue #386) if self.config['USE_CDN']: bootstrap_path = utils.get_asset_path(os.path.join( 'assets', 'css', 'bootstrap.min.css'), self._THEMES) if bootstrap_path and bootstrap_path.split(os.sep)[-4] not in ['bootstrap', 'bootstrap3']: utils.LOGGER.warn('The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.') return self._THEMES THEMES = property(_get_themes) def _get_messages(self): return utils.load_messages(self.THEMES, self.translations, self.default_lang) MESSAGES = property(_get_messages) def _get_global_context(self): """Initialize some parts of GLOBAL_CONTEXT only when it's queried.""" if 'messages' not in self._GLOBAL_CONTEXT: self._GLOBAL_CONTEXT['messages'] = self.MESSAGES if 'has_custom_css' not in self._GLOBAL_CONTEXT: # check if custom css exist and is not empty custom_css_path = utils.get_asset_path( 'assets/css/custom.css', self.THEMES, self.config['FILES_FOLDERS'] ) if custom_css_path and self.file_exists(custom_css_path, not_empty=True): self._GLOBAL_CONTEXT['has_custom_css'] = True else: self._GLOBAL_CONTEXT['has_custom_css'] = False return self._GLOBAL_CONTEXT GLOBAL_CONTEXT = property(_get_global_context) def _get_template_system(self): if self._template_system is None: # Load template plugin template_sys_name = utils.get_template_engine(self.THEMES) pi = self.plugin_manager.getPluginByName( template_sys_name, "TemplateSystem") if pi is None: sys.stderr.write("Error loading {0} template system " "plugin\n".format(template_sys_name)) sys.exit(1) self._template_system = pi.plugin_object lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES] self._template_system.set_directories(lookup_dirs, self.config['CACHE_FOLDER']) return self._template_system template_system = property(_get_template_system) def get_compiler(self, source_name): """Get the correct compiler for a post from `conf.COMPILERS` To make things easier for users, the mapping in conf.py is compiler->[extensions], although this is less convenient for us. The majority of this function is reversing that dictionary and error checking. """ ext = os.path.splitext(source_name)[1] try: compile_html = self.inverse_compilers[ext] except KeyError: # Find the correct compiler for this files extension langs = [lang for lang, exts in list(self.config['COMPILERS'].items()) if ext in exts] if len(langs) != 1: if len(set(langs)) > 1: exit("Your file extension->compiler definition is" "ambiguous.\nPlease remove one of the file extensions" "from 'COMPILERS' in conf.py\n(The error is in" "one of {0})".format(', '.join(langs))) elif len(langs) > 1: langs = langs[:1] else: exit("COMPILERS in conf.py does not tell me how to " "handle '{0}' extensions.".format(ext)) lang = langs[0] compile_html = self.compilers[lang] self.inverse_compilers[ext] = compile_html return compile_html def render_template(self, template_name, output_name, context): local_context = {} local_context["template_name"] = template_name local_context.update(self.GLOBAL_CONTEXT) local_context.update(context) # string, arguments local_context["formatmsg"] = lambda s, *a: s % a data = self.template_system.render_template( template_name, None, local_context) assert output_name.startswith( self.config["OUTPUT_FOLDER"]) url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:] # Treat our site as if output/ is "/" and then make all URLs relative, # making the site "relocatable" src = os.sep + url_part src = os.path.normpath(src) # The os.sep is because normpath will change "/" to "\" on windows src = "/".join(src.split(os.sep)) parsed_src = urlsplit(src) src_elems = parsed_src.path.split('/')[1:] def replacer(dst): # Refuse to replace links that are full URLs. dst_url = urlparse(dst) if dst_url.netloc: if dst_url.scheme == 'link': # Magic link dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'), context['lang']) else: return dst # Refuse to replace links that consist of a fragment only if ((not dst_url.scheme) and (not dst_url.netloc) and (not dst_url.path) and (not dst_url.params) and (not dst_url.query) and dst_url.fragment): return dst # Normalize dst = urljoin(src, dst.lstrip('/')) # Avoid empty links. if src == dst: if self.config.get('URL_TYPE') == 'absolute': dst = urljoin(self.config['BASE_URL'], dst.lstrip('/')) return dst elif self.config.get('URL_TYPE') == 'full_path': return dst else: return "#" # Check that link can be made relative, otherwise return dest parsed_dst = urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: if self.config.get('URL_TYPE') == 'absolute': dst = urljoin(self.config['BASE_URL'], dst.lstrip('/')) return dst if self.config.get('URL_TYPE') in ('full_path', 'absolute'): if self.config.get('URL_TYPE') == 'absolute': dst = urljoin(self.config['BASE_URL'], dst.lstrip('/')) return dst # Now both paths are on the same site and absolute dst_elems = parsed_dst.path.split('/')[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break # Now i is the longest common prefix result = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) if not result: result = "." # Don't forget the fragment (anchor) part of the link if parsed_dst.fragment: result += "#" + parsed_dst.fragment assert result, (src, dst, i, src_elems, dst_elems) return result utils.makedirs(os.path.dirname(output_name)) doc = lxml.html.document_fromstring(data) doc.rewrite_links(replacer) data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8') with open(output_name, "wb+") as post_file: post_file.write(data) def path(self, kind, name, lang=None, is_link=False): """Build the path to a certain kind of page. These are mostly defined by plugins by registering via the register_path_handler method, except for slug and post_path which are defined in this class' init method. Here's some of the others, for historical reasons: * tag_index (name is ignored) * tag (and name is the tag name) * tag_rss (name is the tag name) * category (and name is the category name) * category_rss (and name is the category name) * archive (and name is the year, or None for the main archive index) * index (name is the number in index-number) * rss (name is ignored) * gallery (name is the gallery name) * listing (name is the source code file name) * post_path (name is 1st element in a POSTS/PAGES tuple) * slug (name is the slug of a post or story) The returned value is always a path relative to output, like "categories/whatever.html" If is_link is True, the path is absolute and uses "/" as separator (ex: "/archive/index.html"). If is_link is False, the path is relative to output and uses the platform's separator. (ex: "archive\\index.html") """ if lang is None: lang = utils.LocaleBorg().current_lang path = self.path_handlers[kind](name, lang) if is_link: link = '/' + ('/'.join(path)) index_len = len(self.config['INDEX_FILE']) if self.config['STRIP_INDEXES'] and \ link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']: return link[:-index_len] else: return link else: return os.path.join(*path) def post_path(self, name, lang): """post_path path handler""" return [_f for _f in [self.config['TRANSLATIONS'][lang], os.path.dirname(name), self.config['INDEX_FILE']] if _f] def slug_path(self, name, lang): """slug path handler""" results = [p for p in self.timeline if p.meta('slug') == name] if not results: utils.LOGGER.warning("Can't resolve path request for slug: {0}".format(name)) else: if len(results) > 1: utils.LOGGER.warning('Ambiguous path request for slug: {0}'.format(name)) return [_f for _f in results[0].permalink(lang).split('/') if _f] def register_path_handler(self, kind, f): if kind in self.path_handlers: utils.LOGGER.warning('Conflicting path handlers for kind: {0}'.format(kind)) else: self.path_handlers[kind] = f def link(self, *args): return self.path(*args, is_link=True) def abs_link(self, dst): # Normalize dst = urljoin(self.config['BASE_URL'], dst.lstrip('/')) return urlparse(dst).geturl() def rel_link(self, src, dst): # Normalize try: src = urljoin(self.config['BASE_URL'], src.lstrip('/')) except AttributeError: # sometimes, it’s an Undefined object. src = urljoin(self.config['BASE_URL'], src) try: dst = urljoin(src, dst.lstrip('/')) except AttributeError: dst = urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_src = urlsplit(src) parsed_dst = urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute src_elems = parsed_src.path.split('/')[1:] dst_elems = parsed_dst.path.split('/')[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break else: i += 1 # Now i is the longest common prefix return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) def file_exists(self, path, not_empty=False): """Returns True if the file exists. If not_empty is True, it also has to be not empty.""" exists = os.path.exists(path) if exists and not_empty: exists = os.stat(path).st_size > 0 return exists def clean_task_paths(self, task): """Normalize target paths in the task.""" targets = task.get('targets', None) if targets is not None: task['targets'] = [os.path.normpath(t) for t in targets] return task def gen_tasks(self, name, plugin_category, doc=''): def flatten(task): if isinstance(task, dict): yield task else: for t in task: for ft in flatten(t): yield ft task_dep = [] for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category): for task in flatten(pluginInfo.plugin_object.gen_tasks()): assert 'basename' in task task = self.clean_task_paths(task) yield task for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"): flag = False for task in multi.plugin_object.process(task, name): flag = True yield self.clean_task_paths(task) if flag: task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name)) if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) yield { 'basename': name, 'doc': doc, 'actions': None, 'clean': True, 'task_dep': task_dep } def scan_posts(self): """Scan all the posts.""" if self._scanned: return seen = set([]) print("Scanning posts", end='', file=sys.stderr) lower_case_tags = set([]) for wildcard, destination, template_name, use_in_feeds in \ self.config['post_pages']: print(".", end='', file=sys.stderr) dirname = os.path.dirname(wildcard) for dirpath, _, _ in os.walk(dirname): dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) dest_dir = os.path.normpath(os.path.join(destination, os.path.relpath(dirpath, dirname))) full_list = glob.glob(dir_glob) # Now let's look for things that are not in default_lang for lang in self.config['TRANSLATIONS'].keys(): lang_glob = dir_glob + "." + lang translated_list = glob.glob(lang_glob) for fname in translated_list: orig_name = os.path.splitext(fname)[0] if orig_name in full_list: continue full_list.append(orig_name) # We eliminate from the list the files inside any .ipynb folder full_list = [p for p in full_list if not any([x.startswith('.') for x in p.split(os.sep)])] for base_path in full_list: if base_path in seen: continue else: seen.add(base_path) post = Post( base_path, self.config, dest_dir, use_in_feeds, self.MESSAGES, template_name, self.get_compiler(base_path) ) self.global_data[post.source_path] = post if post.use_in_feeds: self.posts.append(post.source_path) self.posts_per_year[ str(post.date.year)].append(post.source_path) self.posts_per_month[ '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post.source_path) for tag in post.alltags: if tag.lower() in lower_case_tags: if tag not in self.posts_per_tag: # Tags that differ only in case other_tag = [k for k in self.posts_per_tag.keys() if k.lower() == tag.lower()][0] utils.LOGGER.error('You have cases that differ only in upper/lower case: {0} and {1}'.format(tag, other_tag)) utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path)) utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join(self.posts_per_tag[other_tag]))) sys.exit(1) else: lower_case_tags.add(tag.lower()) self.posts_per_tag[tag].append(post.source_path) self.posts_per_category[post.meta('category')].append(post.source_path) else: self.pages.append(post) self.post_per_file[post.destination_path(lang=lang)] = post self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post for name, post in list(self.global_data.items()): self.timeline.append(post) self.timeline.sort(key=lambda p: p.date) self.timeline.reverse() post_timeline = [p for p in self.timeline if p.use_in_feeds] for i, p in enumerate(post_timeline[1:]): p.next_post = post_timeline[i] for i, p in enumerate(post_timeline[:-1]): p.prev_post = post_timeline[i + 1] self._scanned = True print("done!", file=sys.stderr) def generic_page_renderer(self, lang, post, filters): """Render post fragments to final HTML pages.""" context = {} deps = post.deps(lang) + \ self.template_system.template_deps(post.template_name) deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine')) deps = list(filter(None, deps)) context['post'] = post context['lang'] = lang context['title'] = post.title(lang) context['description'] = post.description(lang) context['permalink'] = post.permalink(lang) context['page_list'] = self.pages if post.use_in_feeds: context['enable_comments'] = True else: context['enable_comments'] = self.config['COMMENTS_IN_STORIES'] extension = self.get_compiler(post.source_path).extension() output_name = os.path.join(self.config['OUTPUT_FOLDER'], post.destination_path(lang, extension)) deps_dict = copy(context) deps_dict.pop('post') if post.prev_post: deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)] if post.next_post: deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)] deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER'] deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS'] deps_dict['global'] = self.GLOBAL_CONTEXT deps_dict['comments'] = context['enable_comments'] if post: deps_dict['post_translations'] = post.translated_to task = { 'name': os.path.normpath(output_name), 'file_dep': deps, 'targets': [output_name], 'actions': [(self.render_template, [post.template_name, output_name, context])], 'clean': True, 'uptodate': [config_changed(deps_dict)], } yield utils.apply_filters(task, filters) def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context): """Renders pages with lists of posts.""" deps = self.template_system.template_deps(template_name) for post in posts: deps += post.deps(lang) context = {} context["posts"] = posts context["title"] = self.config['BLOG_TITLE'] context["description"] = self.config['BLOG_DESCRIPTION'] context["lang"] = lang context["prevlink"] = None context["nextlink"] = None context.update(extra_context) deps_context = copy(context) deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in posts] deps_context["global"] = self.GLOBAL_CONTEXT task = { 'name': os.path.normpath(output_name), 'targets': [output_name], 'file_dep': deps, 'actions': [(self.render_template, [template_name, output_name, context])], 'clean': True, 'uptodate': [config_changed(deps_context)] } return utils.apply_filters(task, filters)
class TestDef(object): def __init__(self): # set aside storage for options and cmd line args self.options = {} self.args = [] # record if we have loaded the plugins or # not - this is just a bozo check to ensure # someone doesn't tell us to do it twice self.loaded = False # set aside a spot for a logger object, and # note that it hasn't yet been defined self.logger = None self.modcmd = None self.execmd = None self.harasser = None self.config = None self.stages = None self.tools = None self.utilities = None self.defaults = None self.log = {} def setOptions(self, args): self.options = vars(args) self.args = args # if they want us to clear the scratch, then do so if self.options['clean'] and os.path.isdir(self.options['scratchdir']): shutil.rmtree(self.options['scratchdir']) # setup the scratch directory _mkdir_recursive(self.options['scratchdir']) # private function to convert values def __convert_value(self, opt, inval): if opt is None or type(opt) is str: return 0, inval elif type(opt) is bool: if type(inval) is bool: return 0, inval elif type(inval) is str: if inval.lower() in ['true', '1', 't', 'y', 'yes']: return 0, True else: return 0, False elif type(inval) is int: if 0 == inval: return 0, False else: return 0, True elif is_py2 and type(inval) is unicode: return 0, int(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None elif type(opt) is int: if type(inval) is int: return 0, inval elif type(inval) is str: return 0, int(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None elif type(opt) is float: if type(inval) is float: return 0, inval elif type(inval) is str or type(inval) is int: return 0, float(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None else: return 1, None # scan the key-value pairs obtained from the configuration # parser and compare them with the options defined for a # given plugin. Generate an output dictionary that contains # the updated set of option values, the default value for # any option that wasn't included in the configuration file, # and return an error status plus output identifying any # keys in the configuration file that are not supported # by the list of options # # @log [INPUT] # - a dictionary that will return the status plus # stderr containing strings identifying any # provided keyvals that don't have a corresponding # supported option # @options [INPUT] # - a dictionary of tuples, each consisting of three # entries: # (a) the default value # (b) data type # (c) a help-like description # @keyvals [INPUT] # - a dictionary of key-value pairs obtained from # the configuration parser # @target [OUTPUT] # - the resulting dictionary of key-value pairs def parseOptions(self, log, options, keyvals, target): # parse the incoming keyvals dictionary against the source # options. If a source option isn't provided, then # copy it across to the target. opts = list(options.keys()) kvkeys = list(keyvals.keys()) for opt in opts: found = False for kvkey in kvkeys: if kvkey == opt: # they provided us with an update, so # pass this value into the target - expand # any provided lists if keyvals[kvkey] is None: continue st, outval = self.__convert_value(options[opt][0], keyvals[kvkey]) if 0 == st: target[opt] = outval else: if len(keyvals[kvkey]) == 0: # this indicates they do not want this option found = True break if keyvals[kvkey][0][0] == "[": # they provided a list - remove the brackets val = keyvals[kvkey].replace('[', '') val = val.replace(']', '') # split the input to pickup sets of options newvals = list(val) # convert the values to specified type i = 0 for val in newvals: st, newvals[i] = self.__convert_value( opt[0], val) i = i + 1 target[opt] = newvals else: st, target[opt] = self.__convert_value( opt[0], keyvals[kvkey]) found = True break if not found: # they didn't provide this one, so # transfer only the value across target[opt] = options[opt][0] # add in any default settings that have not # been overridden - anything set by this input # stage will override the default if self.defaults is not None: keys = self.defaults.options.keys() for key in keys: if key not in target: target[key] = self.defaults.options[key][0] # now go thru in the reverse direction to see # if any keyvals they provided aren't supported # as this would be an error stderr = [] for kvkey in kvkeys: # ignore some standard keys if kvkey in ['section', 'plugin']: continue try: if target[kvkey] is not None: pass except KeyError: # some always need to be passed if kvkey in ['parent', 'asis']: target[kvkey] = keyvals[kvkey] else: stderr.append("Option " + kvkey + " is not supported") if stderr: # mark the log with an error status log['status'] = 1 # pass the errors back log['stderr'] = stderr else: log['status'] = 0 log['options'] = target return def loadPlugins(self, basedir, topdir): if self.loaded: print("Cannot load plugins multiple times") exit(1) self.loaded = True # find the loader utility so we can bootstrap ourselves try: m = imp.load_source("LoadClasses", os.path.join(basedir, "LoadClasses.py")) except ImportError: print( "ERROR: unable to load LoadClasses that must contain the class loader object" ) exit(1) cls = getattr(m, "LoadClasses") a = cls() # setup the loader object self.loader = a.__class__() # Setup the array of directories we will search for plugins # Note that we always look at the topdir location by default plugindirs = [] plugindirs.append(topdir) if self.options['plugindir']: # could be a comma-delimited list, so split on commas x = self.options['plugindir'].split(',') for y in x: # prepend so we always look at the given # location first in case the user wants # to "overload/replace" a default MTT # class definition plugindirs.insert(0, y) # Traverse the plugin directory tree and add all # the class definitions we can find for dirPath in plugindirs: try: filez = os.listdir(dirPath) for file in filez: file = os.path.join(dirPath, file) if os.path.isdir(file): self.loader.load(file) except: if not self.options['ignoreloadpatherrs']: print("Plugin directory", dirPath, "not found") sys.exit(1) # Build the stages plugin manager self.stages = PluginManager() # set the location self.stages.setPluginPlaces(plugindirs) # Get a list of all the categories - this corresponds to # the MTT stages that have been defined. Note that we # don't need to formally define the stages here - anyone # can add a new stage, or delete an old one, by simply # adding or removing a plugin directory. self.stages.setCategoriesFilter(self.loader.stages) # Load all plugins we find there self.stages.collectPlugins() # Build the tools plugin manager - tools differ from sections # in that they are plugins we will use to execute the various # sections. For example, the TestRun section clearly needs the # ability to launch jobs. There are many ways to launch jobs # depending on the environment, and sometimes several ways to # start jobs even within one environment (e.g., mpirun vs # direct launch). self.tools = PluginManager() # location is the same self.tools.setPluginPlaces(plugindirs) # Get the list of tools - not every tool will be capable # of executing. For example, a tool that supports direct launch # against a specific resource manager cannot be used on a # system being managed by a different RM. self.tools.setCategoriesFilter(self.loader.tools) # Load all the tool plugins self.tools.collectPlugins() # Tool plugins are required to provide a function we can # probe to determine if they are capable of operating - check # those now and prune those tools that cannot support this # environment # Build the utilities plugins self.utilities = PluginManager() # set the location self.utilities.setPluginPlaces(plugindirs) # Get the list of available utilities. self.utilities.setCategoriesFilter(self.loader.utilities) # Load all the utility plugins self.utilities.collectPlugins() # since we use these all over the place, find the # ExecuteCmd and ModuleCmd plugins and record them availUtil = list(self.loader.utilities.keys()) for util in availUtil: for pluginInfo in self.utilities.getPluginsOfCategory(util): if "ExecuteCmd" == pluginInfo.plugin_object.print_name(): self.execmd = pluginInfo.plugin_object elif "ModuleCmd" == pluginInfo.plugin_object.print_name(): self.modcmd = pluginInfo.plugin_object # initialize this module self.modcmd.setCommand(self.options) if self.execmd is not None and self.modcmd is not None: break if self.execmd is None: print("ExecuteCmd plugin was not found") print("This is a basic capability required") print("for MTT operations - cannot continue") sys.exit(1) print(self.tools.getPluginsOfCategory("Harasser")) for pluginInfo in self.tools.getPluginsOfCategory("Harasser"): print(pluginInfo.plugin_object.print_name()) if "Harasser" == pluginInfo.plugin_object.print_name(): self.harasser = pluginInfo.plugin_object break if self.harasser is None: print("Harasser plugin was not found") print("This is required for all TestRun plugins") print("cannot continue") sys.exit(1) # similarly, capture the highest priority defaults stage here pri = -1 for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"): if pri < pluginInfo.plugin_object.priority(): self.defaults = pluginInfo.plugin_object pri = pluginInfo.plugin_object.priority() return def printInfo(self): # Print the available MTT sections out, if requested if self.options['listsections']: print("Supported MTT stages:") # print them in the default order of execution for stage in self.loader.stageOrder: print(" " + stage) exit(0) # Print the detected plugins for a given stage if self.options['listplugins']: # if the list is '*', print the plugins for every stage if self.options['listplugins'] == "*": sections = self.loader.stageOrder else: sections = self.options['listplugins'].split(',') print() for section in sections: print(section + ":") try: for pluginInfo in self.stages.getPluginsOfCategory( section): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid stage name " + section) print() exit(1) # Print the options for a given plugin if self.options['liststageoptions']: # if the list is '*', print the options for every stage/plugin if self.options['liststageoptions'] == "*": sections = self.loader.stageOrder else: sections = self.options['liststageoptions'].split(',') print() for section in sections: print(section + ":") try: for pluginInfo in self.stages.getPluginsOfCategory( section): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options( self, " ") except KeyError: print(" Invalid stage name " + section) print() exit(1) # Print the available MTT tools out, if requested if self.options['listtools']: print("Available MTT tools:") availTools = list(self.loader.tools.keys()) for tool in availTools: print(" " + tool) exit(0) # Print the detected tool plugins for a given tool type if self.options['listtoolmodules']: # if the list is '*', print the plugins for every type if self.options['listtoolmodules'] == "*": print() availTools = list(self.loader.tools.keys()) else: availTools = self.options['listtoolmodules'].split(',') print() for tool in availTools: print(tool + ":") try: for pluginInfo in self.tools.getPluginsOfCategory(tool): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid tool type name", tool) print() exit(1) # Print the options for a given plugin if self.options['listtooloptions']: # if the list is '*', print the options for every stage/plugin if self.options['listtooloptions'] == "*": availTools = list(self.loader.tools.keys()) else: availTools = self.options['listtooloptions'].split(',') print() for tool in availTools: print(tool + ":") try: for pluginInfo in self.tools.getPluginsOfCategory(tool): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options( self, " ") except KeyError: print(" Invalid tool type name " + tool) print() exit(1) # Print the available MTT utilities out, if requested if self.options['listutils']: print("Available MTT utilities:") availUtils = list(self.loader.utilities.keys()) for util in availUtils: print(" " + util) exit(0) # Print the detected utility plugins for a given tool type if self.options['listutilmodules']: # if the list is '*', print the plugins for every type if self.options['listutilmodules'] == "*": print() availUtils = list(self.loader.utilities.keys()) else: availUtils = self.options['listutilitymodules'].split(',') print() for util in availUtils: print(util + ":") try: for pluginInfo in self.utilities.getPluginsOfCategory( util): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid utility type name") print() exit(1) # Print the options for a given plugin if self.options['listutiloptions']: # if the list is '*', print the options for every stage/plugin if self.options['listutiloptions'] == "*": availUtils = list(self.loader.utilities.keys()) else: availUtils = self.options['listutiloptions'].split(',') print() for util in availUtils: print(util + ":") try: for pluginInfo in self.utilities.getPluginsOfCategory( util): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options( self, " ") except KeyError: print(" Invalid utility type name " + util) print() exit(1) # if they asked for the version info, print it and exit if self.options['version']: for pluginInfo in self.tools.getPluginsOfCategory("Version"): print("MTT Base: " + pluginInfo.plugin_object.getVersion()) print("MTT Client: " + pluginInfo.plugin_object.getClientVersion()) sys.exit(0) def openLogger(self): # there must be a logger utility or we can't do # anything useful if not self.utilities.activatePluginByName("Logger", "Base"): print("Required Logger plugin not found or could not be activated") sys.exit(1) # execute the provided test description self.logger = self.utilities.getPluginByName("Logger", "Base").plugin_object self.logger.open(self) return def configTest(self): # setup the configuration parser self.config = configparser.SafeConfigParser( interpolation=configparser.ExtendedInterpolation()) # Set the config parser to make option names case sensitive. self.config.optionxform = str # fill ENV section with environemt variables self.config.add_section('ENV') for k, v in os.environ.items(): self.config.set('ENV', k, v.replace("$", "$$")) # log the list of files - note that the argument parser # puts the input files in a list, with the first member # being the list of input files self.log['inifiles'] = self.args.ini_files[0] # initialize the list of active sections self.actives = [] # if they specified a list to execute, then use it sections = [] if self.args.section: sections = self.args.section.split(",") skip = False elif self.args.skipsections: sections = self.args.skipsections.split(",") skip = True else: sections = None # cycle thru the input files for testFile in self.log['inifiles']: if not os.path.isfile(testFile): print("Test description file", testFile, "not found!") sys.exit(1) self.config.read(self.log['inifiles']) # Check for ENV input required_env = [] all_file_contents = [] for testFile in self.log['inifiles']: file_contents = open(testFile, "r").read() file_contents = "\n".join([ "%s %d: %s" % (testFile.split("/")[-1], i, l) for i, l in enumerate(file_contents.split("\n")) if not l.lstrip().startswith("#") ]) all_file_contents.append(file_contents) if "${ENV:" in file_contents: required_env.extend([ s.split("}")[0] for s in file_contents.split("${ENV:")[1:] ]) env_not_found = set( [e for e in required_env if e not in os.environ.keys()]) lines_with_env_not_found = [] for file_contents in all_file_contents: lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \ for l in file_contents.split("\n") \ if sum(["${ENV:%s}"%e in l for e in env_not_found])]) if lines_with_env_not_found: print("ERROR: Not all required environment variables are defined.") print("ERROR: Still need:") for l in lines_with_env_not_found: print("ERROR: %s" % l) sys.exit(1) for section in self.config.sections(): if section.startswith("SKIP") or section.startswith("skip"): # users often want to temporarily ignore a section # of their test definition file, but don't want to # remove it lest they forget what it did. So let # them just mark the section as "skip" to be ignored continue # if we are to filter the sections, then do so takeus = True if sections is not None: found = False for sec in sections: if sec == section: found = True sections.remove(sec) if skip: takeus = False break if not found and not skip: takeus = False if takeus: self.actives.append(section) if self.logger is not None: self.logger.verbose_print("SECTION: " + section) self.logger.verbose_print(self.config.items(section)) if sections is not None and 0 != len(sections) and not skip: print( "ERROR: sections were specified for execution and not found:", sections) sys.exit(1) return # Used with combinatorial executor, loads next .ini file to be run with the # sequential executor def configNewTest(self, file): # clear the configuration parser for section in self.config.sections(): self.config.remove_section(section) # read in the file self.config.read(file) for section in self.config.sections(): if section.startswith("SKIP") or section.startswith("skip"): # users often want to temporarily ignore a section # of their test definition file, but don't want to # remove it lest they forget what it did. So let # them just mark the section as "skip" to be ignored continue if self.logger is not None: self.logger.verbose_print("SECTION: " + section) self.logger.verbose_print(self.config.items(section)) return def executeTest(self): if not self.loaded: print("Plugins have not been loaded - cannot execute test") exit(1) if self.config is None: print("No test definition file was parsed - cannot execute test") exit(1) if not self.tools.getPluginByName("sequential", "Executor"): print("Specified executor sequential not found") exit(1) # activate the specified plugin self.tools.activatePluginByName("sequential", "Executor") # execute the provided test description executor = self.tools.getPluginByName("sequential", "Executor") status = executor.plugin_object.execute(self) return status def executeCombinatorial(self): if not self.tools.getPluginByName("combinatorial", "Executor"): print("Specified executor combinatorial not found") exit(1) # activate the specified plugin self.tools.activatePluginByName("combinatorial", "Executor") # execute the provided test description executor = self.tools.getPluginByName("combinatorial", "Executor") status = executor.plugin_object.execute(self) return status def printOptions(self, options): # if the options are empty, report that if not options: lines = ["None"] return lines # create the list of options opts = [] vals = list(options.keys()) for val in vals: opts.append(val) if options[val][0] is None: opts.append("None") elif isinstance(options[val][0], bool): if options[val][0]: opts.append("True") else: opts.append("False") elif isinstance(options[val][0], list): opts.append(" ".join(options[val][0])) elif isinstance(options[val][0], int): opts.append(str(options[val][0])) else: opts.append(options[val][0]) opts.append(options[val][1]) # print the options, their default value, and # the help description in 3 column format max1 = 0 max2 = 0 for i in range(0, len(opts), 3): # we want all the columns to line up # and left-justify, so first find out # the max len of each of the first two # column entries if len(opts[i]) > max1: max1 = len(opts[i]) if type(opts[i + 1]) is not str: optout = str(opts[i + 1]) else: optout = opts[i + 1] if len(optout) > max2: max2 = len(optout) # provide some spacing max1 = max1 + 4 max2 = max2 + 4 # cycle thru again, padding each entry to # align the columns lines = [] sp = " " for i in range(0, len(opts), 3): line = opts[i] + (max1 - len(opts[i])) * sp if type(opts[i + 1]) is not str: optout = str(opts[i + 1]) else: optout = opts[i + 1] line = line + optout + (max2 - len(optout)) * sp # to make this more readable, we will wrap the line at # 130 characters. First, see if the line is going to be # too long if 130 < (len(line) + len(opts[i + 2])): # split the remaining column into individual words words = opts[i + 2].split() first = True for word in words: if (len(line) + len(word)) < 130: if first: line = line + word first = False else: line = line + " " + word else: lines.append(line) line = (max1 + max2) * sp + word if 0 < len(line): lines.append(line) else: # the line is fine - so just add the last piece line = line + opts[i + 2] # append the result lines.append(line) # add one blank line lines.append("") return lines def selectPlugin(self, name, category): if category == "stage": try: availStages = list(self.loader.stages.keys()) for stage in availStages: for pluginInfo in self.stages.getPluginsOfCategory(stage): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None elif category == "tool": try: availTools = list(self.loader.tools.keys()) for tool in availTools: for pluginInfo in self.tools.getPluginsOfCategory(tool): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None elif category == "utility": try: availUtils = list(self.loader.utilities.keys()) for util in availUtils: for pluginInfo in self.utilities.getPluginsOfCategory( util): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None else: print("Unrecognized category:", category) return None
class SimpleTestsCase(unittest.TestCase): """ Test the correct loading of a simple plugin as well as basic commands. """ def setUp(self): """ init """ # create the plugin manager self.simplePluginManager = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # load the plugins that may be found self.simplePluginManager.collectPlugins() # Will be used later self.plugin_info = None def plugin_loading_check(self): """ Test if the correct plugin has been loaded. """ if self.plugin_info is None: # check nb of categories self.assertEqual(len(self.simplePluginManager.getCategories()),1) sole_category = self.simplePluginManager.getCategories()[0] # check the number of plugins self.assertEqual(len(self.simplePluginManager.getPluginsOfCategory(sole_category)),1) self.plugin_info = self.simplePluginManager.getPluginsOfCategory(sole_category)[0] # test that the name of the plugin has been correctly defined self.assertEqual(self.plugin_info.name,"Simple Plugin") self.assertEqual(sole_category,self.plugin_info.category) else: self.assert_(True) def testLoaded(self): """ Test if the correct plugin has been loaded. """ self.plugin_loading_check() def testGetAll(self): """ Test if the correct plugin has been loaded. """ self.plugin_loading_check() self.assertEqual(len(self.simplePluginManager.getAllPlugins()),1) self.assertEqual(self.simplePluginManager.getAllPlugins()[0],self.plugin_info) def testActivationAndDeactivation(self): """ Test if the activation procedure works. """ self.plugin_loading_check() self.assert_(not self.plugin_info.plugin_object.is_activated) self.simplePluginManager.activatePluginByName(self.plugin_info.name, self.plugin_info.category) self.assert_(self.plugin_info.plugin_object.is_activated) self.simplePluginManager.deactivatePluginByName(self.plugin_info.name, self.plugin_info.category) self.assert_(not self.plugin_info.plugin_object.is_activated)
class FakeSite(object): def __init__(self): self.template_system = self self.invariant = False self.config = { 'DISABLED_PLUGINS': [], 'EXTRA_PLUGINS': [], 'DEFAULT_LANG': 'en', 'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'], 'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}', 'LISTINGS_FOLDERS': { 'listings': 'listings' }, } self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS'] self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "CompilerExtension": CompilerExtension, "MarkdownExtension": MarkdownExtension, "RestExtension": RestExtension }) self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8 self.shortcode_registry = {} self.plugin_manager.setPluginInfoExtension('plugin') if sys.version_info[0] == 3: places = [ os.path.join(os.path.dirname(nikola.utils.__file__), 'plugins'), ] else: places = [ os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode('plugins')), ] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() self.compiler_extensions = self._activate_plugins_of_category( "CompilerExtension") self.timeline = [FakePost(title='Fake post', slug='fake-post')] self.debug = True self.rst_transforms = [] self.post_per_input_file = {} # This is to make plugin initialization happy self.template_system = self self.name = 'mako' def _activate_plugins_of_category(self, category): """Activate all the plugins of a given category and return them.""" # this code duplicated in nikola/nikola.py plugins = [] for plugin_info in self.plugin_manager.getPluginsOfCategory(category): if plugin_info.name in self.config.get('DISABLED_PLUGINS'): self.plugin_manager.removePluginFromCategory( plugin_info, category) else: self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugins.append(plugin_info) return plugins def render_template(self, name, _, context): return ('<img src="IMG.jpg">') # this code duplicated in nikola/nikola.py def register_shortcode(self, name, f): """Register function f to handle shortcode "name".""" if name in self.shortcode_registry: nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name) return self.shortcode_registry[name] = f def apply_shortcodes(self, data, *a, **kw): """Apply shortcodes from the registry on data.""" return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry, **kw)
class WUnpack(QtWidgets.QDialog): def __init__(self, parent, plugin): super(WUnpack, self).__init__(parent) self.parent = parent self.plugin = plugin self.oshow = super(WUnpack, self).show root = os.path.dirname(sys.argv[0]) self.ui = PyQt5.uic.loadUi(os.path.join(root, 'unpack.ui'), baseinstance=self) self.ui.setWindowTitle('Decrypt/Encrypt') self.manager = PluginManager( categories_filter={"UnpackPlugin": DecryptPlugin}) root = os.path.dirname(sys.argv[0]) self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'unpack')]) #self.manager.setPluginPlaces(["plugins"]) # Load plugins self.manager.locatePlugins() self.manager.loadPlugins() self.Plugins = {} Formats = [] for plugin in self.manager.getPluginsOfCategory("UnpackPlugin"): # plugin.plugin_object is an instance of the plugin po = plugin.plugin_object if po.init(self.parent.dataModel, self.parent.viewMode): self.Plugins[plugin.name] = po #self.ui.horizontalLayout.addWidget(po.getUI()) print('[+] ' + plugin.name) self.ui.listWidget.addItem(plugin.name) #Formats.append(po) self.ui.listWidget.currentItemChanged.connect(self.item_clicked) self.ui.listWidget.setCurrentRow(0) #self.ui.connect(self.ui.proceed, PyQt5.QtCore.SIGNAL("clicked()"), self.handleProceed) self.ui.proceed.clicked.connect(lambda: self.handleProceed()) self.initUI() def handleProceed(self): item = str(self.ui.listWidget.currentItem().text()) self.Plugins[item].proceed() #self.parent.update() self.parent.viewMode.draw(refresh=True) self.parent.update() def item_clicked(self, current, previous): #item = str(self.ui.listWidget.currentItem().text()) item = str(current.text()) if previous: x = self.ui.horizontalLayout.takeAt(0) while x: x.widget().setParent(None) x = self.ui.horizontalLayout.takeAt(0) prev = str(previous.text()) #print prev #self.ui.horizontalLayout.removeWidget(self.Plugins[prev].getUI()) if item: #print item po = self.Plugins[item] self.ui.horizontalLayout.addWidget(po.getUI()) def show(self): # TODO: remember position? resize plugin windows when parent resize? pwidth = self.parent.parent.size().width() pheight = self.parent.parent.size().height() width = self.ui.size().width() + 15 height = self.ui.size().height() + 15 self.setGeometry(pwidth - width - 15, pheight - height, width, height) self.setFixedSize(width, height) self.oshow() def initUI(self): self.setSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed) shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("F4"), self, self.close, self.close) #QtCore.QObject.connect(self.ui.ok, QtCore.SIGNAL('clicked()'), self.onClicked) def onClicked(self): dataModel = self.parent.dataModel self.close()
class binWidget(QtWidgets.QWidget, Observable): scrolled = QtCore.pyqtSignal(int, name='scroll') # oldscrolled = QtWidgets.SIGNAL('scroll') def __init__(self, parent, source): super(binWidget, self).__init__() Observable.__init__(self) self.parent = parent # offset for text window #self.data = mapped self.dataOffset = 0 self.dataModel = source self.cursor = Cursor(0, 0) # self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self), # HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self)] logging.basicConfig(level=logging.ERROR) self.manager = PluginManager( categories_filter={"FileFormat": FileFormat}) root = os.path.dirname(sys.argv[0]) self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'format')]) #self.manager.setPluginPlaces(["plugins"]) # Load plugins self.manager.locatePlugins() self.manager.loadPlugins() Formats = [] for plugin in self.manager.getPluginsOfCategory("FileFormat"): # plugin.plugin_object is an instance of the plugin po = plugin.plugin_object if po.recognize(self.dataModel): print('[+] ' + po.name) Formats.append(po) # sort plugins by priority Formats = sorted(Formats, key=lambda x: x.priority, reverse=True) po = Formats[0] print('Choosed plugin: ' + po.name) #print QtGui.QFontDatabase.addApplicationFont(os.path.join('terminus-ttf-4.39', 'TerminusTTF-4.39.ttf')) self.multipleViewModes = [ BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po), HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po), DisasmViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po) ] self.viewMode = self.multipleViewModes[0] self.textDecorator = TextDecorator(self.viewMode) self.viewMode.setTransformationEngine(self.textDecorator) self.multipleViewModes[1].setTransformationEngine(self.textDecorator) self.Banners = Banners() #self.Banners.add(BottomBanner(self.dataModel, self.viewMode)) # self.Banners.add(TopBanner(self.dataModel, self.viewMode)) #self.Banners.add(self.banner) # self.filebanner = FileAddrBanner(self.dataModel, self.viewMode) #self.filebanner = PEBanner(self.dataModel, self.viewMode) #self.Banners.add(PEBanner(self.dataModel, self.viewMode)) #self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode)) #self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode)) # self.offsetWindow_h = self.filebanner.getDesiredGeometry()[0] + 25 self.offsetWindow_h = 0 self.offsetWindow_v = 0 self.searchable = Searchable(self.dataModel, self.viewMode) self.initUI() [po.init(viewMode, parent=self) for viewMode in self.multipleViewModes] for banner in po.getBanners(): self.Banners.add(banner) po.registerShortcuts(self) self.po = po #self.scrolled = QtCore.pyqtSignal(int, name='scroll') #self.scrolled.connect(self.scroll_from_outside) self.searchWindow = SearchWindow(self, None, self.searchable) self.addHandler(self.po) self.addHandler(self.searchable) self.addHandler(self.Banners) self.notify(self.viewMode) #self.connect(self, self.oldscrolled, self.scroll_from_outside) #self.scrolled.emit(1) #self.emit(QtCore.SIGNAL('scroll'), 1) def scroll_from_outside(self, i): #print 'slot-signal ' + str(i) #self.scroll_pdown = True self.update() def initUI(self): self.setMinimumSize(1, 30) self.activateWindow() self.setFocus() #self.installEventFilter(self) """ # build thumbnail dwidth = 100 dheight = 1200 factor = dheight/dwidth import math x = int(math.sqrt(len(self.data)/factor)) cols = x pixThumb = QtGui.QPixmap(x*self.viewMode.fontWidth, factor*x*self.viewMode.fontHeight) qp = QtGui.QPainter() qp.begin(pixThumb) qp.setFont(self.viewMode.font) qp.setPen(self.viewMode.textPen) for i,c in enumerate(self.data): self.viewMode.transformText(qp, (i, c), self.data) qp.drawText((i%cols)*self.viewMode.fontWidth, self.viewMode.fontHeight + (i/cols)*self.viewMode.fontHeight, c) qp.end() self.newqpix = pixThumb.scaled(dwidth, dheight, aspectRatioMode = QtCore.Qt.IgnoreAspectRatio, transformMode = QtCore.Qt.FastTransformation) """ """ def getDisplayedData(self): if self.dataOffset < 0: self.dataOffset = 0 return False chrlist = [unichr(cp437ToUnicode[ord(c)]) for c in self.data[self.dataOffset : self.dataOffset + self.viewMode.COLUMNS*self.viewMode.ROWS]] self.text = "".join(chrlist) return True """ def switchViewMode(self): self.multipleViewModes = self.multipleViewModes[1:] + [ self.multipleViewModes[0] ] self.viewMode = self.multipleViewModes[0] # notify obervers self.notify(self.viewMode) def _resize(self): self.Banners.resize(self.size().width() - self.offsetWindow_h, self.size().height() - self.offsetWindow_v) # compute space ocupated by banners offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset() offsetBottom = self.offsetWindow_v + self.Banners.getBottomOffset( ) + self.Banners.getTopOffset() # resize window, substract space ocupated by banners self.viewMode.resize(self.size().width() - offsetLeft, self.size().height() - offsetBottom) # event handlers def resizeEvent(self, e): self._resize() def paintEvent(self, e): qp = QtGui.QPainter() qp.begin(self) qp.setOpacity(1) offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset() offsetBottom = self.offsetWindow_v + self.Banners.getTopOffset() #self.viewMode.draw2(qp, refresh=True) #start = time() qp.drawPixmap(offsetLeft, offsetBottom, self.viewMode.getPixmap()) #print 'Draw ' + str(time() - start) self.Banners.draw(qp, self.offsetWindow_h, self.offsetWindow_v, self.size().height()) # qp.drawPixmap(self.offsetWindow_h, self.size().height() - 50, self.banner.getPixmap()) # qp.drawPixmap(20, 0, self.filebanner.getPixmap()) qp.end() """ def keyPressEvent(self, event): if event.modifiers() & QtCore.Qt.ShiftModifier: if self.viewMode.handleKeyPressEvent(QtCore.Qt.ShiftModifier, event.key()): self.update() def keyReleaseEvent(self, event): if event.key() == QtCore.Qt.Key_Shift: if self.viewMode.handleKeyReleaseEvent(QtCore.Qt.ShiftModifier, event.key()): self.update() """ def eventFilter(self, watched, event): if event.type() == QtCore.QEvent.KeyRelease: key = event.key() modifiers = event.modifiers() if self.viewMode.handleKeyEvent(modifiers, key, event=event): self.update() if event.type() == QtCore.QEvent.KeyPress: #TODO: should we accept only certain keys ? key = event.key() modifiers = event.modifiers() if key == QtCore.Qt.Key_F2: if self.viewMode.isEditable(): if self.viewMode.isInEditMode(): self.viewMode.setEditMode(False) else: self.viewMode.setEditMode(True) self.viewMode.draw(refresh=False) # switch view mode if key == QtCore.Qt.Key_Tab: offs = self.viewMode.getCursorOffsetInPage() base = self.viewMode.getDataModel().getOffset() self.switchViewMode() self._resize() self.viewMode.goTo(base + offs) self.update() import pyperclip if event.modifiers() & QtCore.Qt.ControlModifier: if key == QtCore.Qt.Key_Insert: if self.viewMode.selector.getCurrentSelection(): a, b = self.viewMode.selector.getCurrentSelection() #print a, b hx = '' for s in self.dataModel.getStream(a, b): hx += '{:02x}'.format(s) pyperclip.copy(hx) del pyperclip #print pyperclip.paste() # print 'coppied' if event.modifiers() & QtCore.Qt.ShiftModifier: if key == QtCore.Qt.Key_Insert: raise Exception("Not implemented") """ import re hx = pyperclip.paste() #print hx L = re.findall(r'.{1,2}', hx, re.DOTALL) array = '' for s in L: array += chr(int(s, 16)) #print 'write ' #print 'write' #print array self.dataModel.write(0, array) self.viewMode.draw(True) del pyperclip #print array """ if key == QtCore.Qt.Key_F4: self.unp = WUnpack(self, None) self.unp.show() if key == QtCore.Qt.Key_F10: self.dataModel.flush() import os self.w = WHeaders(self, None) self.w.show() if not self.viewMode.isInEditMode(): if key == QtCore.Qt.Key_Slash: self.searchWindow.show() if key == QtCore.Qt.Key_N: self.searchable.next( self.viewMode.getCursorAbsolutePosition() + 1) if key == QtCore.Qt.Key_B: self.searchable.previous( self.viewMode.getCursorAbsolutePosition()) # handle keys to view plugin if self.viewMode.handleKeyEvent(modifiers, key, event=event): event.accept() self.update() return True return False def setTextViewport(self, qp): qp.setViewport(self.offsetWindow_h, self.offsetWindow_v, self.size().width(), self.size().height()) qp.setWindow(0, 0, self.size().width(), self.size().height()) def needsSave(self): return self.dataModel.isDirty() def save(self): return self.dataModel.flush()
class Nikola(object): """Class that handles site generation. Takes a site config as argument on creation. """ EXTRA_PLUGINS = [ 'planetoid', 'ipynb', 'local_search', 'render_mustache', ] def __init__(self, **config): """Setup proper environment for running tasks.""" self.global_data = {} self.posts_per_year = defaultdict(list) self.posts_per_month = defaultdict(list) self.posts_per_tag = defaultdict(list) self.post_per_file = {} self.timeline = [] self.pages = [] self._scanned = False if not config: self.configured = False else: self.configured = True # This is the default config self.config = { 'ADD_THIS_BUTTONS': True, 'ANALYTICS': '', 'ARCHIVE_PATH': "", 'ARCHIVE_FILENAME': "archive.html", 'CACHE_FOLDER': 'cache', 'CODE_COLOR_SCHEME': 'default', 'COMMENTS_IN_GALLERIES': False, 'COMMENTS_IN_STORIES': False, 'CONTENT_FOOTER': '', 'CREATE_MONTHLY_ARCHIVE': False, 'DATE_FORMAT': '%Y-%m-%d %H:%M', 'DEFAULT_LANG': "en", 'DEPLOY_COMMANDS': [], 'DISABLED_PLUGINS': (), 'DISQUS_FORUM': 'nikolademo', 'ENABLED_EXTRAS': (), 'EXTRA_HEAD_DATA': '', 'FAVICONS': {}, 'FILE_METADATA_REGEXP': None, 'FILES_FOLDERS': {'files': ''}, 'FILTERS': {}, 'GALLERY_PATH': 'galleries', 'GZIP_FILES': False, 'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json'), 'HIDE_UNTRANSLATED_POSTS': False, 'INDEX_DISPLAY_POST_COUNT': 10, 'INDEX_FILE': 'index.html', 'INDEX_TEASERS': False, 'INDEXES_TITLE': "", 'INDEXES_PAGES': "", 'INDEX_PATH': '', 'LICENSE': '', 'LINK_CHECK_WHITELIST': [], 'LISTINGS_FOLDER': 'listings', 'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'], 'MAX_IMAGE_SIZE': 1280, 'MATHJAX_CONFIG': '', 'OLD_THEME_SUPPORT': True, 'OUTPUT_FOLDER': 'output', 'post_compilers': { "rest": ('.txt', '.rst'), "markdown": ('.md', '.mdown', '.markdown'), "textile": ('.textile',), "txt2tags": ('.t2t',), "bbcode": ('.bb',), "wiki": ('.wiki',), "ipynb": ('.ipynb',), "html": ('.html', '.htm') }, 'POST_PAGES': ( ("posts/*.txt", "posts", "post.tmpl", True), ("stories/*.txt", "stories", "story.tmpl", False), ), 'PRETTY_URLS': False, 'REDIRECTIONS': [], 'RSS_LINK': None, 'RSS_PATH': '', 'RSS_TEASERS': True, 'SEARCH_FORM': '', 'SLUG_TAG_PATH': True, 'STORY_INDEX': False, 'STRIP_INDEXES': False, 'SITEMAP_INCLUDE_FILELESS_DIRS': True, 'TAG_PATH': 'categories', 'TAG_PAGES_ARE_INDEXES': False, 'THEME': 'site', 'THEME_REVEAL_CONGIF_SUBTHEME': 'sky', 'THEME_REVEAL_CONGIF_TRANSITION': 'cube', 'THUMBNAIL_SIZE': 180, 'USE_BUNDLES': True, 'USE_CDN': False, 'USE_FILENAME_AS_TITLE': True, 'TIMEZONE': None, } self.config.update(config) # STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES # Port it if only the oldef form is there if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config: print("WARNING: You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML") self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML'] # PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config: self.config['STRIP_INDEXES'] = True self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS', {self.config['DEFAULT_' 'LANG']: ''}) self.THEMES = utils.get_theme_chain(self.config['THEME']) self.MESSAGES = utils.load_messages(self.THEMES, self.config['TRANSLATIONS'], self.config['DEFAULT_LANG']) # SITE_URL is required, but if the deprecated BLOG_URL # is available, use it and warn if 'SITE_URL' not in self.config: if 'BLOG_URL' in self.config: print("WARNING: You should configure SITE_URL instead of BLOG_URL") self.config['SITE_URL'] = self.config['BLOG_URL'] self.default_lang = self.config['DEFAULT_LANG'] self.translations = self.config['TRANSLATIONS'] # BASE_URL defaults to SITE_URL if 'BASE_URL' not in self.config: self.config['BASE_URL'] = self.config.get('SITE_URL') self.plugin_manager = PluginManager(categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, }) self.plugin_manager.setPluginInfoExtension('plugin') if sys.version_info[0] == 3: places = [ os.path.join(os.path.dirname(__file__), 'plugins'), os.path.join(os.getcwd(), 'plugins'), ] else: places = [ os.path.join(os.path.dirname(__file__), utils.sys_encode('plugins')), os.path.join(os.getcwd(), utils.sys_encode('plugins')), ] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() self.commands = {} # Activate all command plugins for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"): if (plugin_info.name in self.config['DISABLED_PLUGINS'] or (plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config['ENABLED_EXTRAS'])): self.plugin_manager.removePluginFromCategory(plugin_info, "Command") continue self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugin_info.plugin_object.short_help = plugin_info.description self.commands[plugin_info.name] = plugin_info.plugin_object # Activate all task plugins for task_type in ["Task", "LateTask"]: for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type): if (plugin_info.name in self.config['DISABLED_PLUGINS'] or (plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config['ENABLED_EXTRAS'])): self.plugin_manager.removePluginFromCategory(plugin_info, task_type) continue self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) # Activate all multiplier plugins for plugin_info in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"): if (plugin_info.name in self.config['DISABLED_PLUGINS'] or (plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config['ENABLED_EXTRAS'])): self.plugin_manager.removePluginFromCategory(plugin_info, task_type) continue self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) # Activate all required compiler plugins for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"): if plugin_info.name in self.config["post_compilers"].keys(): self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) # set global_context for template rendering self.GLOBAL_CONTEXT = { } self.GLOBAL_CONTEXT['messages'] = self.MESSAGES self.GLOBAL_CONTEXT['_link'] = self.link self.GLOBAL_CONTEXT['set_locale'] = s_l self.GLOBAL_CONTEXT['rel_link'] = self.rel_link self.GLOBAL_CONTEXT['abs_link'] = self.abs_link self.GLOBAL_CONTEXT['exists'] = self.file_exists self.GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config[ 'SLUG_TAG_PATH'] self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[ 'ADD_THIS_BUTTONS'] self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[ 'INDEX_DISPLAY_POST_COUNT'] self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES'] self.GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN") self.GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS'] self.GLOBAL_CONTEXT['date_format'] = self.config.get( 'DATE_FORMAT', '%Y-%m-%d %H:%M') self.GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR') self.GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE') self.GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL', self.config.get('BLOG_URL')) self.GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION') self.GLOBAL_CONTEXT['analytics'] = self.config.get('ANALYTICS') self.GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS') self.GLOBAL_CONTEXT['license'] = self.config.get('LICENSE') self.GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM') self.GLOBAL_CONTEXT['disqus_forum'] = self.config.get('DISQUS_FORUM') self.GLOBAL_CONTEXT['mathjax_config'] = self.config.get( 'MATHJAX_CONFIG') self.GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONGIF_SUBTHEME') self.GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONGIF_TRANSITION') self.GLOBAL_CONTEXT['content_footer'] = self.config.get( 'CONTENT_FOOTER') self.GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH') self.GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK') self.GLOBAL_CONTEXT['sidebar_links'] = utils.Functionary(list, self.config['DEFAULT_LANG']) for k, v in self.config.get('SIDEBAR_LINKS', {}).items(): self.GLOBAL_CONTEXT['sidebar_links'][k] = v self.GLOBAL_CONTEXT['twitter_card'] = self.config.get( 'TWITTER_CARD', {}) self.GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA') self.GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {})) # check if custom css exist and is not empty for files_path in list(self.config['FILES_FOLDERS'].keys()): custom_css_path = os.path.join(files_path, 'assets/css/custom.css') if self.file_exists(custom_css_path, not_empty=True): self.GLOBAL_CONTEXT['has_custom_css'] = True break else: self.GLOBAL_CONTEXT['has_custom_css'] = False # Load template plugin template_sys_name = utils.get_template_engine(self.THEMES) pi = self.plugin_manager.getPluginByName( template_sys_name, "TemplateSystem") if pi is None: sys.stderr.write("Error loading {0} template system " "plugin\n".format(template_sys_name)) sys.exit(1) self.template_system = pi.plugin_object lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES] self.template_system.set_directories(lookup_dirs, self.config['CACHE_FOLDER']) # Check consistency of USE_CDN and the current THEME (Issue #386) if self.config['USE_CDN']: bootstrap_path = utils.get_asset_path(os.path.join( 'assets', 'css', 'bootstrap.min.css'), self.THEMES) if bootstrap_path.split(os.sep)[-4] != 'site': warnings.warn('The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.') # Load compiler plugins self.compilers = {} self.inverse_compilers = {} for plugin_info in self.plugin_manager.getPluginsOfCategory( "PageCompiler"): self.compilers[plugin_info.name] = \ plugin_info.plugin_object def get_compiler(self, source_name): """Get the correct compiler for a post from `conf.post_compilers` To make things easier for users, the mapping in conf.py is compiler->[extensions], although this is less convenient for us. The majority of this function is reversing that dictionary and error checking. """ ext = os.path.splitext(source_name)[1] try: compile_html = self.inverse_compilers[ext] except KeyError: # Find the correct compiler for this files extension langs = [lang for lang, exts in list(self.config['post_compilers'].items()) if ext in exts] if len(langs) != 1: if len(set(langs)) > 1: exit("Your file extension->compiler definition is" "ambiguous.\nPlease remove one of the file extensions" "from 'post_compilers' in conf.py\n(The error is in" "one of {0})".format(', '.join(langs))) elif len(langs) > 1: langs = langs[:1] else: exit("post_compilers in conf.py does not tell me how to " "handle '{0}' extensions.".format(ext)) lang = langs[0] compile_html = self.compilers[lang] self.inverse_compilers[ext] = compile_html return compile_html def render_template(self, template_name, output_name, context): local_context = {} local_context["template_name"] = template_name local_context.update(self.GLOBAL_CONTEXT) local_context.update(context) data = self.template_system.render_template( template_name, None, local_context) assert output_name.startswith( self.config["OUTPUT_FOLDER"]) url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:] # Treat our site as if output/ is "/" and then make all URLs relative, # making the site "relocatable" src = os.sep + url_part src = os.path.normpath(src) # The os.sep is because normpath will change "/" to "\" on windows src = "/".join(src.split(os.sep)) parsed_src = urlsplit(src) src_elems = parsed_src.path.split('/')[1:] def replacer(dst): # Refuse to replace links that are full URLs. dst_url = urlparse(dst) if dst_url.netloc: if dst_url.scheme == 'link': # Magic link dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'), context['lang']) else: return dst # Normalize dst = urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_dst = urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute dst_elems = parsed_dst.path.split('/')[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break # Now i is the longest common prefix result = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) if not result: result = "." # Don't forget the fragment (anchor) part of the link if parsed_dst.fragment: result += "#" + parsed_dst.fragment assert result, (src, dst, i, src_elems, dst_elems) return result try: os.makedirs(os.path.dirname(output_name)) except: pass doc = lxml.html.document_fromstring(data) doc.rewrite_links(replacer) data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8') with open(output_name, "wb+") as post_file: post_file.write(data) def current_lang(self): # FIXME: this is duplicated, turn into a mixin """Return the currently set locale, if it's one of the available translations, or default_lang.""" lang = utils.LocaleBorg().current_lang if lang: if lang in self.translations: return lang lang = lang.split('_')[0] if lang in self.translations: return lang # whatever return self.default_lang def path(self, kind, name, lang=None, is_link=False): """Build the path to a certain kind of page. kind is one of: * tag_index (name is ignored) * tag (and name is the tag name) * tag_rss (name is the tag name) * archive (and name is the year, or None for the main archive index) * index (name is the number in index-number) * rss (name is ignored) * gallery (name is the gallery name) * listing (name is the source code file name) * post_path (name is 1st element in a post_pages tuple) The returned value is always a path relative to output, like "categories/whatever.html" If is_link is True, the path is absolute and uses "/" as separator (ex: "/archive/index.html"). If is_link is False, the path is relative to output and uses the platform's separator. (ex: "archive\\index.html") """ if lang is None: lang = self.current_lang() path = [] if kind == "tag_index": path = [_f for _f in [self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'], self.config['INDEX_FILE']] if _f] elif kind == "tag": if self.config['SLUG_TAG_PATH']: name = utils.slugify(name) path = [_f for _f in [self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'], name + ".html"] if _f] elif kind == "tag_rss": if self.config['SLUG_TAG_PATH']: name = utils.slugify(name) path = [_f for _f in [self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'], name + ".xml"] if _f] elif kind == "index": if name not in [None, 0]: path = [_f for _f in [self.config['TRANSLATIONS'][lang], self.config['INDEX_PATH'], 'index-{0}.html'.format(name)] if _f] else: path = [_f for _f in [self.config['TRANSLATIONS'][lang], self.config['INDEX_PATH'], self.config['INDEX_FILE']] if _f] elif kind == "post_path": path = [_f for _f in [self.config['TRANSLATIONS'][lang], os.path.dirname(name), self.config['INDEX_FILE']] if _f] elif kind == "rss": path = [_f for _f in [self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'], 'rss.xml'] if _f] elif kind == "archive": if name: path = [_f for _f in [self.config['TRANSLATIONS'][lang], self.config['ARCHIVE_PATH'], name, self.config['INDEX_FILE']] if _f] else: path = [_f for _f in [self.config['TRANSLATIONS'][lang], self.config['ARCHIVE_PATH'], self.config['ARCHIVE_FILENAME']] if _f] elif kind == "gallery": path = [_f for _f in [self.config['GALLERY_PATH'], name, self.config['INDEX_FILE']] if _f] elif kind == "listing": path = [_f for _f in [self.config['LISTINGS_FOLDER'], name + '.html'] if _f] if is_link: link = '/' + ('/'.join(path)) index_len = len(self.config['INDEX_FILE']) if self.config['STRIP_INDEXES'] and \ link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']: return link[:-index_len] else: return link else: return os.path.join(*path) def link(self, *args): return self.path(*args, is_link=True) def abs_link(self, dst): # Normalize dst = urljoin(self.config['BASE_URL'], dst) return urlparse(dst).path def rel_link(self, src, dst): # Normalize src = urljoin(self.config['BASE_URL'], src) dst = urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_src = urlsplit(src) parsed_dst = urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute src_elems = parsed_src.path.split('/')[1:] dst_elems = parsed_dst.path.split('/')[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break else: i += 1 # Now i is the longest common prefix return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) def file_exists(self, path, not_empty=False): """Returns True if the file exists. If not_empty is True, it also has to be not empty.""" exists = os.path.exists(path) if exists and not_empty: exists = os.stat(path).st_size > 0 return exists def gen_tasks(self, name, plugin_category): def flatten(task): if isinstance(task, dict): yield task else: for t in task: for ft in flatten(t): yield ft task_dep = [] for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category): for task in flatten(pluginInfo.plugin_object.gen_tasks()): yield task for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"): flag = False for task in multi.plugin_object.process(task, name): flag = True yield task if flag: task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name)) if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) yield { 'name': name, 'actions': None, 'clean': True, 'task_dep': task_dep } def scan_posts(self): """Scan all the posts.""" if self._scanned: return print("Scanning posts", end='') tzinfo = None if self.config['TIMEZONE'] is not None: tzinfo = pytz.timezone(self.config['TIMEZONE']) current_time = utils.current_time(tzinfo) targets = set([]) for wildcard, destination, template_name, use_in_feeds in \ self.config['post_pages']: print(".", end='') dirname = os.path.dirname(wildcard) for dirpath, _, _ in os.walk(dirname): dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) dest_dir = os.path.normpath(os.path.join(destination, os.path.relpath(dirpath, dirname))) full_list = glob.glob(dir_glob) # Now let's look for things that are not in default_lang for lang in self.config['TRANSLATIONS'].keys(): lang_glob = dir_glob + "." + lang translated_list = glob.glob(lang_glob) for fname in translated_list: orig_name = os.path.splitext(fname)[0] if orig_name in full_list: continue full_list.append(orig_name) for base_path in full_list: post = Post( base_path, self.config['CACHE_FOLDER'], dest_dir, use_in_feeds, self.config['TRANSLATIONS'], self.config['DEFAULT_LANG'], self.config['BASE_URL'], self.MESSAGES, template_name, self.config['FILE_METADATA_REGEXP'], self.config['STRIP_INDEXES'], self.config['INDEX_FILE'], tzinfo, current_time, self.config['HIDE_UNTRANSLATED_POSTS'], self.config['PRETTY_URLS'], ) for lang, langpath in list( self.config['TRANSLATIONS'].items()): dest = (destination, langpath, dir_glob, post.meta[lang]['slug']) if dest in targets: raise Exception('Duplicated output path {0!r} ' 'in post {1!r}'.format( post.meta[lang]['slug'], base_path)) targets.add(dest) self.global_data[post.post_name] = post if post.use_in_feeds: self.posts_per_year[ str(post.date.year)].append(post.post_name) self.posts_per_month[ '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post.post_name) for tag in post.alltags: self.posts_per_tag[tag].append(post.post_name) else: self.pages.append(post) if self.config['OLD_THEME_SUPPORT']: post._add_old_metadata() self.post_per_file[post.destination_path(lang=lang)] = post self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post for name, post in list(self.global_data.items()): self.timeline.append(post) self.timeline.sort(key=lambda p: p.date) self.timeline.reverse() post_timeline = [p for p in self.timeline if p.use_in_feeds] for i, p in enumerate(post_timeline[1:]): p.next_post = post_timeline[i] for i, p in enumerate(post_timeline[:-1]): p.prev_post = post_timeline[i + 1] self._scanned = True print("done!") def generic_page_renderer(self, lang, post, filters): """Render post fragments to final HTML pages.""" context = {} deps = post.deps(lang) + \ self.template_system.template_deps(post.template_name) context['post'] = post context['lang'] = lang context['title'] = post.title(lang) context['description'] = post.description(lang) context['permalink'] = post.permalink(lang) context['page_list'] = self.pages if post.use_in_feeds: context['enable_comments'] = True else: context['enable_comments'] = self.config['COMMENTS_IN_STORIES'] extension = self.get_compiler(post.source_path).extension() output_name = os.path.join(self.config['OUTPUT_FOLDER'], post.destination_path(lang, extension)) deps_dict = copy(context) deps_dict.pop('post') if post.prev_post: deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)] if post.next_post: deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)] deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER'] deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS'] deps_dict['global'] = self.GLOBAL_CONTEXT deps_dict['comments'] = context['enable_comments'] if post: deps_dict['post_translations'] = post.translated_to task = { 'name': os.path.normpath(output_name), 'file_dep': deps, 'targets': [output_name], 'actions': [(self.render_template, [post.template_name, output_name, context])], 'clean': True, 'uptodate': [config_changed(deps_dict)], } yield utils.apply_filters(task, filters) def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context): """Renders pages with lists of posts.""" deps = self.template_system.template_deps(template_name) for post in posts: deps += post.deps(lang) context = {} context["posts"] = posts context["title"] = self.config['BLOG_TITLE'] context["description"] = self.config['BLOG_DESCRIPTION'] context["lang"] = lang context["prevlink"] = None context["nextlink"] = None context.update(extra_context) deps_context = copy(context) deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in posts] deps_context["global"] = self.GLOBAL_CONTEXT task = { 'name': os.path.normpath(output_name), 'targets': [output_name], 'file_dep': deps, 'actions': [(self.render_template, [template_name, output_name, context])], 'clean': True, 'uptodate': [config_changed(deps_context)] } return utils.apply_filters(task, filters)
class WeaponSystem(rpyc.Service): ''' RPC Services: This is the code that does the actual password cracking and returns the results to orbital control. Currently only supports cracking using rainbow tables (RCrackPy) ''' is_initialized = False mutex = Lock() is_busy = False job_id = None def initialize(self): ''' Initializes variables, this should only be called once ''' logging.info("Weapon system initializing ...") self.plugin_manager = PluginManager() self.plugin_manager.setPluginPlaces(["plugins/"]) self.plugin_manager.setCategoriesFilter(FILTERS) self.plugin_manager.collectPlugins() self.plugins = {} logging.info( "Loaded %d plugin(s)" % len(self.plugin_manager.getAllPlugins()) ) self.__cpu__() logging.info("Weapon system online, good hunting.") @atomic def on_connect(self): ''' Called when successfully connected ''' if not self.is_initialized: self.initialize() self.is_initialized = True logging.info("Uplink to orbital control active") def on_disconnect(self): ''' Called if the connection is lost/disconnected ''' logging.info("Disconnected from orbital command server.") def __cpu__(self): ''' Detects the number of CPU cores on a system (including virtual cores) ''' if cpu_count is not None: try: self.cpu_cores = cpu_count() logging.info("Detected %d CPU core(s)" % self.cpu_cores) except NotImplementedError: logging.error("Could not detect number of processors; assuming 1") self.cpu_cores = 1 else: try: self.cpu_cores = int(sysconf("SC_NPROCESSORS_CONF")) logging.info("Detected %d CPU core(s)" % self.cpu_cores) except ValueError: logging.error("Could not detect number of processors; assuming 1") self.cpu_cores = 1 ############################ [ EXPOSED METHODS ] ############################ @atomic def exposed_crack(self, plugin_name, job_id, hashes, **kwargs): ''' Exposes plugins calls ''' assert plugin_name in self.plugins self.is_busy = True self.job_id = job_id self.plugin_manager.activatePluginByName(plugin_name) plugin = self.plugin_manager.getPluginByName(plugin_name) results = plugin.execute(hashes, **kwargs) self.plugin_manager.deactivatePluginByName(plugin_name) self.job_id = None self.is_busy = False return results def exposed_get_plugin_names(self): ''' Returns what algorithms can be cracked ''' logging.info("Method called: exposed_get_capabilities") plugins = self.plugin_manager.getAllPlugins() return [plugin.name for plugin in plugins] def exposed_get_categories(self): ''' Return categories for which we have plugins ''' categories = [] for category in self.plugin_manager.getCategories(): if 0 < len(self.plugin_manager.getPluginsOfCategory(category)): categories.append(category) return categories def exposed_get_category_plugins(self, category): ''' Get plugin names for a category ''' plugins = self.plugin_manager.getPluginsOfCategory(category) return [plugin.name for plugin in plugins] def exposed_get_plugin_details(self, category, plugin_name): ''' Get plugin based on name details ''' plugin = self.plugin_manager.getPluginByName(plugin_name, category) info = {'name': plugin.name} info['author'] = plugin.details.get('Documentation', 'author') info['website'] = plugin.details.get('Documentation', 'website') info['version'] = plugin.details.get('Documentation', 'version') info['description'] = plugin.details.get('Documentation', 'description') info['copyright'] = plugin.details.get('Documentation', 'copyright') info['precomputation'] = plugin.details.getboolean('Core', 'precomputation') return info def exposed_ping(self): ''' Returns a pong message ''' return "PONG" def exposed_is_busy(self): ''' Returns True/False if the current system is busy (thread safe) ''' return self.is_busy def exposed_current_job_id(self): ''' Returns the current job id (thread safe) ''' return self.job_id def exposed_cpu_count(self): ''' Returns the number of detected cpu cores ''' return self.cpu_cores
class FakeSite(object): def __init__(self): self.template_system = self self.invariant = False self.config = { "DISABLED_PLUGINS": [], "EXTRA_PLUGINS": [], "EXTRA_PLUGINS_DIRS": [extra_v6_plugin_dir], "DEFAULT_LANG": "en", "MARKDOWN_EXTENSIONS": ["fenced_code", "codehilite"], "TRANSLATIONS_PATTERN": "{path}.{lang}.{ext}", "LISTINGS_FOLDERS": {}, } self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"] self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "CompilerExtension": CompilerExtension, "RestExtension": RestExtension, "MarkdownExtension": MarkdownExtension, } ) self.loghandlers = [nikola.utils.STDERR_HANDLER] self.plugin_manager.setPluginInfoExtension("plugin") extra_plugins_dirs = self.config["EXTRA_PLUGINS_DIRS"] if sys.version_info[0] == 3: places = [os.path.join(os.path.dirname(nikola.utils.__file__), "plugins")] + [ path for path in extra_plugins_dirs if path ] else: places = [os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode("plugins"))] + [ nikola.utils.sys_encode(path) for path in extra_plugins_dirs if path ] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension") self.timeline = [FakePost(title="Fake post", slug="fake-post")] self.debug = True self.rst_transforms = [] # This is to make plugin initialization happy self.template_system = self self.name = "mako" def _activate_plugins_of_category(self, category): """Activate all the plugins of a given category and return them.""" # this code duplicated in nikola/nikola.py plugins = [] for plugin_info in self.plugin_manager.getPluginsOfCategory(category): if plugin_info.name in self.config.get("DISABLED_PLUGINS"): self.plugin_manager.removePluginFromCategory(plugin_info, category) else: self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugins.append(plugin_info) return plugins def render_template(self, name, _, context): return '<img src="IMG.jpg">'
class TestDef(object): def __init__(self): # set aside storage for options and cmd line args self.options = {} self.args = [] # record if we have loaded the plugins or # not - this is just a bozo check to ensure # someone doesn't tell us to do it twice self.loaded = False # set aside a spot for a logger object, and # note that it hasn't yet been defined self.logger = None self.modcmd = None self.execmd = None self.harasser = None self.config = None self.stages = None self.tools = None self.utilities = None self.defaults = None self.log = {} self.watchdog = None self.plugin_trans_sem = Semaphore() def setOptions(self, args): self.options = vars(args) self.args = args # private function to convert values def __convert_value(self, opt, inval): if opt is None or type(opt) is str: return 0, inval elif type(opt) is bool: if type(inval) is bool: return 0, inval elif type(inval) is str: if inval.lower() in ['true', '1', 't', 'y', 'yes']: return 0, True else: return 0, False elif type(inval) is int: if 0 == inval: return 0, False else: return 0, True elif is_py2 and type(inval) is unicode: return 0, int(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None elif type(opt) is int: if type(inval) is int: return 0, inval elif type(inval) is str: return 0, int(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None elif type(opt) is float: if type(inval) is float: return 0, inval elif type(inval) is str or type(inval) is int: return 0, float(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None else: return 1, None # scan the key-value pairs obtained from the configuration # parser and compare them with the options defined for a # given plugin. Generate an output dictionary that contains # the updated set of option values, the default value for # any option that wasn't included in the configuration file, # and return an error status plus output identifying any # keys in the configuration file that are not supported # by the list of options # # @log [INPUT] # - a dictionary that will return the status plus # stderr containing strings identifying any # provided keyvals that don't have a corresponding # supported option # @options [INPUT] # - a dictionary of tuples, each consisting of three # entries: # (a) the default value # (b) data type # (c) a help-like description # @keyvals [INPUT] # - a dictionary of key-value pairs obtained from # the configuration parser # @target [OUTPUT] # - the resulting dictionary of key-value pairs def parseOptions(self, log, options, keyvals, target): # parse the incoming keyvals dictionary against the source # options. If a source option isn't provided, then # copy it across to the target. opts = list(options.keys()) kvkeys = list(keyvals.keys()) for opt in opts: found = False for kvkey in kvkeys: if kvkey == opt: # they provided us with an update, so # pass this value into the target - expand # any provided lists if keyvals[kvkey] is None: continue st, outval = self.__convert_value(options[opt][0], keyvals[kvkey]) if 0 == st: target[opt] = outval else: if len(keyvals[kvkey]) == 0: # this indicates they do not want this option found = True break if keyvals[kvkey][0][0] == "[": # they provided a list - remove the brackets val = keyvals[kvkey].replace('[','') val = val.replace(']','') # split the input to pickup sets of options newvals = list(val) # convert the values to specified type i=0 for val in newvals: st, newvals[i] = self.__convert_value(opt[0], val) i = i + 1 target[opt] = newvals else: st, target[opt] = self.__convert_value(opt[0], keyvals[kvkey]) found = True break if not found: # they didn't provide this one, so # transfer only the value across target[opt] = options[opt][0] # add in any default settings that have not # been overridden - anything set by this input # stage will override the default if self.defaults is not None: keys = self.defaults.options.keys() for key in keys: if key not in target: target[key] = self.defaults.options[key][0] # now go thru in the reverse direction to see # if any keyvals they provided aren't supported # as this would be an error unsupported_options = [] for kvkey in kvkeys: # ignore some standard keys if kvkey in ['section', 'plugin']: continue try: if target[kvkey] is not None: pass except KeyError: # some always need to be passed if kvkey in ['parent', 'asis']: target[kvkey] = keyvals[kvkey] else: unsupported_options.append(kvkey) if unsupported_options: sys.exit("ERROR: Unsupported options for section [%s]: %s" % (log['section'], ",".join(unsupported_options))) log['status'] = 0 log['options'] = target return def loadPlugins(self, basedir, topdir): if self.loaded: print("Cannot load plugins multiple times") sys.exit(1) self.loaded = True # find the loader utility so we can bootstrap ourselves try: m = imp.load_source("LoadClasses", os.path.join(basedir, "LoadClasses.py")); except ImportError: print("ERROR: unable to load LoadClasses that must contain the class loader object") sys.exit(1) cls = getattr(m, "LoadClasses") a = cls() # setup the loader object self.loader = a.__class__(); # Setup the array of directories we will search for plugins # Note that we always look at the topdir location by default plugindirs = [] plugindirs.append(topdir) if self.options['plugindir']: # could be a comma-delimited list, so split on commas x = self.options['plugindir'].split(',') for y in x: # prepend so we always look at the given # location first in case the user wants # to "overload/replace" a default MTT # class definition plugindirs.insert(0, y) # Load plugins from each of the specified plugin dirs for dirPath in plugindirs: if not Path(dirPath).exists(): print("Attempted to load plugins from non-existent path:", dirPath) continue try: self.loader.load(dirPath) except Exception as e: print("Exception caught while loading plugins:") print(e) sys.exit(1) # Build plugin managers, # class yapsy.PluginManager.PluginManager(categories_filter=None, # directories_list=None, plugin_info_ext=None, plugin_locator=None) # Build the stages plugin manager self.stages = PluginManager(None, plugindirs, None, None) # Get a list of all the categories - this corresponds to # the MTT stages that have been defined. Note that we # don't need to formally define the stages here - anyone # can add a new stage, or delete an old one, by simply # adding or removing a plugin directory. self.stages.setCategoriesFilter(self.loader.stages) # Load all plugins we find there self.stages.collectPlugins() # Build the tools plugin manager - tools differ from sections # in that they are plugins we will use to execute the various # sections. For example, the TestRun section clearly needs the # ability to launch jobs. There are many ways to launch jobs # depending on the environment, and sometimes several ways to # start jobs even within one environment (e.g., mpirun vs # direct launch). self.tools = PluginManager(None, plugindirs, None, None) # Get the list of tools - not every tool will be capable # of executing. For example, a tool that supports direct launch # against a specific resource manager cannot be used on a # system being managed by a different RM. self.tools.setCategoriesFilter(self.loader.tools) # Load all the tool plugins self.tools.collectPlugins() # Tool plugins are required to provide a function we can # probe to determine if they are capable of operating - check # those now and prune those tools that cannot support this # environment # Build the utilities plugins self.utilities = PluginManager(None, plugindirs, None, None) # Get the list of available utilities. self.utilities.setCategoriesFilter(self.loader.utilities) # Load all the utility plugins self.utilities.collectPlugins() # since we use these all over the place, find the # ExecuteCmd and ModuleCmd plugins and record them availUtil = list(self.loader.utilities.keys()) for util in availUtil: for pluginInfo in self.utilities.getPluginsOfCategory(util): if "ExecuteCmd" == pluginInfo.plugin_object.print_name(): self.execmd = pluginInfo.plugin_object elif "ModuleCmd" == pluginInfo.plugin_object.print_name(): self.modcmd = pluginInfo.plugin_object # initialize this module self.modcmd.setCommand(self.options) elif "Watchdog" == pluginInfo.plugin_object.print_name(): self.watchdog = pluginInfo.plugin_object if self.execmd is not None and self.modcmd is not None and self.watchdog is not None: break if self.execmd is None: print("ExecuteCmd plugin was not found") print("This is a basic capability required") print("for MTT operations - cannot continue") sys.exit(1) # Configure harasser plugin for pluginInfo in self.tools.getPluginsOfCategory("Harasser"): if "Harasser" == pluginInfo.plugin_object.print_name(): self.harasser = pluginInfo.plugin_object break if self.harasser is None: print("Harasser plugin was not found") print("This is required for all TestRun plugins") print("cannot continue") sys.exit(1) # similarly, capture the highest priority defaults stage here pri = -1 for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"): if pri < pluginInfo.plugin_object.priority(): self.defaults = pluginInfo.plugin_object pri = pluginInfo.plugin_object.priority() return def printInfo(self): # Print the available MTT sections out, if requested if self.options['listsections']: print("Supported MTT stages:") # print them in the default order of execution for stage in self.loader.stageOrder: print(" " + stage) sys.exit(0) # Print the detected plugins for a given stage if self.options['listplugins']: # if the list is '*', print the plugins for every stage if self.options['listplugins'] == "*": sections = self.loader.stageOrder else: sections = self.options['listplugins'].split(',') print() for section in sections: print(section + ":") try: for pluginInfo in self.stages.getPluginsOfCategory(section): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid stage name " + section) print() sys.exit(1) # Print the options for a given plugin if self.options['liststageoptions']: # if the list is '*', print the options for every stage/plugin if self.options['liststageoptions'] == "*": sections = self.loader.stageOrder else: sections = self.options['liststageoptions'].split(',') print() for section in sections: print(section + ":") try: for pluginInfo in self.stages.getPluginsOfCategory(section): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options(self, " ") except KeyError: print(" Invalid stage name " + section) print() sys.exit(1) # Print the available MTT tools out, if requested if self.options['listtools']: print("Available MTT tools:") availTools = list(self.loader.tools.keys()) for tool in availTools: print(" " + tool) sys.exit(0) # Print the detected tool plugins for a given tool type if self.options['listtoolmodules']: # if the list is '*', print the plugins for every type if self.options['listtoolmodules'] == "*": print() availTools = list(self.loader.tools.keys()) else: availTools = self.options['listtoolmodules'].split(',') print() for tool in availTools: print(tool + ":") try: for pluginInfo in self.tools.getPluginsOfCategory(tool): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid tool type name",tool) print() sys.exit(1) # Print the options for a given plugin if self.options['listtooloptions']: # if the list is '*', print the options for every stage/plugin if self.options['listtooloptions'] == "*": availTools = list(self.loader.tools.keys()) else: availTools = self.options['listtooloptions'].split(',') print() for tool in availTools: print(tool + ":") try: for pluginInfo in self.tools.getPluginsOfCategory(tool): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options(self, " ") except KeyError: print(" Invalid tool type name " + tool) print() sys.exit(1) # Print the available MTT utilities out, if requested if self.options['listutils']: print("Available MTT utilities:") availUtils = list(self.loader.utilities.keys()) for util in availUtils: print(" " + util) sys.exit(0) # Print the detected utility plugins for a given tool type if self.options['listutilmodules']: # if the list is '*', print the plugins for every type if self.options['listutilmodules'] == "*": print() availUtils = list(self.loader.utilities.keys()) else: availUtils = self.options['listutilitymodules'].split(',') print() for util in availUtils: print(util + ":") try: for pluginInfo in self.utilities.getPluginsOfCategory(util): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid utility type name") print() sys.exit(1) # Print the options for a given plugin if self.options['listutiloptions']: # if the list is '*', print the options for every stage/plugin if self.options['listutiloptions'] == "*": availUtils = list(self.loader.utilities.keys()) else: availUtils = self.options['listutiloptions'].split(',') print() for util in availUtils: print(util + ":") try: for pluginInfo in self.utilities.getPluginsOfCategory(util): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options(self, " ") except KeyError: print(" Invalid utility type name " + util) print() sys.exit(1) # if they asked for the version info, print it and exit if self.options['version']: for pluginInfo in self.tools.getPluginsOfCategory("Version"): print("MTT Base: " + pluginInfo.plugin_object.getVersion()) print("MTT Client: " + pluginInfo.plugin_object.getClientVersion()) sys.exit(0) def openLogger(self): # there must be a logger utility or we can't do # anything useful if not self.utilities.activatePluginByName("Logger", "Base"): print("Required Logger plugin not found or could not be activated") sys.exit(1) # execute the provided test description self.logger = self.utilities.getPluginByName("Logger", "Base").plugin_object self.logger.open(self) return def fill_log_interpolation(self, basestr, sublog): if isinstance(sublog, str): self.config.set("LOG", basestr, sublog.replace("$","$$")) elif isinstance(sublog, dict): for k,v in sublog.items(): self.fill_log_interpolation("%s.%s" % (basestr, k), v) elif isinstance(sublog, list): if sum([((isinstance(t, list) or isinstance(t, tuple)) and len(t) == 2) for t in sublog]) == len(sublog): self.fill_log_interpolation(basestr, {k:v for k,v in sublog}) else: for i,v in enumerate(sublog): self.fill_log_interpolation("%s.%d" % (basestr, i), v) else: self.fill_log_interpolation(basestr, str(sublog)) def expandWildCards(self, sections): expsec = [] cpsections = list(sections) for sec in cpsections: if '*' in sec: modsec = sec.split('*') startswith = modsec[0] endswith = modsec[-1] findsec = modsec[1:-1] allsections = self.config.sections() for s in allsections: if not s.startswith(startswith): continue if not s.endswith(endswith): continue found = True s_tmp = s for f in findsec: if not f in s_tmp: found = False break s_tmp = f.join(s_tmp.split(f)[1:]) if not found: continue expsec.append(s) sections.remove(sec) return sections + expsec def fill_env_hidden_section(self): """fill ENV section with environment variables """ try: self.config.add_section('ENV') except configparser.DuplicateSectionError: pass for k,v in os.environ.items(): self.config.set('ENV', k, v.replace("$","$$")) def fill_log_hidden_section(self): """Add LOG section filled with log results of stages """ try: self.config.add_section('LOG') except configparser.DuplicateSectionError: pass thefulllog = self.logger.getLog(None) for e in thefulllog: self.fill_log_interpolation(e['section'].replace(":","_"), e) def check_for_nondefined_env_variables(self): # Check for ENV input required_env = [] all_file_contents = [] for testFile in self.log['inifiles']: file_contents = open(testFile, "r").read() file_contents = "\n".join(["%s %d: %s" % (testFile.split("/")[-1],i,l) for i,l in enumerate(file_contents.split("\n")) if not l.lstrip().startswith("#")]) all_file_contents.append(file_contents) if "${ENV:" in file_contents: required_env.extend([s.split("}")[0] for s in file_contents.split("${ENV:")[1:]]) env_not_found = set([e for e in required_env if e not in os.environ.keys()]) lines_with_env_not_found = [] for file_contents in all_file_contents: lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \ for l in file_contents.split("\n") \ if sum(["${ENV:%s}"%e in l for e in env_not_found])]) if lines_with_env_not_found: print("ERROR: Not all required environment variables are defined.") print("ERROR: Still need:") for l in lines_with_env_not_found: print("ERROR: %s"%l) sys.exit(1) def configTest(self): # setup the configuration parser self.config = configparser.SafeConfigParser(interpolation=configparser.ExtendedInterpolation()) # Set the config parser to make option names case sensitive. self.config.optionxform = str # fill ENV section with environemt variables self.fill_env_hidden_section() # log the list of files - note that the argument parser # puts the input files in a list, with the first member # being the list of input files self.log['inifiles'] = self.args.ini_files[0] # initialize the list of active sections self.actives = [] # if they specified a list to execute, then use it sections = [] if self.args.section: sections = self.args.section.split(",") skip = False elif self.args.skipsections: sections = self.args.skipsections.split(",") skip = True else: sections = None # cycle thru the input files for testFile in self.log['inifiles']: if not os.path.isfile(testFile): print("Test description file",testFile,"not found!") sys.exit(1) self.config.read(self.log['inifiles']) # Check for ENV input self.check_for_nondefined_env_variables() # find all the sections that match the wild card and expand them # this is simple wild carding, ie *text, text*, *text* and * # should all work if sections is not None: sections = self.expandWildCards(sections) #if sections is not None: # expsec = [] # cpsections = list(sections) # for sec in cpsections: # if '*' in sec: # modsec = sec.replace('*','') # for s in self.config.sections(): # if modsec in s: # expsec.append(s) # sections.remove(sec) # sections = sections + expsec for section in self.config.sections(): if section.startswith("SKIP") or section.startswith("skip"): # users often want to temporarily ignore a section # of their test definition file, but don't want to # remove it lest they forget what it did. So let # them just mark the section as "skip" to be ignored continue # if we are to filter the sections, then do so takeus = True if sections is not None: found = False for sec in sections: if sec == section: found = True sections.remove(sec) if skip: takeus = False break if not found and not skip: takeus = False if takeus: self.actives.append(section) if sections is not None and 0 != len(sections) and not skip: print("ERROR: sections were specified for execution and not found:",sections) sys.exit(1) # set Defaults -command line args supercede .ini args try: if not self.options['scratchdir']: self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratchdir') except: try: self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratch') except: self.options['scratchdir'] = os.path.abspath('./mttscratch') self.options['scratchdir'] = os.path.abspath(self.options['scratchdir']) try: if not self.options['executor']: self.options['executor'] = self.config.get('MTTDefaults', 'executor') except: self.options['executor'] = 'sequential' # if they want us to clear the scratch, then do so if self.options['clean'] and os.path.isdir(self.options['scratchdir']) : shutil.rmtree(self.options['scratchdir']) # setup the scratch directory _mkdir_recursive(self.options['scratchdir']) return # Used with combinatorial executor, loads next .ini file to be run with the # sequential executor def configNewTest(self, file): # clear the configuration parser for section in self.config.sections(): self.config.remove_section(section) # read in the file self.config.read(file) for section in self.config.sections(): if section.startswith("SKIP") or section.startswith("skip"): # users often want to temporarily ignore a section # of their test definition file, but don't want to # remove it lest they forget what it did. So let # them just mark the section as "skip" to be ignored continue if self.logger is not None: self.logger.verbose_print("SECTION: " + section) self.logger.verbose_print(self.config.items(section)) return def executeTest(self, executor="sequential"): self.logger.print_cmdline_args(self) if not self.loaded: print("Plugins have not been loaded - cannot execute test") sys.exit(1) if self.config is None: print("No test definition file was parsed - cannot execute test") sys.exit(1) if not self.tools.getPluginByName(executor, "Executor"): print("Specified executor %s not found" % executor) sys.exit(1) # activate the specified plugin self.tools.activatePluginByName(executor, "Executor") # execute the provided test description executor = self.tools.getPluginByName(executor, "Executor") status = executor.plugin_object.execute(self) if status == 0 and self.options['clean_after'] and os.path.isdir(self.options['scratchdir']): self.logger.verbose_print("Cleaning up scratchdir after successful run") shutil.rmtree(self.options['scratchdir']) return status def printOptions(self, options): # if the options are empty, report that if not options: lines = ["None"] return lines # create the list of options opts = [] vals = list(options.keys()) for val in vals: opts.append(val) if options[val][0] is None: opts.append("None") elif isinstance(options[val][0], bool): if options[val][0]: opts.append("True") else: opts.append("False") elif isinstance(options[val][0], list): opts.append(" ".join(options[val][0])) elif isinstance(options[val][0], int): opts.append(str(options[val][0])) else: opts.append(options[val][0]) opts.append(options[val][1]) # print the options, their default value, and # the help description in 3 column format max1 = 0 max2 = 0 for i in range(0,len(opts),3): # we want all the columns to line up # and left-justify, so first find out # the max len of each of the first two # column entries if len(opts[i]) > max1: max1 = len(opts[i]) if type(opts[i+1]) is not str: optout = str(opts[i+1]) else: optout = opts[i+1] if len(optout) > max2: max2 = len(optout) # provide some spacing max1 = max1 + 4 max2 = max2 + 4 # cycle thru again, padding each entry to # align the columns lines = [] sp = " " for i in range(0,len(opts),3): line = opts[i] + (max1-len(opts[i]))*sp if type(opts[i+1]) is not str: optout = str(opts[i+1]) else: optout = opts[i+1] line = line + optout + (max2-len(optout))*sp # to make this more readable, we will wrap the line at # 130 characters. First, see if the line is going to be # too long if 130 < (len(line) + len(opts[i+2])): # split the remaining column into individual words words = opts[i+2].split() first = True for word in words: if (len(line) + len(word)) < 130: if first: line = line + word first = False else: line = line + " " + word else: lines.append(line) line = (max1 + max2)*sp + word if 0 < len(line): lines.append(line) else: # the line is fine - so just add the last piece line = line + opts[i+2] # append the result lines.append(line) # add one blank line lines.append("") return lines def selectPlugin(self, name, category): if category == "stage": try: availStages = list(self.loader.stages.keys()) for stage in availStages: for pluginInfo in self.stages.getPluginsOfCategory(stage): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None elif category == "tool": try: availTools = list(self.loader.tools.keys()) for tool in availTools: for pluginInfo in self.tools.getPluginsOfCategory(tool): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None elif category == "utility": try: availUtils = list(self.loader.utilities.keys()) for util in availUtils: for pluginInfo in self.utilities.getPluginsOfCategory(util): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None else: print("Unrecognized category:",category) return None
class Nikola(object): """Class that handles site generation. Takes a site config as argument on creation. """ def __init__(self, **config): """Setup proper environment for running tasks.""" self.global_data = {} self.posts_per_year = defaultdict(list) self.posts_per_tag = defaultdict(list) self.timeline = [] self.pages = [] self._scanned = False # This is the default config # TODO: fill it self.config = { 'ARCHIVE_PATH': "", 'ARCHIVE_FILENAME': "archive.html", 'DEFAULT_LANG': "en", 'OUTPUT_FOLDER': 'output', 'FILES_FOLDERS': { 'files': '' }, 'LISTINGS_FOLDER': 'listings', 'ADD_THIS_BUTTONS': True, 'INDEX_DISPLAY_POST_COUNT': 10, 'INDEX_TEASERS': False, 'MAX_IMAGE_SIZE': 1280, 'USE_FILENAME_AS_TITLE': True, 'SLUG_TAG_PATH': False, 'INDEXES_TITLE': "", 'INDEXES_PAGES': "", 'FILTERS': {}, 'USE_BUNDLES': True, 'TAG_PAGES_ARE_INDEXES': False, 'THEME': 'default', 'post_compilers': { "rest": ['.txt', '.rst'], "markdown": ['.md', '.mdown', '.markdown'], "html": ['.html', '.htm'], }, } self.config.update(config) self.config['TRANSLATIONS'] = self.config.get( 'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''}) self.GLOBAL_CONTEXT = self.config.get('GLOBAL_CONTEXT', {}) self.THEMES = utils.get_theme_chain(self.config['THEME']) self.MESSAGES = utils.load_messages(self.THEMES, self.config['TRANSLATIONS']) self.GLOBAL_CONTEXT['messages'] = self.MESSAGES self.GLOBAL_CONTEXT['_link'] = self.link self.GLOBAL_CONTEXT['rel_link'] = self.rel_link self.GLOBAL_CONTEXT['abs_link'] = self.abs_link self.GLOBAL_CONTEXT['exists'] = self.file_exists self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[ 'ADD_THIS_BUTTONS'] self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[ 'INDEX_DISPLAY_POST_COUNT'] self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES'] self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, }) self.plugin_manager.setPluginInfoExtension('plugin') self.plugin_manager.setPluginPlaces([ os.path.join(os.path.dirname(__file__), 'plugins'), os.path.join(os.getcwd(), 'plugins'), ]) self.plugin_manager.collectPlugins() self.commands = {} # Activate all command plugins for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"): self.plugin_manager.activatePluginByName(pluginInfo.name) pluginInfo.plugin_object.set_site(self) pluginInfo.plugin_object.short_help = pluginInfo.description self.commands[pluginInfo.name] = pluginInfo.plugin_object # Activate all task plugins for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"): self.plugin_manager.activatePluginByName(pluginInfo.name) pluginInfo.plugin_object.set_site(self) for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"): self.plugin_manager.activatePluginByName(pluginInfo.name) pluginInfo.plugin_object.set_site(self) # Load template plugin template_sys_name = utils.get_template_engine(self.THEMES) pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem") if pi is None: sys.stderr.write("Error loading %s template system plugin\n" % template_sys_name) sys.exit(1) self.template_system = pi.plugin_object self.template_system.set_directories([ os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES ]) # Load compiler plugins self.compilers = {} self.inverse_compilers = {} for pluginInfo in self.plugin_manager.getPluginsOfCategory( "PageCompiler"): self.compilers[pluginInfo.name] = \ pluginInfo.plugin_object.compile_html def get_compiler(self, source_name): """Get the correct compiler for a post from `conf.post_compilers` To make things easier for users, the mapping in conf.py is compiler->[extensions], although this is less convenient for us. The majority of this function is reversing that dictionary and error checking. """ ext = os.path.splitext(source_name)[1] try: compile_html = self.inverse_compilers[ext] except KeyError: # Find the correct compiler for this files extension langs = [ lang for lang, exts in self.config['post_compilers'].items() if ext in exts ] if len(langs) != 1: if len(set(langs)) > 1: exit("Your file extension->compiler definition is" "ambiguous.\nPlease remove one of the file extensions" "from 'post_compilers' in conf.py\n(The error is in" "one of %s)" % ', '.join(langs)) elif len(langs) > 1: langs = langs[:1] else: exit("post_compilers in conf.py does not tell me how to " "handle '%s' extensions." % ext) lang = langs[0] compile_html = self.compilers[lang] self.inverse_compilers[ext] = compile_html return compile_html def render_template(self, template_name, output_name, context): local_context = {} local_context["template_name"] = template_name local_context.update(self.config['GLOBAL_CONTEXT']) local_context.update(context) data = self.template_system.render_template(template_name, None, local_context) assert output_name.startswith(self.config["OUTPUT_FOLDER"]) url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:] # This is to support windows paths url_part = "/".join(url_part.split(os.sep)) src = urlparse.urljoin(self.config["BLOG_URL"], url_part) parsed_src = urlparse.urlsplit(src) src_elems = parsed_src.path.split('/')[1:] def replacer(dst): # Refuse to replace links that are full URLs. dst_url = urlparse.urlparse(dst) if dst_url.netloc: if dst_url.scheme == 'link': # Magic link dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'), context['lang']) else: return dst # Normalize dst = urlparse.urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_dst = urlparse.urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute dst_elems = parsed_dst.path.split('/')[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break # Now i is the longest common prefix result = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) if not result: result = "." # Don't forget the fragment (anchor) part of the link if parsed_dst.fragment: result += "#" + parsed_dst.fragment assert result, (src, dst, i, src_elems, dst_elems) return result try: os.makedirs(os.path.dirname(output_name)) except: pass doc = lxml.html.document_fromstring(data) doc.rewrite_links(replacer) data = '<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8') with open(output_name, "w+") as post_file: post_file.write(data) def path(self, kind, name, lang, is_link=False): """Build the path to a certain kind of page. kind is one of: * tag_index (name is ignored) * tag (and name is the tag name) * tag_rss (name is the tag name) * archive (and name is the year, or None for the main archive index) * index (name is the number in index-number) * rss (name is ignored) * gallery (name is the gallery name) * listing (name is the source code file name) The returned value is always a path relative to output, like "categories/whatever.html" If is_link is True, the path is absolute and uses "/" as separator (ex: "/archive/index.html"). If is_link is False, the path is relative to output and uses the platform's separator. (ex: "archive\\index.html") """ path = [] if kind == "tag_index": path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'], 'index.html' ]) elif kind == "tag": if self.config['SLUG_TAG_PATH']: name = utils.slugify(name) path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'], name + ".html" ]) elif kind == "tag_rss": if self.config['SLUG_TAG_PATH']: name = utils.slugify(name) path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'], name + ".xml" ]) elif kind == "index": if name > 0: path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['INDEX_PATH'], 'index-%s.html' % name ]) else: path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['INDEX_PATH'], 'index.html' ]) elif kind == "rss": path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'], 'rss.xml' ]) elif kind == "archive": if name: path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['ARCHIVE_PATH'], name, 'index.html' ]) else: path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['ARCHIVE_PATH'], self.config['ARCHIVE_FILENAME'] ]) elif kind == "gallery": path = filter(None, [self.config['GALLERY_PATH'], name, 'index.html']) elif kind == "listing": path = filter(None, [self.config['LISTINGS_FOLDER'], name + '.html']) if is_link: return '/' + ('/'.join(path)) else: return os.path.join(*path) def link(self, *args): return self.path(*args, is_link=True) def abs_link(self, dst): # Normalize dst = urlparse.urljoin(self.config['BLOG_URL'], dst) return urlparse.urlparse(dst).path def rel_link(self, src, dst): # Normalize src = urlparse.urljoin(self.config['BLOG_URL'], src) dst = urlparse.urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_src = urlparse.urlsplit(src) parsed_dst = urlparse.urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute src_elems = parsed_src.path.split('/')[1:] dst_elems = parsed_dst.path.split('/')[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break else: i += 1 # Now i is the longest common prefix return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) def file_exists(self, path, not_empty=False): """Returns True if the file exists. If not_empty is True, it also has to be not empty.""" exists = os.path.exists(path) if exists and not_empty: exists = os.stat(path).st_size > 0 return exists def gen_tasks(self): task_dep = [] for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"): for task in pluginInfo.plugin_object.gen_tasks(): yield task if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"): for task in pluginInfo.plugin_object.gen_tasks(): yield task if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) yield { 'name': 'all', 'actions': None, 'clean': True, 'task_dep': task_dep } def scan_posts(self): """Scan all the posts.""" if not self._scanned: print "Scanning posts ", targets = set([]) for wildcard, destination, _, use_in_feeds in \ self.config['post_pages']: print ".", for base_path in glob.glob(wildcard): post = Post(base_path, destination, use_in_feeds, self.config['TRANSLATIONS'], self.config['DEFAULT_LANG'], self.config['BLOG_URL'], self.MESSAGES) for lang, langpath in self.config['TRANSLATIONS'].items(): dest = (destination, langpath, post.pagenames[lang]) if dest in targets: raise Exception( 'Duplicated output path %r in post %r' % (post.pagenames[lang], base_path)) targets.add(dest) self.global_data[post.post_name] = post if post.use_in_feeds: self.posts_per_year[str(post.date.year)].append( post.post_name) for tag in post.tags: self.posts_per_tag[tag].append(post.post_name) else: self.pages.append(post) for name, post in self.global_data.items(): self.timeline.append(post) self.timeline.sort(cmp=lambda a, b: cmp(a.date, b.date)) self.timeline.reverse() post_timeline = [p for p in self.timeline if p.use_in_feeds] for i, p in enumerate(post_timeline[1:]): p.next_post = post_timeline[i] for i, p in enumerate(post_timeline[:-1]): p.prev_post = post_timeline[i + 1] self._scanned = True print "done!" def generic_page_renderer(self, lang, wildcard, template_name, destination, filters): """Render post fragments to final HTML pages.""" for post in glob.glob(wildcard): post_name = os.path.splitext(post)[0] context = {} post = self.global_data[post_name] deps = post.deps(lang) + \ self.template_system.template_deps(template_name) context['post'] = post context['lang'] = lang context['title'] = post.title(lang) context['description'] = post.description(lang) context['permalink'] = post.permalink(lang) context['page_list'] = self.pages output_name = os.path.join(self.config['OUTPUT_FOLDER'], self.config['TRANSLATIONS'][lang], destination, post.pagenames[lang] + ".html") deps_dict = copy(context) deps_dict.pop('post') if post.prev_post: deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)] if post.next_post: deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)] deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER'] deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS'] deps_dict['global'] = self.config['GLOBAL_CONTEXT'] task = { 'name': output_name.encode('utf-8'), 'file_dep': deps, 'targets': [output_name], 'actions': [(self.render_template, [template_name, output_name, context])], 'clean': True, 'uptodate': [config_changed(deps_dict)], } yield utils.apply_filters(task, filters) def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context): """Renders pages with lists of posts.""" deps = self.template_system.template_deps(template_name) for post in posts: deps += post.deps(lang) context = {} context["posts"] = posts context["title"] = self.config['BLOG_TITLE'] context["description"] = self.config['BLOG_DESCRIPTION'] context["lang"] = lang context["prevlink"] = None context["nextlink"] = None context.update(extra_context) deps_context = copy(context) deps_context["posts"] = [(p.titles[lang], p.permalink(lang)) for p in posts] deps_context["global"] = self.config['GLOBAL_CONTEXT'] task = { 'name': output_name.encode('utf8'), 'targets': [output_name], 'file_dep': deps, 'actions': [(self.render_template, [template_name, output_name, context])], 'clean': True, 'uptodate': [config_changed(deps_context)] } yield utils.apply_filters(task, filters)
class binWidget(QtGui.QWidget, Observable): scrolled = QtCore.pyqtSignal(int, name='scroll') oldscrolled = QtCore.SIGNAL('scroll') def __init__(self, parent, source): super(binWidget, self).__init__() Observable.__init__(self) self.parent = parent # offset for text window #self.data = mapped self.dataOffset = 0 self.dataModel = source self.cursor = Cursor(0, 0) # self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self), # HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self)] logging.basicConfig(level=logging.ERROR) self.manager = PluginManager(categories_filter={ "FileFormat": FileFormat}) root = os.path.dirname(sys.argv[0]) self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'format')]) #self.manager.setPluginPlaces(["plugins"]) # Load plugins self.manager.locatePlugins() self.manager.loadPlugins() Formats = [] for plugin in self.manager.getPluginsOfCategory("FileFormat"): # plugin.plugin_object is an instance of the plugin po = plugin.plugin_object if po.recognize(self.dataModel): print '[+] ' + po.name Formats.append(po) # sort plugins by priority Formats = sorted(Formats, key=lambda x: x.priority, reverse=True) po = Formats[0] print 'Choosed plugin: ' + po.name #print QtGui.QFontDatabase.addApplicationFont(os.path.join('terminus-ttf-4.39', 'TerminusTTF-4.39.ttf')) self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po), HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po), DisasmViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po)] self.viewMode = self.multipleViewModes[0] self.textDecorator = TextDecorator(self.viewMode) self.viewMode.setTransformationEngine(self.textDecorator) self.multipleViewModes[1].setTransformationEngine(self.textDecorator) self.Banners = Banners() #self.Banners.add(BottomBanner(self.dataModel, self.viewMode)) # self.Banners.add(TopBanner(self.dataModel, self.viewMode)) #self.Banners.add(self.banner) # self.filebanner = FileAddrBanner(self.dataModel, self.viewMode) #self.filebanner = PEBanner(self.dataModel, self.viewMode) #self.Banners.add(PEBanner(self.dataModel, self.viewMode)) #self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode)) #self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode)) # self.offsetWindow_h = self.filebanner.getDesiredGeometry()[0] + 25 self.offsetWindow_h = 0 self.offsetWindow_v = 0 self.searchable = Searchable(self.dataModel, self.viewMode) self.initUI() [po.init(viewMode, parent=self) for viewMode in self.multipleViewModes] for banner in po.getBanners(): self.Banners.add(banner) po.registerShortcuts(self) self.po = po #self.scrolled = QtCore.pyqtSignal(int, name='scroll') #self.scrolled.connect(self.scroll_from_outside) self.searchWindow = SearchWindow(self, None, self.searchable) self.addHandler(self.po) self.addHandler(self.searchable) self.addHandler(self.Banners) self.notify(self.viewMode) #self.connect(self, self.oldscrolled, self.scroll_from_outside) #self.scrolled.emit(1) #self.emit(QtCore.SIGNAL('scroll'), 1) def scroll_from_outside(self, i): #print 'slot-signal ' + str(i) #self.scroll_pdown = True self.update() def initUI(self): self.setMinimumSize(1, 30) self.activateWindow() self.setFocus() #self.installEventFilter(self) """ # build thumbnail dwidth = 100 dheight = 1200 factor = dheight/dwidth import math x = int(math.sqrt(len(self.data)/factor)) cols = x pixThumb = QtGui.QPixmap(x*self.viewMode.fontWidth, factor*x*self.viewMode.fontHeight) qp = QtGui.QPainter() qp.begin(pixThumb) qp.setFont(self.viewMode.font) qp.setPen(self.viewMode.textPen) for i,c in enumerate(self.data): self.viewMode.transformText(qp, (i, c), self.data) qp.drawText((i%cols)*self.viewMode.fontWidth, self.viewMode.fontHeight + (i/cols)*self.viewMode.fontHeight, c) qp.end() self.newqpix = pixThumb.scaled(dwidth, dheight, aspectRatioMode = QtCore.Qt.IgnoreAspectRatio, transformMode = QtCore.Qt.FastTransformation) """ """ def getDisplayedData(self): if self.dataOffset < 0: self.dataOffset = 0 return False chrlist = [unichr(cp437ToUnicode[ord(c)]) for c in self.data[self.dataOffset : self.dataOffset + self.viewMode.COLUMNS*self.viewMode.ROWS]] self.text = "".join(chrlist) return True """ def switchViewMode(self): self.multipleViewModes = self.multipleViewModes[1:] + [self.multipleViewModes[0]] self.viewMode = self.multipleViewModes[0] # notify obervers self.notify(self.viewMode) def _resize(self): self.Banners.resize(self.size().width() - self.offsetWindow_h, self.size().height() - self.offsetWindow_v) # compute space ocupated by banners offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset() offsetBottom = self.offsetWindow_v + self.Banners.getBottomOffset() + self.Banners.getTopOffset() # resize window, substract space ocupated by banners self.viewMode.resize(self.size().width() - offsetLeft, self.size().height() - offsetBottom) # event handlers def resizeEvent(self, e): self._resize() def paintEvent(self, e): qp = QtGui.QPainter() qp.begin(self) qp.setOpacity(1) offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset() offsetBottom = self.offsetWindow_v + self.Banners.getTopOffset() #self.viewMode.draw2(qp, refresh=True) #start = time() qp.drawPixmap(offsetLeft, offsetBottom, self.viewMode.getPixmap()) #print 'Draw ' + str(time() - start) self.Banners.draw(qp, self.offsetWindow_h, self.offsetWindow_v, self.size().height()) # qp.drawPixmap(self.offsetWindow_h, self.size().height() - 50, self.banner.getPixmap()) # qp.drawPixmap(20, 0, self.filebanner.getPixmap()) qp.end() """ def keyPressEvent(self, event): if event.modifiers() & QtCore.Qt.ShiftModifier: if self.viewMode.handleKeyPressEvent(QtCore.Qt.ShiftModifier, event.key()): self.update() def keyReleaseEvent(self, event): if event.key() == QtCore.Qt.Key_Shift: if self.viewMode.handleKeyReleaseEvent(QtCore.Qt.ShiftModifier, event.key()): self.update() """ def eventFilter(self, watched, event): if event.type() == QtCore.QEvent.KeyRelease: key = event.key() modifiers = event.modifiers() if self.viewMode.handleKeyEvent(modifiers, key, event=event): self.update() if event.type() == QtCore.QEvent.KeyPress: #TODO: should we accept only certain keys ? key = event.key() modifiers = event.modifiers() if key == QtCore.Qt.Key_F2: if self.viewMode.isEditable(): if self.viewMode.isInEditMode(): self.viewMode.setEditMode(False) else: self.viewMode.setEditMode(True) self.viewMode.draw(refresh=False) # switch view mode if key == QtCore.Qt.Key_Tab: offs = self.viewMode.getCursorOffsetInPage() base = self.viewMode.getDataModel().getOffset() self.switchViewMode() self._resize() self.viewMode.goTo(base + offs) self.update() import pyperclip if event.modifiers() & QtCore.Qt.ControlModifier: if key == QtCore.Qt.Key_Insert: if self.viewMode.selector.getCurrentSelection(): a, b = self.viewMode.selector.getCurrentSelection() #print a, b hx = '' for s in self.dataModel.getStream(a, b): hx += '{:02x}'.format(s) pyperclip.copy(hx) del pyperclip #print pyperclip.paste() # print 'coppied' if event.modifiers() & QtCore.Qt.ShiftModifier: if key == QtCore.Qt.Key_Insert: import re hx = pyperclip.paste() #print hx L = re.findall(r'.{1,2}', hx, re.DOTALL) array = '' for s in L: array += chr(int(s, 16)) #print 'write ' #print 'write' #print array self.dataModel.write(0, array) self.viewMode.draw(True) del pyperclip #print array if key == QtCore.Qt.Key_F4: self.unp = WUnpack(self, None) self.unp.show() if key == QtCore.Qt.Key_F10: self.dataModel.flush() import os self.w = WHeaders(self, None) self.w.show() if not self.viewMode.isInEditMode(): if key == QtCore.Qt.Key_Slash: self.searchWindow.show() if key == QtCore.Qt.Key_N: self.searchable.next(self.viewMode.getCursorAbsolutePosition() + 1) if key == QtCore.Qt.Key_B: self.searchable.previous(self.viewMode.getCursorAbsolutePosition()) # handle keys to view plugin if self.viewMode.handleKeyEvent(modifiers, key, event=event): event.accept() self.update() return True return False def setTextViewport(self, qp): qp.setViewport(self.offsetWindow_h, self.offsetWindow_v, self.size().width(), self.size().height()) qp.setWindow(0, 0, self.size().width(), self.size().height()) def needsSave(self): return self.dataModel.isDirty() def save(self): return self.dataModel.flush()
class WUnpack(QtGui.QDialog): def __init__(self, parent, plugin): super(WUnpack, self).__init__(parent) self.parent = parent self.plugin = plugin self.oshow = super(WUnpack, self).show root = os.path.dirname(sys.argv[0]) self.ui = PyQt4.uic.loadUi(os.path.join(root, 'unpack.ui'), baseinstance=self) self.ui.setWindowTitle('Decrypt/Encrypt') self.manager = PluginManager(categories_filter={ "UnpackPlugin": DecryptPlugin}) root = os.path.dirname(sys.argv[0]) self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'unpack')]) #self.manager.setPluginPlaces(["plugins"]) # Load plugins self.manager.locatePlugins() self.manager.loadPlugins() self.Plugins = {} Formats = [] for plugin in self.manager.getPluginsOfCategory("UnpackPlugin"): # plugin.plugin_object is an instance of the plugin po = plugin.plugin_object if po.init(self.parent.dataModel, self.parent.viewMode): self.Plugins[plugin.name] = po #self.ui.horizontalLayout.addWidget(po.getUI()) print '[+] ' + plugin.name self.ui.listWidget.addItem(plugin.name) #Formats.append(po) self.ui.listWidget.currentItemChanged.connect(self.item_clicked) self.ui.listWidget.setCurrentRow(0) self.ui.connect(self.ui.proceed, PyQt4.QtCore.SIGNAL("clicked()"), self.handleProceed) self.initUI() def handleProceed(self): item = str(self.ui.listWidget.currentItem().text()) self.Plugins[item].proceed() #self.parent.update() self.parent.viewMode.draw(refresh=True) self.parent.update() def item_clicked(self, current, previous): #item = str(self.ui.listWidget.currentItem().text()) item = str(current.text()) if previous: x = self.ui.horizontalLayout.takeAt(0) while x: x.widget().setParent(None) x = self.ui.horizontalLayout.takeAt(0) prev = str(previous.text()) #print prev #self.ui.horizontalLayout.removeWidget(self.Plugins[prev].getUI()) if item: #print item po = self.Plugins[item] self.ui.horizontalLayout.addWidget(po.getUI()) def show(self): # TODO: remember position? resize plugin windows when parent resize? pwidth = self.parent.parent.size().width() pheight = self.parent.parent.size().height() width = self.ui.size().width()+15 height = self.ui.size().height()+15 self.setGeometry(pwidth - width - 15, pheight - height, width, height) self.setFixedSize(width, height) self.oshow() def initUI(self): self.setSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed) shortcut = QtGui.QShortcut(QtGui.QKeySequence("F4"), self, self.close, self.close) #QtCore.QObject.connect(self.ui.ok, QtCore.SIGNAL('clicked()'), self.onClicked) def onClicked(self): dataModel = self.parent.dataModel self.close()
class FakeSite(object): def __init__(self): self.template_system = self self.invariant = False self.config = { 'DISABLED_PLUGINS': [], 'EXTRA_PLUGINS': [], 'DEFAULT_LANG': 'en', 'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'], 'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}', 'LISTINGS_FOLDERS': {'listings': 'listings'}, } self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS'] self.plugin_manager = PluginManager(categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "CompilerExtension": CompilerExtension, "MarkdownExtension": MarkdownExtension, "RestExtension": RestExtension }) self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8 self.shortcode_registry = {} self.plugin_manager.setPluginInfoExtension('plugin') if sys.version_info[0] == 3: places = [ os.path.join(os.path.dirname(nikola.utils.__file__), 'plugins'), ] else: places = [ os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode('plugins')), ] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension") self.timeline = [ FakePost(title='Fake post', slug='fake-post') ] self.debug = True self.rst_transforms = [] self.post_per_input_file = {} # This is to make plugin initialization happy self.template_system = self self.name = 'mako' def _activate_plugins_of_category(self, category): """Activate all the plugins of a given category and return them.""" # this code duplicated in nikola/nikola.py plugins = [] for plugin_info in self.plugin_manager.getPluginsOfCategory(category): if plugin_info.name in self.config.get('DISABLED_PLUGINS'): self.plugin_manager.removePluginFromCategory(plugin_info, category) else: self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugins.append(plugin_info) return plugins def render_template(self, name, _, context): return('<img src="IMG.jpg">') # this code duplicated in nikola/nikola.py def register_shortcode(self, name, f): """Register function f to handle shortcode "name".""" if name in self.shortcode_registry: nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name) return self.shortcode_registry[name] = f def apply_shortcodes(self, data): """Apply shortcodes from the registry on data.""" return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry)
class Nikola(object): """Class that handles site generation. Takes a site config as argument on creation. """ def __init__(self, **config): """Setup proper environment for running tasks.""" self.global_data = {} self.posts_per_year = defaultdict(list) self.posts_per_tag = defaultdict(list) self.timeline = [] self.pages = [] self._scanned = False # This is the default config # TODO: fill it self.config = { "ARCHIVE_PATH": "", "ARCHIVE_FILENAME": "archive.html", "DEFAULT_LANG": "en", "OUTPUT_FOLDER": "output", "CACHE_FOLDER": "cache", "FILES_FOLDERS": {"files": ""}, "LISTINGS_FOLDER": "listings", "ADD_THIS_BUTTONS": True, "INDEX_DISPLAY_POST_COUNT": 10, "INDEX_TEASERS": False, "RSS_TEASERS": True, "MAX_IMAGE_SIZE": 1280, "USE_FILENAME_AS_TITLE": True, "SLUG_TAG_PATH": False, "INDEXES_TITLE": "", "INDEXES_PAGES": "", "FILTERS": {}, "USE_BUNDLES": True, "TAG_PAGES_ARE_INDEXES": False, "THEME": "default", "COMMENTS_IN_GALLERIES": False, "COMMENTS_IN_STORIES": False, "FILE_METADATA_REGEXP": None, "post_compilers": { "rest": [".txt", ".rst"], "markdown": [".md", ".mdown", ".markdown"], "html": [".html", ".htm"], }, } self.config.update(config) self.config["TRANSLATIONS"] = self.config.get("TRANSLATIONS", {self.config["DEFAULT_LANG"]: ""}) self.THEMES = utils.get_theme_chain(self.config["THEME"]) self.MESSAGES = utils.load_messages(self.THEMES, self.config["TRANSLATIONS"]) self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, } ) self.plugin_manager.setPluginInfoExtension("plugin") self.plugin_manager.setPluginPlaces( [str(os.path.join(os.path.dirname(__file__), "plugins")), str(os.path.join(os.getcwd(), "plugins"))] ) self.plugin_manager.collectPlugins() self.commands = {} # Activate all command plugins for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"): self.plugin_manager.activatePluginByName(pluginInfo.name) pluginInfo.plugin_object.set_site(self) pluginInfo.plugin_object.short_help = pluginInfo.description self.commands[pluginInfo.name] = pluginInfo.plugin_object # Activate all task plugins for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"): self.plugin_manager.activatePluginByName(pluginInfo.name) pluginInfo.plugin_object.set_site(self) for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"): self.plugin_manager.activatePluginByName(pluginInfo.name) pluginInfo.plugin_object.set_site(self) # set global_context for template rendering self.GLOBAL_CONTEXT = self.config.get("GLOBAL_CONTEXT", {}) self.GLOBAL_CONTEXT["messages"] = self.MESSAGES self.GLOBAL_CONTEXT["_link"] = self.link self.GLOBAL_CONTEXT["rel_link"] = self.rel_link self.GLOBAL_CONTEXT["abs_link"] = self.abs_link self.GLOBAL_CONTEXT["exists"] = self.file_exists self.GLOBAL_CONTEXT["add_this_buttons"] = self.config["ADD_THIS_BUTTONS"] self.GLOBAL_CONTEXT["index_display_post_count"] = self.config["INDEX_DISPLAY_POST_COUNT"] self.GLOBAL_CONTEXT["use_bundles"] = self.config["USE_BUNDLES"] if "date_format" not in self.GLOBAL_CONTEXT: self.GLOBAL_CONTEXT["date_format"] = "%Y-%m-%d %H:%M" # check if custom css exist and is not empty for files_path in list(self.config["FILES_FOLDERS"].keys()): custom_css_path = os.path.join(files_path, "assets/css/custom.css") if self.file_exists(custom_css_path, not_empty=True): self.GLOBAL_CONTEXT["has_custom_css"] = True break else: self.GLOBAL_CONTEXT["has_custom_css"] = False # Load template plugin template_sys_name = utils.get_template_engine(self.THEMES) pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem") if pi is None: sys.stderr.write("Error loading %s template system plugin\n" % template_sys_name) sys.exit(1) self.template_system = pi.plugin_object lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES] self.template_system.set_directories(lookup_dirs, self.config["CACHE_FOLDER"]) # Load compiler plugins self.compilers = {} self.inverse_compilers = {} for pluginInfo in self.plugin_manager.getPluginsOfCategory("PageCompiler"): self.compilers[pluginInfo.name] = pluginInfo.plugin_object.compile_html def get_compiler(self, source_name): """Get the correct compiler for a post from `conf.post_compilers` To make things easier for users, the mapping in conf.py is compiler->[extensions], although this is less convenient for us. The majority of this function is reversing that dictionary and error checking. """ ext = os.path.splitext(source_name)[1] try: compile_html = self.inverse_compilers[ext] except KeyError: # Find the correct compiler for this files extension langs = [lang for lang, exts in list(self.config["post_compilers"].items()) if ext in exts] if len(langs) != 1: if len(set(langs)) > 1: exit( "Your file extension->compiler definition is" "ambiguous.\nPlease remove one of the file extensions" "from 'post_compilers' in conf.py\n(The error is in" "one of %s)" % ", ".join(langs) ) elif len(langs) > 1: langs = langs[:1] else: exit("post_compilers in conf.py does not tell me how to " "handle '%s' extensions." % ext) lang = langs[0] compile_html = self.compilers[lang] self.inverse_compilers[ext] = compile_html return compile_html def render_template(self, template_name, output_name, context): local_context = {} local_context["template_name"] = template_name local_context.update(self.config["GLOBAL_CONTEXT"]) local_context.update(context) data = self.template_system.render_template(template_name, None, local_context) assert isinstance(output_name, bytes) assert output_name.startswith(self.config["OUTPUT_FOLDER"].encode("utf8")) url_part = output_name.decode("utf8")[len(self.config["OUTPUT_FOLDER"]) + 1 :] # This is to support windows paths url_part = "/".join(url_part.split(os.sep)) src = urljoin(self.config["BLOG_URL"], url_part) parsed_src = urlsplit(src) src_elems = parsed_src.path.split("/")[1:] def replacer(dst): # Refuse to replace links that are full URLs. dst_url = urlparse(dst) if dst_url.netloc: if dst_url.scheme == "link": # Magic link dst = self.link(dst_url.netloc, dst_url.path.lstrip("/"), context["lang"]) else: return dst # Normalize dst = urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_dst = urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute dst_elems = parsed_dst.path.split("/")[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break # Now i is the longest common prefix result = "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:]) if not result: result = "." # Don't forget the fragment (anchor) part of the link if parsed_dst.fragment: result += "#" + parsed_dst.fragment assert result, (src, dst, i, src_elems, dst_elems) return result try: os.makedirs(os.path.dirname(output_name)) except: pass doc = lxml.html.document_fromstring(data) doc.rewrite_links(replacer) data = b"<!DOCTYPE html>" + lxml.html.tostring(doc, encoding="utf8") with open(output_name, "wb+") as post_file: post_file.write(data) def path(self, kind, name, lang, is_link=False): """Build the path to a certain kind of page. kind is one of: * tag_index (name is ignored) * tag (and name is the tag name) * tag_rss (name is the tag name) * archive (and name is the year, or None for the main archive index) * index (name is the number in index-number) * rss (name is ignored) * gallery (name is the gallery name) * listing (name is the source code file name) The returned value is always a path relative to output, like "categories/whatever.html" If is_link is True, the path is absolute and uses "/" as separator (ex: "/archive/index.html"). If is_link is False, the path is relative to output and uses the platform's separator. (ex: "archive\\index.html") """ path = [] if kind == "tag_index": path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], "index.html"] if _f] elif kind == "tag": if self.config["SLUG_TAG_PATH"]: name = utils.slugify(name) path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".html"] if _f] elif kind == "tag_rss": if self.config["SLUG_TAG_PATH"]: name = utils.slugify(name) path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".xml"] if _f] elif kind == "index": if name > 0: path = [ _f for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index-%s.html" % name] if _f ] else: path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index.html"] if _f] elif kind == "rss": path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["RSS_PATH"], "rss.xml"] if _f] elif kind == "archive": if name: path = [ _f for _f in [self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], name, "index.html"] if _f ] else: path = [ _f for _f in [ self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], self.config["ARCHIVE_FILENAME"], ] if _f ] elif kind == "gallery": path = [_f for _f in [self.config["GALLERY_PATH"], name, "index.html"] if _f] elif kind == "listing": path = [_f for _f in [self.config["LISTINGS_FOLDER"], name + ".html"] if _f] if is_link: return "/" + ("/".join(path)) else: return os.path.join(*path) def link(self, *args): return self.path(*args, is_link=True) def abs_link(self, dst): # Normalize dst = urljoin(self.config["BLOG_URL"], dst) return urlparse(dst).path def rel_link(self, src, dst): # Normalize src = urljoin(self.config["BLOG_URL"], src) dst = urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_src = urlsplit(src) parsed_dst = urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute src_elems = parsed_src.path.split("/")[1:] dst_elems = parsed_dst.path.split("/")[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break else: i += 1 # Now i is the longest common prefix return "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:]) def file_exists(self, path, not_empty=False): """Returns True if the file exists. If not_empty is True, it also has to be not empty.""" exists = os.path.exists(path) if exists and not_empty: exists = os.stat(path).st_size > 0 return exists def gen_tasks(self): task_dep = [] for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"): for task in pluginInfo.plugin_object.gen_tasks(): yield task if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"): for task in pluginInfo.plugin_object.gen_tasks(): yield task if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) yield {"name": b"all", "actions": None, "clean": True, "task_dep": task_dep} def scan_posts(self): """Scan all the posts.""" if not self._scanned: print("Scanning posts", end="") targets = set([]) for wildcard, destination, template_name, use_in_feeds in self.config["post_pages"]: print(".", end="") base_len = len(destination.split(os.sep)) dirname = os.path.dirname(wildcard) for dirpath, _, _ in os.walk(dirname): dir_glob = os.path.join(dirpath, os.path.basename(wildcard)) dest_dir = os.path.join(*([destination] + dirpath.split(os.sep)[base_len:])) for base_path in glob.glob(dir_glob): post = Post( base_path, self.config["CACHE_FOLDER"], dest_dir, use_in_feeds, self.config["TRANSLATIONS"], self.config["DEFAULT_LANG"], self.config["BLOG_URL"], self.MESSAGES, template_name, self.config["FILE_METADATA_REGEXP"], ) for lang, langpath in list(self.config["TRANSLATIONS"].items()): dest = (destination, langpath, dir_glob, post.pagenames[lang]) if dest in targets: raise Exception( "Duplicated output path %r in post %r" % (post.pagenames[lang], base_path) ) targets.add(dest) self.global_data[post.post_name] = post if post.use_in_feeds: self.posts_per_year[str(post.date.year)].append(post.post_name) for tag in post.tags: self.posts_per_tag[tag].append(post.post_name) else: self.pages.append(post) for name, post in list(self.global_data.items()): self.timeline.append(post) self.timeline.sort(key=lambda p: p.date) self.timeline.reverse() post_timeline = [p for p in self.timeline if p.use_in_feeds] for i, p in enumerate(post_timeline[1:]): p.next_post = post_timeline[i] for i, p in enumerate(post_timeline[:-1]): p.prev_post = post_timeline[i + 1] self._scanned = True print("done!") def generic_page_renderer(self, lang, post, filters): """Render post fragments to final HTML pages.""" context = {} deps = post.deps(lang) + self.template_system.template_deps(post.template_name) context["post"] = post context["lang"] = lang context["title"] = post.title(lang) context["description"] = post.description(lang) context["permalink"] = post.permalink(lang) context["page_list"] = self.pages if post.use_in_feeds: context["enable_comments"] = True else: context["enable_comments"] = self.config["COMMENTS_IN_STORIES"] output_name = os.path.join(self.config["OUTPUT_FOLDER"], post.destination_path(lang)).encode("utf8") deps_dict = copy(context) deps_dict.pop("post") if post.prev_post: deps_dict["PREV_LINK"] = [post.prev_post.permalink(lang)] if post.next_post: deps_dict["NEXT_LINK"] = [post.next_post.permalink(lang)] deps_dict["OUTPUT_FOLDER"] = self.config["OUTPUT_FOLDER"] deps_dict["TRANSLATIONS"] = self.config["TRANSLATIONS"] deps_dict["global"] = self.config["GLOBAL_CONTEXT"] deps_dict["comments"] = context["enable_comments"] task = { "name": output_name, "file_dep": deps, "targets": [output_name], "actions": [(self.render_template, [post.template_name, output_name, context])], "clean": True, "uptodate": [config_changed(deps_dict)], } yield utils.apply_filters(task, filters) def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context): """Renders pages with lists of posts.""" # This is a name on disk, has to be bytes assert isinstance(output_name, bytes) deps = self.template_system.template_deps(template_name) for post in posts: deps += post.deps(lang) context = {} context["posts"] = posts context["title"] = self.config["BLOG_TITLE"] context["description"] = self.config["BLOG_DESCRIPTION"] context["lang"] = lang context["prevlink"] = None context["nextlink"] = None context.update(extra_context) deps_context = copy(context) deps_context["posts"] = [(p.titles[lang], p.permalink(lang)) for p in posts] deps_context["global"] = self.config["GLOBAL_CONTEXT"] task = { "name": output_name, "targets": [output_name], "file_dep": deps, "actions": [(self.render_template, [template_name, output_name, context])], "clean": True, "uptodate": [config_changed(deps_context)], } return utils.apply_filters(task, filters)
class ContentModelListener(ConnectionListener): ''' classdocs ''' def __init__(self, content_models, host='localhost', port=61613, user='', passcode='', fedora_url=''): ''' Constructor ''' self.conn = Connection([(host, port)], user, passcode) self.conn.set_listener('', self) self.conn.start() logging.info('Connecting to STOMP server %(host)s on port %(port)s.' % {'host': host, 'port': port}) self.transaction_id = None logging.info("Connecting to Fedora server at %(url)s" % {'url': fedora_url}) self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode) self.client = FedoraClient(self.fc) self.fedora_url = fedora_url self.username = user self.password = passcode # Create plugin manager self.manager = PluginManager(categories_filter = {"FedoraMicroService": FedoraMicroService}) plugin_path = os.path.dirname(__file__) self.manager.setPluginPlaces([plugin_path + "/plugins"]) logging.debug("Plugin path: " + plugin_path + "/plugins") # Load plugins. self.manager.locatePlugins() self.manager.loadPlugins() self.contentModels = {} for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"): # plugin.plugin_object is an instance of the plubin logging.info("Loading plugin: %(name)s for content model %(cmodel)s." % {'name': plugin.plugin_object.name, 'cmodel': plugin.plugin_object.content_model}) plugin.plugin_object.config = config if type(plugin.plugin_object.content_model) == types.StringType: content_models = [plugin.plugin_object.content_model] else: content_models = plugin.plugin_object.content_model for content_model in content_models: if content_model in self.contentModels: self.contentModels[content_model].append(plugin.plugin_object) else: self.contentModels[content_model] = [plugin.plugin_object] def __print_async(self, frame_type, headers, body): """ Utility function for printing messages. """ #logging.debug("\r \r", end='') logging.debug(frame_type) for header_key in headers.keys(): logging.debug('%s: %s' % (header_key, headers[header_key])) logging.debug(body) def on_connecting(self, host_and_port): """ \see ConnectionListener::on_connecting """ self.conn.connect(wait=True) def on_disconnected(self): """ \see ConnectionListener::on_disconnected """ logging.error("lost connection reconnect in %d sec..." % reconnect_wait) signal.alarm(reconnect_wait) def on_message(self, headers, body): """ \see ConnectionListener::on_message """ try: global TOPIC_PREFIX self.__print_async('MESSAGE', headers, body) pid = headers['pid'] dsid = headers['dsid'] obj = self.client.getObject(pid) content_model = headers['destination'][len(TOPIC_PREFIX):] if content_model in self.contentModels: logging.info('Running rules for %(pid)s from %(cmodel)s.' % {'pid': obj.pid, 'cmodel': content_model} ) for plugin in self.contentModels[content_model]: plugin.runRules(obj, dsid, body) except FedoraConnectionException: logging.warning('Object %s was not found.' % (pid)) except: logging.error("an exception occurred: " + str(sys.exc_info()[0])) def on_error(self, headers, body): """ \see ConnectionListener::on_error """ self.__print_async("ERROR", headers, body) def on_connected(self, headers, body): """ \see ConnectionListener::on_connected """ self.__print_async("CONNECTED", headers, body) def ack(self, args): """ Required Parameters: message-id - the id of the message being acknowledged Description: Acknowledge consumption of a message from a subscription using client acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client received from that destination will not be considered to have been consumed (by the server) until the message has been acknowledged. """ if not self.transaction_id: self.conn.ack(headers = { 'message-id' : args[1]}) else: self.conn.ack(headers = { 'message-id' : args[1]}, transaction=self.transaction_id) def abort(self, args): """ Description: Roll back a transaction in progress. """ if self.transaction_id: self.conn.abort(transaction=self.transaction_id) self.transaction_id = None def begin(self, args): """ Description Start a transaction. Transactions in this case apply to sending and acknowledging any messages sent or acknowledged during a transaction will be handled atomically based on teh transaction. """ if not self.transaction_id: self.transaction_id = self.conn.begin() def commit(self, args): """ Description: Commit a transaction in progress. """ if self.transaction_id: self.conn.commit(transaction=self.transaction_id) self.transaction_id = None def disconnect(self, args): """ Description: Gracefully disconnect from the server. """ try: self.conn.disconnect() except NotConnectedException: pass def send(self, destination, correlation_id, message): """ Required Parametes: destination - where to send the message message - the content to send Description: Sends a message to a destination in the message system. """ self.conn.send(destination=destination, message=message, headers={'correlation-id': correlation_id}) def subscribe(self, destination, ack='auto'): """ Required Parameters: destination - the name to subscribe to Optional Parameters: ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client) Description Register to listen to a given destination. Like send, the subscribe command requires a destination header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto. """ self.conn.subscribe(destination=destination, ack=ack) def unsubscribe(self, destination): """ Required Parameters: destination - the name to unsubscribe from Description: Remove an existing subscription - so that the client no longer receives messages from that destination. """ self.conn.unsubscribe(destination) def connect(self): self.conn.start() self.fc = fcrepo.connection.Connection(self.fedora_url, username = self.username, password = self.password) self.client = FedoraClient(self.fc)