def testTwoStepsLoad(self): """ Test loading the plugins in two steps in order to collect more deltailed informations. """ spm = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # trigger the first step to look up for plugins spm.locatePlugins() # make full use of the "feedback" the loadPlugins can give # - set-up the callback function that will be called *before* # loading each plugin callback_infos = [] def preload_cbk(plugin_info): callback_infos.append(plugin_info) # - gather infos about the processed plugins (loaded or not) loadedPlugins = spm.loadPlugins(callback=preload_cbk) self.assertEqual(len(loadedPlugins),1) self.assertEqual(len(callback_infos),1) self.assertEqual(loadedPlugins[0].error,None) self.assertEqual(loadedPlugins[0],callback_infos[0]) # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1) plugin_info = spm.getPluginsOfCategory(sole_category)[0] # try to remove it and check that is worked spm.removePluginFromCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),0) # now re-add this plugin the to same category spm.appendPluginToCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1)
def initialize_plugins(self):
self.__log.info("Collecting and loading plugins")
try:
plugin_manager = PluginManager()
# TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT)
plugin_manager.setCategoriesFilter({
CARTRIDGE_AGENT_PLUGIN:
ICartridgeAgentPlugin,
ARTIFACT_MGT_PLUGIN:
IArtifactManagementPlugin
})
plugin_manager.setPluginPlaces(
[self.__config.read_property(constants.PLUGINS_DIR)])
plugin_manager.collectPlugins()
# activate cartridge agent plugins
plugins = plugin_manager.getPluginsOfCategory(
CARTRIDGE_AGENT_PLUGIN)
grouped_plugins = {}
for plugin_info in plugins:
self.__log.debug("Found plugin [%s] at [%s]" %
(plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated plugin [%s]" % plugin_info.name)
mapped_events = plugin_info.description.split(",")
for mapped_event in mapped_events:
if mapped_event.strip() != "":
if grouped_plugins.get(mapped_event) is None:
grouped_plugins[mapped_event] = []
grouped_plugins[mapped_event].append(plugin_info)
# activate artifact management plugins
artifact_mgt_plugins = plugin_manager.getPluginsOfCategory(
ARTIFACT_MGT_PLUGIN)
for plugin_info in artifact_mgt_plugins:
self.__log.debug(
"Found artifact management plugin [%s] at [%s]" %
(plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated artifact management plugin [%s]" %
plugin_info.name)
return plugin_manager, grouped_plugins, artifact_mgt_plugins
except ParameterNotFoundException as e:
self.__log.exception(
"Could not load plugins. Plugins directory not set: %s" % e)
return None, None, None
except Exception as e:
self.__log.exception("Error while loading plugin: %s" % e)
return None, None, None
class MUCBot(sleekxmpp.ClientXMPP):
def __init__(self, jid, password, room, nick):
sleekxmpp.ClientXMPP.__init__(self, jid, password)
self.room = room
self.nick = nick
# Load plugins
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["plugins_enabled"])
self.plugin_manager.setCategoriesFilter({
"Message" : MessagePlugin,
"Presence": PresencePlugin
})
self.plugin_manager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in self.plugin_manager.getAllPlugins():
self.plugin_manager.activatePluginByName(pluginInfo.name)
self.add_event_handler("session_start", self.start)
self.add_event_handler("message", self.message)
self.add_event_handler("groupchat_message", self.muc_message)
self.add_event_handler("muc::%s::got_online" % self.room, self.muc_online)
self.add_event_handler("muc::%s::got_offline" % self.room, self.muc_offline)
def start(self, event):
self.get_roster()
self.send_presence()
if self.room and len(self.room) > 0:
self.plugin['xep_0045'].joinMUC(self.room,
self.nick,
wait=True)
def message(self, msg):
if msg['type'] in ('chat', 'normal'):
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"):
pluginInfo.plugin_object.message_received(msg)
def muc_message(self, msg):
if msg['mucnick'] != self.nick:
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"):
pluginInfo.plugin_object.message_received(msg, nick=self.nick)
def muc_online(self, presence):
if presence['muc']['nick'] != self.nick:
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"):
pluginInfo.plugin_object.got_online(self, presence)
def muc_offline(self, presence):
if presence['muc']['nick'] != self.nick:
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"):
pluginInfo.plugin_object.got_offline(self, presence)
class ExtInfoTest(unittest.TestCase):
def setUp(self):
# create the plugin manager - use the base plugin manager to
# remove any unecessary dependencies
self.simplePluginManager = PluginManager(directories_list=[
os.path.join(
os.path.dirname(os.path.abspath(__file__)),"Plugins")
],
plugin_info_ext="mm-plugin",
)
#Set mode to test info class
self.simplePluginManager.setPluginInfoClass(ExtensionInfo)
# load the plugins that may be found
self.simplePluginManager.collectPlugins()
# Will be used later
self.plugin_info = None
def loading_check(self):
"""
Test the plugins load.
"""
if self.plugin_info is None:
# check nb of categories
self.assertEqual(len(self.simplePluginManager.getCategories()),1)
sole_category = self.simplePluginManager.getCategories()[0]
# check the number of plugins
self.assertEqual(len(self.simplePluginManager.getPluginsOfCategory(sole_category)),2)
self.plugin_info = [ None , None ]
self.plugin_info[0] = self.simplePluginManager.getPluginsOfCategory(sole_category)[0]
self.plugin_info[1] = self.simplePluginManager.getPluginsOfCategory(sole_category)[1]
# test that the name of the plugin has been correctly defined
self.assertTrue("FirstPlugin" in [x.name for x in self.plugin_info])
self.assertEqual(sole_category,self.plugin_info[0].category)
else:
self.assert_(True)
def testBasic(self):
self.loading_check()
def testKnownHash(self):
"""
Test hash can be fetched thru ExtensionInfo
"""
self.loading_check()
hash1=ExtensionSecureID.fromPathName(self.plugin_info[0].path+".py")
self.assertEquals(hash1,self.plugin_info[0].getSecureID())
self.assertNotEquals(self.plugin_info[0].getSecureID(),self.plugin_info[1].getSecureID())
def main():
# Load the plugins from the plugin directory.
manager = PluginManager()
manager.setPluginPlaces(["plugins\\plugins1", "plugins\\plugins2"])
manager.setCategoriesFilter({
"PluginOne": category.PluginOne,
"PluginTwo": category.PluginTwo,
})
print { "PluginOne": category.PluginOne, "PluginTwo": category.PluginTwo }
manager.collectPlugins()
print manager.getPluginsOfCategory("PluginTwo")
for plugin in manager.getPluginsOfCategory("PluginTwo"):
plugin.plugin_object.print_name()
class DownloadManager(object):
"""DownloadManager will be used to download any kind of resource from a \
source system. DownloadManager will locate all plugins of\
type :class:`IDownloader` and will provide it back to the\
requesting object
"""
__single_download_manager = None
def __new__(cls, *args, **kwargs):
"""Singleton class constructor
"""
if cls != type(cls.__single_download_manager):
cls.__single_download_manager = object.__new__(cls, *args, **kwargs)
return cls.__single_download_manager
def __init__(self):
"""Constructor for DownloadManager
"""
self.plugins_path = constants.DOWNLOADER_PLUGINS_PATH
self.save_downloads_to = constants.SAVE_DOWNLOADS_TO_PATH
self.plugin_manager = PluginManager(categories_filter={constants.DOWNLOAD_PLUGIN_FILTER: iplugins.IDownloader})
self.plugin_manager.setPluginPlaces([self.plugins_path])
self.plugin_manager.locatePlugins()
self.plugin_manager.loadPlugins()
def downloaders_list(self):
"""Provides a list of all downloader plugins available
"""
return self.plugin_manager.getPluginsOfCategory(constants.DOWNLOAD_PLUGIN_FILTER)
def get_downloader(self, downloader_name=None):
"""Returns a downloader plugin instance by finding through name
"""
return self.plugin_manager.getPluginByName(downloader_name, constants.DOWNLOAD_PLUGIN_FILTER)
class MarinheiroCmdApp(cmd2.Cmd):
def __init__(self):
super().__init__()
self.manager = PluginManager(
plugin_info_ext="marinheiro-test",
directories_list=[os.path.join(os.path.dirname(__file__), "tests")],
categories_filter={
"Basic" : marinheiro.tests.BasicTest,
"Web" : marinheiro.tests.WebTest,
})
self.manager.collectPlugins()
@cmd2.options([cmd2.make_option("-c", "--category", action="store",
help="Specify test category.")])
def do_list(self, arg, opts=None):
if opts.category:
categories = self.manager.getPluginsOfCategory(opts.category)
else:
categories = self.manager.getAllPlugins()
for plugin in categories:
print("{}: {p.name}".format(self.colorize("Name", "bold"), p=plugin))
print("{}: {p.description}".format(self.colorize("Description", "bold"), p=plugin))
print()
@cmd2.options([cmd2.make_option("-c", "--category", action="store",
help="Specify test category.")])
def do_run(self, arg, opts=None):
if opts.category:
categories = self.manager.getPluginsOfCategory(opts.category)
else:
categories = self.manager.getAllPlugins()
for plugin in categories:
print("{}: {p.name}".format(self.colorize("Name", "bold"), p=plugin))
print("{}: {p.description}".format(self.colorize("Description", "bold"), p=plugin))
try:
output = plugin.plugin_object.run()
except marinheiro.exceptions.FailedTest as err:
status = self.colorize("FAILED", "red")
output = err.msg
else:
status = self.colorize("PASSED", "green")
finally:
print("{}: {}".format(self.colorize("Status", "bold"), status))
print("{}: {}".format(self.colorize("Output", "bold"), output))
print()
def test_json_reporting_plugin_found():
"""Test that the plugin manager finds the JSON plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__),
'plugins')])
manager.setCategoriesFilter({
"Reporting": ReportingPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "upload_risk_assessment"
assert any(plugin_info.plugin_object.get_name() == 'upload_risk_assessment' for
plugin_info in manager.getPluginsOfCategory("Reporting"))
# While we're at it, verify that a plugin is named JSON Risk Assessment Reporting Plugin
assert any(plugin_info.name == 'JSON Risk Assessment Reporting Plugin' for
plugin_info in manager.getPluginsOfCategory("Reporting"))
def test_maven_discovery_plugin_found():
"""Test that the plugin manager finds the Maven discovery plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")])
manager.setCategoriesFilter({
"Discovery": DiscoveryPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "java"
assert any(plugin_info.plugin_object.get_name() == "maven"
for plugin_info in manager.getPluginsOfCategory("Discovery"))
# While we're at it, verify that a plugin is named Maven Discovery Plugin
assert any(plugin_info.name == "Maven Discovery Plugin"
for plugin_info in manager.getPluginsOfCategory("Discovery"))
def test_clang_tidy_tool_plugin_found():
"""Test that the plugin manager can find the clang-tidy plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__),
'plugins')])
manager.setCategoriesFilter({
"Tool": ToolPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "clang_tidy"
assert any(plugin_info.plugin_object.get_name() == 'clang-tidy' for
plugin_info in manager.getPluginsOfCategory("Tool"))
# While we're at it, verify that a plugin is named ClangTidy Tool Plugin
assert any(plugin_info.name == 'clang-tidy Tool Plugin' for
plugin_info in manager.getPluginsOfCategory("Tool"))
def test_cccc_tool_plugin_found():
"""Test that the CCCC tool plugin is detected by the plugin system."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")])
manager.setCategoriesFilter({
"Tool": ToolPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "cccc"
assert any(plugin_info.plugin_object.get_name() == "cccc"
for plugin_info in manager.getPluginsOfCategory("Tool"))
# While we're at it, verify that a plugin is named CCCC Tool Plugin
assert any(plugin_info.name == "CCCC Tool Plugin"
for plugin_info in manager.getPluginsOfCategory("Tool"))
def test_console_reporting_plugin_found():
"""Test that the plugin manager finds the console reporting plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), 'plugins')])
manager.setCategoriesFilter({
"Reporting": ReportingPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "c"
assert any(plugin_info.plugin_object.get_name() == 'print_to_console'
for plugin_info in manager.getPluginsOfCategory("Reporting"))
# While we're at it, verify that a plugin is named C Discovery Plugin
assert any(plugin_info.name == 'Print To Console Reporting Plugin'
for plugin_info in manager.getPluginsOfCategory("Reporting"))
def test_yaml_discovery_plugin_found():
"""Test that the YAML discovery plugin is detected by the plugin system."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces([os.path.join(os.path.dirname(statick_tool.__file__),
'plugins')])
manager.setCategoriesFilter({
"Discovery": DiscoveryPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "yaml"
assert any(plugin_info.plugin_object.get_name() == 'yaml' for
plugin_info in manager.getPluginsOfCategory("Discovery"))
# While we're at it, verify that a plugin is named YAML Discovery Plugin
assert any(plugin_info.name == 'YAML Discovery Plugin' for
plugin_info in manager.getPluginsOfCategory("Discovery"))
class Plugins():
'''
Plugins
'''
def __init__(self):
'''
Constructor
'''
super(Plugins, self).__init__()
self.write_tab_dock_plugin_dict = {}
self.write_panel_dock_plugin_dict = {}
# Build the manager
self._plugin_manager = PluginManager()
# List all sub-directories of "plugins"
plugin_path = os.path.sep.join([os.getcwd(), "plugins"])
plugin_places = [plugin_path]
for dirname, dirnames, filenames in os.walk(plugin_path):
# print path to all subdirectories first.
for subdirname in dirnames:
plugin_places.append(os.path.join(dirname, subdirname))
# Tell it the default place(s) where to find plugins
self._plugin_manager.setPluginPlaces(plugin_places)
sys.path.append(plugin_path)
# Define the various categories corresponding to the different
# kinds of plugins you have defined
self._plugin_manager.setCategoriesFilter({
"CoreWriteTabDockPlugin":
CoreWriteTabDockPlugin,
"CoreWritePanelDockPlugin":
CoreWritePanelDockPlugin
})
self._plugin_manager.collectPlugins()
self.load_plugins(
["CoreWriteTabDockPlugin", "CoreWritePanelDockPlugin"])
def load_plugins(self, categories):
'''
function:: load_plugins(categories)
:param categories: list
'''
for category in categories:
for pluginInfo in self._plugin_manager.getPluginsOfCategory(
category):
setattr(self,
pluginInfo.plugin_object.core_class().__name__,
pluginInfo.plugin_object.core_class())
if category is "CoreWriteTabDockPlugin":
self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.core_class().dock_name] \
= pluginInfo.plugin_object.core_class()
if category is "CoreWritePanelDockPlugin":
self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.core_class().dock_name] \
= pluginInfo.plugin_object.core_class()
def initialize_plugins(self):
self.__log.info("Collecting and loading plugins")
try:
plugin_manager = PluginManager()
# TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT)
plugin_manager.setCategoriesFilter({
CARTRIDGE_AGENT_PLUGIN: ICartridgeAgentPlugin,
ARTIFACT_MGT_PLUGIN: IArtifactManagementPlugin
})
plugin_manager.setPluginPlaces([self.__config.read_property(constants.PLUGINS_DIR)])
plugin_manager.collectPlugins()
# activate cartridge agent plugins
plugins = plugin_manager.getPluginsOfCategory(CARTRIDGE_AGENT_PLUGIN)
grouped_plugins = {}
for plugin_info in plugins:
self.__log.debug("Found plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated plugin [%s]" % plugin_info.name)
mapped_events = plugin_info.description.split(",")
for mapped_event in mapped_events:
if mapped_event.strip() != "":
if grouped_plugins.get(mapped_event) is None:
grouped_plugins[mapped_event] = []
grouped_plugins[mapped_event].append(plugin_info)
# activate artifact management plugins
artifact_mgt_plugins = plugin_manager.getPluginsOfCategory(ARTIFACT_MGT_PLUGIN)
for plugin_info in artifact_mgt_plugins:
self.__log.debug("Found artifact management plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated artifact management plugin [%s]" % plugin_info.name)
return plugin_manager, grouped_plugins, artifact_mgt_plugins
except ParameterNotFoundException as e:
self.__log.exception("Could not load plugins. Plugins directory not set: %s" % e)
return None, None, None
except Exception as e:
self.__log.exception("Error while loading plugin: %s" % e)
return None, None, None
class Plugins():
'''
Plugins
'''
def __init__(self):
'''
Constructor
'''
super(Plugins, self).__init__()
self.write_tab_dock_plugin_dict = {}
self.write_panel_dock_plugin_dict = {}
# Build the manager
self._plugin_manager = PluginManager()
# List all sub-directories of "plugins"
plugin_path = os.path.sep.join([os.getcwd(), "plugins"])
plugin_places = [plugin_path]
for dirname, dirnames, filenames in os.walk(plugin_path):
# print path to all subdirectories first.
for subdirname in dirnames:
plugin_places.append(os.path.join(dirname, subdirname))
# Tell it the default place(s) where to find plugins
self._plugin_manager.setPluginPlaces(plugin_places)
sys.path.append(plugin_path)
# Define the various categories corresponding to the different
# kinds of plugins you have defined
self._plugin_manager.setCategoriesFilter({
"GuiWriteTabDockPlugin": GuiWriteTabDockPlugin,
"GuiWritePanelDockPlugin": GuiWritePanelDockPlugin
})
self._plugin_manager.collectPlugins()
self.load_plugins(
["GuiWriteTabDockPlugin", "GuiWritePanelDockPlugin"])
def load_plugins(self, categories):
'''
function:: load_plugins(categories)
:param categories: list
'''
for category in categories:
for pluginInfo in self._plugin_manager.getPluginsOfCategory(category):
setattr(self, pluginInfo.plugin_object.gui_class(
).__name__, pluginInfo.plugin_object.gui_class())
if category is "GuiWriteTabDockPlugin":
self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \
= pluginInfo.plugin_object.gui_class()
if category is "GuiWritePanelDockPlugin":
self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \
= pluginInfo.plugin_object.gui_class()
def test_write_jenkins_warnings_ng_reporting_plugin_found():
"""Test that the plugin manager finds the file writing plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")])
manager.setCategoriesFilter({
"Reporting": ReportingPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "c"
assert any(
plugin_info.plugin_object.get_name() == "write_jenkins_warnings_ng"
for plugin_info in manager.getPluginsOfCategory("Reporting"))
# While we're at it, verify that a plugin is named C Discovery Plugin
assert any(plugin_info.name == "Write Jenkins Warnings NG Reporting Plugin"
for plugin_info in manager.getPluginsOfCategory("Reporting"))
def testTwoStepsLoad(self):
"""
Test loading the plugins in two steps in order to collect more
deltailed informations.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# trigger the first step to look up for plugins
spm.locatePlugins()
# make full use of the "feedback" the loadPlugins can give
# - set-up the callback function that will be called *before*
# loading each plugin
callback_infos = []
def preload_cbk(plugin_info):
callback_infos.append(plugin_info)
callback_after_infos = []
def postload_cbk(plugin_info):
callback_after_infos.append(plugin_info)
# - gather infos about the processed plugins (loaded or not)
loadedPlugins = spm.loadPlugins(callback=preload_cbk,
callback_after=postload_cbk)
self.assertEqual(len(loadedPlugins), 1)
self.assertEqual(len(callback_infos), 1)
self.assertEqual(loadedPlugins[0].error, None)
self.assertEqual(loadedPlugins[0], callback_infos[0])
self.assertEqual(len(callback_after_infos), 1)
self.assertEqual(loadedPlugins[0], callback_infos[0])
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
plugin_info = spm.getPluginsOfCategory(sole_category)[0]
# try to remove it and check that is worked
spm.removePluginFromCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
# now re-add this plugin the to same category
spm.appendPluginToCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
class PluginHandler:
def __init__(self):
self._plugin_manager = PluginManager()
self._category_active = {"Modifier": False, "Analyzer": False, "Comparator": False}
self._plugin_from_category = {"Modifier": [], "Analyzer": [], "Comparator": []}
self._modifier_plugins = []
self._analyzer_plugins = []
self._collect_all_plugins()
def _collect_all_plugins(self):
self._plugin_manager.setPluginPlaces(
["hugin/analyze/modifier", "hugin/analyze/analyzer", "hugin/analyze/comparator"]
)
# setting filter categories for pluginmanager
self._plugin_manager.setCategoriesFilter(
{
# movie metadata provider
"Modifier": IModifier,
# movie metadata provider
"Comparator": IComparator,
# sub metadata provider
"Analyzer": IAnalyzer,
}
)
self._plugin_manager.collectPlugins()
def activate_plugins_by_category(self, category):
self._toggle_activate_plugins_by_category(category)
def deactivate_plugins_by_category(self, category):
self._toggle_activate_plugins_by_category(category)
def _toggle_activate_plugins_by_category(self, category):
plugins = self._plugin_manager.getPluginsOfCategory(category)
is_active = self._category_active[category]
for pluginInfo in plugins:
if is_active:
self._plugin_manager.deactivatePluginByName(name=pluginInfo.name, category=category)
self._plugin_from_category[category].remove(pluginInfo)
else:
self._plugin_manager.activatePluginByName(name=pluginInfo.name, category=category)
self._plugin_from_category[category].append(pluginInfo)
self._category_active[category] = not is_active
def get_plugins_from_category(self, category):
plugins = []
for plugin in self._plugin_from_category[category]:
plugin.plugin_object.name = plugin.name
plugin.plugin_object.description = plugin.description
plugins.append(plugin.plugin_object)
return plugins
def is_activated(self, category):
""" True if category is activated. """
return self._category_active[category]
def test_isort_tool_plugin_found():
"""Test that the plugin manager can find the Isort plugin."""
if sys.version_info.major == 3 and sys.version_info.minor < 6:
pytest.skip("isort is only available for Python 3.6+, unable to test")
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")])
manager.setCategoriesFilter({
"Tool": ToolPlugin,
})
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "isort"
assert any(plugin_info.plugin_object.get_name() == "isort"
for plugin_info in manager.getPluginsOfCategory("Tool"))
# While we're at it, verify that a plugin is named Isort Tool Plugin
assert any(plugin_info.name == "Isort Tool Plugin"
for plugin_info in manager.getPluginsOfCategory("Tool"))
def __init__(self, manager=None, name=None):
"""
Create the plugin object
:param manager: Manager instance
:type manager: :class:`biomajmanager.manager.Manager`
:param name: Name of the plugin to load. [DEFAULT: load all plugins]
:type name: String
:raises SystemExit: If 'manager' arg is not given
:raises SystemExit: If 'PLUGINS' section not found in :py:data:`manager.properties`
:raises SystemExit: If 'plugins.dir' not set in :py:data:`manager.properties`
:raises SystemExit: If 'plugins.list' not set in :py:data:`manager.properties`
:raises SystemExit: If 'plugins.dir' does not exist
"""
self.pm = None
self.name = None
self.config = None
self.manager = None
if not manager:
Utils.error("'manager' is required")
self.manager = manager
self.config = self.manager.config
if not self.config.has_section('PLUGINS'):
Utils.error("Can't load plugins, no section found!")
if not self.config.has_option('MANAGER', 'plugins.dir'):
Utils.error("plugins.dir not defined!")
if not self.config.has_option('PLUGINS', 'plugins.list'):
Utils.error("plugins.list is not defined!")
if not os.path.isdir(self.config.get('MANAGER', 'plugins.dir')):
Utils.error("Can't find plugins.dir")
plugin_manager = PluginManager(directories_list=[self.config.get('MANAGER', 'plugins.dir')],
categories_filter={Plugins.CATEGORY: BMPlugin})
plugin_manager.collectPlugins()
self.pm = plugin_manager
self.name = name
user_plugins = []
# Load user wanted plugin(s)
for plugin in self.config.get('PLUGINS', 'plugins.list').split(','):
plugin.strip()
# We need to lower the plugin name
user_plugins.append(plugin)
# This means that all plugins must inherits from BMPlugin
for pluginInfo in plugin_manager.getPluginsOfCategory(Plugins.CATEGORY):
Utils.verbose("[manager] plugin name => %s" % pluginInfo.name)
if pluginInfo.name in user_plugins:
if not pluginInfo.is_activated:
Utils.verbose("[manager] plugin %s activated" % pluginInfo.name)
plugin_manager.activatePluginByName(pluginInfo.name)
setattr(self, pluginInfo.name, pluginInfo.plugin_object)
pluginInfo.plugin_object.set_config(self.config)
pluginInfo.plugin_object.set_manager(self.manager)
def testTwoStepsLoadWithError(self):
"""
Test loading the plugins in two steps in order to collect more
deltailed informations and take care of an erroneous plugin.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
],
plugin_info_ext="yapsy-error-plugin")
# trigger the first step to look up for plugins
spm.locatePlugins()
# make full use of the "feedback" the loadPlugins can give
# - set-up the callback function that will be called *before*
# loading each plugin
callback_infos = []
def preload_cbk(i_plugin_info):
callback_infos.append(i_plugin_info)
callback_after_infos = []
def postload_cbk(i_plugin_info):
callback_after_infos.append(i_plugin_info)
# - gather infos about the processed plugins (loaded or not)
# and for the test, monkey patch the logger
originalLogLevel = log.getEffectiveLevel()
log.setLevel(logging.ERROR)
errorLogCallFlag = [False]
def errorMock(*args, **kwargs):
errorLogCallFlag[0] = True
originalErrorMethod = log.error
log.error = errorMock
try:
loadedPlugins = spm.loadPlugins(callback=preload_cbk,
callback_after=postload_cbk)
finally:
log.setLevel(originalLogLevel)
log.error = originalErrorMethod
self.assertTrue(errorLogCallFlag[0])
self.assertEqual(len(loadedPlugins), 1)
self.assertEqual(len(callback_infos), 1)
self.assertTrue(isinstance(callback_infos[0].error, tuple))
self.assertEqual(loadedPlugins[0], callback_infos[0])
self.assertTrue(issubclass(callback_infos[0].error[0], ImportError))
self.assertEqual(len(callback_after_infos), 0)
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
def testCategoryManipulation(self):
"""
Test querying, removing and adding plugins from/to a category.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# load the plugins that may be found
spm.collectPlugins()
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
plugin_info = spm.getPluginsOfCategory(sole_category)[0]
# try to remove it and check that is worked
spm.removePluginFromCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
# now re-add this plugin the to same category
spm.appendPluginToCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
def testCategoryManipulation(self): """ Test querying, removing and adding plugins from/to a category. """ spm = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1) plugin_info = spm.getPluginsOfCategory(sole_category)[0] # try to remove it and check that is worked spm.removePluginFromCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),0) # now re-add this plugin the to same category spm.appendPluginToCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1)
def testMultipleCategoriesForASamePlugin(self):
"""
Test that associating a plugin to multiple categories works as expected.
"""
class AnotherPluginIfce(object):
def __init__(self):
pass
def activate(self):
pass
def deactivate(self):
pass
spm = PluginManager(categories_filter={
"Default": IPlugin,
"IP": IPlugin,
"Other": AnotherPluginIfce,
},
directories_list=[
os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"plugins")
])
# load the plugins that may be found
spm.collectPlugins()
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 3)
categories = spm.getCategories()
self.assertTrue("Default" in categories)
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory("Default")), 1)
plugin_info = spm.getPluginsOfCategory("Default")[0]
self.assertTrue("Default" in plugin_info.categories)
self.assertTrue("IP" in plugin_info.categories)
self.assertTrue("IP" in categories)
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1)
self.assertTrue("Other" in categories)
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory("Other")), 0)
# try to remove the plugin from one category and check the
# other category
spm.removePluginFromCategory(plugin_info, "Default")
self.assertEqual(len(spm.getPluginsOfCategory("Default")), 0)
self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1)
# now re-add this plugin the to same category
spm.appendPluginToCategory(plugin_info, "Default")
self.assertEqual(len(spm.getPluginsOfCategory("Default")), 1)
self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1)
def test_print_json_reporting_plugin_found():
"""Test that the plugin manager finds the print json reporting plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]
)
manager.setCategoriesFilter(
{
"Reporting": ReportingPlugin,
}
)
manager.collectPlugins()
assert any(
plugin_info.plugin_object.get_name() == "print_json"
for plugin_info in manager.getPluginsOfCategory("Reporting")
)
assert any(
plugin_info.name == "Print JSON Reporting Plugin"
for plugin_info in manager.getPluginsOfCategory("Reporting")
)
def modelBehaviorImplementations(paths):
"""
Return an iterable of plugin-info for every locatable implementation of this interface on a given path.
PyDev for Eclipse reports a compilation error here on a line that is actually legal Python due to
the schedule upon which module resolution and imports happen.
"""
manager = PluginManager()
manager.setPluginPlaces(paths)
from fakeDataGenerator.model import IModelBehavior as foreignModelBehavior
# hey PyDev: the previous line is weird, but is actually legal
manager.setCategoriesFilter({"ModelBehavior": foreignModelBehavior})
manager.collectPlugins()
return manager.getPluginsOfCategory("ModelBehavior")
def testMultipleCategoriesForASamePlugin(self):
"""
Test that associating a plugin to multiple categories works as expected.
"""
class AnotherPluginIfce(object):
def __init__(self):
pass
def activate(self):
pass
def deactivate(self):
pass
spm = PluginManager(
categories_filter = {
"Default": IPlugin,
"IP": IPlugin,
"Other": AnotherPluginIfce,
},
directories_list=[
os.path.join(
os.path.dirname(os.path.abspath(__file__)),"plugins")])
# load the plugins that may be found
spm.collectPlugins()
# check that the getCategories works
self.assertEqual(len(spm.getCategories()),3)
categories = spm.getCategories()
self.assertTrue("Default" in categories)
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory("Default")), 1)
plugin_info = spm.getPluginsOfCategory("Default")[0]
self.assertTrue("Default" in plugin_info.categories)
self.assertTrue("IP" in plugin_info.categories)
self.assertTrue("IP" in categories)
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory("IP")),1)
self.assertTrue("Other" in categories)
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory("Other")),0)
# try to remove the plugin from one category and check the
# other category
spm.removePluginFromCategory(plugin_info, "Default")
self.assertEqual(len(spm.getPluginsOfCategory("Default")), 0)
self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1)
# now re-add this plugin the to same category
spm.appendPluginToCategory(plugin_info, "Default")
self.assertEqual(len(spm.getPluginsOfCategory("Default")),1)
self.assertEqual(len(spm.getPluginsOfCategory("IP")),1)
def testRecursivePluginlocation(self):
"""
Test detection of plugins which by default must be
recusrive. Here we give the test directory as a plugin place
whereas we expect the plugins to be in test/plugins.
"""
spm = PluginManager(
directories_list=[os.path.dirname(os.path.abspath(__file__))])
# load the plugins that may be found
spm.collectPlugins()
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
def test_groovylint_tool_plugin_found():
"""Test that the plugin manager can find the groovylint plugin."""
manager = PluginManager()
# Get the path to statick_tool/__init__.py, get the directory part, and
# add 'plugins' to that to get the standard plugins dir
manager.setPluginPlaces(
[os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]
)
manager.setCategoriesFilter(
{
"Tool": ToolPlugin,
}
)
manager.collectPlugins()
# Verify that a plugin's get_name() function returns "groovylint"
assert any(
plugin_info.plugin_object.get_name() == "groovylint"
for plugin_info in manager.getPluginsOfCategory("Tool")
)
# While we're at it, verify that a plugin is named NPM Groovy Lint Tool Plugin
assert any(
plugin_info.name == "NPM Groovy Lint Tool Plugin"
for plugin_info in manager.getPluginsOfCategory("Tool")
)
def testRecursivePluginlocation(self): """ Test detection of plugins which by default must be recusrive. Here we give the test directory as a plugin place whereas we expect the plugins to be in test/plugins. """ spm = PluginManager(directories_list=[ os.path.dirname(os.path.abspath(__file__))]) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1)
def modelBehaviorImplementations(paths):
"""
Return an iterable of plugin-info for every locatable implementation of this interface on a given path.
PyDev for Eclipse reports a compilation error here on a line that is actually legal Python due to
the schedule upon which module resolution and imports happen.
"""
manager = PluginManager()
manager.setPluginPlaces(paths)
from fakeDataGenerator.model import IModelBehavior as foreignModelBehavior
#hey PyDev: the previous line is weird, but is actually legal
manager.setCategoriesFilter({
"ModelBehavior" : foreignModelBehavior,
})
manager.collectPlugins()
return manager.getPluginsOfCategory("ModelBehavior")
def testTwoStepsLoadWithError(self):
"""
Test loading the plugins in two steps in order to collect more
deltailed informations and take care of an erroneous plugin.
"""
spm = PluginManager(
directories_list=[os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")],
plugin_info_ext="yapsy-error-plugin",
)
# trigger the first step to look up for plugins
spm.locatePlugins()
# make full use of the "feedback" the loadPlugins can give
# - set-up the callback function that will be called *before*
# loading each plugin
callback_infos = []
def preload_cbk(i_plugin_info):
callback_infos.append(i_plugin_info)
# - gather infos about the processed plugins (loaded or not)
# and for the test, monkey patch the logger
originalLogLevel = log.getEffectiveLevel()
log.setLevel(logging.ERROR)
errorLogCallFlag = [False]
def errorMock(*args, **kwargs):
errorLogCallFlag[0] = True
originalErrorMethod = log.error
log.error = errorMock
try:
loadedPlugins = spm.loadPlugins(callback=preload_cbk)
finally:
log.setLevel(originalLogLevel)
log.error = originalErrorMethod
self.assertTrue(errorLogCallFlag[0])
self.assertEqual(len(loadedPlugins), 1)
self.assertEqual(len(callback_infos), 1)
self.assertTrue(isinstance(callback_infos[0].error, tuple))
self.assertEqual(loadedPlugins[0], callback_infos[0])
self.assertEqual(callback_infos[0].error[0], ImportError)
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
def testNonRecursivePluginlocationNotFound(self): """ Test detection of plugins when the detection is non recursive. Here we test that it cannot look into subdirectories of the test directory. """ pluginLocator = PluginFileLocator() pluginLocator.setPluginPlaces([ os.path.dirname(os.path.abspath(__file__))]) pluginLocator.disableRecursiveScan() spm = PluginManager() spm.setPluginLocator(pluginLocator) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),0)
class FakeNikola(object):
def __init__(self):
self.config = {'DISABLED_PLUGINS': []}
self.debug = False
self.loghandlers = []
self.timeline = []
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
"SignalHandler": SignalHandler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = ''
places = [
resource_filename('nikola', utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
os.path.expanduser('~/.nikola/plugins'),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
#self.pug = None
#for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
# if plugin_info.name == 'rest':
# self.plugin_manager.activatePluginByName(plugin_info.name)
# plugin_info.plugin_object.set_site(self)
# self.pug = plugin_info
# Now we have our pug
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"RestExtension"):
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
def testNonRecursivePluginlocationNotFound(self):
"""
Test detection of plugins when the detection is non recursive.
Here we test that it cannot look into subdirectories of the
test directory.
"""
pluginLocator = PluginFileLocator()
pluginLocator.setPluginPlaces(
[os.path.dirname(os.path.abspath(__file__))])
pluginLocator.disableRecursiveScan()
spm = PluginManager()
spm.setPluginLocator(pluginLocator)
# load the plugins that may be found
spm.collectPlugins()
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
def testDisablingRecursivePluginLocationAllowsFindingTopLevelPlugins(self): """ Test detection of plugins when the detection is non recursive. Here we test that if we give test/plugin as the directory to scan it can find the plugin. """ pluginLocator = PluginFileLocator() pluginLocator.setPluginPlaces([ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) pluginLocator.disableRecursiveScan() spm = PluginManager() spm.setPluginLocator(pluginLocator) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1)
def testDisablingRecursivePluginLocationAllowsFindingTopLevelPlugins(self):
"""
Test detection of plugins when the detection is non
recursive. Here we test that if we give test/plugin as the
directory to scan it can find the plugin.
"""
pluginLocator = PluginFileLocator()
pluginLocator.setPluginPlaces([
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
pluginLocator.disableRecursiveScan()
spm = PluginManager()
spm.setPluginLocator(pluginLocator)
# load the plugins that may be found
spm.collectPlugins()
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
def testChangingCategoriesFilter(self):
"""
Test the effect of setting a new category filer.
"""
spm = PluginManager(directories_list=[
os.path.join(
os.path.dirname(os.path.abspath(__file__)),"plugins")])
# load the plugins that may be found
spm.collectPlugins()
newCategory = "Mouf"
# Pre-requisite for the test
previousCategories = spm.getCategories()
self.assertTrue(len(previousCategories) >= 1)
self.assertTrue(newCategory not in previousCategories)
# change the category and see what's happening
spm.setCategoriesFilter({newCategory: IPlugin})
spm.collectPlugins()
for categoryName in previousCategories:
self.assertRaises(KeyError, spm.getPluginsOfCategory, categoryName)
self.assertTrue(len(spm.getPluginsOfCategory(newCategory)) >= 1)
def testChangingCategoriesFilter(self):
"""
Test the effect of setting a new category filer.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# load the plugins that may be found
spm.collectPlugins()
newCategory = "Mouf"
# Pre-requisite for the test
previousCategories = spm.getCategories()
self.assertTrue(len(previousCategories) >= 1)
self.assertTrue(newCategory not in previousCategories)
# change the category and see what's happening
spm.setCategoriesFilter({newCategory: IPlugin})
spm.collectPlugins()
for categoryName in previousCategories:
self.assertRaises(KeyError, spm.getPluginsOfCategory, categoryName)
self.assertTrue(len(spm.getPluginsOfCategory(newCategory)) >= 1)
class FakeNikola(object):
def __init__(self):
self.config = {'DISABLED_PLUGINS': []}
self.debug = False
self.loghandlers = []
self.timeline = []
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
"SignalHandler": SignalHandler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = ''
places = [
resource_filename('nikola', utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
os.path.expanduser('~/.nikola/plugins'),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
#self.pug = None
#for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
# if plugin_info.name == 'rest':
# self.plugin_manager.activatePluginByName(plugin_info.name)
# plugin_info.plugin_object.set_site(self)
# self.pug = plugin_info
# Now we have our pug
for plugin_info in self.plugin_manager.getPluginsOfCategory("RestExtension"):
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
def main():
configpath = ".animedb.config"
config = Config(configpath)
if config['error']:
print "No Config file found - using default config"
print "please edit %s to include your anidb username and pass" % config['path']
config=DefaultConfig(configpath)
config.save()
raise IOError
manager = PluginManager(categories_filter={"Parsers": Parser}, plugin_info_ext="parser")
manager.setPluginPlaces(["plugins"])
manager.collectPlugins()
parsers = []
for plugin in manager.getPluginsOfCategory("Parsers"):
parsers.append(plugin.plugin_object)
parsers.append(MiscParser())
for parser in parsers:
parser.setdb("db.sqlite", "sqlite")
parser.setconfig(config)
for root,dirs,files in os.walk(os.path.abspath(config['ScanFolder'])):
for filename in files:
name = os.path.join(root,filename)
print "Parsing: %s" % name
for parser in parsers:
catalogue = []
Info = parser.ScanFile(name)
if Info:
location = parser.SortFile(Info, name)
catalogue.append((location, Info))
if catalogue:
parser.Catalogue(catalogue)
break
for parser in parsers:
parser.commit()
class SimpleTestsCase(unittest.TestCase):
"""
Test the correct loading of a simple plugin as well as basic
commands.
"""
def setUp(self):
"""
init
"""
# create the plugin manager
self.simplePluginManager = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# load the plugins that may be found
self.simplePluginManager.collectPlugins()
# Will be used later
self.plugin_info = None
def plugin_loading_check(self):
"""
Test if the correct plugin has been loaded.
"""
if self.plugin_info is None:
# check nb of categories
self.assertEqual(len(self.simplePluginManager.getCategories()), 1)
sole_category = self.simplePluginManager.getCategories()[0]
# check the number of plugins
self.assertEqual(
len(
self.simplePluginManager.getPluginsOfCategory(
sole_category)), 1)
self.plugin_info = self.simplePluginManager.getPluginsOfCategory(
sole_category)[0]
# test that the name of the plugin has been correctly defined
self.assertEqual(self.plugin_info.name, "Simple Plugin")
self.assertEqual(sole_category, self.plugin_info.category)
else:
self.assert_(True)
def testLoaded(self):
"""
Test if the correct plugin has been loaded.
"""
self.plugin_loading_check()
def testGetAll(self):
"""
Test if the correct plugin has been loaded.
"""
self.plugin_loading_check()
self.assertEqual(len(self.simplePluginManager.getAllPlugins()), 1)
self.assertEqual(self.simplePluginManager.getAllPlugins()[0],
self.plugin_info)
def testActivationAndDeactivation(self):
"""
Test if the activation procedure works.
"""
self.plugin_loading_check()
self.assert_(not self.plugin_info.plugin_object.is_activated)
self.simplePluginManager.activatePluginByName(
self.plugin_info.name, self.plugin_info.category)
self.assert_(self.plugin_info.plugin_object.is_activated)
self.simplePluginManager.deactivatePluginByName(
self.plugin_info.name, self.plugin_info.category)
self.assert_(not self.plugin_info.plugin_object.is_activated)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = ["planetoid", "ipynb", "local_search", "render_mustache"]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
"ADD_THIS_BUTTONS": True,
"ANALYTICS": "",
"ARCHIVE_PATH": "",
"ARCHIVE_FILENAME": "archive.html",
"CACHE_FOLDER": "cache",
"CODE_COLOR_SCHEME": "default",
"COMMENTS_IN_GALLERIES": False,
"COMMENTS_IN_STORIES": False,
"CONTENT_FOOTER": "",
"CREATE_MONTHLY_ARCHIVE": False,
"DATE_FORMAT": "%Y-%m-%d %H:%M",
"DEFAULT_LANG": "en",
"DEPLOY_COMMANDS": [],
"DISABLED_PLUGINS": (),
"DISQUS_FORUM": "nikolademo",
"ENABLED_EXTRAS": (),
"EXTRA_HEAD_DATA": "",
"FAVICONS": {},
"FILE_METADATA_REGEXP": None,
"FILES_FOLDERS": {"files": ""},
"FILTERS": {},
"GALLERY_PATH": "galleries",
"GZIP_FILES": False,
"GZIP_EXTENSIONS": (".txt", ".htm", ".html", ".css", ".js", ".json"),
"HIDE_UNTRANSLATED_POSTS": False,
"INDEX_DISPLAY_POST_COUNT": 10,
"INDEX_TEASERS": False,
"INDEXES_TITLE": "",
"INDEXES_PAGES": "",
"INDEX_PATH": "",
"LICENSE": "",
"LISTINGS_FOLDER": "listings",
"MAX_IMAGE_SIZE": 1280,
"MATHJAX_CONFIG": "",
"OLD_THEME_SUPPORT": True,
"OUTPUT_FOLDER": "output",
"post_compilers": {
"rest": (".txt", ".rst"),
"markdown": (".md", ".mdown", ".markdown"),
"textile": (".textile",),
"txt2tags": (".t2t",),
"bbcode": (".bb",),
"wiki": (".wiki",),
"ipynb": (".ipynb",),
"html": (".html", ".htm"),
},
"POST_PAGES": (
("posts/*.txt", "posts", "post.tmpl", True),
("stories/*.txt", "stories", "story.tmpl", False),
),
"REDIRECTIONS": [],
"RSS_LINK": None,
"RSS_PATH": "",
"RSS_TEASERS": True,
"SEARCH_FORM": "",
"SLUG_TAG_PATH": True,
"STORY_INDEX": False,
"STRIP_INDEX_HTML": False,
"TAG_PATH": "categories",
"TAG_PAGES_ARE_INDEXES": False,
"THEME": "site",
"THEME_REVEAL_CONGIF_SUBTHEME": "sky",
"THEME_REVEAL_CONGIF_TRANSITION": "cube",
"THUMBNAIL_SIZE": 180,
"USE_BUNDLES": True,
"USE_CDN": False,
"USE_FILENAME_AS_TITLE": True,
"TIMEZONE": None,
}
self.config.update(config)
self.config["TRANSLATIONS"] = self.config.get("TRANSLATIONS", {self.config["DEFAULT_" "LANG"]: ""})
self.THEMES = utils.get_theme_chain(self.config["THEME"])
self.MESSAGES = utils.load_messages(self.THEMES, self.config["TRANSLATIONS"], self.config["DEFAULT_LANG"])
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
if "SITE_URL" not in self.config:
if "BLOG_URL" in self.config:
print("WARNING: You should configure SITE_URL instead of BLOG_URL")
self.config["SITE_URL"] = self.config["BLOG_URL"]
self.default_lang = self.config["DEFAULT_LANG"]
self.translations = self.config["TRANSLATIONS"]
# BASE_URL defaults to SITE_URL
if "BASE_URL" not in self.config:
self.config["BASE_URL"] = self.config.get("SITE_URL")
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
}
)
self.plugin_manager.setPluginInfoExtension("plugin")
self.plugin_manager.setPluginPlaces(
[str(os.path.join(os.path.dirname(__file__), "plugins")), str(os.path.join(os.getcwd(), "plugins"))]
)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if plugin_info.name in self.config["DISABLED_PLUGINS"] or (
plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"]
):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if plugin_info.name in self.config["DISABLED_PLUGINS"] or (
plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"]
):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = {}
self.GLOBAL_CONTEXT["messages"] = self.MESSAGES
self.GLOBAL_CONTEXT["_link"] = self.link
self.GLOBAL_CONTEXT["set_locale"] = s_l
self.GLOBAL_CONTEXT["rel_link"] = self.rel_link
self.GLOBAL_CONTEXT["abs_link"] = self.abs_link
self.GLOBAL_CONTEXT["exists"] = self.file_exists
self.GLOBAL_CONTEXT["SLUG_TAG_PATH"] = self.config["SLUG_TAG_PATH"]
self.GLOBAL_CONTEXT["add_this_buttons"] = self.config["ADD_THIS_BUTTONS"]
self.GLOBAL_CONTEXT["index_display_post_count"] = self.config["INDEX_DISPLAY_POST_COUNT"]
self.GLOBAL_CONTEXT["use_bundles"] = self.config["USE_BUNDLES"]
self.GLOBAL_CONTEXT["use_cdn"] = self.config.get("USE_CDN")
self.GLOBAL_CONTEXT["favicons"] = self.config["FAVICONS"]
self.GLOBAL_CONTEXT["date_format"] = self.config.get("DATE_FORMAT", "%Y-%m-%d %H:%M")
self.GLOBAL_CONTEXT["blog_author"] = self.config.get("BLOG_AUTHOR")
self.GLOBAL_CONTEXT["blog_title"] = self.config.get("BLOG_TITLE")
self.GLOBAL_CONTEXT["blog_url"] = self.config.get("SITE_URL", self.config.get("BLOG_URL"))
self.GLOBAL_CONTEXT["blog_desc"] = self.config.get("BLOG_DESCRIPTION")
self.GLOBAL_CONTEXT["analytics"] = self.config.get("ANALYTICS")
self.GLOBAL_CONTEXT["translations"] = self.config.get("TRANSLATIONS")
self.GLOBAL_CONTEXT["license"] = self.config.get("LICENSE")
self.GLOBAL_CONTEXT["search_form"] = self.config.get("SEARCH_FORM")
self.GLOBAL_CONTEXT["disqus_forum"] = self.config.get("DISQUS_FORUM")
self.GLOBAL_CONTEXT["mathjax_config"] = self.config.get("MATHJAX_CONFIG")
self.GLOBAL_CONTEXT["subtheme"] = self.config.get("THEME_REVEAL_CONGIF_SUBTHEME")
self.GLOBAL_CONTEXT["transition"] = self.config.get("THEME_REVEAL_CONGIF_TRANSITION")
self.GLOBAL_CONTEXT["content_footer"] = self.config.get("CONTENT_FOOTER")
self.GLOBAL_CONTEXT["rss_path"] = self.config.get("RSS_PATH")
self.GLOBAL_CONTEXT["rss_link"] = self.config.get("RSS_LINK")
self.GLOBAL_CONTEXT["sidebar_links"] = utils.Functionary(list, self.config["DEFAULT_LANG"])
for k, v in self.config.get("SIDEBAR_LINKS", {}).items():
self.GLOBAL_CONTEXT["sidebar_links"][k] = v
self.GLOBAL_CONTEXT["twitter_card"] = self.config.get("TWITTER_CARD", {})
self.GLOBAL_CONTEXT["extra_head_data"] = self.config.get("EXTRA_HEAD_DATA")
self.GLOBAL_CONTEXT.update(self.config.get("GLOBAL_CONTEXT", {}))
# check if custom css exist and is not empty
for files_path in list(self.config["FILES_FOLDERS"].keys()):
custom_css_path = os.path.join(files_path, "assets/css/custom.css")
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT["has_custom_css"] = True
break
else:
self.GLOBAL_CONTEXT["has_custom_css"] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system " "plugin\n".format(template_sys_name))
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES]
self.template_system.set_directories(lookup_dirs, self.config["CACHE_FOLDER"])
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config["USE_CDN"]:
bootstrap_path = utils.get_asset_path(os.path.join("assets", "css", "bootstrap.min.css"), self.THEMES)
if bootstrap_path.split(os.sep)[-4] != "site":
warnings.warn(
"The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap."
)
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
self.compilers[plugin_info.name] = plugin_info.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in list(self.config["post_compilers"].items()) if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit(
"Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of {0})".format(", ".join(langs))
)
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to " "handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(template_name, None, local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1 :]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split("/")[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == "link": # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip("/"), context["lang"])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b"<!DOCTYPE html>" + lxml.html.tostring(doc, encoding="utf8")
with open(output_name, "wb+") as post_file:
post_file.write(data)
def current_lang(self): # FIXME: this is duplicated, turn into a mixin
"""Return the currently set locale, if it's one of the
available translations, or default_lang."""
lang = utils.LocaleBorg().current_lang
if lang:
if lang in self.translations:
return lang
lang = lang.split("_")[0]
if lang in self.translations:
return lang
# whatever
return self.default_lang
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a post_pages tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = self.current_lang()
path = []
if kind == "tag_index":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], "index.html"] if _f]
elif kind == "tag":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".html"] if _f]
elif kind == "tag_rss":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".xml"] if _f]
elif kind == "index":
if name not in [None, 0]:
path = [
_f
for _f in [
self.config["TRANSLATIONS"][lang],
self.config["INDEX_PATH"],
"index-{0}.html".format(name),
]
if _f
]
else:
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index.html"] if _f]
elif kind == "post_path":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], os.path.dirname(name), "index.html"] if _f]
elif kind == "rss":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["RSS_PATH"], "rss.xml"] if _f]
elif kind == "archive":
if name:
path = [
_f
for _f in [self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], name, "index.html"]
if _f
]
else:
path = [
_f
for _f in [
self.config["TRANSLATIONS"][lang],
self.config["ARCHIVE_PATH"],
self.config["ARCHIVE_FILENAME"],
]
if _f
]
elif kind == "gallery":
path = [_f for _f in [self.config["GALLERY_PATH"], name, "index.html"] if _f]
elif kind == "listing":
path = [_f for _f in [self.config["LISTINGS_FOLDER"], name + ".html"] if _f]
if is_link:
link = "/" + ("/".join(path))
if self.config["STRIP_INDEX_HTML"] and link.endswith("/index.html"):
return link[:-10]
else:
return link
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config["BASE_URL"], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config["BASE_URL"], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split("/")[1:]
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
def create_gzipped_copy(in_path, out_path):
with gzip.GzipFile(out_path, "wb+") as outf:
with open(in_path, "rb") as inf:
outf.write(inf.read())
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
def add_gzipped_copies(task):
if not self.config["GZIP_FILES"]:
return None
if task.get("name") is None:
return None
gzip_task = {
"file_dep": [],
"targets": [],
"actions": [],
"basename": "gzip",
"name": task.get("name") + ".gz",
"clean": True,
}
targets = task.get("targets", [])
flag = False
for target in targets:
ext = os.path.splitext(target)[1]
if ext.lower() in self.config["GZIP_EXTENSIONS"] and target.startswith(self.config["OUTPUT_FOLDER"]):
flag = True
gzipped = target + ".gz"
gzip_task["file_dep"].append(target)
gzip_task["targets"].append(gzipped)
gzip_task["actions"].append((create_gzipped_copy, (target, gzipped)))
if not flag:
return None
return gzip_task
if self.config["GZIP_FILES"]:
task_dep = ["gzip"]
else:
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
gztask = add_gzipped_copies(task)
if gztask:
yield gztask
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
gztask = add_gzipped_copies(task)
if gztask:
yield gztask
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {"name": b"all", "actions": None, "clean": True, "task_dep": task_dep}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
print("Scanning posts", end="")
tzinfo = None
if self.config["TIMEZONE"] is not None:
tzinfo = pytz.timezone(self.config["TIMEZONE"])
targets = set([])
for wildcard, destination, template_name, use_in_feeds in self.config["post_pages"]:
print(".", end="")
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination, os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config["TRANSLATIONS"].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
for base_path in full_list:
post = Post(
base_path,
self.config["CACHE_FOLDER"],
dest_dir,
use_in_feeds,
self.config["TRANSLATIONS"],
self.config["DEFAULT_LANG"],
self.config["BASE_URL"],
self.MESSAGES,
template_name,
self.config["FILE_METADATA_REGEXP"],
self.config["STRIP_INDEX_HTML"],
tzinfo,
self.config["HIDE_UNTRANSLATED_POSTS"],
)
for lang, langpath in list(self.config["TRANSLATIONS"].items()):
dest = (destination, langpath, dir_glob, post.meta[lang]["slug"])
if dest in targets:
raise Exception(
"Duplicated output path {0!r} "
"in post {1!r}".format(post.meta[lang]["slug"], base_path)
)
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(post.post_name)
self.posts_per_month["{0}/{1:02d}".format(post.date.year, post.date.month)].append(
post.post_name
)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
if self.config["OLD_THEME_SUPPORT"]:
post._add_old_metadata()
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + self.template_system.template_deps(post.template_name)
context["post"] = post
context["lang"] = lang
context["title"] = post.title(lang)
context["description"] = post.description(lang)
context["permalink"] = post.permalink(lang)
context["page_list"] = self.pages
if post.use_in_feeds:
context["enable_comments"] = True
else:
context["enable_comments"] = self.config["COMMENTS_IN_STORIES"]
output_name = os.path.join(self.config["OUTPUT_FOLDER"], post.destination_path(lang))
deps_dict = copy(context)
deps_dict.pop("post")
if post.prev_post:
deps_dict["PREV_LINK"] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict["NEXT_LINK"] = [post.next_post.permalink(lang)]
deps_dict["OUTPUT_FOLDER"] = self.config["OUTPUT_FOLDER"]
deps_dict["TRANSLATIONS"] = self.config["TRANSLATIONS"]
deps_dict["global"] = self.GLOBAL_CONTEXT
deps_dict["comments"] = context["enable_comments"]
task = {
"name": os.path.normpath(output_name),
"file_dep": deps,
"targets": [output_name],
"actions": [(self.render_template, [post.template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config["BLOG_TITLE"]
context["description"] = self.config["BLOG_DESCRIPTION"]
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]["title"], p.permalink(lang)) for p in posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
"name": os.path.normpath(output_name),
"targets": [output_name],
"file_dep": deps,
"actions": [(self.render_template, [template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_context)],
}
return utils.apply_filters(task, filters)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = [
'planetoid',
'ipynb',
'local_search',
'render_mustache',
]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
# Register our own path handlers
self.path_handlers = {
'slug': self.slug_path,
'post_path': self.post_path,
}
self.strict = False
self.global_data = {}
self.posts = []
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.posts_per_category = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
self._template_system = None
self._THEMES = None
self.debug = DEBUG
self.loghandlers = []
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ANNOTATIONS': False,
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'BLOG_TITLE': 'Default Title',
'BLOG_DESCRIPTION': 'Default Description',
'BODY_END': "",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENT_SYSTEM': 'disqus',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'COMPILERS': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile',),
"txt2tags": ('.t2t',),
"bbcode": ('.bb',),
"wiki": ('.wiki',),
"ipynb": ('.ipynb',),
"html": ('.html', '.htm')
},
'CONTENT_FOOTER': '',
'COPY_SOURCES': True,
'CREATE_MONTHLY_ARCHIVE': False,
'CREATE_SINGLE_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'EXTRA_PLUGINS_DIRS': [],
'COMMENT_SYSTEM_ID': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FEED_LENGTH': 10,
'FILE_METADATA_REGEXP': None,
'ADDITIONAL_METADATA': {},
'FILES_FOLDERS': {'files': ''},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GALLERY_SORT_BY_DATE': True,
'GZIP_COMMAND': None,
'GZIP_FILES': False,
'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json', '.xml'),
'HIDE_SOURCELINK': False,
'HIDE_UNTRANSLATED_POSTS': False,
'HYPHENATE': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEXES_PAGES_MAIN': False,
'INDEX_PATH': '',
'IPYNB_CONFIG': {},
'LESS_COMPILER': 'lessc',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'NAVIGATION_LINKS': None,
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'POSTS': (("posts/*.txt", "posts", "post.tmpl"),),
'PAGES': (("stories/*.txt", "stories", "story.tmpl"),),
'PRETTY_URLS': False,
'FUTURE_IS_NOW': False,
'READ_MORE_LINK': '<p class="more"><a href="{link}">{read_more}…</a></p>',
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SASS_COMPILER': 'sass',
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'SOCIAL_BUTTONS_CODE': SOCIAL_BUTTONS_CODE,
'SITE_URL': 'http://getnikola.com/',
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'bootstrap',
'THEME_REVEAL_CONFIG_SUBTHEME': 'sky',
'THEME_REVEAL_CONFIG_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'URL_TYPE': 'rel_path',
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': 'UTC',
'DEPLOY_DRAFTS': True,
'DEPLOY_FUTURE': False,
'SCHEDULE_ALL': False,
'SCHEDULE_RULE': '',
'SCHEDULE_FORCE_TODAY': False,
'LOGGING_HANDLERS': {'stderr': {'loglevel': 'WARNING', 'bubble': True}},
'DEMOTE_HEADERS': 1,
}
self.config.update(config)
# Make sure we have pyphen installed if we are using it
if self.config.get('HYPHENATE') and pyphen is None:
utils.LOGGER.warn('To use the hyphenation, you have to install '
'the "pyphen" package.')
utils.LOGGER.warn('Setting HYPHENATE to False.')
self.config['HYPHENATE'] = False
# Deprecating post_compilers
# TODO: remove on v7
if 'post_compilers' in config:
utils.LOGGER.warn('The post_compilers option is deprecated, use COMPILERS instead.')
if 'COMPILERS' in config:
utils.LOGGER.warn('COMPILERS conflicts with post_compilers, ignoring post_compilers.')
else:
self.config['COMPILERS'] = config['post_compilers']
# Deprecating post_pages
# TODO: remove on v7
if 'post_pages' in config:
utils.LOGGER.warn('The post_pages option is deprecated, use POSTS and PAGES instead.')
if 'POSTS' in config or 'PAGES' in config:
utils.LOGGER.warn('POSTS and PAGES conflict with post_pages, ignoring post_pages.')
else:
self.config['POSTS'] = [item[:3] for item in config['post_pages'] if item[-1]]
self.config['PAGES'] = [item[:3] for item in config['post_pages'] if not item[-1]]
# FIXME: Internally, we still use post_pages because it's a pain to change it
self.config['post_pages'] = []
for i1, i2, i3 in self.config['POSTS']:
self.config['post_pages'].append([i1, i2, i3, True])
for i1, i2, i3 in self.config['PAGES']:
self.config['post_pages'].append([i1, i2, i3, False])
# Deprecating DISQUS_FORUM
# TODO: remove on v7
if 'DISQUS_FORUM' in config:
utils.LOGGER.warn('The DISQUS_FORUM option is deprecated, use COMMENT_SYSTEM_ID instead.')
if 'COMMENT_SYSTEM_ID' in config:
utils.LOGGER.warn('DISQUS_FORUM conflicts with COMMENT_SYSTEM_ID, ignoring DISQUS_FORUM.')
else:
self.config['COMMENT_SYSTEM_ID'] = config['DISQUS_FORUM']
# Deprecating the ANALYTICS option
# TODO: remove on v7
if 'ANALYTICS' in config:
utils.LOGGER.warn('The ANALYTICS option is deprecated, use BODY_END instead.')
if 'BODY_END' in config:
utils.LOGGER.warn('ANALYTICS conflicts with BODY_END, ignoring ANALYTICS.')
else:
self.config['BODY_END'] = config['ANALYTICS']
# Deprecating the SIDEBAR_LINKS option
# TODO: remove on v7
if 'SIDEBAR_LINKS' in config:
utils.LOGGER.warn('The SIDEBAR_LINKS option is deprecated, use NAVIGATION_LINKS instead.')
if 'NAVIGATION_LINKS' in config:
utils.LOGGER.warn('The SIDEBAR_LINKS conflicts with NAVIGATION_LINKS, ignoring SIDEBAR_LINKS.')
else:
self.config['NAVIGATION_LINKS'] = config['SIDEBAR_LINKS']
# Compatibility alias
self.config['SIDEBAR_LINKS'] = self.config['NAVIGATION_LINKS']
if self.config['NAVIGATION_LINKS'] in (None, {}):
self.config['NAVIGATION_LINKS'] = {self.config['DEFAULT_LANG']: ()}
# Deprecating the ADD_THIS_BUTTONS option
# TODO: remove on v7
if 'ADD_THIS_BUTTONS' in config:
utils.LOGGER.warn('The ADD_THIS_BUTTONS option is deprecated, use SOCIAL_BUTTONS_CODE instead.')
if not config['ADD_THIS_BUTTONS']:
utils.LOGGER.warn('Setting SOCIAL_BUTTONS_CODE to empty because ADD_THIS_BUTTONS is False.')
self.config['SOCIAL_BUTTONS_CODE'] = ''
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
# TODO: remove on v7
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
utils.LOGGER.warn('You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML')
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
if config.get('COPY_SOURCES') and not self.config['HIDE_SOURCELINK']:
self.config['HIDE_SOURCELINK'] = True
self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
{self.config['DEFAULT_LANG']: ''})
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
# TODO: remove on v7
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
utils.LOGGER.warn('You should configure SITE_URL instead of BLOG_URL')
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
locale_fallback, locale_default, locales = sanitized_locales(
self.config.get('LOCALE_FALLBACK', None),
self.config.get('LOCALE_DEFAULT', None),
self.config.get('LOCALES', {}),
self.translations) # NOQA
utils.LocaleBorg.initialize(locales, self.default_lang)
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
# BASE_URL should *always* end in /
if self.config['BASE_URL'] and self.config['BASE_URL'][-1] != '/':
utils.LOGGER.warn("Your BASE_URL doesn't end in / -- adding it.")
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"SignalHandler": SignalHandler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
] + [path for path in extra_plugins_dirs if path]
else:
places = [
os.path.join(os.path.dirname(__file__), utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
# Activate all required SignalHandler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("SignalHandler"):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(plugin_info, "SignalHandler")
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Emit signal for SignalHandlers which need to start running immediately.
signal('sighandlers_loaded').send(self)
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
if plugin_info.name in self.config["COMPILERS"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self._GLOBAL_CONTEXT = {}
self._GLOBAL_CONTEXT['_link'] = self.link
self._GLOBAL_CONTEXT['set_locale'] = utils.LocaleBorg().set_locale
self._GLOBAL_CONTEXT['rel_link'] = self.rel_link
self._GLOBAL_CONTEXT['abs_link'] = self.abs_link
self._GLOBAL_CONTEXT['exists'] = self.file_exists
self._GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config['SLUG_TAG_PATH']
self._GLOBAL_CONTEXT['annotations'] = self.config['ANNOTATIONS']
self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self._GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self._GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
# TODO: remove fallback in v7
self._GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL', self.config.get('BLOG_URL'))
self._GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self._GLOBAL_CONTEXT['body_end'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['analytics'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['add_this_buttons'] = self.config.get('SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['social_buttons_code'] = self.config.get('SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self._GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self._GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self._GLOBAL_CONTEXT['comment_system'] = self.config.get('COMMENT_SYSTEM')
self._GLOBAL_CONTEXT['comment_system_id'] = self.config.get('COMMENT_SYSTEM_ID')
# TODO: remove in v7
self._GLOBAL_CONTEXT['disqus_forum'] = self.config.get('COMMENT_SYSTEM_ID')
self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self._GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONFIG_SUBTHEME')
self._GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONFIG_TRANSITION')
self._GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self._GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self._GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self._GLOBAL_CONTEXT['navigation_links'] = utils.Functionary(list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('NAVIGATION_LINKS', {}).items():
self._GLOBAL_CONTEXT['navigation_links'][k] = v
# TODO: remove on v7
# Compatibility alias
self._GLOBAL_CONTEXT['sidebar_links'] = self._GLOBAL_CONTEXT['navigation_links']
self._GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self._GLOBAL_CONTEXT['hide_sourcelink'] = self.config.get(
'HIDE_SOURCELINK')
self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA')
self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
signal('configured').send(self)
def _get_themes(self):
if self._THEMES is None:
# Check for old theme names (Issue #650) TODO: remove in v7
theme_replacements = {
'site': 'bootstrap',
'orphan': 'base',
'default': 'oldfashioned',
}
if self.config['THEME'] in theme_replacements:
utils.LOGGER.warn('You are using the old theme "{0}", using "{1}" instead.'.format(
self.config['THEME'], theme_replacements[self.config['THEME']]))
self.config['THEME'] = theme_replacements[self.config['THEME']]
if self.config['THEME'] == 'oldfashioned':
utils.LOGGER.warn('''You may need to install the "oldfashioned" theme '''
'''from themes.nikola.ralsina.com.ar because it's not '''
'''shipped by default anymore.''')
utils.LOGGER.warn('Please change your THEME setting.')
try:
self._THEMES = utils.get_theme_chain(self.config['THEME'])
except Exception:
utils.LOGGER.warn('''Can't load theme "{0}", using 'bootstrap' instead.'''.format(self.config['THEME']))
self.config['THEME'] = 'bootstrap'
return self._get_themes()
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config['USE_CDN']:
bootstrap_path = utils.get_asset_path(os.path.join(
'assets', 'css', 'bootstrap.min.css'), self._THEMES)
if bootstrap_path and bootstrap_path.split(os.sep)[-4] not in ['bootstrap', 'bootstrap3']:
utils.LOGGER.warn('The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.')
return self._THEMES
THEMES = property(_get_themes)
def _get_messages(self):
return utils.load_messages(self.THEMES,
self.translations,
self.default_lang)
MESSAGES = property(_get_messages)
def _get_global_context(self):
"""Initialize some parts of GLOBAL_CONTEXT only when it's queried."""
if 'messages' not in self._GLOBAL_CONTEXT:
self._GLOBAL_CONTEXT['messages'] = self.MESSAGES
if 'has_custom_css' not in self._GLOBAL_CONTEXT:
# check if custom css exist and is not empty
custom_css_path = utils.get_asset_path(
'assets/css/custom.css',
self.THEMES,
self.config['FILES_FOLDERS']
)
if custom_css_path and self.file_exists(custom_css_path, not_empty=True):
self._GLOBAL_CONTEXT['has_custom_css'] = True
else:
self._GLOBAL_CONTEXT['has_custom_css'] = False
return self._GLOBAL_CONTEXT
GLOBAL_CONTEXT = property(_get_global_context)
def _get_template_system(self):
if self._template_system is None:
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(
template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self._template_system = pi.plugin_object
lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES]
self._template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
return self._template_system
template_system = property(_get_template_system)
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.COMPILERS`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in
list(self.config['COMPILERS'].items())
if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'COMPILERS' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("COMPILERS in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
# string, arguments
local_context["formatmsg"] = lambda s, *a: s % a
data = self.template_system.render_template(
template_name, None, local_context)
assert output_name.startswith(
self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Refuse to replace links that consist of a fragment only
if ((not dst_url.scheme) and (not dst_url.netloc) and
(not dst_url.path) and (not dst_url.params) and
(not dst_url.query) and dst_url.fragment):
return dst
# Normalize
dst = urljoin(src, dst.lstrip('/'))
# Avoid empty links.
if src == dst:
if self.config.get('URL_TYPE') == 'absolute':
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return dst
elif self.config.get('URL_TYPE') == 'full_path':
return dst
else:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
if self.config.get('URL_TYPE') == 'absolute':
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return dst
if self.config.get('URL_TYPE') in ('full_path', 'absolute'):
if self.config.get('URL_TYPE') == 'absolute':
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
utils.makedirs(os.path.dirname(output_name))
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
These are mostly defined by plugins by registering via
the register_path_handler method, except for slug and
post_path which are defined in this class' init method.
Here's some of the others, for historical reasons:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* category (and name is the category name)
* category_rss (and name is the category name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a POSTS/PAGES tuple)
* slug (name is the slug of a post or story)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = utils.LocaleBorg().current_lang
path = self.path_handlers[kind](name, lang)
if is_link:
link = '/' + ('/'.join(path))
index_len = len(self.config['INDEX_FILE'])
if self.config['STRIP_INDEXES'] and \
link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def post_path(self, name, lang):
"""post_path path handler"""
return [_f for _f in [self.config['TRANSLATIONS'][lang],
os.path.dirname(name),
self.config['INDEX_FILE']] if _f]
def slug_path(self, name, lang):
"""slug path handler"""
results = [p for p in self.timeline if p.meta('slug') == name]
if not results:
utils.LOGGER.warning("Can't resolve path request for slug: {0}".format(name))
else:
if len(results) > 1:
utils.LOGGER.warning('Ambiguous path request for slug: {0}'.format(name))
return [_f for _f in results[0].permalink(lang).split('/') if _f]
def register_path_handler(self, kind, f):
if kind in self.path_handlers:
utils.LOGGER.warning('Conflicting path handlers for kind: {0}'.format(kind))
else:
self.path_handlers[kind] = f
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return urlparse(dst).geturl()
def rel_link(self, src, dst):
# Normalize
try:
src = urljoin(self.config['BASE_URL'], src.lstrip('/'))
except AttributeError:
# sometimes, it’s an Undefined object.
src = urljoin(self.config['BASE_URL'], src)
try:
dst = urljoin(src, dst.lstrip('/'))
except AttributeError:
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def clean_task_paths(self, task):
"""Normalize target paths in the task."""
targets = task.get('targets', None)
if targets is not None:
task['targets'] = [os.path.normpath(t) for t in targets]
return task
def gen_tasks(self, name, plugin_category, doc=''):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
assert 'basename' in task
task = self.clean_task_paths(task)
yield task
for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield self.clean_task_paths(task)
if flag:
task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'basename': name,
'doc': doc,
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
seen = set([])
print("Scanning posts", end='', file=sys.stderr)
lower_case_tags = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='', file=sys.stderr)
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination,
os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config['TRANSLATIONS'].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
# We eliminate from the list the files inside any .ipynb folder
full_list = [p for p in full_list
if not any([x.startswith('.')
for x in p.split(os.sep)])]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.config,
dest_dir,
use_in_feeds,
self.MESSAGES,
template_name,
self.get_compiler(base_path)
)
self.global_data[post.source_path] = post
if post.use_in_feeds:
self.posts.append(post.source_path)
self.posts_per_year[
str(post.date.year)].append(post.source_path)
self.posts_per_month[
'{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post.source_path)
for tag in post.alltags:
if tag.lower() in lower_case_tags:
if tag not in self.posts_per_tag:
# Tags that differ only in case
other_tag = [k for k in self.posts_per_tag.keys() if k.lower() == tag.lower()][0]
utils.LOGGER.error('You have cases that differ only in upper/lower case: {0} and {1}'.format(tag, other_tag))
utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join(self.posts_per_tag[other_tag])))
sys.exit(1)
else:
lower_case_tags.add(tag.lower())
self.posts_per_tag[tag].append(post.source_path)
self.posts_per_category[post.meta('category')].append(post.source_path)
else:
self.pages.append(post)
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!", file=sys.stderr)
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine'))
deps = list(filter(None, deps))
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
if post:
deps_dict['post_translations'] = post.translated_to
task = {
'name': os.path.normpath(output_name),
'file_dep': deps,
'targets': [output_name],
'actions': [(self.render_template, [post.template_name,
output_name, context])],
'clean': True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in
posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name': os.path.normpath(output_name),
'targets': [output_name],
'file_dep': deps,
'actions': [(self.render_template, [template_name, output_name,
context])],
'clean': True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
class TestDef(object):
def __init__(self):
# set aside storage for options and cmd line args
self.options = {}
self.args = []
# record if we have loaded the plugins or
# not - this is just a bozo check to ensure
# someone doesn't tell us to do it twice
self.loaded = False
# set aside a spot for a logger object, and
# note that it hasn't yet been defined
self.logger = None
self.modcmd = None
self.execmd = None
self.harasser = None
self.config = None
self.stages = None
self.tools = None
self.utilities = None
self.defaults = None
self.log = {}
def setOptions(self, args):
self.options = vars(args)
self.args = args
# if they want us to clear the scratch, then do so
if self.options['clean'] and os.path.isdir(self.options['scratchdir']):
shutil.rmtree(self.options['scratchdir'])
# setup the scratch directory
_mkdir_recursive(self.options['scratchdir'])
# private function to convert values
def __convert_value(self, opt, inval):
if opt is None or type(opt) is str:
return 0, inval
elif type(opt) is bool:
if type(inval) is bool:
return 0, inval
elif type(inval) is str:
if inval.lower() in ['true', '1', 't', 'y', 'yes']:
return 0, True
else:
return 0, False
elif type(inval) is int:
if 0 == inval:
return 0, False
else:
return 0, True
elif is_py2 and type(inval) is unicode:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is int:
if type(inval) is int:
return 0, inval
elif type(inval) is str:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is float:
if type(inval) is float:
return 0, inval
elif type(inval) is str or type(inval) is int:
return 0, float(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
else:
return 1, None
# scan the key-value pairs obtained from the configuration
# parser and compare them with the options defined for a
# given plugin. Generate an output dictionary that contains
# the updated set of option values, the default value for
# any option that wasn't included in the configuration file,
# and return an error status plus output identifying any
# keys in the configuration file that are not supported
# by the list of options
#
# @log [INPUT]
# - a dictionary that will return the status plus
# stderr containing strings identifying any
# provided keyvals that don't have a corresponding
# supported option
# @options [INPUT]
# - a dictionary of tuples, each consisting of three
# entries:
# (a) the default value
# (b) data type
# (c) a help-like description
# @keyvals [INPUT]
# - a dictionary of key-value pairs obtained from
# the configuration parser
# @target [OUTPUT]
# - the resulting dictionary of key-value pairs
def parseOptions(self, log, options, keyvals, target):
# parse the incoming keyvals dictionary against the source
# options. If a source option isn't provided, then
# copy it across to the target.
opts = list(options.keys())
kvkeys = list(keyvals.keys())
for opt in opts:
found = False
for kvkey in kvkeys:
if kvkey == opt:
# they provided us with an update, so
# pass this value into the target - expand
# any provided lists
if keyvals[kvkey] is None:
continue
st, outval = self.__convert_value(options[opt][0],
keyvals[kvkey])
if 0 == st:
target[opt] = outval
else:
if len(keyvals[kvkey]) == 0:
# this indicates they do not want this option
found = True
break
if keyvals[kvkey][0][0] == "[":
# they provided a list - remove the brackets
val = keyvals[kvkey].replace('[', '')
val = val.replace(']', '')
# split the input to pickup sets of options
newvals = list(val)
# convert the values to specified type
i = 0
for val in newvals:
st, newvals[i] = self.__convert_value(
opt[0], val)
i = i + 1
target[opt] = newvals
else:
st, target[opt] = self.__convert_value(
opt[0], keyvals[kvkey])
found = True
break
if not found:
# they didn't provide this one, so
# transfer only the value across
target[opt] = options[opt][0]
# add in any default settings that have not
# been overridden - anything set by this input
# stage will override the default
if self.defaults is not None:
keys = self.defaults.options.keys()
for key in keys:
if key not in target:
target[key] = self.defaults.options[key][0]
# now go thru in the reverse direction to see
# if any keyvals they provided aren't supported
# as this would be an error
stderr = []
for kvkey in kvkeys:
# ignore some standard keys
if kvkey in ['section', 'plugin']:
continue
try:
if target[kvkey] is not None:
pass
except KeyError:
# some always need to be passed
if kvkey in ['parent', 'asis']:
target[kvkey] = keyvals[kvkey]
else:
stderr.append("Option " + kvkey + " is not supported")
if stderr:
# mark the log with an error status
log['status'] = 1
# pass the errors back
log['stderr'] = stderr
else:
log['status'] = 0
log['options'] = target
return
def loadPlugins(self, basedir, topdir):
if self.loaded:
print("Cannot load plugins multiple times")
exit(1)
self.loaded = True
# find the loader utility so we can bootstrap ourselves
try:
m = imp.load_source("LoadClasses",
os.path.join(basedir, "LoadClasses.py"))
except ImportError:
print(
"ERROR: unable to load LoadClasses that must contain the class loader object"
)
exit(1)
cls = getattr(m, "LoadClasses")
a = cls()
# setup the loader object
self.loader = a.__class__()
# Setup the array of directories we will search for plugins
# Note that we always look at the topdir location by default
plugindirs = []
plugindirs.append(topdir)
if self.options['plugindir']:
# could be a comma-delimited list, so split on commas
x = self.options['plugindir'].split(',')
for y in x:
# prepend so we always look at the given
# location first in case the user wants
# to "overload/replace" a default MTT
# class definition
plugindirs.insert(0, y)
# Traverse the plugin directory tree and add all
# the class definitions we can find
for dirPath in plugindirs:
try:
filez = os.listdir(dirPath)
for file in filez:
file = os.path.join(dirPath, file)
if os.path.isdir(file):
self.loader.load(file)
except:
if not self.options['ignoreloadpatherrs']:
print("Plugin directory", dirPath, "not found")
sys.exit(1)
# Build the stages plugin manager
self.stages = PluginManager()
# set the location
self.stages.setPluginPlaces(plugindirs)
# Get a list of all the categories - this corresponds to
# the MTT stages that have been defined. Note that we
# don't need to formally define the stages here - anyone
# can add a new stage, or delete an old one, by simply
# adding or removing a plugin directory.
self.stages.setCategoriesFilter(self.loader.stages)
# Load all plugins we find there
self.stages.collectPlugins()
# Build the tools plugin manager - tools differ from sections
# in that they are plugins we will use to execute the various
# sections. For example, the TestRun section clearly needs the
# ability to launch jobs. There are many ways to launch jobs
# depending on the environment, and sometimes several ways to
# start jobs even within one environment (e.g., mpirun vs
# direct launch).
self.tools = PluginManager()
# location is the same
self.tools.setPluginPlaces(plugindirs)
# Get the list of tools - not every tool will be capable
# of executing. For example, a tool that supports direct launch
# against a specific resource manager cannot be used on a
# system being managed by a different RM.
self.tools.setCategoriesFilter(self.loader.tools)
# Load all the tool plugins
self.tools.collectPlugins()
# Tool plugins are required to provide a function we can
# probe to determine if they are capable of operating - check
# those now and prune those tools that cannot support this
# environment
# Build the utilities plugins
self.utilities = PluginManager()
# set the location
self.utilities.setPluginPlaces(plugindirs)
# Get the list of available utilities.
self.utilities.setCategoriesFilter(self.loader.utilities)
# Load all the utility plugins
self.utilities.collectPlugins()
# since we use these all over the place, find the
# ExecuteCmd and ModuleCmd plugins and record them
availUtil = list(self.loader.utilities.keys())
for util in availUtil:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if "ExecuteCmd" == pluginInfo.plugin_object.print_name():
self.execmd = pluginInfo.plugin_object
elif "ModuleCmd" == pluginInfo.plugin_object.print_name():
self.modcmd = pluginInfo.plugin_object
# initialize this module
self.modcmd.setCommand(self.options)
if self.execmd is not None and self.modcmd is not None:
break
if self.execmd is None:
print("ExecuteCmd plugin was not found")
print("This is a basic capability required")
print("for MTT operations - cannot continue")
sys.exit(1)
print(self.tools.getPluginsOfCategory("Harasser"))
for pluginInfo in self.tools.getPluginsOfCategory("Harasser"):
print(pluginInfo.plugin_object.print_name())
if "Harasser" == pluginInfo.plugin_object.print_name():
self.harasser = pluginInfo.plugin_object
break
if self.harasser is None:
print("Harasser plugin was not found")
print("This is required for all TestRun plugins")
print("cannot continue")
sys.exit(1)
# similarly, capture the highest priority defaults stage here
pri = -1
for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"):
if pri < pluginInfo.plugin_object.priority():
self.defaults = pluginInfo.plugin_object
pri = pluginInfo.plugin_object.priority()
return
def printInfo(self):
# Print the available MTT sections out, if requested
if self.options['listsections']:
print("Supported MTT stages:")
# print them in the default order of execution
for stage in self.loader.stageOrder:
print(" " + stage)
exit(0)
# Print the detected plugins for a given stage
if self.options['listplugins']:
# if the list is '*', print the plugins for every stage
if self.options['listplugins'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['listplugins'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(
section):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid stage name " + section)
print()
exit(1)
# Print the options for a given plugin
if self.options['liststageoptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['liststageoptions'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['liststageoptions'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(
section):
print(" " + pluginInfo.plugin_object.print_name() +
":")
pluginInfo.plugin_object.print_options(
self, " ")
except KeyError:
print(" Invalid stage name " + section)
print()
exit(1)
# Print the available MTT tools out, if requested
if self.options['listtools']:
print("Available MTT tools:")
availTools = list(self.loader.tools.keys())
for tool in availTools:
print(" " + tool)
exit(0)
# Print the detected tool plugins for a given tool type
if self.options['listtoolmodules']:
# if the list is '*', print the plugins for every type
if self.options['listtoolmodules'] == "*":
print()
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtoolmodules'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid tool type name", tool)
print()
exit(1)
# Print the options for a given plugin
if self.options['listtooloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listtooloptions'] == "*":
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtooloptions'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name() +
":")
pluginInfo.plugin_object.print_options(
self, " ")
except KeyError:
print(" Invalid tool type name " + tool)
print()
exit(1)
# Print the available MTT utilities out, if requested
if self.options['listutils']:
print("Available MTT utilities:")
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
print(" " + util)
exit(0)
# Print the detected utility plugins for a given tool type
if self.options['listutilmodules']:
# if the list is '*', print the plugins for every type
if self.options['listutilmodules'] == "*":
print()
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutilitymodules'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(
util):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid utility type name")
print()
exit(1)
# Print the options for a given plugin
if self.options['listutiloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listutiloptions'] == "*":
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutiloptions'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(
util):
print(" " + pluginInfo.plugin_object.print_name() +
":")
pluginInfo.plugin_object.print_options(
self, " ")
except KeyError:
print(" Invalid utility type name " + util)
print()
exit(1)
# if they asked for the version info, print it and exit
if self.options['version']:
for pluginInfo in self.tools.getPluginsOfCategory("Version"):
print("MTT Base: " + pluginInfo.plugin_object.getVersion())
print("MTT Client: " +
pluginInfo.plugin_object.getClientVersion())
sys.exit(0)
def openLogger(self):
# there must be a logger utility or we can't do
# anything useful
if not self.utilities.activatePluginByName("Logger", "Base"):
print("Required Logger plugin not found or could not be activated")
sys.exit(1)
# execute the provided test description
self.logger = self.utilities.getPluginByName("Logger",
"Base").plugin_object
self.logger.open(self)
return
def configTest(self):
# setup the configuration parser
self.config = configparser.SafeConfigParser(
interpolation=configparser.ExtendedInterpolation())
# Set the config parser to make option names case sensitive.
self.config.optionxform = str
# fill ENV section with environemt variables
self.config.add_section('ENV')
for k, v in os.environ.items():
self.config.set('ENV', k, v.replace("$", "$$"))
# log the list of files - note that the argument parser
# puts the input files in a list, with the first member
# being the list of input files
self.log['inifiles'] = self.args.ini_files[0]
# initialize the list of active sections
self.actives = []
# if they specified a list to execute, then use it
sections = []
if self.args.section:
sections = self.args.section.split(",")
skip = False
elif self.args.skipsections:
sections = self.args.skipsections.split(",")
skip = True
else:
sections = None
# cycle thru the input files
for testFile in self.log['inifiles']:
if not os.path.isfile(testFile):
print("Test description file", testFile, "not found!")
sys.exit(1)
self.config.read(self.log['inifiles'])
# Check for ENV input
required_env = []
all_file_contents = []
for testFile in self.log['inifiles']:
file_contents = open(testFile, "r").read()
file_contents = "\n".join([
"%s %d: %s" % (testFile.split("/")[-1], i, l)
for i, l in enumerate(file_contents.split("\n"))
if not l.lstrip().startswith("#")
])
all_file_contents.append(file_contents)
if "${ENV:" in file_contents:
required_env.extend([
s.split("}")[0] for s in file_contents.split("${ENV:")[1:]
])
env_not_found = set(
[e for e in required_env if e not in os.environ.keys()])
lines_with_env_not_found = []
for file_contents in all_file_contents:
lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \
for l in file_contents.split("\n") \
if sum(["${ENV:%s}"%e in l for e in env_not_found])])
if lines_with_env_not_found:
print("ERROR: Not all required environment variables are defined.")
print("ERROR: Still need:")
for l in lines_with_env_not_found:
print("ERROR: %s" % l)
sys.exit(1)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
# if we are to filter the sections, then do so
takeus = True
if sections is not None:
found = False
for sec in sections:
if sec == section:
found = True
sections.remove(sec)
if skip:
takeus = False
break
if not found and not skip:
takeus = False
if takeus:
self.actives.append(section)
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
if sections is not None and 0 != len(sections) and not skip:
print(
"ERROR: sections were specified for execution and not found:",
sections)
sys.exit(1)
return
# Used with combinatorial executor, loads next .ini file to be run with the
# sequential executor
def configNewTest(self, file):
# clear the configuration parser
for section in self.config.sections():
self.config.remove_section(section)
# read in the file
self.config.read(file)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
return
def executeTest(self):
if not self.loaded:
print("Plugins have not been loaded - cannot execute test")
exit(1)
if self.config is None:
print("No test definition file was parsed - cannot execute test")
exit(1)
if not self.tools.getPluginByName("sequential", "Executor"):
print("Specified executor sequential not found")
exit(1)
# activate the specified plugin
self.tools.activatePluginByName("sequential", "Executor")
# execute the provided test description
executor = self.tools.getPluginByName("sequential", "Executor")
status = executor.plugin_object.execute(self)
return status
def executeCombinatorial(self):
if not self.tools.getPluginByName("combinatorial", "Executor"):
print("Specified executor combinatorial not found")
exit(1)
# activate the specified plugin
self.tools.activatePluginByName("combinatorial", "Executor")
# execute the provided test description
executor = self.tools.getPluginByName("combinatorial", "Executor")
status = executor.plugin_object.execute(self)
return status
def printOptions(self, options):
# if the options are empty, report that
if not options:
lines = ["None"]
return lines
# create the list of options
opts = []
vals = list(options.keys())
for val in vals:
opts.append(val)
if options[val][0] is None:
opts.append("None")
elif isinstance(options[val][0], bool):
if options[val][0]:
opts.append("True")
else:
opts.append("False")
elif isinstance(options[val][0], list):
opts.append(" ".join(options[val][0]))
elif isinstance(options[val][0], int):
opts.append(str(options[val][0]))
else:
opts.append(options[val][0])
opts.append(options[val][1])
# print the options, their default value, and
# the help description in 3 column format
max1 = 0
max2 = 0
for i in range(0, len(opts), 3):
# we want all the columns to line up
# and left-justify, so first find out
# the max len of each of the first two
# column entries
if len(opts[i]) > max1:
max1 = len(opts[i])
if type(opts[i + 1]) is not str:
optout = str(opts[i + 1])
else:
optout = opts[i + 1]
if len(optout) > max2:
max2 = len(optout)
# provide some spacing
max1 = max1 + 4
max2 = max2 + 4
# cycle thru again, padding each entry to
# align the columns
lines = []
sp = " "
for i in range(0, len(opts), 3):
line = opts[i] + (max1 - len(opts[i])) * sp
if type(opts[i + 1]) is not str:
optout = str(opts[i + 1])
else:
optout = opts[i + 1]
line = line + optout + (max2 - len(optout)) * sp
# to make this more readable, we will wrap the line at
# 130 characters. First, see if the line is going to be
# too long
if 130 < (len(line) + len(opts[i + 2])):
# split the remaining column into individual words
words = opts[i + 2].split()
first = True
for word in words:
if (len(line) + len(word)) < 130:
if first:
line = line + word
first = False
else:
line = line + " " + word
else:
lines.append(line)
line = (max1 + max2) * sp + word
if 0 < len(line):
lines.append(line)
else:
# the line is fine - so just add the last piece
line = line + opts[i + 2]
# append the result
lines.append(line)
# add one blank line
lines.append("")
return lines
def selectPlugin(self, name, category):
if category == "stage":
try:
availStages = list(self.loader.stages.keys())
for stage in availStages:
for pluginInfo in self.stages.getPluginsOfCategory(stage):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "tool":
try:
availTools = list(self.loader.tools.keys())
for tool in availTools:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "utility":
try:
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
for pluginInfo in self.utilities.getPluginsOfCategory(
util):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
else:
print("Unrecognized category:", category)
return None
class SimpleTestsCase(unittest.TestCase): """ Test the correct loading of a simple plugin as well as basic commands. """ def setUp(self): """ init """ # create the plugin manager self.simplePluginManager = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # load the plugins that may be found self.simplePluginManager.collectPlugins() # Will be used later self.plugin_info = None def plugin_loading_check(self): """ Test if the correct plugin has been loaded. """ if self.plugin_info is None: # check nb of categories self.assertEqual(len(self.simplePluginManager.getCategories()),1) sole_category = self.simplePluginManager.getCategories()[0] # check the number of plugins self.assertEqual(len(self.simplePluginManager.getPluginsOfCategory(sole_category)),1) self.plugin_info = self.simplePluginManager.getPluginsOfCategory(sole_category)[0] # test that the name of the plugin has been correctly defined self.assertEqual(self.plugin_info.name,"Simple Plugin") self.assertEqual(sole_category,self.plugin_info.category) else: self.assert_(True) def testLoaded(self): """ Test if the correct plugin has been loaded. """ self.plugin_loading_check() def testGetAll(self): """ Test if the correct plugin has been loaded. """ self.plugin_loading_check() self.assertEqual(len(self.simplePluginManager.getAllPlugins()),1) self.assertEqual(self.simplePluginManager.getAllPlugins()[0],self.plugin_info) def testActivationAndDeactivation(self): """ Test if the activation procedure works. """ self.plugin_loading_check() self.assert_(not self.plugin_info.plugin_object.is_activated) self.simplePluginManager.activatePluginByName(self.plugin_info.name, self.plugin_info.category) self.assert_(self.plugin_info.plugin_object.is_activated) self.simplePluginManager.deactivatePluginByName(self.plugin_info.name, self.plugin_info.category) self.assert_(not self.plugin_info.plugin_object.is_activated)
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {
'listings': 'listings'
},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension
})
self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
nikola.utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category(
"CompilerExtension")
self.timeline = [FakePost(title='Fake post', slug='fake-post')]
self.debug = True
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(
plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return ('<img src="IMG.jpg">')
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data, *a, **kw):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data,
self.shortcode_registry,
**kw)
class WUnpack(QtWidgets.QDialog):
def __init__(self, parent, plugin):
super(WUnpack, self).__init__(parent)
self.parent = parent
self.plugin = plugin
self.oshow = super(WUnpack, self).show
root = os.path.dirname(sys.argv[0])
self.ui = PyQt5.uic.loadUi(os.path.join(root, 'unpack.ui'),
baseinstance=self)
self.ui.setWindowTitle('Decrypt/Encrypt')
self.manager = PluginManager(
categories_filter={"UnpackPlugin": DecryptPlugin})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'unpack')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
self.Plugins = {}
Formats = []
for plugin in self.manager.getPluginsOfCategory("UnpackPlugin"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.init(self.parent.dataModel, self.parent.viewMode):
self.Plugins[plugin.name] = po
#self.ui.horizontalLayout.addWidget(po.getUI())
print('[+] ' + plugin.name)
self.ui.listWidget.addItem(plugin.name)
#Formats.append(po)
self.ui.listWidget.currentItemChanged.connect(self.item_clicked)
self.ui.listWidget.setCurrentRow(0)
#self.ui.connect(self.ui.proceed, PyQt5.QtCore.SIGNAL("clicked()"), self.handleProceed)
self.ui.proceed.clicked.connect(lambda: self.handleProceed())
self.initUI()
def handleProceed(self):
item = str(self.ui.listWidget.currentItem().text())
self.Plugins[item].proceed()
#self.parent.update()
self.parent.viewMode.draw(refresh=True)
self.parent.update()
def item_clicked(self, current, previous):
#item = str(self.ui.listWidget.currentItem().text())
item = str(current.text())
if previous:
x = self.ui.horizontalLayout.takeAt(0)
while x:
x.widget().setParent(None)
x = self.ui.horizontalLayout.takeAt(0)
prev = str(previous.text())
#print prev
#self.ui.horizontalLayout.removeWidget(self.Plugins[prev].getUI())
if item:
#print item
po = self.Plugins[item]
self.ui.horizontalLayout.addWidget(po.getUI())
def show(self):
# TODO: remember position? resize plugin windows when parent resize?
pwidth = self.parent.parent.size().width()
pheight = self.parent.parent.size().height()
width = self.ui.size().width() + 15
height = self.ui.size().height() + 15
self.setGeometry(pwidth - width - 15, pheight - height, width, height)
self.setFixedSize(width, height)
self.oshow()
def initUI(self):
self.setSizePolicy(QtWidgets.QSizePolicy.Fixed,
QtWidgets.QSizePolicy.Fixed)
shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("F4"), self,
self.close, self.close)
#QtCore.QObject.connect(self.ui.ok, QtCore.SIGNAL('clicked()'), self.onClicked)
def onClicked(self):
dataModel = self.parent.dataModel
self.close()
class binWidget(QtWidgets.QWidget, Observable):
scrolled = QtCore.pyqtSignal(int, name='scroll')
# oldscrolled = QtWidgets.SIGNAL('scroll')
def __init__(self, parent, source):
super(binWidget, self).__init__()
Observable.__init__(self)
self.parent = parent
# offset for text window
#self.data = mapped
self.dataOffset = 0
self.dataModel = source
self.cursor = Cursor(0, 0)
# self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self),
# HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self)]
logging.basicConfig(level=logging.ERROR)
self.manager = PluginManager(
categories_filter={"FileFormat": FileFormat})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'format')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
Formats = []
for plugin in self.manager.getPluginsOfCategory("FileFormat"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.recognize(self.dataModel):
print('[+] ' + po.name)
Formats.append(po)
# sort plugins by priority
Formats = sorted(Formats, key=lambda x: x.priority, reverse=True)
po = Formats[0]
print('Choosed plugin: ' + po.name)
#print QtGui.QFontDatabase.addApplicationFont(os.path.join('terminus-ttf-4.39', 'TerminusTTF-4.39.ttf'))
self.multipleViewModes = [
BinViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po),
HexViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po),
DisasmViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po)
]
self.viewMode = self.multipleViewModes[0]
self.textDecorator = TextDecorator(self.viewMode)
self.viewMode.setTransformationEngine(self.textDecorator)
self.multipleViewModes[1].setTransformationEngine(self.textDecorator)
self.Banners = Banners()
#self.Banners.add(BottomBanner(self.dataModel, self.viewMode))
# self.Banners.add(TopBanner(self.dataModel, self.viewMode))
#self.Banners.add(self.banner)
# self.filebanner = FileAddrBanner(self.dataModel, self.viewMode)
#self.filebanner = PEBanner(self.dataModel, self.viewMode)
#self.Banners.add(PEBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
# self.offsetWindow_h = self.filebanner.getDesiredGeometry()[0] + 25
self.offsetWindow_h = 0
self.offsetWindow_v = 0
self.searchable = Searchable(self.dataModel, self.viewMode)
self.initUI()
[po.init(viewMode, parent=self) for viewMode in self.multipleViewModes]
for banner in po.getBanners():
self.Banners.add(banner)
po.registerShortcuts(self)
self.po = po
#self.scrolled = QtCore.pyqtSignal(int, name='scroll')
#self.scrolled.connect(self.scroll_from_outside)
self.searchWindow = SearchWindow(self, None, self.searchable)
self.addHandler(self.po)
self.addHandler(self.searchable)
self.addHandler(self.Banners)
self.notify(self.viewMode)
#self.connect(self, self.oldscrolled, self.scroll_from_outside)
#self.scrolled.emit(1)
#self.emit(QtCore.SIGNAL('scroll'), 1)
def scroll_from_outside(self, i):
#print 'slot-signal ' + str(i)
#self.scroll_pdown = True
self.update()
def initUI(self):
self.setMinimumSize(1, 30)
self.activateWindow()
self.setFocus()
#self.installEventFilter(self)
"""
# build thumbnail
dwidth = 100
dheight = 1200
factor = dheight/dwidth
import math
x = int(math.sqrt(len(self.data)/factor))
cols = x
pixThumb = QtGui.QPixmap(x*self.viewMode.fontWidth, factor*x*self.viewMode.fontHeight)
qp = QtGui.QPainter()
qp.begin(pixThumb)
qp.setFont(self.viewMode.font)
qp.setPen(self.viewMode.textPen)
for i,c in enumerate(self.data):
self.viewMode.transformText(qp, (i, c), self.data)
qp.drawText((i%cols)*self.viewMode.fontWidth, self.viewMode.fontHeight + (i/cols)*self.viewMode.fontHeight, c)
qp.end()
self.newqpix = pixThumb.scaled(dwidth, dheight, aspectRatioMode = QtCore.Qt.IgnoreAspectRatio, transformMode = QtCore.Qt.FastTransformation)
"""
"""
def getDisplayedData(self):
if self.dataOffset < 0:
self.dataOffset = 0
return False
chrlist = [unichr(cp437ToUnicode[ord(c)]) for c in self.data[self.dataOffset : self.dataOffset + self.viewMode.COLUMNS*self.viewMode.ROWS]]
self.text = "".join(chrlist)
return True
"""
def switchViewMode(self):
self.multipleViewModes = self.multipleViewModes[1:] + [
self.multipleViewModes[0]
]
self.viewMode = self.multipleViewModes[0]
# notify obervers
self.notify(self.viewMode)
def _resize(self):
self.Banners.resize(self.size().width() - self.offsetWindow_h,
self.size().height() - self.offsetWindow_v)
# compute space ocupated by banners
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getBottomOffset(
) + self.Banners.getTopOffset()
# resize window, substract space ocupated by banners
self.viewMode.resize(self.size().width() - offsetLeft,
self.size().height() - offsetBottom)
# event handlers
def resizeEvent(self, e):
self._resize()
def paintEvent(self, e):
qp = QtGui.QPainter()
qp.begin(self)
qp.setOpacity(1)
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getTopOffset()
#self.viewMode.draw2(qp, refresh=True)
#start = time()
qp.drawPixmap(offsetLeft, offsetBottom, self.viewMode.getPixmap())
#print 'Draw ' + str(time() - start)
self.Banners.draw(qp, self.offsetWindow_h, self.offsetWindow_v,
self.size().height())
# qp.drawPixmap(self.offsetWindow_h, self.size().height() - 50, self.banner.getPixmap())
# qp.drawPixmap(20, 0, self.filebanner.getPixmap())
qp.end()
"""
def keyPressEvent(self, event):
if event.modifiers() & QtCore.Qt.ShiftModifier:
if self.viewMode.handleKeyPressEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
def keyReleaseEvent(self, event):
if event.key() == QtCore.Qt.Key_Shift:
if self.viewMode.handleKeyReleaseEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
"""
def eventFilter(self, watched, event):
if event.type() == QtCore.QEvent.KeyRelease:
key = event.key()
modifiers = event.modifiers()
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
self.update()
if event.type() == QtCore.QEvent.KeyPress:
#TODO: should we accept only certain keys ?
key = event.key()
modifiers = event.modifiers()
if key == QtCore.Qt.Key_F2:
if self.viewMode.isEditable():
if self.viewMode.isInEditMode():
self.viewMode.setEditMode(False)
else:
self.viewMode.setEditMode(True)
self.viewMode.draw(refresh=False)
# switch view mode
if key == QtCore.Qt.Key_Tab:
offs = self.viewMode.getCursorOffsetInPage()
base = self.viewMode.getDataModel().getOffset()
self.switchViewMode()
self._resize()
self.viewMode.goTo(base + offs)
self.update()
import pyperclip
if event.modifiers() & QtCore.Qt.ControlModifier:
if key == QtCore.Qt.Key_Insert:
if self.viewMode.selector.getCurrentSelection():
a, b = self.viewMode.selector.getCurrentSelection()
#print a, b
hx = ''
for s in self.dataModel.getStream(a, b):
hx += '{:02x}'.format(s)
pyperclip.copy(hx)
del pyperclip
#print pyperclip.paste()
# print 'coppied'
if event.modifiers() & QtCore.Qt.ShiftModifier:
if key == QtCore.Qt.Key_Insert:
raise Exception("Not implemented")
"""
import re
hx = pyperclip.paste()
#print hx
L = re.findall(r'.{1,2}', hx, re.DOTALL)
array = ''
for s in L:
array += chr(int(s, 16))
#print 'write '
#print 'write'
#print array
self.dataModel.write(0, array)
self.viewMode.draw(True)
del pyperclip
#print array
"""
if key == QtCore.Qt.Key_F4:
self.unp = WUnpack(self, None)
self.unp.show()
if key == QtCore.Qt.Key_F10:
self.dataModel.flush()
import os
self.w = WHeaders(self, None)
self.w.show()
if not self.viewMode.isInEditMode():
if key == QtCore.Qt.Key_Slash:
self.searchWindow.show()
if key == QtCore.Qt.Key_N:
self.searchable.next(
self.viewMode.getCursorAbsolutePosition() + 1)
if key == QtCore.Qt.Key_B:
self.searchable.previous(
self.viewMode.getCursorAbsolutePosition())
# handle keys to view plugin
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
event.accept()
self.update()
return True
return False
def setTextViewport(self, qp):
qp.setViewport(self.offsetWindow_h, self.offsetWindow_v,
self.size().width(),
self.size().height())
qp.setWindow(0, 0, self.size().width(), self.size().height())
def needsSave(self):
return self.dataModel.isDirty()
def save(self):
return self.dataModel.flush()
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = [
'planetoid',
'ipynb',
'local_search',
'render_mustache',
]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ANALYTICS': '',
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'CONTENT_FOOTER': '',
'CREATE_MONTHLY_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'DISQUS_FORUM': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FILE_METADATA_REGEXP': None,
'FILES_FOLDERS': {'files': ''},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GZIP_FILES': False,
'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json'),
'HIDE_UNTRANSLATED_POSTS': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEX_PATH': '',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'post_compilers': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile',),
"txt2tags": ('.t2t',),
"bbcode": ('.bb',),
"wiki": ('.wiki',),
"ipynb": ('.ipynb',),
"html": ('.html', '.htm')
},
'POST_PAGES': (
("posts/*.txt", "posts", "post.tmpl", True),
("stories/*.txt", "stories", "story.tmpl", False),
),
'PRETTY_URLS': False,
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'site',
'THEME_REVEAL_CONGIF_SUBTHEME': 'sky',
'THEME_REVEAL_CONGIF_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': None,
}
self.config.update(config)
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
print("WARNING: You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML")
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
{self.config['DEFAULT_'
'LANG']: ''})
self.THEMES = utils.get_theme_chain(self.config['THEME'])
self.MESSAGES = utils.load_messages(self.THEMES,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'])
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
print("WARNING: You should configure SITE_URL instead of BLOG_URL")
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(__file__), utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
if plugin_info.name in self.config["post_compilers"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = {
}
self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
self.GLOBAL_CONTEXT['_link'] = self.link
self.GLOBAL_CONTEXT['set_locale'] = s_l
self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
self.GLOBAL_CONTEXT['exists'] = self.file_exists
self.GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config[
'SLUG_TAG_PATH']
self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
'ADD_THIS_BUTTONS']
self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self.GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self.GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self.GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self.GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self.GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
self.GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL', self.config.get('BLOG_URL'))
self.GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self.GLOBAL_CONTEXT['analytics'] = self.config.get('ANALYTICS')
self.GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self.GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self.GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self.GLOBAL_CONTEXT['disqus_forum'] = self.config.get('DISQUS_FORUM')
self.GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self.GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONGIF_SUBTHEME')
self.GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONGIF_TRANSITION')
self.GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self.GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self.GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self.GLOBAL_CONTEXT['sidebar_links'] = utils.Functionary(list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('SIDEBAR_LINKS', {}).items():
self.GLOBAL_CONTEXT['sidebar_links'][k] = v
self.GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self.GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA')
self.GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# check if custom css exist and is not empty
for files_path in list(self.config['FILES_FOLDERS'].keys()):
custom_css_path = os.path.join(files_path, 'assets/css/custom.css')
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT['has_custom_css'] = True
break
else:
self.GLOBAL_CONTEXT['has_custom_css'] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(
template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES]
self.template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config['USE_CDN']:
bootstrap_path = utils.get_asset_path(os.path.join(
'assets', 'css', 'bootstrap.min.css'), self.THEMES)
if bootstrap_path.split(os.sep)[-4] != 'site':
warnings.warn('The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.')
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in
list(self.config['post_compilers'].items())
if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(
template_name, None, local_context)
assert output_name.startswith(
self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def current_lang(self): # FIXME: this is duplicated, turn into a mixin
"""Return the currently set locale, if it's one of the
available translations, or default_lang."""
lang = utils.LocaleBorg().current_lang
if lang:
if lang in self.translations:
return lang
lang = lang.split('_')[0]
if lang in self.translations:
return lang
# whatever
return self.default_lang
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a post_pages tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = self.current_lang()
path = []
if kind == "tag_index":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'],
self.config['INDEX_FILE']] if _f]
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".html"] if
_f]
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".xml"] if
_f]
elif kind == "index":
if name not in [None, 0]:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
'index-{0}.html'.format(name)] if _f]
else:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
self.config['INDEX_FILE']]
if _f]
elif kind == "post_path":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
os.path.dirname(name),
self.config['INDEX_FILE']] if _f]
elif kind == "rss":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['RSS_PATH'], 'rss.xml'] if _f]
elif kind == "archive":
if name:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], name,
self.config['INDEX_FILE']] if _f]
else:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'],
self.config['ARCHIVE_FILENAME']] if _f]
elif kind == "gallery":
path = [_f for _f in [self.config['GALLERY_PATH'], name,
self.config['INDEX_FILE']] if _f]
elif kind == "listing":
path = [_f for _f in [self.config['LISTINGS_FOLDER'], name +
'.html'] if _f]
if is_link:
link = '/' + ('/'.join(path))
index_len = len(self.config['INDEX_FILE'])
if self.config['STRIP_INDEXES'] and \
link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BASE_URL'], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config['BASE_URL'], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self, name, plugin_category):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
yield task
for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield task
if flag:
task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': name,
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
print("Scanning posts", end='')
tzinfo = None
if self.config['TIMEZONE'] is not None:
tzinfo = pytz.timezone(self.config['TIMEZONE'])
current_time = utils.current_time(tzinfo)
targets = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='')
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination,
os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config['TRANSLATIONS'].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
for base_path in full_list:
post = Post(
base_path,
self.config['CACHE_FOLDER'],
dest_dir,
use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BASE_URL'],
self.MESSAGES,
template_name,
self.config['FILE_METADATA_REGEXP'],
self.config['STRIP_INDEXES'],
self.config['INDEX_FILE'],
tzinfo,
current_time,
self.config['HIDE_UNTRANSLATED_POSTS'],
self.config['PRETTY_URLS'],
)
for lang, langpath in list(
self.config['TRANSLATIONS'].items()):
dest = (destination, langpath, dir_glob,
post.meta[lang]['slug'])
if dest in targets:
raise Exception('Duplicated output path {0!r} '
'in post {1!r}'.format(
post.meta[lang]['slug'],
base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[
str(post.date.year)].append(post.post_name)
self.posts_per_month[
'{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post.post_name)
for tag in post.alltags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
if self.config['OLD_THEME_SUPPORT']:
post._add_old_metadata()
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
if post:
deps_dict['post_translations'] = post.translated_to
task = {
'name': os.path.normpath(output_name),
'file_dep': deps,
'targets': [output_name],
'actions': [(self.render_template, [post.template_name,
output_name, context])],
'clean': True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in
posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name': os.path.normpath(output_name),
'targets': [output_name],
'file_dep': deps,
'actions': [(self.render_template, [template_name, output_name,
context])],
'clean': True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
class WeaponSystem(rpyc.Service):
'''
RPC Services: This is the code that does the actual password cracking
and returns the results to orbital control. Currently only supports
cracking using rainbow tables (RCrackPy)
'''
is_initialized = False
mutex = Lock()
is_busy = False
job_id = None
def initialize(self):
''' Initializes variables, this should only be called once '''
logging.info("Weapon system initializing ...")
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["plugins/"])
self.plugin_manager.setCategoriesFilter(FILTERS)
self.plugin_manager.collectPlugins()
self.plugins = {}
logging.info(
"Loaded %d plugin(s)" % len(self.plugin_manager.getAllPlugins())
)
self.__cpu__()
logging.info("Weapon system online, good hunting.")
@atomic
def on_connect(self):
''' Called when successfully connected '''
if not self.is_initialized:
self.initialize()
self.is_initialized = True
logging.info("Uplink to orbital control active")
def on_disconnect(self):
''' Called if the connection is lost/disconnected '''
logging.info("Disconnected from orbital command server.")
def __cpu__(self):
''' Detects the number of CPU cores on a system (including virtual cores) '''
if cpu_count is not None:
try:
self.cpu_cores = cpu_count()
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except NotImplementedError:
logging.error("Could not detect number of processors; assuming 1")
self.cpu_cores = 1
else:
try:
self.cpu_cores = int(sysconf("SC_NPROCESSORS_CONF"))
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except ValueError:
logging.error("Could not detect number of processors; assuming 1")
self.cpu_cores = 1
############################ [ EXPOSED METHODS ] ############################
@atomic
def exposed_crack(self, plugin_name, job_id, hashes, **kwargs):
''' Exposes plugins calls '''
assert plugin_name in self.plugins
self.is_busy = True
self.job_id = job_id
self.plugin_manager.activatePluginByName(plugin_name)
plugin = self.plugin_manager.getPluginByName(plugin_name)
results = plugin.execute(hashes, **kwargs)
self.plugin_manager.deactivatePluginByName(plugin_name)
self.job_id = None
self.is_busy = False
return results
def exposed_get_plugin_names(self):
''' Returns what algorithms can be cracked '''
logging.info("Method called: exposed_get_capabilities")
plugins = self.plugin_manager.getAllPlugins()
return [plugin.name for plugin in plugins]
def exposed_get_categories(self):
''' Return categories for which we have plugins '''
categories = []
for category in self.plugin_manager.getCategories():
if 0 < len(self.plugin_manager.getPluginsOfCategory(category)):
categories.append(category)
return categories
def exposed_get_category_plugins(self, category):
''' Get plugin names for a category '''
plugins = self.plugin_manager.getPluginsOfCategory(category)
return [plugin.name for plugin in plugins]
def exposed_get_plugin_details(self, category, plugin_name):
''' Get plugin based on name details '''
plugin = self.plugin_manager.getPluginByName(plugin_name, category)
info = {'name': plugin.name}
info['author'] = plugin.details.get('Documentation', 'author')
info['website'] = plugin.details.get('Documentation', 'website')
info['version'] = plugin.details.get('Documentation', 'version')
info['description'] = plugin.details.get('Documentation', 'description')
info['copyright'] = plugin.details.get('Documentation', 'copyright')
info['precomputation'] = plugin.details.getboolean('Core', 'precomputation')
return info
def exposed_ping(self):
''' Returns a pong message '''
return "PONG"
def exposed_is_busy(self):
''' Returns True/False if the current system is busy (thread safe) '''
return self.is_busy
def exposed_current_job_id(self):
''' Returns the current job id (thread safe) '''
return self.job_id
def exposed_cpu_count(self):
''' Returns the number of detected cpu cores '''
return self.cpu_cores
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
"DISABLED_PLUGINS": [],
"EXTRA_PLUGINS": [],
"EXTRA_PLUGINS_DIRS": [extra_v6_plugin_dir],
"DEFAULT_LANG": "en",
"MARKDOWN_EXTENSIONS": ["fenced_code", "codehilite"],
"TRANSLATIONS_PATTERN": "{path}.{lang}.{ext}",
"LISTINGS_FOLDERS": {},
}
self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"]
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
}
)
self.loghandlers = [nikola.utils.STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension("plugin")
extra_plugins_dirs = self.config["EXTRA_PLUGINS_DIRS"]
if sys.version_info[0] == 3:
places = [os.path.join(os.path.dirname(nikola.utils.__file__), "plugins")] + [
path for path in extra_plugins_dirs if path
]
else:
places = [os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode("plugins"))] + [
nikola.utils.sys_encode(path) for path in extra_plugins_dirs if path
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension")
self.timeline = [FakePost(title="Fake post", slug="fake-post")]
self.debug = True
self.rst_transforms = []
# This is to make plugin initialization happy
self.template_system = self
self.name = "mako"
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get("DISABLED_PLUGINS"):
self.plugin_manager.removePluginFromCategory(plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return '<img src="IMG.jpg">'
class TestDef(object):
def __init__(self):
# set aside storage for options and cmd line args
self.options = {}
self.args = []
# record if we have loaded the plugins or
# not - this is just a bozo check to ensure
# someone doesn't tell us to do it twice
self.loaded = False
# set aside a spot for a logger object, and
# note that it hasn't yet been defined
self.logger = None
self.modcmd = None
self.execmd = None
self.harasser = None
self.config = None
self.stages = None
self.tools = None
self.utilities = None
self.defaults = None
self.log = {}
self.watchdog = None
self.plugin_trans_sem = Semaphore()
def setOptions(self, args):
self.options = vars(args)
self.args = args
# private function to convert values
def __convert_value(self, opt, inval):
if opt is None or type(opt) is str:
return 0, inval
elif type(opt) is bool:
if type(inval) is bool:
return 0, inval
elif type(inval) is str:
if inval.lower() in ['true', '1', 't', 'y', 'yes']:
return 0, True
else:
return 0, False
elif type(inval) is int:
if 0 == inval:
return 0, False
else:
return 0, True
elif is_py2 and type(inval) is unicode:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is int:
if type(inval) is int:
return 0, inval
elif type(inval) is str:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is float:
if type(inval) is float:
return 0, inval
elif type(inval) is str or type(inval) is int:
return 0, float(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
else:
return 1, None
# scan the key-value pairs obtained from the configuration
# parser and compare them with the options defined for a
# given plugin. Generate an output dictionary that contains
# the updated set of option values, the default value for
# any option that wasn't included in the configuration file,
# and return an error status plus output identifying any
# keys in the configuration file that are not supported
# by the list of options
#
# @log [INPUT]
# - a dictionary that will return the status plus
# stderr containing strings identifying any
# provided keyvals that don't have a corresponding
# supported option
# @options [INPUT]
# - a dictionary of tuples, each consisting of three
# entries:
# (a) the default value
# (b) data type
# (c) a help-like description
# @keyvals [INPUT]
# - a dictionary of key-value pairs obtained from
# the configuration parser
# @target [OUTPUT]
# - the resulting dictionary of key-value pairs
def parseOptions(self, log, options, keyvals, target):
# parse the incoming keyvals dictionary against the source
# options. If a source option isn't provided, then
# copy it across to the target.
opts = list(options.keys())
kvkeys = list(keyvals.keys())
for opt in opts:
found = False
for kvkey in kvkeys:
if kvkey == opt:
# they provided us with an update, so
# pass this value into the target - expand
# any provided lists
if keyvals[kvkey] is None:
continue
st, outval = self.__convert_value(options[opt][0], keyvals[kvkey])
if 0 == st:
target[opt] = outval
else:
if len(keyvals[kvkey]) == 0:
# this indicates they do not want this option
found = True
break
if keyvals[kvkey][0][0] == "[":
# they provided a list - remove the brackets
val = keyvals[kvkey].replace('[','')
val = val.replace(']','')
# split the input to pickup sets of options
newvals = list(val)
# convert the values to specified type
i=0
for val in newvals:
st, newvals[i] = self.__convert_value(opt[0], val)
i = i + 1
target[opt] = newvals
else:
st, target[opt] = self.__convert_value(opt[0], keyvals[kvkey])
found = True
break
if not found:
# they didn't provide this one, so
# transfer only the value across
target[opt] = options[opt][0]
# add in any default settings that have not
# been overridden - anything set by this input
# stage will override the default
if self.defaults is not None:
keys = self.defaults.options.keys()
for key in keys:
if key not in target:
target[key] = self.defaults.options[key][0]
# now go thru in the reverse direction to see
# if any keyvals they provided aren't supported
# as this would be an error
unsupported_options = []
for kvkey in kvkeys:
# ignore some standard keys
if kvkey in ['section', 'plugin']:
continue
try:
if target[kvkey] is not None:
pass
except KeyError:
# some always need to be passed
if kvkey in ['parent', 'asis']:
target[kvkey] = keyvals[kvkey]
else:
unsupported_options.append(kvkey)
if unsupported_options:
sys.exit("ERROR: Unsupported options for section [%s]: %s" % (log['section'], ",".join(unsupported_options)))
log['status'] = 0
log['options'] = target
return
def loadPlugins(self, basedir, topdir):
if self.loaded:
print("Cannot load plugins multiple times")
sys.exit(1)
self.loaded = True
# find the loader utility so we can bootstrap ourselves
try:
m = imp.load_source("LoadClasses", os.path.join(basedir, "LoadClasses.py"));
except ImportError:
print("ERROR: unable to load LoadClasses that must contain the class loader object")
sys.exit(1)
cls = getattr(m, "LoadClasses")
a = cls()
# setup the loader object
self.loader = a.__class__();
# Setup the array of directories we will search for plugins
# Note that we always look at the topdir location by default
plugindirs = []
plugindirs.append(topdir)
if self.options['plugindir']:
# could be a comma-delimited list, so split on commas
x = self.options['plugindir'].split(',')
for y in x:
# prepend so we always look at the given
# location first in case the user wants
# to "overload/replace" a default MTT
# class definition
plugindirs.insert(0, y)
# Load plugins from each of the specified plugin dirs
for dirPath in plugindirs:
if not Path(dirPath).exists():
print("Attempted to load plugins from non-existent path:", dirPath)
continue
try:
self.loader.load(dirPath)
except Exception as e:
print("Exception caught while loading plugins:")
print(e)
sys.exit(1)
# Build plugin managers,
# class yapsy.PluginManager.PluginManager(categories_filter=None,
# directories_list=None, plugin_info_ext=None, plugin_locator=None)
# Build the stages plugin manager
self.stages = PluginManager(None, plugindirs, None, None)
# Get a list of all the categories - this corresponds to
# the MTT stages that have been defined. Note that we
# don't need to formally define the stages here - anyone
# can add a new stage, or delete an old one, by simply
# adding or removing a plugin directory.
self.stages.setCategoriesFilter(self.loader.stages)
# Load all plugins we find there
self.stages.collectPlugins()
# Build the tools plugin manager - tools differ from sections
# in that they are plugins we will use to execute the various
# sections. For example, the TestRun section clearly needs the
# ability to launch jobs. There are many ways to launch jobs
# depending on the environment, and sometimes several ways to
# start jobs even within one environment (e.g., mpirun vs
# direct launch).
self.tools = PluginManager(None, plugindirs, None, None)
# Get the list of tools - not every tool will be capable
# of executing. For example, a tool that supports direct launch
# against a specific resource manager cannot be used on a
# system being managed by a different RM.
self.tools.setCategoriesFilter(self.loader.tools)
# Load all the tool plugins
self.tools.collectPlugins()
# Tool plugins are required to provide a function we can
# probe to determine if they are capable of operating - check
# those now and prune those tools that cannot support this
# environment
# Build the utilities plugins
self.utilities = PluginManager(None, plugindirs, None, None)
# Get the list of available utilities.
self.utilities.setCategoriesFilter(self.loader.utilities)
# Load all the utility plugins
self.utilities.collectPlugins()
# since we use these all over the place, find the
# ExecuteCmd and ModuleCmd plugins and record them
availUtil = list(self.loader.utilities.keys())
for util in availUtil:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if "ExecuteCmd" == pluginInfo.plugin_object.print_name():
self.execmd = pluginInfo.plugin_object
elif "ModuleCmd" == pluginInfo.plugin_object.print_name():
self.modcmd = pluginInfo.plugin_object
# initialize this module
self.modcmd.setCommand(self.options)
elif "Watchdog" == pluginInfo.plugin_object.print_name():
self.watchdog = pluginInfo.plugin_object
if self.execmd is not None and self.modcmd is not None and self.watchdog is not None:
break
if self.execmd is None:
print("ExecuteCmd plugin was not found")
print("This is a basic capability required")
print("for MTT operations - cannot continue")
sys.exit(1)
# Configure harasser plugin
for pluginInfo in self.tools.getPluginsOfCategory("Harasser"):
if "Harasser" == pluginInfo.plugin_object.print_name():
self.harasser = pluginInfo.plugin_object
break
if self.harasser is None:
print("Harasser plugin was not found")
print("This is required for all TestRun plugins")
print("cannot continue")
sys.exit(1)
# similarly, capture the highest priority defaults stage here
pri = -1
for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"):
if pri < pluginInfo.plugin_object.priority():
self.defaults = pluginInfo.plugin_object
pri = pluginInfo.plugin_object.priority()
return
def printInfo(self):
# Print the available MTT sections out, if requested
if self.options['listsections']:
print("Supported MTT stages:")
# print them in the default order of execution
for stage in self.loader.stageOrder:
print(" " + stage)
sys.exit(0)
# Print the detected plugins for a given stage
if self.options['listplugins']:
# if the list is '*', print the plugins for every stage
if self.options['listplugins'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['listplugins'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(section):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid stage name " + section)
print()
sys.exit(1)
# Print the options for a given plugin
if self.options['liststageoptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['liststageoptions'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['liststageoptions'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(section):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid stage name " + section)
print()
sys.exit(1)
# Print the available MTT tools out, if requested
if self.options['listtools']:
print("Available MTT tools:")
availTools = list(self.loader.tools.keys())
for tool in availTools:
print(" " + tool)
sys.exit(0)
# Print the detected tool plugins for a given tool type
if self.options['listtoolmodules']:
# if the list is '*', print the plugins for every type
if self.options['listtoolmodules'] == "*":
print()
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtoolmodules'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid tool type name",tool)
print()
sys.exit(1)
# Print the options for a given plugin
if self.options['listtooloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listtooloptions'] == "*":
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtooloptions'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid tool type name " + tool)
print()
sys.exit(1)
# Print the available MTT utilities out, if requested
if self.options['listutils']:
print("Available MTT utilities:")
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
print(" " + util)
sys.exit(0)
# Print the detected utility plugins for a given tool type
if self.options['listutilmodules']:
# if the list is '*', print the plugins for every type
if self.options['listutilmodules'] == "*":
print()
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutilitymodules'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid utility type name")
print()
sys.exit(1)
# Print the options for a given plugin
if self.options['listutiloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listutiloptions'] == "*":
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutiloptions'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid utility type name " + util)
print()
sys.exit(1)
# if they asked for the version info, print it and exit
if self.options['version']:
for pluginInfo in self.tools.getPluginsOfCategory("Version"):
print("MTT Base: " + pluginInfo.plugin_object.getVersion())
print("MTT Client: " + pluginInfo.plugin_object.getClientVersion())
sys.exit(0)
def openLogger(self):
# there must be a logger utility or we can't do
# anything useful
if not self.utilities.activatePluginByName("Logger", "Base"):
print("Required Logger plugin not found or could not be activated")
sys.exit(1)
# execute the provided test description
self.logger = self.utilities.getPluginByName("Logger", "Base").plugin_object
self.logger.open(self)
return
def fill_log_interpolation(self, basestr, sublog):
if isinstance(sublog, str):
self.config.set("LOG", basestr, sublog.replace("$","$$"))
elif isinstance(sublog, dict):
for k,v in sublog.items():
self.fill_log_interpolation("%s.%s" % (basestr, k), v)
elif isinstance(sublog, list):
if sum([((isinstance(t, list) or isinstance(t, tuple)) and len(t) == 2) for t in sublog]) == len(sublog):
self.fill_log_interpolation(basestr, {k:v for k,v in sublog})
else:
for i,v in enumerate(sublog):
self.fill_log_interpolation("%s.%d" % (basestr, i), v)
else:
self.fill_log_interpolation(basestr, str(sublog))
def expandWildCards(self, sections):
expsec = []
cpsections = list(sections)
for sec in cpsections:
if '*' in sec:
modsec = sec.split('*')
startswith = modsec[0]
endswith = modsec[-1]
findsec = modsec[1:-1]
allsections = self.config.sections()
for s in allsections:
if not s.startswith(startswith):
continue
if not s.endswith(endswith):
continue
found = True
s_tmp = s
for f in findsec:
if not f in s_tmp:
found = False
break
s_tmp = f.join(s_tmp.split(f)[1:])
if not found:
continue
expsec.append(s)
sections.remove(sec)
return sections + expsec
def fill_env_hidden_section(self):
"""fill ENV section with environment variables
"""
try:
self.config.add_section('ENV')
except configparser.DuplicateSectionError:
pass
for k,v in os.environ.items():
self.config.set('ENV', k, v.replace("$","$$"))
def fill_log_hidden_section(self):
"""Add LOG section filled with log results of stages
"""
try:
self.config.add_section('LOG')
except configparser.DuplicateSectionError:
pass
thefulllog = self.logger.getLog(None)
for e in thefulllog:
self.fill_log_interpolation(e['section'].replace(":","_"), e)
def check_for_nondefined_env_variables(self):
# Check for ENV input
required_env = []
all_file_contents = []
for testFile in self.log['inifiles']:
file_contents = open(testFile, "r").read()
file_contents = "\n".join(["%s %d: %s" % (testFile.split("/")[-1],i,l) for i,l in enumerate(file_contents.split("\n")) if not l.lstrip().startswith("#")])
all_file_contents.append(file_contents)
if "${ENV:" in file_contents:
required_env.extend([s.split("}")[0] for s in file_contents.split("${ENV:")[1:]])
env_not_found = set([e for e in required_env if e not in os.environ.keys()])
lines_with_env_not_found = []
for file_contents in all_file_contents:
lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \
for l in file_contents.split("\n") \
if sum(["${ENV:%s}"%e in l for e in env_not_found])])
if lines_with_env_not_found:
print("ERROR: Not all required environment variables are defined.")
print("ERROR: Still need:")
for l in lines_with_env_not_found:
print("ERROR: %s"%l)
sys.exit(1)
def configTest(self):
# setup the configuration parser
self.config = configparser.SafeConfigParser(interpolation=configparser.ExtendedInterpolation())
# Set the config parser to make option names case sensitive.
self.config.optionxform = str
# fill ENV section with environemt variables
self.fill_env_hidden_section()
# log the list of files - note that the argument parser
# puts the input files in a list, with the first member
# being the list of input files
self.log['inifiles'] = self.args.ini_files[0]
# initialize the list of active sections
self.actives = []
# if they specified a list to execute, then use it
sections = []
if self.args.section:
sections = self.args.section.split(",")
skip = False
elif self.args.skipsections:
sections = self.args.skipsections.split(",")
skip = True
else:
sections = None
# cycle thru the input files
for testFile in self.log['inifiles']:
if not os.path.isfile(testFile):
print("Test description file",testFile,"not found!")
sys.exit(1)
self.config.read(self.log['inifiles'])
# Check for ENV input
self.check_for_nondefined_env_variables()
# find all the sections that match the wild card and expand them
# this is simple wild carding, ie *text, text*, *text* and *
# should all work
if sections is not None:
sections = self.expandWildCards(sections)
#if sections is not None:
# expsec = []
# cpsections = list(sections)
# for sec in cpsections:
# if '*' in sec:
# modsec = sec.replace('*','')
# for s in self.config.sections():
# if modsec in s:
# expsec.append(s)
# sections.remove(sec)
# sections = sections + expsec
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
# if we are to filter the sections, then do so
takeus = True
if sections is not None:
found = False
for sec in sections:
if sec == section:
found = True
sections.remove(sec)
if skip:
takeus = False
break
if not found and not skip:
takeus = False
if takeus:
self.actives.append(section)
if sections is not None and 0 != len(sections) and not skip:
print("ERROR: sections were specified for execution and not found:",sections)
sys.exit(1)
# set Defaults -command line args supercede .ini args
try:
if not self.options['scratchdir']:
self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratchdir')
except:
try:
self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratch')
except:
self.options['scratchdir'] = os.path.abspath('./mttscratch')
self.options['scratchdir'] = os.path.abspath(self.options['scratchdir'])
try:
if not self.options['executor']:
self.options['executor'] = self.config.get('MTTDefaults', 'executor')
except:
self.options['executor'] = 'sequential'
# if they want us to clear the scratch, then do so
if self.options['clean'] and os.path.isdir(self.options['scratchdir']) :
shutil.rmtree(self.options['scratchdir'])
# setup the scratch directory
_mkdir_recursive(self.options['scratchdir'])
return
# Used with combinatorial executor, loads next .ini file to be run with the
# sequential executor
def configNewTest(self, file):
# clear the configuration parser
for section in self.config.sections():
self.config.remove_section(section)
# read in the file
self.config.read(file)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
return
def executeTest(self, executor="sequential"):
self.logger.print_cmdline_args(self)
if not self.loaded:
print("Plugins have not been loaded - cannot execute test")
sys.exit(1)
if self.config is None:
print("No test definition file was parsed - cannot execute test")
sys.exit(1)
if not self.tools.getPluginByName(executor, "Executor"):
print("Specified executor %s not found" % executor)
sys.exit(1)
# activate the specified plugin
self.tools.activatePluginByName(executor, "Executor")
# execute the provided test description
executor = self.tools.getPluginByName(executor, "Executor")
status = executor.plugin_object.execute(self)
if status == 0 and self.options['clean_after'] and os.path.isdir(self.options['scratchdir']):
self.logger.verbose_print("Cleaning up scratchdir after successful run")
shutil.rmtree(self.options['scratchdir'])
return status
def printOptions(self, options):
# if the options are empty, report that
if not options:
lines = ["None"]
return lines
# create the list of options
opts = []
vals = list(options.keys())
for val in vals:
opts.append(val)
if options[val][0] is None:
opts.append("None")
elif isinstance(options[val][0], bool):
if options[val][0]:
opts.append("True")
else:
opts.append("False")
elif isinstance(options[val][0], list):
opts.append(" ".join(options[val][0]))
elif isinstance(options[val][0], int):
opts.append(str(options[val][0]))
else:
opts.append(options[val][0])
opts.append(options[val][1])
# print the options, their default value, and
# the help description in 3 column format
max1 = 0
max2 = 0
for i in range(0,len(opts),3):
# we want all the columns to line up
# and left-justify, so first find out
# the max len of each of the first two
# column entries
if len(opts[i]) > max1:
max1 = len(opts[i])
if type(opts[i+1]) is not str:
optout = str(opts[i+1])
else:
optout = opts[i+1]
if len(optout) > max2:
max2 = len(optout)
# provide some spacing
max1 = max1 + 4
max2 = max2 + 4
# cycle thru again, padding each entry to
# align the columns
lines = []
sp = " "
for i in range(0,len(opts),3):
line = opts[i] + (max1-len(opts[i]))*sp
if type(opts[i+1]) is not str:
optout = str(opts[i+1])
else:
optout = opts[i+1]
line = line + optout + (max2-len(optout))*sp
# to make this more readable, we will wrap the line at
# 130 characters. First, see if the line is going to be
# too long
if 130 < (len(line) + len(opts[i+2])):
# split the remaining column into individual words
words = opts[i+2].split()
first = True
for word in words:
if (len(line) + len(word)) < 130:
if first:
line = line + word
first = False
else:
line = line + " " + word
else:
lines.append(line)
line = (max1 + max2)*sp + word
if 0 < len(line):
lines.append(line)
else:
# the line is fine - so just add the last piece
line = line + opts[i+2]
# append the result
lines.append(line)
# add one blank line
lines.append("")
return lines
def selectPlugin(self, name, category):
if category == "stage":
try:
availStages = list(self.loader.stages.keys())
for stage in availStages:
for pluginInfo in self.stages.getPluginsOfCategory(stage):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "tool":
try:
availTools = list(self.loader.tools.keys())
for tool in availTools:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "utility":
try:
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
else:
print("Unrecognized category:",category)
return None
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
# This is the default config
# TODO: fill it
self.config = {
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'DEFAULT_LANG': "en",
'OUTPUT_FOLDER': 'output',
'FILES_FOLDERS': {
'files': ''
},
'LISTINGS_FOLDER': 'listings',
'ADD_THIS_BUTTONS': True,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_TEASERS': False,
'MAX_IMAGE_SIZE': 1280,
'USE_FILENAME_AS_TITLE': True,
'SLUG_TAG_PATH': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'FILTERS': {},
'USE_BUNDLES': True,
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'default',
'post_compilers': {
"rest": ['.txt', '.rst'],
"markdown": ['.md', '.mdown', '.markdown'],
"html": ['.html', '.htm'],
},
}
self.config.update(config)
self.config['TRANSLATIONS'] = self.config.get(
'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''})
self.GLOBAL_CONTEXT = self.config.get('GLOBAL_CONTEXT', {})
self.THEMES = utils.get_theme_chain(self.config['THEME'])
self.MESSAGES = utils.load_messages(self.THEMES,
self.config['TRANSLATIONS'])
self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
self.GLOBAL_CONTEXT['_link'] = self.link
self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
self.GLOBAL_CONTEXT['exists'] = self.file_exists
self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
'ADD_THIS_BUTTONS']
self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
self.plugin_manager.setPluginPlaces([
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
])
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
pluginInfo.plugin_object.short_help = pluginInfo.description
self.commands[pluginInfo.name] = pluginInfo.plugin_object
# Activate all task plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name,
"TemplateSystem")
if pi is None:
sys.stderr.write("Error loading %s template system plugin\n" %
template_sys_name)
sys.exit(1)
self.template_system = pi.plugin_object
self.template_system.set_directories([
os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES
])
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for pluginInfo in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[pluginInfo.name] = \
pluginInfo.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [
lang for lang, exts in self.config['post_compilers'].items()
if ext in exts
]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of %s)" % ', '.join(langs))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to "
"handle '%s' extensions." % ext)
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.config['GLOBAL_CONTEXT'])
local_context.update(context)
data = self.template_system.render_template(template_name, None,
local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# This is to support windows paths
url_part = "/".join(url_part.split(os.sep))
src = urlparse.urljoin(self.config["BLOG_URL"], url_part)
parsed_src = urlparse.urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse.urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = '<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "w+") as post_file:
post_file.write(data)
def path(self, kind, name, lang, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
path = []
if kind == "tag_index":
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
'index.html'
])
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".html"
])
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".xml"
])
elif kind == "index":
if name > 0:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
'index-%s.html' % name
])
else:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], 'index.html'
])
elif kind == "rss":
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'],
'rss.xml'
])
elif kind == "archive":
if name:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], name, 'index.html'
])
else:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'],
self.config['ARCHIVE_FILENAME']
])
elif kind == "gallery":
path = filter(None,
[self.config['GALLERY_PATH'], name, 'index.html'])
elif kind == "listing":
path = filter(None,
[self.config['LISTINGS_FOLDER'], name + '.html'])
if is_link:
return '/' + ('/'.join(path))
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urlparse.urljoin(self.config['BLOG_URL'], dst)
return urlparse.urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urlparse.urljoin(self.config['BLOG_URL'], src)
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlparse.urlsplit(src)
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': 'all',
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if not self._scanned:
print "Scanning posts ",
targets = set([])
for wildcard, destination, _, use_in_feeds in \
self.config['post_pages']:
print ".",
for base_path in glob.glob(wildcard):
post = Post(base_path, destination, use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BLOG_URL'], self.MESSAGES)
for lang, langpath in self.config['TRANSLATIONS'].items():
dest = (destination, langpath, post.pagenames[lang])
if dest in targets:
raise Exception(
'Duplicated output path %r in post %r' %
(post.pagenames[lang], base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(
post.post_name)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
for name, post in self.global_data.items():
self.timeline.append(post)
self.timeline.sort(cmp=lambda a, b: cmp(a.date, b.date))
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print "done!"
def generic_page_renderer(self, lang, wildcard, template_name, destination,
filters):
"""Render post fragments to final HTML pages."""
for post in glob.glob(wildcard):
post_name = os.path.splitext(post)[0]
context = {}
post = self.global_data[post_name]
deps = post.deps(lang) + \
self.template_system.template_deps(template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
self.config['TRANSLATIONS'][lang],
destination,
post.pagenames[lang] + ".html")
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.config['GLOBAL_CONTEXT']
task = {
'name':
output_name.encode('utf-8'),
'file_dep':
deps,
'targets': [output_name],
'actions':
[(self.render_template, [template_name, output_name,
context])],
'clean':
True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.titles[lang], p.permalink(lang))
for p in posts]
deps_context["global"] = self.config['GLOBAL_CONTEXT']
task = {
'name':
output_name.encode('utf8'),
'targets': [output_name],
'file_dep':
deps,
'actions':
[(self.render_template, [template_name, output_name, context])],
'clean':
True,
'uptodate': [config_changed(deps_context)]
}
yield utils.apply_filters(task, filters)
class binWidget(QtGui.QWidget, Observable):
scrolled = QtCore.pyqtSignal(int, name='scroll')
oldscrolled = QtCore.SIGNAL('scroll')
def __init__(self, parent, source):
super(binWidget, self).__init__()
Observable.__init__(self)
self.parent = parent
# offset for text window
#self.data = mapped
self.dataOffset = 0
self.dataModel = source
self.cursor = Cursor(0, 0)
# self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self),
# HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self)]
logging.basicConfig(level=logging.ERROR)
self.manager = PluginManager(categories_filter={ "FileFormat": FileFormat})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'format')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
Formats = []
for plugin in self.manager.getPluginsOfCategory("FileFormat"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.recognize(self.dataModel):
print '[+] ' + po.name
Formats.append(po)
# sort plugins by priority
Formats = sorted(Formats, key=lambda x: x.priority, reverse=True)
po = Formats[0]
print 'Choosed plugin: ' + po.name
#print QtGui.QFontDatabase.addApplicationFont(os.path.join('terminus-ttf-4.39', 'TerminusTTF-4.39.ttf'))
self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po),
HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po),
DisasmViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po)]
self.viewMode = self.multipleViewModes[0]
self.textDecorator = TextDecorator(self.viewMode)
self.viewMode.setTransformationEngine(self.textDecorator)
self.multipleViewModes[1].setTransformationEngine(self.textDecorator)
self.Banners = Banners()
#self.Banners.add(BottomBanner(self.dataModel, self.viewMode))
# self.Banners.add(TopBanner(self.dataModel, self.viewMode))
#self.Banners.add(self.banner)
# self.filebanner = FileAddrBanner(self.dataModel, self.viewMode)
#self.filebanner = PEBanner(self.dataModel, self.viewMode)
#self.Banners.add(PEBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
# self.offsetWindow_h = self.filebanner.getDesiredGeometry()[0] + 25
self.offsetWindow_h = 0
self.offsetWindow_v = 0
self.searchable = Searchable(self.dataModel, self.viewMode)
self.initUI()
[po.init(viewMode, parent=self) for viewMode in self.multipleViewModes]
for banner in po.getBanners():
self.Banners.add(banner)
po.registerShortcuts(self)
self.po = po
#self.scrolled = QtCore.pyqtSignal(int, name='scroll')
#self.scrolled.connect(self.scroll_from_outside)
self.searchWindow = SearchWindow(self, None, self.searchable)
self.addHandler(self.po)
self.addHandler(self.searchable)
self.addHandler(self.Banners)
self.notify(self.viewMode)
#self.connect(self, self.oldscrolled, self.scroll_from_outside)
#self.scrolled.emit(1)
#self.emit(QtCore.SIGNAL('scroll'), 1)
def scroll_from_outside(self, i):
#print 'slot-signal ' + str(i)
#self.scroll_pdown = True
self.update()
def initUI(self):
self.setMinimumSize(1, 30)
self.activateWindow()
self.setFocus()
#self.installEventFilter(self)
"""
# build thumbnail
dwidth = 100
dheight = 1200
factor = dheight/dwidth
import math
x = int(math.sqrt(len(self.data)/factor))
cols = x
pixThumb = QtGui.QPixmap(x*self.viewMode.fontWidth, factor*x*self.viewMode.fontHeight)
qp = QtGui.QPainter()
qp.begin(pixThumb)
qp.setFont(self.viewMode.font)
qp.setPen(self.viewMode.textPen)
for i,c in enumerate(self.data):
self.viewMode.transformText(qp, (i, c), self.data)
qp.drawText((i%cols)*self.viewMode.fontWidth, self.viewMode.fontHeight + (i/cols)*self.viewMode.fontHeight, c)
qp.end()
self.newqpix = pixThumb.scaled(dwidth, dheight, aspectRatioMode = QtCore.Qt.IgnoreAspectRatio, transformMode = QtCore.Qt.FastTransformation)
"""
"""
def getDisplayedData(self):
if self.dataOffset < 0:
self.dataOffset = 0
return False
chrlist = [unichr(cp437ToUnicode[ord(c)]) for c in self.data[self.dataOffset : self.dataOffset + self.viewMode.COLUMNS*self.viewMode.ROWS]]
self.text = "".join(chrlist)
return True
"""
def switchViewMode(self):
self.multipleViewModes = self.multipleViewModes[1:] + [self.multipleViewModes[0]]
self.viewMode = self.multipleViewModes[0]
# notify obervers
self.notify(self.viewMode)
def _resize(self):
self.Banners.resize(self.size().width() - self.offsetWindow_h, self.size().height() - self.offsetWindow_v)
# compute space ocupated by banners
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getBottomOffset() + self.Banners.getTopOffset()
# resize window, substract space ocupated by banners
self.viewMode.resize(self.size().width() - offsetLeft, self.size().height() - offsetBottom)
# event handlers
def resizeEvent(self, e):
self._resize()
def paintEvent(self, e):
qp = QtGui.QPainter()
qp.begin(self)
qp.setOpacity(1)
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getTopOffset()
#self.viewMode.draw2(qp, refresh=True)
#start = time()
qp.drawPixmap(offsetLeft, offsetBottom, self.viewMode.getPixmap())
#print 'Draw ' + str(time() - start)
self.Banners.draw(qp, self.offsetWindow_h, self.offsetWindow_v, self.size().height())
# qp.drawPixmap(self.offsetWindow_h, self.size().height() - 50, self.banner.getPixmap())
# qp.drawPixmap(20, 0, self.filebanner.getPixmap())
qp.end()
"""
def keyPressEvent(self, event):
if event.modifiers() & QtCore.Qt.ShiftModifier:
if self.viewMode.handleKeyPressEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
def keyReleaseEvent(self, event):
if event.key() == QtCore.Qt.Key_Shift:
if self.viewMode.handleKeyReleaseEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
"""
def eventFilter(self, watched, event):
if event.type() == QtCore.QEvent.KeyRelease:
key = event.key()
modifiers = event.modifiers()
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
self.update()
if event.type() == QtCore.QEvent.KeyPress:
#TODO: should we accept only certain keys ?
key = event.key()
modifiers = event.modifiers()
if key == QtCore.Qt.Key_F2:
if self.viewMode.isEditable():
if self.viewMode.isInEditMode():
self.viewMode.setEditMode(False)
else:
self.viewMode.setEditMode(True)
self.viewMode.draw(refresh=False)
# switch view mode
if key == QtCore.Qt.Key_Tab:
offs = self.viewMode.getCursorOffsetInPage()
base = self.viewMode.getDataModel().getOffset()
self.switchViewMode()
self._resize()
self.viewMode.goTo(base + offs)
self.update()
import pyperclip
if event.modifiers() & QtCore.Qt.ControlModifier:
if key == QtCore.Qt.Key_Insert:
if self.viewMode.selector.getCurrentSelection():
a, b = self.viewMode.selector.getCurrentSelection()
#print a, b
hx = ''
for s in self.dataModel.getStream(a, b):
hx += '{:02x}'.format(s)
pyperclip.copy(hx)
del pyperclip
#print pyperclip.paste()
# print 'coppied'
if event.modifiers() & QtCore.Qt.ShiftModifier:
if key == QtCore.Qt.Key_Insert:
import re
hx = pyperclip.paste()
#print hx
L = re.findall(r'.{1,2}', hx, re.DOTALL)
array = ''
for s in L:
array += chr(int(s, 16))
#print 'write '
#print 'write'
#print array
self.dataModel.write(0, array)
self.viewMode.draw(True)
del pyperclip
#print array
if key == QtCore.Qt.Key_F4:
self.unp = WUnpack(self, None)
self.unp.show()
if key == QtCore.Qt.Key_F10:
self.dataModel.flush()
import os
self.w = WHeaders(self, None)
self.w.show()
if not self.viewMode.isInEditMode():
if key == QtCore.Qt.Key_Slash:
self.searchWindow.show()
if key == QtCore.Qt.Key_N:
self.searchable.next(self.viewMode.getCursorAbsolutePosition() + 1)
if key == QtCore.Qt.Key_B:
self.searchable.previous(self.viewMode.getCursorAbsolutePosition())
# handle keys to view plugin
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
event.accept()
self.update()
return True
return False
def setTextViewport(self, qp):
qp.setViewport(self.offsetWindow_h, self.offsetWindow_v, self.size().width(), self.size().height())
qp.setWindow(0, 0, self.size().width(), self.size().height())
def needsSave(self):
return self.dataModel.isDirty()
def save(self):
return self.dataModel.flush()
class WUnpack(QtGui.QDialog):
def __init__(self, parent, plugin):
super(WUnpack, self).__init__(parent)
self.parent = parent
self.plugin = plugin
self.oshow = super(WUnpack, self).show
root = os.path.dirname(sys.argv[0])
self.ui = PyQt4.uic.loadUi(os.path.join(root, 'unpack.ui'), baseinstance=self)
self.ui.setWindowTitle('Decrypt/Encrypt')
self.manager = PluginManager(categories_filter={ "UnpackPlugin": DecryptPlugin})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'unpack')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
self.Plugins = {}
Formats = []
for plugin in self.manager.getPluginsOfCategory("UnpackPlugin"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.init(self.parent.dataModel, self.parent.viewMode):
self.Plugins[plugin.name] = po
#self.ui.horizontalLayout.addWidget(po.getUI())
print '[+] ' + plugin.name
self.ui.listWidget.addItem(plugin.name)
#Formats.append(po)
self.ui.listWidget.currentItemChanged.connect(self.item_clicked)
self.ui.listWidget.setCurrentRow(0)
self.ui.connect(self.ui.proceed, PyQt4.QtCore.SIGNAL("clicked()"), self.handleProceed)
self.initUI()
def handleProceed(self):
item = str(self.ui.listWidget.currentItem().text())
self.Plugins[item].proceed()
#self.parent.update()
self.parent.viewMode.draw(refresh=True)
self.parent.update()
def item_clicked(self, current, previous):
#item = str(self.ui.listWidget.currentItem().text())
item = str(current.text())
if previous:
x = self.ui.horizontalLayout.takeAt(0)
while x:
x.widget().setParent(None)
x = self.ui.horizontalLayout.takeAt(0)
prev = str(previous.text())
#print prev
#self.ui.horizontalLayout.removeWidget(self.Plugins[prev].getUI())
if item:
#print item
po = self.Plugins[item]
self.ui.horizontalLayout.addWidget(po.getUI())
def show(self):
# TODO: remember position? resize plugin windows when parent resize?
pwidth = self.parent.parent.size().width()
pheight = self.parent.parent.size().height()
width = self.ui.size().width()+15
height = self.ui.size().height()+15
self.setGeometry(pwidth - width - 15, pheight - height, width, height)
self.setFixedSize(width, height)
self.oshow()
def initUI(self):
self.setSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
shortcut = QtGui.QShortcut(QtGui.QKeySequence("F4"), self, self.close, self.close)
#QtCore.QObject.connect(self.ui.ok, QtCore.SIGNAL('clicked()'), self.onClicked)
def onClicked(self):
dataModel = self.parent.dataModel
self.close()
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {'listings': 'listings'},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension
})
self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension")
self.timeline = [
FakePost(title='Fake post',
slug='fake-post')
]
self.debug = True
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return('<img src="IMG.jpg">')
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
# This is the default config
# TODO: fill it
self.config = {
"ARCHIVE_PATH": "",
"ARCHIVE_FILENAME": "archive.html",
"DEFAULT_LANG": "en",
"OUTPUT_FOLDER": "output",
"CACHE_FOLDER": "cache",
"FILES_FOLDERS": {"files": ""},
"LISTINGS_FOLDER": "listings",
"ADD_THIS_BUTTONS": True,
"INDEX_DISPLAY_POST_COUNT": 10,
"INDEX_TEASERS": False,
"RSS_TEASERS": True,
"MAX_IMAGE_SIZE": 1280,
"USE_FILENAME_AS_TITLE": True,
"SLUG_TAG_PATH": False,
"INDEXES_TITLE": "",
"INDEXES_PAGES": "",
"FILTERS": {},
"USE_BUNDLES": True,
"TAG_PAGES_ARE_INDEXES": False,
"THEME": "default",
"COMMENTS_IN_GALLERIES": False,
"COMMENTS_IN_STORIES": False,
"FILE_METADATA_REGEXP": None,
"post_compilers": {
"rest": [".txt", ".rst"],
"markdown": [".md", ".mdown", ".markdown"],
"html": [".html", ".htm"],
},
}
self.config.update(config)
self.config["TRANSLATIONS"] = self.config.get("TRANSLATIONS", {self.config["DEFAULT_LANG"]: ""})
self.THEMES = utils.get_theme_chain(self.config["THEME"])
self.MESSAGES = utils.load_messages(self.THEMES, self.config["TRANSLATIONS"])
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
}
)
self.plugin_manager.setPluginInfoExtension("plugin")
self.plugin_manager.setPluginPlaces(
[str(os.path.join(os.path.dirname(__file__), "plugins")), str(os.path.join(os.getcwd(), "plugins"))]
)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
pluginInfo.plugin_object.short_help = pluginInfo.description
self.commands[pluginInfo.name] = pluginInfo.plugin_object
# Activate all task plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = self.config.get("GLOBAL_CONTEXT", {})
self.GLOBAL_CONTEXT["messages"] = self.MESSAGES
self.GLOBAL_CONTEXT["_link"] = self.link
self.GLOBAL_CONTEXT["rel_link"] = self.rel_link
self.GLOBAL_CONTEXT["abs_link"] = self.abs_link
self.GLOBAL_CONTEXT["exists"] = self.file_exists
self.GLOBAL_CONTEXT["add_this_buttons"] = self.config["ADD_THIS_BUTTONS"]
self.GLOBAL_CONTEXT["index_display_post_count"] = self.config["INDEX_DISPLAY_POST_COUNT"]
self.GLOBAL_CONTEXT["use_bundles"] = self.config["USE_BUNDLES"]
if "date_format" not in self.GLOBAL_CONTEXT:
self.GLOBAL_CONTEXT["date_format"] = "%Y-%m-%d %H:%M"
# check if custom css exist and is not empty
for files_path in list(self.config["FILES_FOLDERS"].keys()):
custom_css_path = os.path.join(files_path, "assets/css/custom.css")
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT["has_custom_css"] = True
break
else:
self.GLOBAL_CONTEXT["has_custom_css"] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading %s template system plugin\n" % template_sys_name)
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES]
self.template_system.set_directories(lookup_dirs, self.config["CACHE_FOLDER"])
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for pluginInfo in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
self.compilers[pluginInfo.name] = pluginInfo.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in list(self.config["post_compilers"].items()) if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit(
"Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of %s)" % ", ".join(langs)
)
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to " "handle '%s' extensions." % ext)
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.config["GLOBAL_CONTEXT"])
local_context.update(context)
data = self.template_system.render_template(template_name, None, local_context)
assert isinstance(output_name, bytes)
assert output_name.startswith(self.config["OUTPUT_FOLDER"].encode("utf8"))
url_part = output_name.decode("utf8")[len(self.config["OUTPUT_FOLDER"]) + 1 :]
# This is to support windows paths
url_part = "/".join(url_part.split(os.sep))
src = urljoin(self.config["BLOG_URL"], url_part)
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split("/")[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == "link": # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip("/"), context["lang"])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b"<!DOCTYPE html>" + lxml.html.tostring(doc, encoding="utf8")
with open(output_name, "wb+") as post_file:
post_file.write(data)
def path(self, kind, name, lang, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
path = []
if kind == "tag_index":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], "index.html"] if _f]
elif kind == "tag":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".html"] if _f]
elif kind == "tag_rss":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".xml"] if _f]
elif kind == "index":
if name > 0:
path = [
_f
for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index-%s.html" % name]
if _f
]
else:
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index.html"] if _f]
elif kind == "rss":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["RSS_PATH"], "rss.xml"] if _f]
elif kind == "archive":
if name:
path = [
_f
for _f in [self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], name, "index.html"]
if _f
]
else:
path = [
_f
for _f in [
self.config["TRANSLATIONS"][lang],
self.config["ARCHIVE_PATH"],
self.config["ARCHIVE_FILENAME"],
]
if _f
]
elif kind == "gallery":
path = [_f for _f in [self.config["GALLERY_PATH"], name, "index.html"] if _f]
elif kind == "listing":
path = [_f for _f in [self.config["LISTINGS_FOLDER"], name + ".html"] if _f]
if is_link:
return "/" + ("/".join(path))
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config["BLOG_URL"], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config["BLOG_URL"], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split("/")[1:]
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {"name": b"all", "actions": None, "clean": True, "task_dep": task_dep}
def scan_posts(self):
"""Scan all the posts."""
if not self._scanned:
print("Scanning posts", end="")
targets = set([])
for wildcard, destination, template_name, use_in_feeds in self.config["post_pages"]:
print(".", end="")
base_len = len(destination.split(os.sep))
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.join(*([destination] + dirpath.split(os.sep)[base_len:]))
for base_path in glob.glob(dir_glob):
post = Post(
base_path,
self.config["CACHE_FOLDER"],
dest_dir,
use_in_feeds,
self.config["TRANSLATIONS"],
self.config["DEFAULT_LANG"],
self.config["BLOG_URL"],
self.MESSAGES,
template_name,
self.config["FILE_METADATA_REGEXP"],
)
for lang, langpath in list(self.config["TRANSLATIONS"].items()):
dest = (destination, langpath, dir_glob, post.pagenames[lang])
if dest in targets:
raise Exception(
"Duplicated output path %r in post %r" % (post.pagenames[lang], base_path)
)
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(post.post_name)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + self.template_system.template_deps(post.template_name)
context["post"] = post
context["lang"] = lang
context["title"] = post.title(lang)
context["description"] = post.description(lang)
context["permalink"] = post.permalink(lang)
context["page_list"] = self.pages
if post.use_in_feeds:
context["enable_comments"] = True
else:
context["enable_comments"] = self.config["COMMENTS_IN_STORIES"]
output_name = os.path.join(self.config["OUTPUT_FOLDER"], post.destination_path(lang)).encode("utf8")
deps_dict = copy(context)
deps_dict.pop("post")
if post.prev_post:
deps_dict["PREV_LINK"] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict["NEXT_LINK"] = [post.next_post.permalink(lang)]
deps_dict["OUTPUT_FOLDER"] = self.config["OUTPUT_FOLDER"]
deps_dict["TRANSLATIONS"] = self.config["TRANSLATIONS"]
deps_dict["global"] = self.config["GLOBAL_CONTEXT"]
deps_dict["comments"] = context["enable_comments"]
task = {
"name": output_name,
"file_dep": deps,
"targets": [output_name],
"actions": [(self.render_template, [post.template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context):
"""Renders pages with lists of posts."""
# This is a name on disk, has to be bytes
assert isinstance(output_name, bytes)
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config["BLOG_TITLE"]
context["description"] = self.config["BLOG_DESCRIPTION"]
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.titles[lang], p.permalink(lang)) for p in posts]
deps_context["global"] = self.config["GLOBAL_CONTEXT"]
task = {
"name": output_name,
"targets": [output_name],
"file_dep": deps,
"actions": [(self.render_template, [template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_context)],
}
return utils.apply_filters(task, filters)
class ContentModelListener(ConnectionListener):
'''
classdocs
'''
def __init__(self, content_models, host='localhost', port=61613, user='', passcode='', fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' % {'host': host, 'port': port})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" % {'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.username = user
self.password = passcode
# Create plugin manager
self.manager = PluginManager(categories_filter = {"FedoraMicroService": FedoraMicroService})
plugin_path = os.path.dirname(__file__)
self.manager.setPluginPlaces([plugin_path + "/plugins"])
logging.debug("Plugin path: " + plugin_path + "/plugins")
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info("Loading plugin: %(name)s for content model %(cmodel)s." % {'name': plugin.plugin_object.name, 'cmodel': plugin.plugin_object.content_model})
plugin.plugin_object.config = config
if type(plugin.plugin_object.content_model) == types.StringType:
content_models = [plugin.plugin_object.content_model]
else:
content_models = plugin.plugin_object.content_model
for content_model in content_models:
if content_model in self.contentModels:
self.contentModels[content_model].append(plugin.plugin_object)
else:
self.contentModels[content_model] = [plugin.plugin_object]
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
#logging.debug("\r \r", end='')
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
logging.error("lost connection reconnect in %d sec..." % reconnect_wait)
signal.alarm(reconnect_wait)
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
try:
global TOPIC_PREFIX
self.__print_async('MESSAGE', headers, body)
pid = headers['pid']
dsid = headers['dsid']
obj = self.client.getObject(pid)
content_model = headers['destination'][len(TOPIC_PREFIX):]
if content_model in self.contentModels:
logging.info('Running rules for %(pid)s from %(cmodel)s.' % {'pid': obj.pid, 'cmodel': content_model} )
for plugin in self.contentModels[content_model]:
plugin.runRules(obj, dsid, body)
except FedoraConnectionException:
logging.warning('Object %s was not found.' % (pid))
except:
logging.error("an exception occurred: " + str(sys.exc_info()[0]))
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers = { 'message-id' : args[1]})
else:
self.conn.ack(headers = { 'message-id' : args[1]}, transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, correlation_id, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination, message=message, headers={'correlation-id': correlation_id})
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url, username = self.username, password = self.password)
self.client = FedoraClient(self.fc)
