def run():
logging.basicConfig(level=logging.INFO)
os.chdir('/home/mjolnir/git/PURIKURA')
# H A N D L E P L U G I N S
pm = PluginManager()
pm.setPluginPlaces(['./pyrikura/plugins'])
pm.collectPlugins()
for pi in pm.getAllPlugins():
logging.info('loading plugin %s', pi.name)
pm.activatePluginByName(pi.name)
brokers = {}
nodes = build()
head = nodes[0]
for node in nodes:
brokers[node] = node.load(pm)
for node, broker in brokers.items():
for other in node._listening:
broker.subscribe(brokers[other])
start = time.time()
last_time = 0
shots = 0
last_trigger = 0
for broker in itertools.cycle(brokers.values()):
broker.update()
def loadPlugins(root_folder, plugin_name, categories_filter):
from yapsy.PluginManager import PluginManager
plugin_dirs = []
plugin_dirs.append(configuration.fabfile_basedir + '/.fabalicious/plugins')
plugin_dirs.append(expanduser("~") + '/.fabalicious/plugins')
plugin_dirs.append(root_folder + '/plugins')
log.debug("Looking for %s-plugins in %s" % (plugin_name, ", ".join(plugin_dirs)))
manager = PluginManager()
manager.setPluginPlaces(plugin_dirs)
manager.setCategoriesFilter(categories_filter)
manager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in manager.getAllPlugins():
manager.activatePluginByName(pluginInfo.name)
result = {}
for plugin in manager.getAllPlugins():
if hasattr(plugin.plugin_object, 'aliases') and isinstance(plugin.plugin_object.aliases, list):
for alias in plugin.plugin_object.aliases:
result[alias] = plugin.plugin_object
elif hasattr(plugin.plugin_object, 'alias'):
result[plugin.plugin_object.alias] = plugin.plugin_object
else:
result[plugin.name] = plugin.plugin_object
return result
def execute(route, plugin, payload):
logging.basicConfig()
# Build the manager
simplePluginManager = PluginManager()
# Tell it the default place(s) where to find plugins
print route
simplePluginManager.setPluginPlaces(route)
# Load all plugins
simplePluginManager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in simplePluginManager.getAllPlugins():
simplePluginManager.activatePluginByName(pluginInfo.name)
# for pluginInfo in simplePluginManager.getAllPlugins():
# var1,var2 = pluginInfo.plugin_object.execute()
simplePluginManager.activatePluginByName(plugin)
pluginInfo = simplePluginManager.getPluginByName(plugin)
if pluginInfo is None:
current_app.logger.error(
"Lo sentimos pero no se ha podido cargar el plugin o el mismo no se ha encontrado"
)
raise Exception("No se ha podido cargar el plugin")
return pluginInfo.plugin_object.execute(payload)
def main():
# Read configuration
config = SafeConfigParser(
defaults = {'port':'15915',
'plugins_directory':'./plugins',
'plugins_enabled':'',
})
config.read(['./conf/cm15d.conf',
'/etc/cm15d.conf',
'/etc/cm15d/cm15d.conf',
'/etc/cm15d/conf.d/local.conf',
])
# Activate enabled plugins
plugins = PluginManager()
plugins.setPluginPlaces(config.get('cm15d', 'plugins_directory').split(','))
plugins.collectPlugins()
plugins_enabled = config.get('cm15d', 'plugins_enabled').split(',')
for plugin in plugins.getAllPlugins():
if plugin.name in plugins_enabled:
plugins.activatePluginByName(plugin.name)
print("Plugin %s enabled" % plugin.name)
# Start server
port = int(config.get('cm15d', 'port'))
endpoint = TCP4ServerEndpoint(reactor, port)
endpoint.listen(CM15DaemonFactory(plugins))
print("Server listening on port %s" % port)
reactor.run()
def create(message):
pluginManager = PluginManager()
pluginManager.setPluginPlaces(
[current_app.config["PLUGINS_PATH"] + os.sep + "validators"])
pluginManager.collectPlugins()
for pluginInfo in pluginManager.getAllPlugins():
pluginManager.activatePluginByName(pluginInfo.name)
pluginInfo = pluginManager.getPluginByName(message['name'])
if pluginInfo != None:
raise "Lo sentimos pero el nombre del plugin que ya esta siendo utilizado"
plugin_info_file = open(
current_app.config["PLUGINS_PATH"] + os.sep + "validators" + os.sep +
str(message["name"]) + ".yapsy-plugin", "w")
plugin_file = open(
current_app.config["PLUGINS_PATH"] + os.sep + "validators" + os.sep +
str(message["name"]) + ".py", "w")
plugin_info_file.write(
str(message["file_info"]).decode("base64", errors="strict"))
plugin_file.write(str(message["data"]).decode("base64"))
plugin_info_file.write(
str('Organization = ' + current_app.config['ORGANIZATION_CONTEXT_ID'] +
' - ' + current_app.config['ORGANIZATION_CONTEXT_DESC']))
plugin_info_file.close()
plugin_file.close()
def init_request_validators(gn_env: GNEnvironment) -> None:
from yapsy.PluginManager import PluginManager
logging.getLogger('yapsy').setLevel(
gn_env.config.get(ConfigKeys.LOG_LEVEL, logging.INFO))
plugin_manager = PluginManager()
plugin_manager.setPluginPlaces(['dino/validation/events'])
plugin_manager.collectPlugins()
for pluginInfo in plugin_manager.getAllPlugins():
plugin_manager.activatePluginByName(pluginInfo.name)
gn_env.event_validators[pluginInfo.name] = pluginInfo.plugin_object
validation = gn_env.config.get(ConfigKeys.VALIDATION, None)
if validation is None:
return
for key in validation.keys():
if key not in gn_env.event_validator_map:
gn_env.event_validator_map[key] = list()
plugins = validation[key].copy()
validation[key] = dict()
for plugin_info in plugins:
plugin_name = plugin_info.get('name')
validation[key][plugin_name] = plugin_info
try:
gn_env.event_validator_map[key].append(
gn_env.event_validators[plugin_name])
except KeyError:
raise KeyError('specified plugin "%s" does not exist' % key)
gn_env.config.set(ConfigKeys.VALIDATION, validation)
for pluginInfo in plugin_manager.getAllPlugins():
pluginInfo.plugin_object.setup(gn_env)
def main():
# Yapsy uses Python’s standard logging module to record most important events and especially plugin loading failures.
logging.basicConfig(level=logging.INFO)
logging.getLogger('yapsy').setLevel(logging.DEBUG) # Enable DEBUG logging during development
# Build the manager
simplePluginManager = PluginManager()
# Tell it the directories it should look in to find plugins
simplePluginManager.setPluginPlaces(['plugins'])
# Tell it the file extension to use for PluginInfo files (INI format)
simplePluginManager.setPluginInfoExtension('plug')
# Load all plugins
simplePluginManager.collectPlugins()
# Activate all loaded plugins (calls activate method)
for pluginInfo in simplePluginManager.getAllPlugins():
# pluginInfo is a plugin_info object, which is typically an instance of IPlugin
simplePluginManager.activatePluginByName(pluginInfo.name)
# Loop through the plugins and call a custom method to print their names
for plugin in simplePluginManager.getAllPlugins():
plugin.plugin_object.print_name()
def run():
logging.basicConfig(level=logging.INFO)
os.chdir('/home/mjolnir/git/PURIKURA')
# H A N D L E P L U G I N S
pm = PluginManager()
pm.setPluginPlaces(['./pyrikura/plugins'])
pm.collectPlugins()
for pi in pm.getAllPlugins():
logging.info('loading plugin %s', pi.name)
pm.activatePluginByName(pi.name)
brokers = {}
nodes = build()
head = nodes[0]
for node in nodes:
brokers[node] = node.load(pm)
for node, broker in brokers.items():
for other in node._listening:
broker.subscribe(brokers[other])
start = time.time()
last_time = 0
shots = 0
last_trigger = 0
for broker in itertools.cycle(brokers.values()):
broker.update()
def get_plugins():
plugin_analyzer = PluginFileAnalyzerWithInfoFile(name="torrentstausanalyzer", extensions="plugin-manifest")
plugin_locator = PluginFileLocator(analyzers=[plugin_analyzer])
extra_plugins_dir = config.getSettingsAsDict()["extra_plugins_dir"]
plugin_extras = os.path.join(get_config_dir(), extra_plugins_dir)
directories = [os.path.join(os.path.dirname(os.path.realpath(__file__)), "plugins")]
##
## Allow user defined plugins
## These plugins should be named as
## name.function.{py, plugin-manifest}
## Example:
## MyPlugin.onstart.py
## MyPlguin.onstart.plugin-manifest
##
if os.path.exists(plugin_extras):
directories.append(plugin_extras)
manager = PluginManager(directories_list=directories, plugin_locator=plugin_locator)
manager.collectPlugins()
plugins = manager.getAllPlugins()
# Activate all loaded plugins
for pluginInfo in plugins:
manager.activatePluginByName(pluginInfo.name)
return plugins
def main():
# Read configuration
config = SafeConfigParser(defaults={
'port': '15915',
'plugins_directory': './plugins',
'plugins_enabled': '',
})
config.read([
'./conf/cm15d.conf',
'/etc/cm15d.conf',
'/etc/cm15d/cm15d.conf',
'/etc/cm15d/conf.d/local.conf',
])
# Activate enabled plugins
plugins = PluginManager()
plugins.setPluginPlaces(
config.get('cm15d', 'plugins_directory').split(','))
plugins.collectPlugins()
plugins_enabled = config.get('cm15d', 'plugins_enabled').split(',')
for plugin in plugins.getAllPlugins():
if plugin.name in plugins_enabled:
plugins.activatePluginByName(plugin.name)
print("Plugin %s enabled" % plugin.name)
# Start server
port = int(config.get('cm15d', 'port'))
endpoint = TCP4ServerEndpoint(reactor, port)
endpoint.listen(CM15DaemonFactory(plugins))
print("Server listening on port %s" % port)
reactor.run()
class AlgorithmManager():
def __init__(self):
self._pluginManager = PluginManager()
self._pluginManager.setPluginPlaces(config.PLUGIN_DIR)
self._pluginManager.collectPlugins()
for pluginInfo in self._pluginManager.getAllPlugins():
self._pluginManager.activatePluginByName(pluginInfo.name)
def get_alg_names(self):
for plugin in self._pluginManager.getAllPlugins():
plugin.plugin_object.print_name()
def execute_calc(self, data):
run = []
if not self.__is_sequence(data):
raise Exception('Data is no array!')
for alg in self._pluginManager.getAllPlugins():
result, prop = alg.plugin_object.calc(data)
run.append({alg.name : (result, prop)})
return run
def __is_sequence(self, arg):
return (not hasattr(arg, "strip") and
hasattr(arg, "__getitem__") or
hasattr(arg, "__iter__"))
def loadPlugins(root_folder, plugin_name, categories_filter):
from yapsy.PluginManager import PluginManager
plugin_dirs = []
plugin_dirs.append(configuration.fabfile_basedir + '/.fabalicious/plugins')
plugin_dirs.append(expanduser("~") + '/.fabalicious/plugins')
plugin_dirs.append(root_folder + '/plugins')
log.debug("Looking for %s-plugins in %s" %
(plugin_name, ", ".join(plugin_dirs)))
manager = PluginManager()
manager.setPluginPlaces(plugin_dirs)
manager.setCategoriesFilter(categories_filter)
manager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in manager.getAllPlugins():
manager.activatePluginByName(pluginInfo.name)
result = {}
for plugin in manager.getAllPlugins():
if hasattr(plugin.plugin_object, 'aliases') and isinstance(
plugin.plugin_object.aliases, list):
for alias in plugin.plugin_object.aliases:
result[alias] = plugin.plugin_object
elif hasattr(plugin.plugin_object, 'alias'):
result[plugin.plugin_object.alias] = plugin.plugin_object
else:
result[plugin.name] = plugin.plugin_object
return result
class PluginHandler:
def __init__(self):
self._plugin_manager = PluginManager()
self._category_active = {"Modifier": False, "Analyzer": False, "Comparator": False}
self._plugin_from_category = {"Modifier": [], "Analyzer": [], "Comparator": []}
self._modifier_plugins = []
self._analyzer_plugins = []
self._collect_all_plugins()
def _collect_all_plugins(self):
self._plugin_manager.setPluginPlaces(
["hugin/analyze/modifier", "hugin/analyze/analyzer", "hugin/analyze/comparator"]
)
# setting filter categories for pluginmanager
self._plugin_manager.setCategoriesFilter(
{
# movie metadata provider
"Modifier": IModifier,
# movie metadata provider
"Comparator": IComparator,
# sub metadata provider
"Analyzer": IAnalyzer,
}
)
self._plugin_manager.collectPlugins()
def activate_plugins_by_category(self, category):
self._toggle_activate_plugins_by_category(category)
def deactivate_plugins_by_category(self, category):
self._toggle_activate_plugins_by_category(category)
def _toggle_activate_plugins_by_category(self, category):
plugins = self._plugin_manager.getPluginsOfCategory(category)
is_active = self._category_active[category]
for pluginInfo in plugins:
if is_active:
self._plugin_manager.deactivatePluginByName(name=pluginInfo.name, category=category)
self._plugin_from_category[category].remove(pluginInfo)
else:
self._plugin_manager.activatePluginByName(name=pluginInfo.name, category=category)
self._plugin_from_category[category].append(pluginInfo)
self._category_active[category] = not is_active
def get_plugins_from_category(self, category):
plugins = []
for plugin in self._plugin_from_category[category]:
plugin.plugin_object.name = plugin.name
plugin.plugin_object.description = plugin.description
plugins.append(plugin.plugin_object)
return plugins
def is_activated(self, category):
""" True if category is activated. """
return self._category_active[category]
def initialize_plugins(self):
self.__log.info("Collecting and loading plugins")
try:
plugin_manager = PluginManager()
# TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT)
plugin_manager.setCategoriesFilter({
CARTRIDGE_AGENT_PLUGIN:
ICartridgeAgentPlugin,
ARTIFACT_MGT_PLUGIN:
IArtifactManagementPlugin
})
plugin_manager.setPluginPlaces(
[self.__config.read_property(constants.PLUGINS_DIR)])
plugin_manager.collectPlugins()
# activate cartridge agent plugins
plugins = plugin_manager.getPluginsOfCategory(
CARTRIDGE_AGENT_PLUGIN)
grouped_plugins = {}
for plugin_info in plugins:
self.__log.debug("Found plugin [%s] at [%s]" %
(plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated plugin [%s]" % plugin_info.name)
mapped_events = plugin_info.description.split(",")
for mapped_event in mapped_events:
if mapped_event.strip() != "":
if grouped_plugins.get(mapped_event) is None:
grouped_plugins[mapped_event] = []
grouped_plugins[mapped_event].append(plugin_info)
# activate artifact management plugins
artifact_mgt_plugins = plugin_manager.getPluginsOfCategory(
ARTIFACT_MGT_PLUGIN)
for plugin_info in artifact_mgt_plugins:
self.__log.debug(
"Found artifact management plugin [%s] at [%s]" %
(plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated artifact management plugin [%s]" %
plugin_info.name)
return plugin_manager, grouped_plugins, artifact_mgt_plugins
except ParameterNotFoundException as e:
self.__log.exception(
"Could not load plugins. Plugins directory not set: %s" % e)
return None, None, None
except Exception as e:
self.__log.exception("Error while loading plugin: %s" % e)
return None, None, None
def __init__(self, manager=None, name=None):
"""
Create the plugin object
:param manager: Manager instance
:type manager: :class:`biomajmanager.manager.Manager`
:param name: Name of the plugin to load. [DEFAULT: load all plugins]
:type name: String
:raises SystemExit: If 'manager' arg is not given
:raises SystemExit: If 'PLUGINS' section not found in :py:data:`manager.properties`
:raises SystemExit: If 'plugins.dir' not set in :py:data:`manager.properties`
:raises SystemExit: If 'plugins.list' not set in :py:data:`manager.properties`
:raises SystemExit: If 'plugins.dir' does not exist
"""
self.pm = None
self.name = None
self.config = None
self.manager = None
if not manager:
Utils.error("'manager' is required")
self.manager = manager
self.config = self.manager.config
if not self.config.has_section('PLUGINS'):
Utils.error("Can't load plugins, no section found!")
if not self.config.has_option('MANAGER', 'plugins.dir'):
Utils.error("plugins.dir not defined!")
if not self.config.has_option('PLUGINS', 'plugins.list'):
Utils.error("plugins.list is not defined!")
if not os.path.isdir(self.config.get('MANAGER', 'plugins.dir')):
Utils.error("Can't find plugins.dir")
plugin_manager = PluginManager(directories_list=[self.config.get('MANAGER', 'plugins.dir')],
categories_filter={Plugins.CATEGORY: BMPlugin})
plugin_manager.collectPlugins()
self.pm = plugin_manager
self.name = name
user_plugins = []
# Load user wanted plugin(s)
for plugin in self.config.get('PLUGINS', 'plugins.list').split(','):
plugin.strip()
# We need to lower the plugin name
user_plugins.append(plugin)
# This means that all plugins must inherits from BMPlugin
for pluginInfo in plugin_manager.getPluginsOfCategory(Plugins.CATEGORY):
Utils.verbose("[manager] plugin name => %s" % pluginInfo.name)
if pluginInfo.name in user_plugins:
if not pluginInfo.is_activated:
Utils.verbose("[manager] plugin %s activated" % pluginInfo.name)
plugin_manager.activatePluginByName(pluginInfo.name)
setattr(self, pluginInfo.name, pluginInfo.plugin_object)
pluginInfo.plugin_object.set_config(self.config)
pluginInfo.plugin_object.set_manager(self.manager)
class MUCBot(sleekxmpp.ClientXMPP):
def __init__(self, jid, password, room, nick):
sleekxmpp.ClientXMPP.__init__(self, jid, password)
self.room = room
self.nick = nick
# Load plugins
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["plugins_enabled"])
self.plugin_manager.setCategoriesFilter({
"Message" : MessagePlugin,
"Presence": PresencePlugin
})
self.plugin_manager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in self.plugin_manager.getAllPlugins():
self.plugin_manager.activatePluginByName(pluginInfo.name)
self.add_event_handler("session_start", self.start)
self.add_event_handler("message", self.message)
self.add_event_handler("groupchat_message", self.muc_message)
self.add_event_handler("muc::%s::got_online" % self.room, self.muc_online)
self.add_event_handler("muc::%s::got_offline" % self.room, self.muc_offline)
def start(self, event):
self.get_roster()
self.send_presence()
if self.room and len(self.room) > 0:
self.plugin['xep_0045'].joinMUC(self.room,
self.nick,
wait=True)
def message(self, msg):
if msg['type'] in ('chat', 'normal'):
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"):
pluginInfo.plugin_object.message_received(msg)
def muc_message(self, msg):
if msg['mucnick'] != self.nick:
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Message"):
pluginInfo.plugin_object.message_received(msg, nick=self.nick)
def muc_online(self, presence):
if presence['muc']['nick'] != self.nick:
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"):
pluginInfo.plugin_object.got_online(self, presence)
def muc_offline(self, presence):
if presence['muc']['nick'] != self.nick:
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Presence"):
pluginInfo.plugin_object.got_offline(self, presence)
def load_plugins():
"""Loads plugins from the puglins folder"""
plugins = PluginManager()
plugins_folder = os.path.join(os.environ['LOOKDEVTOOLS'], 'python',
'ldtplugins')
print plugins_folder
plugins.setPluginPlaces([plugins_folder])
plugins.collectPlugins()
plugins.locatePlugins()
logger.info('Plugin candidates %s' % plugins.getPluginCandidates())
for pluginInfo in plugins.getAllPlugins():
plugins.activatePluginByName(pluginInfo.name)
logger.info('Plugin activated %s' %
plugins.activatePluginByName(pluginInfo.name))
return plugins
def main():
fps = 30 # FPS for the generated video(s)
set_size = 500 # Size for the sorting set
# Parse the arguments
parser = argparse.ArgumentParser()
parser.add_argument("-f", "--fps", help="The desired FPS of the video to be outputted.", type=int)
parser.add_argument("-s", "--size", help="The desired size of the sorting set.", type=int)
parser.add_argument("sorters", help="The sorters to run. Type 'all' to run all available sorters.")
args = parser.parse_args()
if args.fps:
fps = args.fps
if args.size:
set_size = args.size
# Get a manager
plugin_manager = PluginManager()
# Give it a folder to look in, sorters
plugin_manager.setPluginPlaces(["sorters"])
# Load 'em
plugin_manager.collectPlugins()
# TODO: Add flag for how many numbers it should sort
# Activate 'em and run 'em
for plugin in plugin_manager.getAllPlugins():
if "all" not in sys.argv:
if plugin.name not in sys.argv:
continue
plugin_manager.activatePluginByName(plugin.name)
folder_name = plugin.name
temp, _ = os.path.split(os.path.abspath(__file__))
folder_path = os.path.join(temp, folder_name)
try:
os.makedirs(folder_path)
except:
shutil.rmtree(folder_name) # Remove the folder, we're going to remake it
os.makedirs(folder_path)
a = range(set_size)
shuffle(a)
plot = Plotter()
plugin.plugin_object.prepare(a, partial(plot.plot, folder_path))
plugin.plugin_object.sort()
create_movie(folder_name, plugin.plugin_object.counter, fps)
class fbbot(fbchat.Client):
pluginManager = None
user_keyword_vector = {}
admin_keyword_vector = {}
def __init__(self, email, password, debug=True, user_agent=None):
fbchat.Client.__init__(self, email, password, debug, user_agent)
self.pluginManager = PluginManager()
self.pluginManager.setPluginPlaces(['plugins'])
self.pluginManager.collectPlugins()
#Register all the keywords and the name of their respective plugin
for plugin in self.pluginManager.getAllPlugins():
try:
self.pluginManager.activatePluginByName(plugin.name)
for key in plugin.plugin_object.user_keyword_vector.keys():
self.user_keyword_vector[key] = plugin.name
for key in plugin.plugin_object.admin_keyword_vector.keys():
self.admin_keyword_vector[key] = plugin.name
except Exception as ex:
print(ex)
def on_message(self, mid, author_id, author_name, message, metadata):
self.markAsDelivered(author_id, mid)
self.markAsRead(author_id)
#Check for self author
if str(author_id) != str(self.uid):
#trigger word
if message.startswith("/bot"):
words = message.split()
if words[1] in self.user_keyword_vector.keys():
plugin = self.pluginManager.getPluginByName(
self.user_keyword_vector[words[1]])
response = plugin.plugin_object.user_keyword_vector[
words[1]](words)
else:
response = "Unrecognized Command"
#Send the response
sent = self.send(author_id, response)
if sent:
print("Sent", response, "to", author_id)
else:
print("Failed", response, "to", author_id)
def initialize_plugins(self):
self.__log.info("Collecting and loading plugins")
try:
plugin_manager = PluginManager()
# TODO: change plugin descriptor extensions, plugin_manager.setPluginInfoExtension(AGENT_PLUGIN_EXT)
plugin_manager.setCategoriesFilter({
CARTRIDGE_AGENT_PLUGIN: ICartridgeAgentPlugin,
ARTIFACT_MGT_PLUGIN: IArtifactManagementPlugin
})
plugin_manager.setPluginPlaces([self.__config.read_property(constants.PLUGINS_DIR)])
plugin_manager.collectPlugins()
# activate cartridge agent plugins
plugins = plugin_manager.getPluginsOfCategory(CARTRIDGE_AGENT_PLUGIN)
grouped_plugins = {}
for plugin_info in plugins:
self.__log.debug("Found plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated plugin [%s]" % plugin_info.name)
mapped_events = plugin_info.description.split(",")
for mapped_event in mapped_events:
if mapped_event.strip() != "":
if grouped_plugins.get(mapped_event) is None:
grouped_plugins[mapped_event] = []
grouped_plugins[mapped_event].append(plugin_info)
# activate artifact management plugins
artifact_mgt_plugins = plugin_manager.getPluginsOfCategory(ARTIFACT_MGT_PLUGIN)
for plugin_info in artifact_mgt_plugins:
self.__log.debug("Found artifact management plugin [%s] at [%s]" % (plugin_info.name, plugin_info.path))
plugin_manager.activatePluginByName(plugin_info.name)
self.__log.info("Activated artifact management plugin [%s]" % plugin_info.name)
return plugin_manager, grouped_plugins, artifact_mgt_plugins
except ParameterNotFoundException as e:
self.__log.exception("Could not load plugins. Plugins directory not set: %s" % e)
return None, None, None
except Exception as e:
self.__log.exception("Error while loading plugin: %s" % e)
return None, None, None
def _tryLoad(self, schema_name, num_colors):
plugin_manager = PluginManager()
plugin_manager.getPluginLocator().setPluginInfoExtension("ini")
plugin_manager.setPluginPlaces([paths.pluginsDir("user", "syntaxcolors"), paths.pluginsDir("system", "syntaxcolors")])
plugin_manager.collectPlugins()
for plugin_info in plugin_manager.getAllPlugins():
# Useless, but let's activate them
plugin_manager.activatePluginByName(plugin_info.name)
plugin_object = plugin_info.plugin_object
if plugin_object.name() == schema_name and plugin_object.supportsNumColors(num_colors):
self._color_map.update(_parseColorSchema(plugin_object.colorSchema(num_colors)))
return True
return False
class FakeNikola(object):
def __init__(self):
self.config = {'DISABLED_PLUGINS': []}
self.debug = False
self.loghandlers = []
self.timeline = []
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
"SignalHandler": SignalHandler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = ''
places = [
resource_filename('nikola', utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
os.path.expanduser('~/.nikola/plugins'),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
#self.pug = None
#for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
# if plugin_info.name == 'rest':
# self.plugin_manager.activatePluginByName(plugin_info.name)
# plugin_info.plugin_object.set_site(self)
# self.pug = plugin_info
# Now we have our pug
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"RestExtension"):
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
def main(target_dir=None):
if not target_dir:
target_dir = os.path.abspath(os.curdir)
user_config_dir = os.path.expanduser('~/.grumpy')
user_config_path = os.path.join(user_config_dir, 'conf')
user_plugin_base_dir = os.path.join(user_config_dir, 'plugins')
user_plugin_dirs = None
try:
user_plugin_dirs = [os.path.join(user_plugin_base_dir, plugin_dir)
for plugin_dir in os.listdir(user_plugin_base_dir)]
except FileNotFoundError:
pass
default_plugin_base_dir = os.path.join(os.path.dirname(__file__), "plugins")
default_plugin_dirs = [os.path.join(default_plugin_base_dir, plugin_dir)
for plugin_dir in os.listdir(default_plugin_base_dir)]
plugin_dirs = user_plugin_dirs + default_plugin_dirs
print(plugin_dirs)
conf = Configuration.parse_file(user_config_path)
# Build the manager
simplePluginManager = PluginManager()
# Tell it the default place(s) where to find plugins
simplePluginManager.setPluginPlaces(plugin_dirs)
# Load all plugins
simplePluginManager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in simplePluginManager.getAllPlugins():
simplePluginManager.activatePluginByName(pluginInfo.name)
# pipeline = [simplePluginManager.getPluginByName(name).plugin_object.run for name in parse_pipeline(conf.get("Pipeline", "Default"))]
# output = 0
# output = reduce(lambda x,y: y(x), pipeline, output)
# print(next(output))
print(os.path.dirname(__file__))
class FakeNikola(object):
def __init__(self):
self.config = {'DISABLED_PLUGINS': []}
self.debug = False
self.loghandlers = []
self.timeline = []
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
"SignalHandler": SignalHandler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = ''
places = [
resource_filename('nikola', utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
os.path.expanduser('~/.nikola/plugins'),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
#self.pug = None
#for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
# if plugin_info.name == 'rest':
# self.plugin_manager.activatePluginByName(plugin_info.name)
# plugin_info.plugin_object.set_site(self)
# self.pug = plugin_info
# Now we have our pug
for plugin_info in self.plugin_manager.getPluginsOfCategory("RestExtension"):
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
class App(QtGui.QApplication):
def __init__(self, argv):
super(App, self).__init__(argv)
parser = optparse.OptionParser()
parser.add_option('-c', '--config', help='Path to a configuration', default='/etc/bonehead.cfg')
(options, args) = parser.parse_args()
logging.basicConfig(level=logging.DEBUG)
self._conf = ConfigParser()
self._conf.read(options.config)
self._plugins = PluginManager()
self._plugins.setPluginPlaces([
'/usr/lib/bonehead/pages',
'./pages/'
])
self._plugins.collectPlugins()
for plugin in self._plugins.getAllPlugins():
self._plugins.activatePluginByName(plugin.name)
self.__ui = KioskUI()
pages = self._conf.get('general', 'pages', []).split(',')
for pageName in pages:
pageConfig = {}
for k,v in self._conf.items("page:%s"%(pageName)):
pageConfig[k] = v
pagePlugin = self._plugins.getPluginByName(pageConfig['plugin'])
page = pagePlugin.plugin_object.newPage(pageName, pageConfig, self.__ui)
self.__ui.addPage(page)
self.__ui.showFullScreen()
def notify(self, object, event):
if event.type() == QtCore.QEvent.KeyPress or event.type() == QtCore.QEvent.MouseButtonPress:
self.__ui.resetTimeout()
return super(App, self).notify(object, event)
def save(message):
pluginManager = PluginManager()
pluginManager.setPluginPlaces(
[current_app.config["PLUGINS_PATH"] + os.sep + "validators"])
pluginManager.collectPlugins()
for pluginInfo in pluginManager.getAllPlugins():
pluginManager.activatePluginByName(pluginInfo.name)
pluginInfo = pluginManager.getPluginByName(message['name'])
plugin_info_file = open(
current_app.config["PLUGINS_PATH"] + os.sep + "validators" + os.sep +
str(message["name"]).replace(" ", "") + ".yapsy-plugin", "w")
plugin_file = open(
current_app.config["PLUGINS_PATH"] + os.sep + "validators" + os.sep +
str(message["name"]).replace(" ", "") + ".py", "w")
plugin_info_file.write(
str(message["file_info"]).decode("base64", errors="strict"))
plugin_file.write(str(message["data"]).decode("base64"))
plugin_info_file.close()
plugin_file.close()
class CM15DaemonFactory(Factory):
def buildProtocol(self, addr):
return CM15Daemon(self)
def startFactory(self):
self.loadPlugins()
self.cm15 = cm15.CM15()
self.cm15.open()
for plugin in self.pluginManager.getAllPlugins():
self.cm15.subscribeToEvent("dataReceived", plugin.plugin_object.cm15DataReceivedHandler)
self.cm15.subscribeToEvent("dataWritten", plugin.plugin_object.cm15DataWrittenHandler)
self.cm15.startListening()
def stopFactory(self):
self.cm15.stopListening()
self.cm15.close()
def loadPlugins(self):
self.pluginManager = PluginManager()
self.pluginManager.setPluginPlaces(["./plugins"])
self.pluginManager.collectPlugins()
for plugin in self.pluginManager.getAllPlugins():
self.pluginManager.activatePluginByName(plugin.name)
class ServerPluginManager():
def __init__(self):
self.plugin_manager = PluginManager(plugin_locator=PluginFileLocator(analyzers=[PluginFileAnalyzerMathingRegex("regex_matcher",('^.*%s%s$' % (PLUGIN_SUFFIX,PLUGIN_EXT)))]))
self.plugin_manager.setPluginPlaces([PLUGIN_PATH])
self.dummy_plugin = None
self.plugin_map = {}
self.init_plugins()
def init_plugins(self):
# Clear plugin map
self.plugin_map = {}
# Collect plugins, activate them and add them to the map
print("LOADING PLUGINS ...")
self.plugin_manager.collectPlugins()
for plugin in self.plugin_manager.getAllPlugins():
name = re.sub(PLUGIN_SUFFIX,'',plugin.name)
print("... %s"%name)
# set plugin name
plugin.plugin_object._init_plugin(name,self)
print(name)
if name == PLUGIN_DUMMY_NAME:
self.dummy_plugin = plugin
else:
self.plugin_manager.activatePluginByName(plugin.name)
self.plugin_map[name] = plugin
def get_plugin_info(self,name):
try:
return self.plugin_map[name]
except Exception:
return self.dummy_plugin
def get_plugin(self,name):
return self.get_plugin_info(name).plugin_object
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = [
'planetoid',
'ipynb',
'local_search',
'render_mustache',
]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
# Register our own path handlers
self.path_handlers = {
'slug': self.slug_path,
'post_path': self.post_path,
}
self.strict = False
self.global_data = {}
self.posts = []
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.posts_per_category = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
self._template_system = None
self._THEMES = None
self.debug = DEBUG
self.loghandlers = []
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ANNOTATIONS': False,
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'BLOG_TITLE': 'Default Title',
'BLOG_DESCRIPTION': 'Default Description',
'BODY_END': "",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENT_SYSTEM': 'disqus',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'COMPILERS': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile',),
"txt2tags": ('.t2t',),
"bbcode": ('.bb',),
"wiki": ('.wiki',),
"ipynb": ('.ipynb',),
"html": ('.html', '.htm')
},
'CONTENT_FOOTER': '',
'COPY_SOURCES': True,
'CREATE_MONTHLY_ARCHIVE': False,
'CREATE_SINGLE_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'EXTRA_PLUGINS_DIRS': [],
'COMMENT_SYSTEM_ID': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FEED_LENGTH': 10,
'FILE_METADATA_REGEXP': None,
'ADDITIONAL_METADATA': {},
'FILES_FOLDERS': {'files': ''},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GALLERY_SORT_BY_DATE': True,
'GZIP_COMMAND': None,
'GZIP_FILES': False,
'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json', '.xml'),
'HIDE_SOURCELINK': False,
'HIDE_UNTRANSLATED_POSTS': False,
'HYPHENATE': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEXES_PAGES_MAIN': False,
'INDEX_PATH': '',
'IPYNB_CONFIG': {},
'LESS_COMPILER': 'lessc',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'NAVIGATION_LINKS': None,
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'POSTS': (("posts/*.txt", "posts", "post.tmpl"),),
'PAGES': (("stories/*.txt", "stories", "story.tmpl"),),
'PRETTY_URLS': False,
'FUTURE_IS_NOW': False,
'READ_MORE_LINK': '<p class="more"><a href="{link}">{read_more}…</a></p>',
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SASS_COMPILER': 'sass',
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'SOCIAL_BUTTONS_CODE': SOCIAL_BUTTONS_CODE,
'SITE_URL': 'http://getnikola.com/',
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'bootstrap',
'THEME_REVEAL_CONFIG_SUBTHEME': 'sky',
'THEME_REVEAL_CONFIG_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'URL_TYPE': 'rel_path',
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': 'UTC',
'DEPLOY_DRAFTS': True,
'DEPLOY_FUTURE': False,
'SCHEDULE_ALL': False,
'SCHEDULE_RULE': '',
'SCHEDULE_FORCE_TODAY': False,
'LOGGING_HANDLERS': {'stderr': {'loglevel': 'WARNING', 'bubble': True}},
'DEMOTE_HEADERS': 1,
}
self.config.update(config)
# Make sure we have pyphen installed if we are using it
if self.config.get('HYPHENATE') and pyphen is None:
utils.LOGGER.warn('To use the hyphenation, you have to install '
'the "pyphen" package.')
utils.LOGGER.warn('Setting HYPHENATE to False.')
self.config['HYPHENATE'] = False
# Deprecating post_compilers
# TODO: remove on v7
if 'post_compilers' in config:
utils.LOGGER.warn('The post_compilers option is deprecated, use COMPILERS instead.')
if 'COMPILERS' in config:
utils.LOGGER.warn('COMPILERS conflicts with post_compilers, ignoring post_compilers.')
else:
self.config['COMPILERS'] = config['post_compilers']
# Deprecating post_pages
# TODO: remove on v7
if 'post_pages' in config:
utils.LOGGER.warn('The post_pages option is deprecated, use POSTS and PAGES instead.')
if 'POSTS' in config or 'PAGES' in config:
utils.LOGGER.warn('POSTS and PAGES conflict with post_pages, ignoring post_pages.')
else:
self.config['POSTS'] = [item[:3] for item in config['post_pages'] if item[-1]]
self.config['PAGES'] = [item[:3] for item in config['post_pages'] if not item[-1]]
# FIXME: Internally, we still use post_pages because it's a pain to change it
self.config['post_pages'] = []
for i1, i2, i3 in self.config['POSTS']:
self.config['post_pages'].append([i1, i2, i3, True])
for i1, i2, i3 in self.config['PAGES']:
self.config['post_pages'].append([i1, i2, i3, False])
# Deprecating DISQUS_FORUM
# TODO: remove on v7
if 'DISQUS_FORUM' in config:
utils.LOGGER.warn('The DISQUS_FORUM option is deprecated, use COMMENT_SYSTEM_ID instead.')
if 'COMMENT_SYSTEM_ID' in config:
utils.LOGGER.warn('DISQUS_FORUM conflicts with COMMENT_SYSTEM_ID, ignoring DISQUS_FORUM.')
else:
self.config['COMMENT_SYSTEM_ID'] = config['DISQUS_FORUM']
# Deprecating the ANALYTICS option
# TODO: remove on v7
if 'ANALYTICS' in config:
utils.LOGGER.warn('The ANALYTICS option is deprecated, use BODY_END instead.')
if 'BODY_END' in config:
utils.LOGGER.warn('ANALYTICS conflicts with BODY_END, ignoring ANALYTICS.')
else:
self.config['BODY_END'] = config['ANALYTICS']
# Deprecating the SIDEBAR_LINKS option
# TODO: remove on v7
if 'SIDEBAR_LINKS' in config:
utils.LOGGER.warn('The SIDEBAR_LINKS option is deprecated, use NAVIGATION_LINKS instead.')
if 'NAVIGATION_LINKS' in config:
utils.LOGGER.warn('The SIDEBAR_LINKS conflicts with NAVIGATION_LINKS, ignoring SIDEBAR_LINKS.')
else:
self.config['NAVIGATION_LINKS'] = config['SIDEBAR_LINKS']
# Compatibility alias
self.config['SIDEBAR_LINKS'] = self.config['NAVIGATION_LINKS']
if self.config['NAVIGATION_LINKS'] in (None, {}):
self.config['NAVIGATION_LINKS'] = {self.config['DEFAULT_LANG']: ()}
# Deprecating the ADD_THIS_BUTTONS option
# TODO: remove on v7
if 'ADD_THIS_BUTTONS' in config:
utils.LOGGER.warn('The ADD_THIS_BUTTONS option is deprecated, use SOCIAL_BUTTONS_CODE instead.')
if not config['ADD_THIS_BUTTONS']:
utils.LOGGER.warn('Setting SOCIAL_BUTTONS_CODE to empty because ADD_THIS_BUTTONS is False.')
self.config['SOCIAL_BUTTONS_CODE'] = ''
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
# TODO: remove on v7
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
utils.LOGGER.warn('You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML')
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
if config.get('COPY_SOURCES') and not self.config['HIDE_SOURCELINK']:
self.config['HIDE_SOURCELINK'] = True
self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
{self.config['DEFAULT_LANG']: ''})
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
# TODO: remove on v7
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
utils.LOGGER.warn('You should configure SITE_URL instead of BLOG_URL')
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
locale_fallback, locale_default, locales = sanitized_locales(
self.config.get('LOCALE_FALLBACK', None),
self.config.get('LOCALE_DEFAULT', None),
self.config.get('LOCALES', {}),
self.translations) # NOQA
utils.LocaleBorg.initialize(locales, self.default_lang)
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
# BASE_URL should *always* end in /
if self.config['BASE_URL'] and self.config['BASE_URL'][-1] != '/':
utils.LOGGER.warn("Your BASE_URL doesn't end in / -- adding it.")
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"SignalHandler": SignalHandler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
] + [path for path in extra_plugins_dirs if path]
else:
places = [
os.path.join(os.path.dirname(__file__), utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
# Activate all required SignalHandler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("SignalHandler"):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(plugin_info, "SignalHandler")
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Emit signal for SignalHandlers which need to start running immediately.
signal('sighandlers_loaded').send(self)
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
if plugin_info.name in self.config["COMPILERS"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self._GLOBAL_CONTEXT = {}
self._GLOBAL_CONTEXT['_link'] = self.link
self._GLOBAL_CONTEXT['set_locale'] = utils.LocaleBorg().set_locale
self._GLOBAL_CONTEXT['rel_link'] = self.rel_link
self._GLOBAL_CONTEXT['abs_link'] = self.abs_link
self._GLOBAL_CONTEXT['exists'] = self.file_exists
self._GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config['SLUG_TAG_PATH']
self._GLOBAL_CONTEXT['annotations'] = self.config['ANNOTATIONS']
self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self._GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self._GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
# TODO: remove fallback in v7
self._GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL', self.config.get('BLOG_URL'))
self._GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self._GLOBAL_CONTEXT['body_end'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['analytics'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['add_this_buttons'] = self.config.get('SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['social_buttons_code'] = self.config.get('SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self._GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self._GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self._GLOBAL_CONTEXT['comment_system'] = self.config.get('COMMENT_SYSTEM')
self._GLOBAL_CONTEXT['comment_system_id'] = self.config.get('COMMENT_SYSTEM_ID')
# TODO: remove in v7
self._GLOBAL_CONTEXT['disqus_forum'] = self.config.get('COMMENT_SYSTEM_ID')
self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self._GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONFIG_SUBTHEME')
self._GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONFIG_TRANSITION')
self._GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self._GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self._GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self._GLOBAL_CONTEXT['navigation_links'] = utils.Functionary(list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('NAVIGATION_LINKS', {}).items():
self._GLOBAL_CONTEXT['navigation_links'][k] = v
# TODO: remove on v7
# Compatibility alias
self._GLOBAL_CONTEXT['sidebar_links'] = self._GLOBAL_CONTEXT['navigation_links']
self._GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self._GLOBAL_CONTEXT['hide_sourcelink'] = self.config.get(
'HIDE_SOURCELINK')
self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA')
self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
signal('configured').send(self)
def _get_themes(self):
if self._THEMES is None:
# Check for old theme names (Issue #650) TODO: remove in v7
theme_replacements = {
'site': 'bootstrap',
'orphan': 'base',
'default': 'oldfashioned',
}
if self.config['THEME'] in theme_replacements:
utils.LOGGER.warn('You are using the old theme "{0}", using "{1}" instead.'.format(
self.config['THEME'], theme_replacements[self.config['THEME']]))
self.config['THEME'] = theme_replacements[self.config['THEME']]
if self.config['THEME'] == 'oldfashioned':
utils.LOGGER.warn('''You may need to install the "oldfashioned" theme '''
'''from themes.nikola.ralsina.com.ar because it's not '''
'''shipped by default anymore.''')
utils.LOGGER.warn('Please change your THEME setting.')
try:
self._THEMES = utils.get_theme_chain(self.config['THEME'])
except Exception:
utils.LOGGER.warn('''Can't load theme "{0}", using 'bootstrap' instead.'''.format(self.config['THEME']))
self.config['THEME'] = 'bootstrap'
return self._get_themes()
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config['USE_CDN']:
bootstrap_path = utils.get_asset_path(os.path.join(
'assets', 'css', 'bootstrap.min.css'), self._THEMES)
if bootstrap_path and bootstrap_path.split(os.sep)[-4] not in ['bootstrap', 'bootstrap3']:
utils.LOGGER.warn('The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.')
return self._THEMES
THEMES = property(_get_themes)
def _get_messages(self):
return utils.load_messages(self.THEMES,
self.translations,
self.default_lang)
MESSAGES = property(_get_messages)
def _get_global_context(self):
"""Initialize some parts of GLOBAL_CONTEXT only when it's queried."""
if 'messages' not in self._GLOBAL_CONTEXT:
self._GLOBAL_CONTEXT['messages'] = self.MESSAGES
if 'has_custom_css' not in self._GLOBAL_CONTEXT:
# check if custom css exist and is not empty
custom_css_path = utils.get_asset_path(
'assets/css/custom.css',
self.THEMES,
self.config['FILES_FOLDERS']
)
if custom_css_path and self.file_exists(custom_css_path, not_empty=True):
self._GLOBAL_CONTEXT['has_custom_css'] = True
else:
self._GLOBAL_CONTEXT['has_custom_css'] = False
return self._GLOBAL_CONTEXT
GLOBAL_CONTEXT = property(_get_global_context)
def _get_template_system(self):
if self._template_system is None:
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(
template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self._template_system = pi.plugin_object
lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES]
self._template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
return self._template_system
template_system = property(_get_template_system)
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.COMPILERS`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in
list(self.config['COMPILERS'].items())
if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'COMPILERS' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("COMPILERS in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
# string, arguments
local_context["formatmsg"] = lambda s, *a: s % a
data = self.template_system.render_template(
template_name, None, local_context)
assert output_name.startswith(
self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Refuse to replace links that consist of a fragment only
if ((not dst_url.scheme) and (not dst_url.netloc) and
(not dst_url.path) and (not dst_url.params) and
(not dst_url.query) and dst_url.fragment):
return dst
# Normalize
dst = urljoin(src, dst.lstrip('/'))
# Avoid empty links.
if src == dst:
if self.config.get('URL_TYPE') == 'absolute':
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return dst
elif self.config.get('URL_TYPE') == 'full_path':
return dst
else:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
if self.config.get('URL_TYPE') == 'absolute':
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return dst
if self.config.get('URL_TYPE') in ('full_path', 'absolute'):
if self.config.get('URL_TYPE') == 'absolute':
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
utils.makedirs(os.path.dirname(output_name))
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
These are mostly defined by plugins by registering via
the register_path_handler method, except for slug and
post_path which are defined in this class' init method.
Here's some of the others, for historical reasons:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* category (and name is the category name)
* category_rss (and name is the category name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a POSTS/PAGES tuple)
* slug (name is the slug of a post or story)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = utils.LocaleBorg().current_lang
path = self.path_handlers[kind](name, lang)
if is_link:
link = '/' + ('/'.join(path))
index_len = len(self.config['INDEX_FILE'])
if self.config['STRIP_INDEXES'] and \
link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def post_path(self, name, lang):
"""post_path path handler"""
return [_f for _f in [self.config['TRANSLATIONS'][lang],
os.path.dirname(name),
self.config['INDEX_FILE']] if _f]
def slug_path(self, name, lang):
"""slug path handler"""
results = [p for p in self.timeline if p.meta('slug') == name]
if not results:
utils.LOGGER.warning("Can't resolve path request for slug: {0}".format(name))
else:
if len(results) > 1:
utils.LOGGER.warning('Ambiguous path request for slug: {0}'.format(name))
return [_f for _f in results[0].permalink(lang).split('/') if _f]
def register_path_handler(self, kind, f):
if kind in self.path_handlers:
utils.LOGGER.warning('Conflicting path handlers for kind: {0}'.format(kind))
else:
self.path_handlers[kind] = f
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return urlparse(dst).geturl()
def rel_link(self, src, dst):
# Normalize
try:
src = urljoin(self.config['BASE_URL'], src.lstrip('/'))
except AttributeError:
# sometimes, it’s an Undefined object.
src = urljoin(self.config['BASE_URL'], src)
try:
dst = urljoin(src, dst.lstrip('/'))
except AttributeError:
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def clean_task_paths(self, task):
"""Normalize target paths in the task."""
targets = task.get('targets', None)
if targets is not None:
task['targets'] = [os.path.normpath(t) for t in targets]
return task
def gen_tasks(self, name, plugin_category, doc=''):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
assert 'basename' in task
task = self.clean_task_paths(task)
yield task
for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield self.clean_task_paths(task)
if flag:
task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'basename': name,
'doc': doc,
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
seen = set([])
print("Scanning posts", end='', file=sys.stderr)
lower_case_tags = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='', file=sys.stderr)
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination,
os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config['TRANSLATIONS'].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
# We eliminate from the list the files inside any .ipynb folder
full_list = [p for p in full_list
if not any([x.startswith('.')
for x in p.split(os.sep)])]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.config,
dest_dir,
use_in_feeds,
self.MESSAGES,
template_name,
self.get_compiler(base_path)
)
self.global_data[post.source_path] = post
if post.use_in_feeds:
self.posts.append(post.source_path)
self.posts_per_year[
str(post.date.year)].append(post.source_path)
self.posts_per_month[
'{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post.source_path)
for tag in post.alltags:
if tag.lower() in lower_case_tags:
if tag not in self.posts_per_tag:
# Tags that differ only in case
other_tag = [k for k in self.posts_per_tag.keys() if k.lower() == tag.lower()][0]
utils.LOGGER.error('You have cases that differ only in upper/lower case: {0} and {1}'.format(tag, other_tag))
utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join(self.posts_per_tag[other_tag])))
sys.exit(1)
else:
lower_case_tags.add(tag.lower())
self.posts_per_tag[tag].append(post.source_path)
self.posts_per_category[post.meta('category')].append(post.source_path)
else:
self.pages.append(post)
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!", file=sys.stderr)
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine'))
deps = list(filter(None, deps))
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
if post:
deps_dict['post_translations'] = post.translated_to
task = {
'name': os.path.normpath(output_name),
'file_dep': deps,
'targets': [output_name],
'actions': [(self.render_template, [post.template_name,
output_name, context])],
'clean': True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in
posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name': os.path.normpath(output_name),
'targets': [output_name],
'file_dep': deps,
'actions': [(self.render_template, [template_name, output_name,
context])],
'clean': True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
import logging
from yapsy.PluginManager import PluginManager
logging.basicConfig()
manager = PluginManager()
manager.setPluginPlaces(["./plugins"])
manager.collectPlugins()
for plugin in manager.getAllPlugins():
print("Activating: " + plugin.name)
manager.activatePluginByName(plugin.name)
class ChemaBot(irc.IRCClient):
"""The main IRC bot class.
ChemaBot mainly handles message dispatching
to plugins and other general bot functions.
"""
def __init__(self, nickname):
self.nickname = nickname
logging.basicConfig(level=logging.DEBUG)
self.plugins_init()
#TODO: database location should be taken fron config file.
self.db_manager = DatabaseManager("bot.db")
def plugins_init(self, is_reloading = False):
if is_reloading:
logging.info("Deactivating All Plugins")
for pluginInfo in self.pm.getAllPlugins():
self.pm.deactivatePluginByName(pluginInfo.name)
self.pm = PluginManager(
categories_filter = {
"BaseActions" : BaseActionPlugin,
"TextActions" : TextTriggerPlugin,
},
directories_list=["plugins"],)
self.pm.collectPlugins()
for pluginInfo in self.pm.getAllPlugins():
self.pm.activatePluginByName(pluginInfo.name)
logging.info("Plugin {0} activated".format(pluginInfo.name))
# TODO: create a list of localized ("_()") plugin triggers
# Create a dictionary of the names of all the plugins
self.action_plugins = {}
# List of the regexes and plugins
self.text_trigger_plugins = []
# TODO: specify categories of plugins with each trigger http://yapsy.sourceforge.net/PluginManager.html
for pluginInfo in self.pm.getPluginsOfCategory("BaseActions"):
self.action_plugins[pluginInfo.name] = pluginInfo.plugin_object
logging.debug("Action plugins: {0}".format(self.action_plugins))
for pluginInfo in self.pm.getPluginsOfCategory("TextActions"):
self.text_trigger_plugins.append((pluginInfo.plugin_object.trigger, pluginInfo.plugin_object))
logging.debug("Regex plugins: {0}".format(self.text_trigger_plugins))
# Useful for debugging
def listPlugins(self):
print 'Plugins:'
for item in self.plugins:
print item
def signedOn(self):
"""Called when bot has succesfully signed on to server."""
self.join(self.factory.channel)
def joined(self, channel):
"""This will get called when the bot joins the channel."""
pass
def emitMessage(self, message, channel = None):
"""A function to abstract message emission."""
if channel:
self.msg(channel, message.render().encode('utf-8'))
## Handling private messages.
elif message.channel == self.nickname:
self.msg(message.user.split("!")[0], message.render().encode('utf-8'))
else:
self.msg(message.channel, message.render().encode('utf-8'))
def _parseAndExecute(self, ircm):
"""Recieves an IRCMessage, detects the command and triggers the appropiate plugin."""
message = ircm.msg
for (text_trigger, plugin) in self.text_trigger_plugins:
if isinstance(text_trigger, type(re.compile(''))):
result = text_trigger.findall(message)
#TODO: specify the info sent to the trigger.
else:
#trigger.fire(message, *args, **kwargs)
result = text_trigger.fire(ircm)
if result:
d = threads.deferToThread(plugin.execute, ircm, None, result)
d.addCallback(self.emitMessage)
## Main Command trigger is commonly '!'
trigger = self.factory.main_trigger
if message.startswith(trigger) or message.startswith(self.nickname):
word_list = message.split(' ')
if message.startswith(trigger):
command = word_list[0].lstrip(trigger)
elif message.startswith(self.nickname):
try:
command = word_list[1].strip()
except IndexError:
logging.warning('Invalid call: "{0}"'.format(ircm.render()))
return
## TODO: Reload should be dependant on userRole
## TODO: Localize
if command == "reload":
self.plugins_init(is_reloading=True)
return
## TODO: Consider sending the split word list.
try:
plugin = self.action_plugins[command]
except KeyError:
logging.warning("Command {0} missing".format(command))
return
if plugin.synchronous:
d = defer.maybeDeferred(plugin.execute, ircm, None)
else:
d = threads.deferToThread(plugin.execute, ircm, None, connection = self.db_manager)
d.addCallback(self.emitMessage)
return
def privmsg(self, user, channel, msg):
"""Gets called when the bot receives a message in a channel or via PM.
Here is contained the main logic of tio_chema. Should dispatch messages
to the plugins registered, depending if they register for a global or
a specific trigger keyword.
It blocks, so it should not include heavy or slow logic.
Args:
user: A string containing the origin user.
channel: A string with the originating channel or PM channel.
msg: A string containing the message recieved.
"""
message = IRCMessage(channel, msg, user)
#TODO: add logging
#print message
#TODO: add channel trigger plugins (user defined actions)
self._parseAndExecute(message)
self.config_version_db=None
plugin_dir = os.path.expanduser("~/.repacked/plugins")
if not os.path.exists(plugin_dir):
plugin_dir = os.path.join(os.path.dirname(__file__),'../../repacked/plugins')
pkg_plugins = {}
pluginMgr = PluginManager(plugin_info_ext="plugin")
pluginMgr.setPluginPlaces([plugin_dir])
pluginMgr.locatePlugins()
pluginMgr.loadPlugins()
for pluginInfo in pluginMgr.getAllPlugins():
pluginMgr.activatePluginByName(pluginInfo.name)
def parse_spec(filename):
"""
Loads the YAML file into a Python object for parsing
and returns it
"""
fp = open(filename, 'r')
spec = yaml.safe_load("\n".join(fp.readlines()))
return spec
def update_dist_hook(config, spec):
if config.update_dist_hook:
logger.debug ("Update Dist hook script at: "+config.update_dist_hook)
class Manager(object):
"""
This is the main ``octo`` application class.
Normally, you would call `octo.main` instead of creating an instance of this
class directly, as `octo.main` will make it available globally as `octo.instance`
so plugins may interact with it.
"""
def __init__(self, plugin_dirs=[]):
logging.info(
"Initializing with plugin directories: {!r}".format(plugin_dirs))
self.plugin_manager = PluginManager(directories_list=plugin_dirs,
plugin_info_ext='octoplugin')
self.plugin_manager.collectPlugins()
for plugin in self.get_plugins(include_inactive=True).values():
# Bind the plugin object so the plugin can refer to it via self
plugin.plugin_object.plugin_object = plugin
# And bind it's configparser object separately as well for a cleaner API
plugin.plugin_object.plugin_config = plugin.details
def get_plugins(self, include_inactive=False):
"""
Return a dictionary of loaded plugins
Keys will consist of plugin names, with their values being the plugin
instances (yapsy.PluginInfo.PluginInfo objects).
When ``include_inactive`` is True, all collected plugins will be
returned, otherwise only the activated plugins will be returned.
"""
if include_inactive:
plugins = self.plugin_manager.getAllPlugins()
else:
plugins = [
plugin for plugin in self.plugin_manager.getAllPlugins()
if hasattr(plugin, 'is_activated') and plugin.is_activated
]
return dict(zip([plugin.name for plugin in plugins], plugins))
def activate_plugin(self, plugin_name):
"""
Activate the given plugin
plugin_name should be the name of the plugin to be activated.
"""
self.plugin_manager.activatePluginByName(plugin_name)
def deactivate_plugin(self, plugin_name):
"""
Deactivate the given plugin
plugin_name should be the name of the plugin to be deactivated.
"""
self.plugin_manager.deactivatePluginByName(plugin_name)
def call(self, plugin_name, func, args=[], kwargs={}):
"""
Call the given function on the given plugin object (specifed by plugin name)
"""
for plugin in self.get_plugins().values():
if plugin.name == plugin_name:
return getattr(plugin.plugin_object, func)(*args, **kwargs)
raise octo.exceptions.NoSuchPluginError(
"The specified plugin isn't active or doesn't exist")
def call_many(self, func, args=[], kwargs={}):
"""
Call the given function on all active plugins and return results as a dictionary
The returned dictionary will have the form of {'plugin name': <function_result>}
"""
results = {}
for plugin in self.get_plugins().values():
try:
logging.debug("Calling {} on plugin '{}'".format(
func, plugin.name))
results[plugin.name] = getattr(plugin.plugin_object,
func)(*args, **kwargs)
except AttributeError as e:
logging.debug("'{}' has no attribute {}".format(
plugin.name, func))
except Exception as e:
logging.exception(
"Exception while calling '{}' on '{}'".format(
func, plugin.name))
results[plugin.name] = e
return results
def start(self):
"""Start and activate collected plugins
A plugin will be activated when it has a config item 'Enable'
under the section 'Config' with a value of True"""
logging.debug("Activating plugins")
for plugin in self.get_plugins(include_inactive=True).values():
try:
should_activate = plugin.details.getboolean('Config', 'Enable')
except configparser.NoSectionError:
should_activate = False
if should_activate:
logging.debug("Activating plugin {}".format(plugin.name))
self.activate_plugin(plugin.name)
else:
logging.debug(
"Plugin {} not activated because config item Enable "
"is not True".format(plugin.name))
logging.debug("Plugin activation done")
return self
def stop(self):
"""Stop and deactivate loaded plugins"""
logging.debug("Deactivating plugins")
for plugin in self.get_plugins().values():
logging.debug("Deactivating plugin {}".format(plugin.name))
self.deactivate_plugin(plugin.name)
logging.debug("Plugin deactivation done")
return self
class GiantCommands(object):
_client_category = 'Client'
_server_category = 'Server'
def __init__(self):
self.client_plugins = None
self.server_plugins = None
self._plugin_manager = PluginManager(plugin_info_ext='giant')
self._plugin_manager.setCategoriesFilter({
GiantCommands._client_category : BaseGiantClient,
GiantCommands._server_category: BaseGiantServer,
})
path = os.path.dirname(os.path.realpath(__file__))
self._plugin_manager.setPluginPlaces([path + '/plugins'])
self._plugin_manager.collectPlugins()
self.plugin_infos = self._plugin_manager.getAllPlugins()
def new_plugin(self, args):
import getpass
username = getpass.getuser()
safe_language = (args.language
.replace('#', 'Sharp')
.replace('+', 'Plus')
.replace(' ', '')
.replace('-', '_')
.replace('!', 'Exclamation')
.replace('/', 'Slash')
.replace('*', 'Star'))
repo_name = safe_language.lower()+'_'+args.framework.lower()
cookiecutter('gh:lixar/giant-plugin', extra_context={
'full_name': username,
'project_language': args.language,
'project_framework': args.framework,
'project_type': args.type,
'repo_name': repo_name,
'command': repo_name.replace('_', '-')
})
def install_plugin(self, args):
import glob
input_path = args.input_plugin
plugin_file = glob.glob(input_path + '/*.giant')
with open(plugin_file[0], 'r') as plugin:
for line in plugin.readlines():
if line.startswith('Name'):
plugin_name = line.split(' = ')[1].strip()
if line.startswith('Type'):
plugin_type = line.split(' = ')[1].strip()
path = os.path.dirname(os.path.abspath(__file__))
plugins_dir = os.path.join(path, 'plugins')
plugins_dir = os.path.join(plugins_dir, plugin_type.lower())
try:
os.makedirs(plugins_dir)
except:
pass
plugins_dir = os.path.join(plugins_dir, plugin_name)
if args.symlink:
try:
os.symlink(os.path.abspath(args.input_plugin), plugins_dir)
except:
logging.error('Plugin with this name already exists.')
else:
import shutil
shutil.copytree(os.path.abspath(args.input_plugin), plugins_dir)
# TODO Update giant-1.0.0.dist-info/RECORD to have new files added.
def uninstall_plugin(self, args):
import shutil
plugin = self._get_plugin(args.command_name)
path = plugin.path
while not path.endswith('client') and not path.endswith('server'):
path, plugin_dirname = os.path.split(path)
shutil.rmtree(os.path.join(path, plugin_dirname))
def get_client_plugins(self):
if self.client_plugins == None:
self.client_plugins = self._plugin_manager.getPluginsOfCategory(GiantCommands._client_category)
return self.client_plugins
def get_server_plugins(self):
if self.server_plugins == None:
self.server_plugins = self._plugin_manager.getPluginsOfCategory(GiantCommands._server_category)
return self.server_plugins
def get_all_plugins(self):
return self.get_client_plugins() + self.get_server_plugins()
def _get_plugin(self, command_name):
return next(plugin for plugin in self.get_all_plugins() if plugin.details.get('Details', 'Command') == command_name)
def generate_client(self, args, name):
plugin_info = next(plugin_info for plugin_info in self.client_plugins if plugin_info.details.get('Details', 'Command') == name)
self._plugin_manager.activatePluginByName(plugin_info.name)
self._generate_project(args, plugin_info.plugin_object)
def generate_server(self, args, name):
plugin_info = next(plugin_info for plugin_info in self.server_plugins if plugin_info.details.get('Details', 'Command') == name)
plugin = self._plugin_manager.activatePluginByName(plugin_info.name)
self._generate_project(args, plugin_info.plugin_object)
def _generate_project(self, args, plugin):
if args.swagger_files == None:
import requests
response = requests.get(args.swagger_url)
try:
swagger = response.json()
except:
import yaml
try:
swagger = yaml.load(response.text)
except:
logging.critical("Failed to parse server response to JSON or YAML.")
exit(1)
else:
swagger = {}
for f in args.swagger_files:
if os.path.isdir(f):
files = os.listdir(f)
else:
files = [f]
for swagger_file_path in files:
with open(swagger_file_path, 'r') as swagger_file:
try:
api = json.load(swagger_file)
except:
swagger_file.seek(0)
import yaml
api = yaml.load(swagger_file)
paths = {}
for path_name, path in api['paths'].iteritems():
if len(args.swagger_files) > 1:
paths[api['host'] + path_name] = path
else:
paths[path_name] = path
api['paths'] = paths
tag = f.split('.')[0]
self.mergedicts(swagger, api, conflicts=['paths', 'definitions'], conflict_tag=tag)
plugin.setup(swagger, args.output_dir, args.force_overwrite)
print('Generating Project...')
plugin.generate()
def mergedicts(self, a, b, path=None, conflicts=[], conflict_tag=None):
"merges b into a"
if path is None: path = []
for key in b:
if key in a:
if isinstance(a[key], dict) and isinstance(b[key], dict):
self.mergedicts(a[key], b[key], path + [str(key)], conflicts=conflicts, conflict_tag=conflict_tag)
elif a[key] == b[key]:
pass # same leaf value
else:
conflict_path = '.'.join(path + [str(key)])
if any(conflict in conflicts for conflict in conflict_path.split('.')):
if conflict_tag is not None and key + conflict_tag not in a.keys():
a[key + conflict_tag] = b[key]
continue # resolve conflict using tag.
raise Exception('Conflict at ' + '.'.join(path + [str(key)]))
else:
print('Ignoring conflict ' + conflict_path)
else:
a[key] = b[key]
return a
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = ["planetoid", "ipynb", "local_search", "render_mustache"]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
"ADD_THIS_BUTTONS": True,
"ANALYTICS": "",
"ARCHIVE_PATH": "",
"ARCHIVE_FILENAME": "archive.html",
"CACHE_FOLDER": "cache",
"CODE_COLOR_SCHEME": "default",
"COMMENTS_IN_GALLERIES": False,
"COMMENTS_IN_STORIES": False,
"CONTENT_FOOTER": "",
"CREATE_MONTHLY_ARCHIVE": False,
"DATE_FORMAT": "%Y-%m-%d %H:%M",
"DEFAULT_LANG": "en",
"DEPLOY_COMMANDS": [],
"DISABLED_PLUGINS": (),
"DISQUS_FORUM": "nikolademo",
"ENABLED_EXTRAS": (),
"EXTRA_HEAD_DATA": "",
"FAVICONS": {},
"FILE_METADATA_REGEXP": None,
"FILES_FOLDERS": {"files": ""},
"FILTERS": {},
"GALLERY_PATH": "galleries",
"GZIP_FILES": False,
"GZIP_EXTENSIONS": (".txt", ".htm", ".html", ".css", ".js", ".json"),
"HIDE_UNTRANSLATED_POSTS": False,
"INDEX_DISPLAY_POST_COUNT": 10,
"INDEX_TEASERS": False,
"INDEXES_TITLE": "",
"INDEXES_PAGES": "",
"INDEX_PATH": "",
"LICENSE": "",
"LISTINGS_FOLDER": "listings",
"MAX_IMAGE_SIZE": 1280,
"MATHJAX_CONFIG": "",
"OLD_THEME_SUPPORT": True,
"OUTPUT_FOLDER": "output",
"post_compilers": {
"rest": (".txt", ".rst"),
"markdown": (".md", ".mdown", ".markdown"),
"textile": (".textile",),
"txt2tags": (".t2t",),
"bbcode": (".bb",),
"wiki": (".wiki",),
"ipynb": (".ipynb",),
"html": (".html", ".htm"),
},
"POST_PAGES": (
("posts/*.txt", "posts", "post.tmpl", True),
("stories/*.txt", "stories", "story.tmpl", False),
),
"REDIRECTIONS": [],
"RSS_LINK": None,
"RSS_PATH": "",
"RSS_TEASERS": True,
"SEARCH_FORM": "",
"SLUG_TAG_PATH": True,
"STORY_INDEX": False,
"STRIP_INDEX_HTML": False,
"TAG_PATH": "categories",
"TAG_PAGES_ARE_INDEXES": False,
"THEME": "site",
"THEME_REVEAL_CONGIF_SUBTHEME": "sky",
"THEME_REVEAL_CONGIF_TRANSITION": "cube",
"THUMBNAIL_SIZE": 180,
"USE_BUNDLES": True,
"USE_CDN": False,
"USE_FILENAME_AS_TITLE": True,
"TIMEZONE": None,
}
self.config.update(config)
self.config["TRANSLATIONS"] = self.config.get("TRANSLATIONS", {self.config["DEFAULT_" "LANG"]: ""})
self.THEMES = utils.get_theme_chain(self.config["THEME"])
self.MESSAGES = utils.load_messages(self.THEMES, self.config["TRANSLATIONS"], self.config["DEFAULT_LANG"])
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
if "SITE_URL" not in self.config:
if "BLOG_URL" in self.config:
print("WARNING: You should configure SITE_URL instead of BLOG_URL")
self.config["SITE_URL"] = self.config["BLOG_URL"]
self.default_lang = self.config["DEFAULT_LANG"]
self.translations = self.config["TRANSLATIONS"]
# BASE_URL defaults to SITE_URL
if "BASE_URL" not in self.config:
self.config["BASE_URL"] = self.config.get("SITE_URL")
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
}
)
self.plugin_manager.setPluginInfoExtension("plugin")
self.plugin_manager.setPluginPlaces(
[str(os.path.join(os.path.dirname(__file__), "plugins")), str(os.path.join(os.getcwd(), "plugins"))]
)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if plugin_info.name in self.config["DISABLED_PLUGINS"] or (
plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"]
):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if plugin_info.name in self.config["DISABLED_PLUGINS"] or (
plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"]
):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = {}
self.GLOBAL_CONTEXT["messages"] = self.MESSAGES
self.GLOBAL_CONTEXT["_link"] = self.link
self.GLOBAL_CONTEXT["set_locale"] = s_l
self.GLOBAL_CONTEXT["rel_link"] = self.rel_link
self.GLOBAL_CONTEXT["abs_link"] = self.abs_link
self.GLOBAL_CONTEXT["exists"] = self.file_exists
self.GLOBAL_CONTEXT["SLUG_TAG_PATH"] = self.config["SLUG_TAG_PATH"]
self.GLOBAL_CONTEXT["add_this_buttons"] = self.config["ADD_THIS_BUTTONS"]
self.GLOBAL_CONTEXT["index_display_post_count"] = self.config["INDEX_DISPLAY_POST_COUNT"]
self.GLOBAL_CONTEXT["use_bundles"] = self.config["USE_BUNDLES"]
self.GLOBAL_CONTEXT["use_cdn"] = self.config.get("USE_CDN")
self.GLOBAL_CONTEXT["favicons"] = self.config["FAVICONS"]
self.GLOBAL_CONTEXT["date_format"] = self.config.get("DATE_FORMAT", "%Y-%m-%d %H:%M")
self.GLOBAL_CONTEXT["blog_author"] = self.config.get("BLOG_AUTHOR")
self.GLOBAL_CONTEXT["blog_title"] = self.config.get("BLOG_TITLE")
self.GLOBAL_CONTEXT["blog_url"] = self.config.get("SITE_URL", self.config.get("BLOG_URL"))
self.GLOBAL_CONTEXT["blog_desc"] = self.config.get("BLOG_DESCRIPTION")
self.GLOBAL_CONTEXT["analytics"] = self.config.get("ANALYTICS")
self.GLOBAL_CONTEXT["translations"] = self.config.get("TRANSLATIONS")
self.GLOBAL_CONTEXT["license"] = self.config.get("LICENSE")
self.GLOBAL_CONTEXT["search_form"] = self.config.get("SEARCH_FORM")
self.GLOBAL_CONTEXT["disqus_forum"] = self.config.get("DISQUS_FORUM")
self.GLOBAL_CONTEXT["mathjax_config"] = self.config.get("MATHJAX_CONFIG")
self.GLOBAL_CONTEXT["subtheme"] = self.config.get("THEME_REVEAL_CONGIF_SUBTHEME")
self.GLOBAL_CONTEXT["transition"] = self.config.get("THEME_REVEAL_CONGIF_TRANSITION")
self.GLOBAL_CONTEXT["content_footer"] = self.config.get("CONTENT_FOOTER")
self.GLOBAL_CONTEXT["rss_path"] = self.config.get("RSS_PATH")
self.GLOBAL_CONTEXT["rss_link"] = self.config.get("RSS_LINK")
self.GLOBAL_CONTEXT["sidebar_links"] = utils.Functionary(list, self.config["DEFAULT_LANG"])
for k, v in self.config.get("SIDEBAR_LINKS", {}).items():
self.GLOBAL_CONTEXT["sidebar_links"][k] = v
self.GLOBAL_CONTEXT["twitter_card"] = self.config.get("TWITTER_CARD", {})
self.GLOBAL_CONTEXT["extra_head_data"] = self.config.get("EXTRA_HEAD_DATA")
self.GLOBAL_CONTEXT.update(self.config.get("GLOBAL_CONTEXT", {}))
# check if custom css exist and is not empty
for files_path in list(self.config["FILES_FOLDERS"].keys()):
custom_css_path = os.path.join(files_path, "assets/css/custom.css")
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT["has_custom_css"] = True
break
else:
self.GLOBAL_CONTEXT["has_custom_css"] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system " "plugin\n".format(template_sys_name))
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES]
self.template_system.set_directories(lookup_dirs, self.config["CACHE_FOLDER"])
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config["USE_CDN"]:
bootstrap_path = utils.get_asset_path(os.path.join("assets", "css", "bootstrap.min.css"), self.THEMES)
if bootstrap_path.split(os.sep)[-4] != "site":
warnings.warn(
"The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap."
)
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
self.compilers[plugin_info.name] = plugin_info.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in list(self.config["post_compilers"].items()) if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit(
"Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of {0})".format(", ".join(langs))
)
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to " "handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(template_name, None, local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1 :]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split("/")[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == "link": # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip("/"), context["lang"])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b"<!DOCTYPE html>" + lxml.html.tostring(doc, encoding="utf8")
with open(output_name, "wb+") as post_file:
post_file.write(data)
def current_lang(self): # FIXME: this is duplicated, turn into a mixin
"""Return the currently set locale, if it's one of the
available translations, or default_lang."""
lang = utils.LocaleBorg().current_lang
if lang:
if lang in self.translations:
return lang
lang = lang.split("_")[0]
if lang in self.translations:
return lang
# whatever
return self.default_lang
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a post_pages tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = self.current_lang()
path = []
if kind == "tag_index":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], "index.html"] if _f]
elif kind == "tag":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".html"] if _f]
elif kind == "tag_rss":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".xml"] if _f]
elif kind == "index":
if name not in [None, 0]:
path = [
_f
for _f in [
self.config["TRANSLATIONS"][lang],
self.config["INDEX_PATH"],
"index-{0}.html".format(name),
]
if _f
]
else:
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index.html"] if _f]
elif kind == "post_path":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], os.path.dirname(name), "index.html"] if _f]
elif kind == "rss":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["RSS_PATH"], "rss.xml"] if _f]
elif kind == "archive":
if name:
path = [
_f
for _f in [self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], name, "index.html"]
if _f
]
else:
path = [
_f
for _f in [
self.config["TRANSLATIONS"][lang],
self.config["ARCHIVE_PATH"],
self.config["ARCHIVE_FILENAME"],
]
if _f
]
elif kind == "gallery":
path = [_f for _f in [self.config["GALLERY_PATH"], name, "index.html"] if _f]
elif kind == "listing":
path = [_f for _f in [self.config["LISTINGS_FOLDER"], name + ".html"] if _f]
if is_link:
link = "/" + ("/".join(path))
if self.config["STRIP_INDEX_HTML"] and link.endswith("/index.html"):
return link[:-10]
else:
return link
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config["BASE_URL"], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config["BASE_URL"], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split("/")[1:]
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
def create_gzipped_copy(in_path, out_path):
with gzip.GzipFile(out_path, "wb+") as outf:
with open(in_path, "rb") as inf:
outf.write(inf.read())
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
def add_gzipped_copies(task):
if not self.config["GZIP_FILES"]:
return None
if task.get("name") is None:
return None
gzip_task = {
"file_dep": [],
"targets": [],
"actions": [],
"basename": "gzip",
"name": task.get("name") + ".gz",
"clean": True,
}
targets = task.get("targets", [])
flag = False
for target in targets:
ext = os.path.splitext(target)[1]
if ext.lower() in self.config["GZIP_EXTENSIONS"] and target.startswith(self.config["OUTPUT_FOLDER"]):
flag = True
gzipped = target + ".gz"
gzip_task["file_dep"].append(target)
gzip_task["targets"].append(gzipped)
gzip_task["actions"].append((create_gzipped_copy, (target, gzipped)))
if not flag:
return None
return gzip_task
if self.config["GZIP_FILES"]:
task_dep = ["gzip"]
else:
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
gztask = add_gzipped_copies(task)
if gztask:
yield gztask
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
gztask = add_gzipped_copies(task)
if gztask:
yield gztask
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {"name": b"all", "actions": None, "clean": True, "task_dep": task_dep}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
print("Scanning posts", end="")
tzinfo = None
if self.config["TIMEZONE"] is not None:
tzinfo = pytz.timezone(self.config["TIMEZONE"])
targets = set([])
for wildcard, destination, template_name, use_in_feeds in self.config["post_pages"]:
print(".", end="")
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination, os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config["TRANSLATIONS"].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
for base_path in full_list:
post = Post(
base_path,
self.config["CACHE_FOLDER"],
dest_dir,
use_in_feeds,
self.config["TRANSLATIONS"],
self.config["DEFAULT_LANG"],
self.config["BASE_URL"],
self.MESSAGES,
template_name,
self.config["FILE_METADATA_REGEXP"],
self.config["STRIP_INDEX_HTML"],
tzinfo,
self.config["HIDE_UNTRANSLATED_POSTS"],
)
for lang, langpath in list(self.config["TRANSLATIONS"].items()):
dest = (destination, langpath, dir_glob, post.meta[lang]["slug"])
if dest in targets:
raise Exception(
"Duplicated output path {0!r} "
"in post {1!r}".format(post.meta[lang]["slug"], base_path)
)
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(post.post_name)
self.posts_per_month["{0}/{1:02d}".format(post.date.year, post.date.month)].append(
post.post_name
)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
if self.config["OLD_THEME_SUPPORT"]:
post._add_old_metadata()
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + self.template_system.template_deps(post.template_name)
context["post"] = post
context["lang"] = lang
context["title"] = post.title(lang)
context["description"] = post.description(lang)
context["permalink"] = post.permalink(lang)
context["page_list"] = self.pages
if post.use_in_feeds:
context["enable_comments"] = True
else:
context["enable_comments"] = self.config["COMMENTS_IN_STORIES"]
output_name = os.path.join(self.config["OUTPUT_FOLDER"], post.destination_path(lang))
deps_dict = copy(context)
deps_dict.pop("post")
if post.prev_post:
deps_dict["PREV_LINK"] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict["NEXT_LINK"] = [post.next_post.permalink(lang)]
deps_dict["OUTPUT_FOLDER"] = self.config["OUTPUT_FOLDER"]
deps_dict["TRANSLATIONS"] = self.config["TRANSLATIONS"]
deps_dict["global"] = self.GLOBAL_CONTEXT
deps_dict["comments"] = context["enable_comments"]
task = {
"name": os.path.normpath(output_name),
"file_dep": deps,
"targets": [output_name],
"actions": [(self.render_template, [post.template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config["BLOG_TITLE"]
context["description"] = self.config["BLOG_DESCRIPTION"]
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]["title"], p.permalink(lang)) for p in posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
"name": os.path.normpath(output_name),
"targets": [output_name],
"file_dep": deps,
"actions": [(self.render_template, [template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_context)],
}
return utils.apply_filters(task, filters)
class WeaponSystem(rpyc.Service):
'''
RPC Services: This is the code that does the actual password cracking
and returns the results to orbital control. Currently only supports
cracking using rainbow tables (RCrackPy)
'''
is_initialized = False
mutex = Lock()
is_busy = False
job_id = None
def initialize(self):
''' Initializes variables, this should only be called once '''
logging.info("Weapon system initializing ...")
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["plugins/"])
self.plugin_manager.setCategoriesFilter(FILTERS)
self.plugin_manager.collectPlugins()
self.plugins = {}
logging.info(
"Loaded %d plugin(s)" % len(self.plugin_manager.getAllPlugins())
)
self.__cpu__()
logging.info("Weapon system online, good hunting.")
@atomic
def on_connect(self):
''' Called when successfully connected '''
if not self.is_initialized:
self.initialize()
self.is_initialized = True
logging.info("Uplink to orbital control active")
def on_disconnect(self):
''' Called if the connection is lost/disconnected '''
logging.info("Disconnected from orbital command server.")
def __cpu__(self):
''' Detects the number of CPU cores on a system (including virtual cores) '''
if cpu_count is not None:
try:
self.cpu_cores = cpu_count()
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except NotImplementedError:
logging.error("Could not detect number of processors; assuming 1")
self.cpu_cores = 1
else:
try:
self.cpu_cores = int(sysconf("SC_NPROCESSORS_CONF"))
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except ValueError:
logging.error("Could not detect number of processors; assuming 1")
self.cpu_cores = 1
############################ [ EXPOSED METHODS ] ############################
@atomic
def exposed_crack(self, plugin_name, job_id, hashes, **kwargs):
''' Exposes plugins calls '''
assert plugin_name in self.plugins
self.is_busy = True
self.job_id = job_id
self.plugin_manager.activatePluginByName(plugin_name)
plugin = self.plugin_manager.getPluginByName(plugin_name)
results = plugin.execute(hashes, **kwargs)
self.plugin_manager.deactivatePluginByName(plugin_name)
self.job_id = None
self.is_busy = False
return results
def exposed_get_plugin_names(self):
''' Returns what algorithms can be cracked '''
logging.info("Method called: exposed_get_capabilities")
plugins = self.plugin_manager.getAllPlugins()
return [plugin.name for plugin in plugins]
def exposed_get_categories(self):
''' Return categories for which we have plugins '''
categories = []
for category in self.plugin_manager.getCategories():
if 0 < len(self.plugin_manager.getPluginsOfCategory(category)):
categories.append(category)
return categories
def exposed_get_category_plugins(self, category):
''' Get plugin names for a category '''
plugins = self.plugin_manager.getPluginsOfCategory(category)
return [plugin.name for plugin in plugins]
def exposed_get_plugin_details(self, category, plugin_name):
''' Get plugin based on name details '''
plugin = self.plugin_manager.getPluginByName(plugin_name, category)
info = {'name': plugin.name}
info['author'] = plugin.details.get('Documentation', 'author')
info['website'] = plugin.details.get('Documentation', 'website')
info['version'] = plugin.details.get('Documentation', 'version')
info['description'] = plugin.details.get('Documentation', 'description')
info['copyright'] = plugin.details.get('Documentation', 'copyright')
info['precomputation'] = plugin.details.getboolean('Core', 'precomputation')
return info
def exposed_ping(self):
''' Returns a pong message '''
return "PONG"
def exposed_is_busy(self):
''' Returns True/False if the current system is busy (thread safe) '''
return self.is_busy
def exposed_current_job_id(self):
''' Returns the current job id (thread safe) '''
return self.job_id
def exposed_cpu_count(self):
''' Returns the number of detected cpu cores '''
return self.cpu_cores
class TestDef(object):
def __init__(self):
# set aside storage for options and cmd line args
self.options = {}
self.args = []
# record if we have loaded the plugins or
# not - this is just a bozo check to ensure
# someone doesn't tell us to do it twice
self.loaded = False
# set aside a spot for a logger object, and
# note that it hasn't yet been defined
self.logger = None
self.modcmd = None
self.execmd = None
self.harasser = None
self.config = None
self.stages = None
self.tools = None
self.utilities = None
self.defaults = None
self.log = {}
def setOptions(self, args):
self.options = vars(args)
self.args = args
# if they want us to clear the scratch, then do so
if self.options['clean'] and os.path.isdir(self.options['scratchdir']):
shutil.rmtree(self.options['scratchdir'])
# setup the scratch directory
_mkdir_recursive(self.options['scratchdir'])
# private function to convert values
def __convert_value(self, opt, inval):
if opt is None or type(opt) is str:
return 0, inval
elif type(opt) is bool:
if type(inval) is bool:
return 0, inval
elif type(inval) is str:
if inval.lower() in ['true', '1', 't', 'y', 'yes']:
return 0, True
else:
return 0, False
elif type(inval) is int:
if 0 == inval:
return 0, False
else:
return 0, True
elif is_py2 and type(inval) is unicode:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is int:
if type(inval) is int:
return 0, inval
elif type(inval) is str:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is float:
if type(inval) is float:
return 0, inval
elif type(inval) is str or type(inval) is int:
return 0, float(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
else:
return 1, None
# scan the key-value pairs obtained from the configuration
# parser and compare them with the options defined for a
# given plugin. Generate an output dictionary that contains
# the updated set of option values, the default value for
# any option that wasn't included in the configuration file,
# and return an error status plus output identifying any
# keys in the configuration file that are not supported
# by the list of options
#
# @log [INPUT]
# - a dictionary that will return the status plus
# stderr containing strings identifying any
# provided keyvals that don't have a corresponding
# supported option
# @options [INPUT]
# - a dictionary of tuples, each consisting of three
# entries:
# (a) the default value
# (b) data type
# (c) a help-like description
# @keyvals [INPUT]
# - a dictionary of key-value pairs obtained from
# the configuration parser
# @target [OUTPUT]
# - the resulting dictionary of key-value pairs
def parseOptions(self, log, options, keyvals, target):
# parse the incoming keyvals dictionary against the source
# options. If a source option isn't provided, then
# copy it across to the target.
opts = list(options.keys())
kvkeys = list(keyvals.keys())
for opt in opts:
found = False
for kvkey in kvkeys:
if kvkey == opt:
# they provided us with an update, so
# pass this value into the target - expand
# any provided lists
if keyvals[kvkey] is None:
continue
st, outval = self.__convert_value(options[opt][0],
keyvals[kvkey])
if 0 == st:
target[opt] = outval
else:
if len(keyvals[kvkey]) == 0:
# this indicates they do not want this option
found = True
break
if keyvals[kvkey][0][0] == "[":
# they provided a list - remove the brackets
val = keyvals[kvkey].replace('[', '')
val = val.replace(']', '')
# split the input to pickup sets of options
newvals = list(val)
# convert the values to specified type
i = 0
for val in newvals:
st, newvals[i] = self.__convert_value(
opt[0], val)
i = i + 1
target[opt] = newvals
else:
st, target[opt] = self.__convert_value(
opt[0], keyvals[kvkey])
found = True
break
if not found:
# they didn't provide this one, so
# transfer only the value across
target[opt] = options[opt][0]
# add in any default settings that have not
# been overridden - anything set by this input
# stage will override the default
if self.defaults is not None:
keys = self.defaults.options.keys()
for key in keys:
if key not in target:
target[key] = self.defaults.options[key][0]
# now go thru in the reverse direction to see
# if any keyvals they provided aren't supported
# as this would be an error
stderr = []
for kvkey in kvkeys:
# ignore some standard keys
if kvkey in ['section', 'plugin']:
continue
try:
if target[kvkey] is not None:
pass
except KeyError:
# some always need to be passed
if kvkey in ['parent', 'asis']:
target[kvkey] = keyvals[kvkey]
else:
stderr.append("Option " + kvkey + " is not supported")
if stderr:
# mark the log with an error status
log['status'] = 1
# pass the errors back
log['stderr'] = stderr
else:
log['status'] = 0
log['options'] = target
return
def loadPlugins(self, basedir, topdir):
if self.loaded:
print("Cannot load plugins multiple times")
exit(1)
self.loaded = True
# find the loader utility so we can bootstrap ourselves
try:
m = imp.load_source("LoadClasses",
os.path.join(basedir, "LoadClasses.py"))
except ImportError:
print(
"ERROR: unable to load LoadClasses that must contain the class loader object"
)
exit(1)
cls = getattr(m, "LoadClasses")
a = cls()
# setup the loader object
self.loader = a.__class__()
# Setup the array of directories we will search for plugins
# Note that we always look at the topdir location by default
plugindirs = []
plugindirs.append(topdir)
if self.options['plugindir']:
# could be a comma-delimited list, so split on commas
x = self.options['plugindir'].split(',')
for y in x:
# prepend so we always look at the given
# location first in case the user wants
# to "overload/replace" a default MTT
# class definition
plugindirs.insert(0, y)
# Traverse the plugin directory tree and add all
# the class definitions we can find
for dirPath in plugindirs:
try:
filez = os.listdir(dirPath)
for file in filez:
file = os.path.join(dirPath, file)
if os.path.isdir(file):
self.loader.load(file)
except:
if not self.options['ignoreloadpatherrs']:
print("Plugin directory", dirPath, "not found")
sys.exit(1)
# Build the stages plugin manager
self.stages = PluginManager()
# set the location
self.stages.setPluginPlaces(plugindirs)
# Get a list of all the categories - this corresponds to
# the MTT stages that have been defined. Note that we
# don't need to formally define the stages here - anyone
# can add a new stage, or delete an old one, by simply
# adding or removing a plugin directory.
self.stages.setCategoriesFilter(self.loader.stages)
# Load all plugins we find there
self.stages.collectPlugins()
# Build the tools plugin manager - tools differ from sections
# in that they are plugins we will use to execute the various
# sections. For example, the TestRun section clearly needs the
# ability to launch jobs. There are many ways to launch jobs
# depending on the environment, and sometimes several ways to
# start jobs even within one environment (e.g., mpirun vs
# direct launch).
self.tools = PluginManager()
# location is the same
self.tools.setPluginPlaces(plugindirs)
# Get the list of tools - not every tool will be capable
# of executing. For example, a tool that supports direct launch
# against a specific resource manager cannot be used on a
# system being managed by a different RM.
self.tools.setCategoriesFilter(self.loader.tools)
# Load all the tool plugins
self.tools.collectPlugins()
# Tool plugins are required to provide a function we can
# probe to determine if they are capable of operating - check
# those now and prune those tools that cannot support this
# environment
# Build the utilities plugins
self.utilities = PluginManager()
# set the location
self.utilities.setPluginPlaces(plugindirs)
# Get the list of available utilities.
self.utilities.setCategoriesFilter(self.loader.utilities)
# Load all the utility plugins
self.utilities.collectPlugins()
# since we use these all over the place, find the
# ExecuteCmd and ModuleCmd plugins and record them
availUtil = list(self.loader.utilities.keys())
for util in availUtil:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if "ExecuteCmd" == pluginInfo.plugin_object.print_name():
self.execmd = pluginInfo.plugin_object
elif "ModuleCmd" == pluginInfo.plugin_object.print_name():
self.modcmd = pluginInfo.plugin_object
# initialize this module
self.modcmd.setCommand(self.options)
if self.execmd is not None and self.modcmd is not None:
break
if self.execmd is None:
print("ExecuteCmd plugin was not found")
print("This is a basic capability required")
print("for MTT operations - cannot continue")
sys.exit(1)
print(self.tools.getPluginsOfCategory("Harasser"))
for pluginInfo in self.tools.getPluginsOfCategory("Harasser"):
print(pluginInfo.plugin_object.print_name())
if "Harasser" == pluginInfo.plugin_object.print_name():
self.harasser = pluginInfo.plugin_object
break
if self.harasser is None:
print("Harasser plugin was not found")
print("This is required for all TestRun plugins")
print("cannot continue")
sys.exit(1)
# similarly, capture the highest priority defaults stage here
pri = -1
for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"):
if pri < pluginInfo.plugin_object.priority():
self.defaults = pluginInfo.plugin_object
pri = pluginInfo.plugin_object.priority()
return
def printInfo(self):
# Print the available MTT sections out, if requested
if self.options['listsections']:
print("Supported MTT stages:")
# print them in the default order of execution
for stage in self.loader.stageOrder:
print(" " + stage)
exit(0)
# Print the detected plugins for a given stage
if self.options['listplugins']:
# if the list is '*', print the plugins for every stage
if self.options['listplugins'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['listplugins'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(
section):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid stage name " + section)
print()
exit(1)
# Print the options for a given plugin
if self.options['liststageoptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['liststageoptions'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['liststageoptions'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(
section):
print(" " + pluginInfo.plugin_object.print_name() +
":")
pluginInfo.plugin_object.print_options(
self, " ")
except KeyError:
print(" Invalid stage name " + section)
print()
exit(1)
# Print the available MTT tools out, if requested
if self.options['listtools']:
print("Available MTT tools:")
availTools = list(self.loader.tools.keys())
for tool in availTools:
print(" " + tool)
exit(0)
# Print the detected tool plugins for a given tool type
if self.options['listtoolmodules']:
# if the list is '*', print the plugins for every type
if self.options['listtoolmodules'] == "*":
print()
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtoolmodules'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid tool type name", tool)
print()
exit(1)
# Print the options for a given plugin
if self.options['listtooloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listtooloptions'] == "*":
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtooloptions'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name() +
":")
pluginInfo.plugin_object.print_options(
self, " ")
except KeyError:
print(" Invalid tool type name " + tool)
print()
exit(1)
# Print the available MTT utilities out, if requested
if self.options['listutils']:
print("Available MTT utilities:")
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
print(" " + util)
exit(0)
# Print the detected utility plugins for a given tool type
if self.options['listutilmodules']:
# if the list is '*', print the plugins for every type
if self.options['listutilmodules'] == "*":
print()
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutilitymodules'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(
util):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid utility type name")
print()
exit(1)
# Print the options for a given plugin
if self.options['listutiloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listutiloptions'] == "*":
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutiloptions'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(
util):
print(" " + pluginInfo.plugin_object.print_name() +
":")
pluginInfo.plugin_object.print_options(
self, " ")
except KeyError:
print(" Invalid utility type name " + util)
print()
exit(1)
# if they asked for the version info, print it and exit
if self.options['version']:
for pluginInfo in self.tools.getPluginsOfCategory("Version"):
print("MTT Base: " + pluginInfo.plugin_object.getVersion())
print("MTT Client: " +
pluginInfo.plugin_object.getClientVersion())
sys.exit(0)
def openLogger(self):
# there must be a logger utility or we can't do
# anything useful
if not self.utilities.activatePluginByName("Logger", "Base"):
print("Required Logger plugin not found or could not be activated")
sys.exit(1)
# execute the provided test description
self.logger = self.utilities.getPluginByName("Logger",
"Base").plugin_object
self.logger.open(self)
return
def configTest(self):
# setup the configuration parser
self.config = configparser.SafeConfigParser(
interpolation=configparser.ExtendedInterpolation())
# Set the config parser to make option names case sensitive.
self.config.optionxform = str
# fill ENV section with environemt variables
self.config.add_section('ENV')
for k, v in os.environ.items():
self.config.set('ENV', k, v.replace("$", "$$"))
# log the list of files - note that the argument parser
# puts the input files in a list, with the first member
# being the list of input files
self.log['inifiles'] = self.args.ini_files[0]
# initialize the list of active sections
self.actives = []
# if they specified a list to execute, then use it
sections = []
if self.args.section:
sections = self.args.section.split(",")
skip = False
elif self.args.skipsections:
sections = self.args.skipsections.split(",")
skip = True
else:
sections = None
# cycle thru the input files
for testFile in self.log['inifiles']:
if not os.path.isfile(testFile):
print("Test description file", testFile, "not found!")
sys.exit(1)
self.config.read(self.log['inifiles'])
# Check for ENV input
required_env = []
all_file_contents = []
for testFile in self.log['inifiles']:
file_contents = open(testFile, "r").read()
file_contents = "\n".join([
"%s %d: %s" % (testFile.split("/")[-1], i, l)
for i, l in enumerate(file_contents.split("\n"))
if not l.lstrip().startswith("#")
])
all_file_contents.append(file_contents)
if "${ENV:" in file_contents:
required_env.extend([
s.split("}")[0] for s in file_contents.split("${ENV:")[1:]
])
env_not_found = set(
[e for e in required_env if e not in os.environ.keys()])
lines_with_env_not_found = []
for file_contents in all_file_contents:
lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \
for l in file_contents.split("\n") \
if sum(["${ENV:%s}"%e in l for e in env_not_found])])
if lines_with_env_not_found:
print("ERROR: Not all required environment variables are defined.")
print("ERROR: Still need:")
for l in lines_with_env_not_found:
print("ERROR: %s" % l)
sys.exit(1)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
# if we are to filter the sections, then do so
takeus = True
if sections is not None:
found = False
for sec in sections:
if sec == section:
found = True
sections.remove(sec)
if skip:
takeus = False
break
if not found and not skip:
takeus = False
if takeus:
self.actives.append(section)
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
if sections is not None and 0 != len(sections) and not skip:
print(
"ERROR: sections were specified for execution and not found:",
sections)
sys.exit(1)
return
# Used with combinatorial executor, loads next .ini file to be run with the
# sequential executor
def configNewTest(self, file):
# clear the configuration parser
for section in self.config.sections():
self.config.remove_section(section)
# read in the file
self.config.read(file)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
return
def executeTest(self):
if not self.loaded:
print("Plugins have not been loaded - cannot execute test")
exit(1)
if self.config is None:
print("No test definition file was parsed - cannot execute test")
exit(1)
if not self.tools.getPluginByName("sequential", "Executor"):
print("Specified executor sequential not found")
exit(1)
# activate the specified plugin
self.tools.activatePluginByName("sequential", "Executor")
# execute the provided test description
executor = self.tools.getPluginByName("sequential", "Executor")
status = executor.plugin_object.execute(self)
return status
def executeCombinatorial(self):
if not self.tools.getPluginByName("combinatorial", "Executor"):
print("Specified executor combinatorial not found")
exit(1)
# activate the specified plugin
self.tools.activatePluginByName("combinatorial", "Executor")
# execute the provided test description
executor = self.tools.getPluginByName("combinatorial", "Executor")
status = executor.plugin_object.execute(self)
return status
def printOptions(self, options):
# if the options are empty, report that
if not options:
lines = ["None"]
return lines
# create the list of options
opts = []
vals = list(options.keys())
for val in vals:
opts.append(val)
if options[val][0] is None:
opts.append("None")
elif isinstance(options[val][0], bool):
if options[val][0]:
opts.append("True")
else:
opts.append("False")
elif isinstance(options[val][0], list):
opts.append(" ".join(options[val][0]))
elif isinstance(options[val][0], int):
opts.append(str(options[val][0]))
else:
opts.append(options[val][0])
opts.append(options[val][1])
# print the options, their default value, and
# the help description in 3 column format
max1 = 0
max2 = 0
for i in range(0, len(opts), 3):
# we want all the columns to line up
# and left-justify, so first find out
# the max len of each of the first two
# column entries
if len(opts[i]) > max1:
max1 = len(opts[i])
if type(opts[i + 1]) is not str:
optout = str(opts[i + 1])
else:
optout = opts[i + 1]
if len(optout) > max2:
max2 = len(optout)
# provide some spacing
max1 = max1 + 4
max2 = max2 + 4
# cycle thru again, padding each entry to
# align the columns
lines = []
sp = " "
for i in range(0, len(opts), 3):
line = opts[i] + (max1 - len(opts[i])) * sp
if type(opts[i + 1]) is not str:
optout = str(opts[i + 1])
else:
optout = opts[i + 1]
line = line + optout + (max2 - len(optout)) * sp
# to make this more readable, we will wrap the line at
# 130 characters. First, see if the line is going to be
# too long
if 130 < (len(line) + len(opts[i + 2])):
# split the remaining column into individual words
words = opts[i + 2].split()
first = True
for word in words:
if (len(line) + len(word)) < 130:
if first:
line = line + word
first = False
else:
line = line + " " + word
else:
lines.append(line)
line = (max1 + max2) * sp + word
if 0 < len(line):
lines.append(line)
else:
# the line is fine - so just add the last piece
line = line + opts[i + 2]
# append the result
lines.append(line)
# add one blank line
lines.append("")
return lines
def selectPlugin(self, name, category):
if category == "stage":
try:
availStages = list(self.loader.stages.keys())
for stage in availStages:
for pluginInfo in self.stages.getPluginsOfCategory(stage):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "tool":
try:
availTools = list(self.loader.tools.keys())
for tool in availTools:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "utility":
try:
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
for pluginInfo in self.utilities.getPluginsOfCategory(
util):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
else:
print("Unrecognized category:", category)
return None
class TestDef(object):
def __init__(self):
# set aside storage for options and cmd line args
self.options = {}
self.args = []
# record if we have loaded the plugins or
# not - this is just a bozo check to ensure
# someone doesn't tell us to do it twice
self.loaded = False
# set aside a spot for a logger object, and
# note that it hasn't yet been defined
self.logger = None
self.modcmd = None
self.execmd = None
self.harasser = None
self.config = None
self.stages = None
self.tools = None
self.utilities = None
self.defaults = None
self.log = {}
self.watchdog = None
self.plugin_trans_sem = Semaphore()
def setOptions(self, args):
self.options = vars(args)
self.args = args
# private function to convert values
def __convert_value(self, opt, inval):
if opt is None or type(opt) is str:
return 0, inval
elif type(opt) is bool:
if type(inval) is bool:
return 0, inval
elif type(inval) is str:
if inval.lower() in ['true', '1', 't', 'y', 'yes']:
return 0, True
else:
return 0, False
elif type(inval) is int:
if 0 == inval:
return 0, False
else:
return 0, True
elif is_py2 and type(inval) is unicode:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is int:
if type(inval) is int:
return 0, inval
elif type(inval) is str:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is float:
if type(inval) is float:
return 0, inval
elif type(inval) is str or type(inval) is int:
return 0, float(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
else:
return 1, None
# scan the key-value pairs obtained from the configuration
# parser and compare them with the options defined for a
# given plugin. Generate an output dictionary that contains
# the updated set of option values, the default value for
# any option that wasn't included in the configuration file,
# and return an error status plus output identifying any
# keys in the configuration file that are not supported
# by the list of options
#
# @log [INPUT]
# - a dictionary that will return the status plus
# stderr containing strings identifying any
# provided keyvals that don't have a corresponding
# supported option
# @options [INPUT]
# - a dictionary of tuples, each consisting of three
# entries:
# (a) the default value
# (b) data type
# (c) a help-like description
# @keyvals [INPUT]
# - a dictionary of key-value pairs obtained from
# the configuration parser
# @target [OUTPUT]
# - the resulting dictionary of key-value pairs
def parseOptions(self, log, options, keyvals, target):
# parse the incoming keyvals dictionary against the source
# options. If a source option isn't provided, then
# copy it across to the target.
opts = list(options.keys())
kvkeys = list(keyvals.keys())
for opt in opts:
found = False
for kvkey in kvkeys:
if kvkey == opt:
# they provided us with an update, so
# pass this value into the target - expand
# any provided lists
if keyvals[kvkey] is None:
continue
st, outval = self.__convert_value(options[opt][0], keyvals[kvkey])
if 0 == st:
target[opt] = outval
else:
if len(keyvals[kvkey]) == 0:
# this indicates they do not want this option
found = True
break
if keyvals[kvkey][0][0] == "[":
# they provided a list - remove the brackets
val = keyvals[kvkey].replace('[','')
val = val.replace(']','')
# split the input to pickup sets of options
newvals = list(val)
# convert the values to specified type
i=0
for val in newvals:
st, newvals[i] = self.__convert_value(opt[0], val)
i = i + 1
target[opt] = newvals
else:
st, target[opt] = self.__convert_value(opt[0], keyvals[kvkey])
found = True
break
if not found:
# they didn't provide this one, so
# transfer only the value across
target[opt] = options[opt][0]
# add in any default settings that have not
# been overridden - anything set by this input
# stage will override the default
if self.defaults is not None:
keys = self.defaults.options.keys()
for key in keys:
if key not in target:
target[key] = self.defaults.options[key][0]
# now go thru in the reverse direction to see
# if any keyvals they provided aren't supported
# as this would be an error
unsupported_options = []
for kvkey in kvkeys:
# ignore some standard keys
if kvkey in ['section', 'plugin']:
continue
try:
if target[kvkey] is not None:
pass
except KeyError:
# some always need to be passed
if kvkey in ['parent', 'asis']:
target[kvkey] = keyvals[kvkey]
else:
unsupported_options.append(kvkey)
if unsupported_options:
sys.exit("ERROR: Unsupported options for section [%s]: %s" % (log['section'], ",".join(unsupported_options)))
log['status'] = 0
log['options'] = target
return
def loadPlugins(self, basedir, topdir):
if self.loaded:
print("Cannot load plugins multiple times")
sys.exit(1)
self.loaded = True
# find the loader utility so we can bootstrap ourselves
try:
m = imp.load_source("LoadClasses", os.path.join(basedir, "LoadClasses.py"));
except ImportError:
print("ERROR: unable to load LoadClasses that must contain the class loader object")
sys.exit(1)
cls = getattr(m, "LoadClasses")
a = cls()
# setup the loader object
self.loader = a.__class__();
# Setup the array of directories we will search for plugins
# Note that we always look at the topdir location by default
plugindirs = []
plugindirs.append(topdir)
if self.options['plugindir']:
# could be a comma-delimited list, so split on commas
x = self.options['plugindir'].split(',')
for y in x:
# prepend so we always look at the given
# location first in case the user wants
# to "overload/replace" a default MTT
# class definition
plugindirs.insert(0, y)
# Load plugins from each of the specified plugin dirs
for dirPath in plugindirs:
if not Path(dirPath).exists():
print("Attempted to load plugins from non-existent path:", dirPath)
continue
try:
self.loader.load(dirPath)
except Exception as e:
print("Exception caught while loading plugins:")
print(e)
sys.exit(1)
# Build plugin managers,
# class yapsy.PluginManager.PluginManager(categories_filter=None,
# directories_list=None, plugin_info_ext=None, plugin_locator=None)
# Build the stages plugin manager
self.stages = PluginManager(None, plugindirs, None, None)
# Get a list of all the categories - this corresponds to
# the MTT stages that have been defined. Note that we
# don't need to formally define the stages here - anyone
# can add a new stage, or delete an old one, by simply
# adding or removing a plugin directory.
self.stages.setCategoriesFilter(self.loader.stages)
# Load all plugins we find there
self.stages.collectPlugins()
# Build the tools plugin manager - tools differ from sections
# in that they are plugins we will use to execute the various
# sections. For example, the TestRun section clearly needs the
# ability to launch jobs. There are many ways to launch jobs
# depending on the environment, and sometimes several ways to
# start jobs even within one environment (e.g., mpirun vs
# direct launch).
self.tools = PluginManager(None, plugindirs, None, None)
# Get the list of tools - not every tool will be capable
# of executing. For example, a tool that supports direct launch
# against a specific resource manager cannot be used on a
# system being managed by a different RM.
self.tools.setCategoriesFilter(self.loader.tools)
# Load all the tool plugins
self.tools.collectPlugins()
# Tool plugins are required to provide a function we can
# probe to determine if they are capable of operating - check
# those now and prune those tools that cannot support this
# environment
# Build the utilities plugins
self.utilities = PluginManager(None, plugindirs, None, None)
# Get the list of available utilities.
self.utilities.setCategoriesFilter(self.loader.utilities)
# Load all the utility plugins
self.utilities.collectPlugins()
# since we use these all over the place, find the
# ExecuteCmd and ModuleCmd plugins and record them
availUtil = list(self.loader.utilities.keys())
for util in availUtil:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if "ExecuteCmd" == pluginInfo.plugin_object.print_name():
self.execmd = pluginInfo.plugin_object
elif "ModuleCmd" == pluginInfo.plugin_object.print_name():
self.modcmd = pluginInfo.plugin_object
# initialize this module
self.modcmd.setCommand(self.options)
elif "Watchdog" == pluginInfo.plugin_object.print_name():
self.watchdog = pluginInfo.plugin_object
if self.execmd is not None and self.modcmd is not None and self.watchdog is not None:
break
if self.execmd is None:
print("ExecuteCmd plugin was not found")
print("This is a basic capability required")
print("for MTT operations - cannot continue")
sys.exit(1)
# Configure harasser plugin
for pluginInfo in self.tools.getPluginsOfCategory("Harasser"):
if "Harasser" == pluginInfo.plugin_object.print_name():
self.harasser = pluginInfo.plugin_object
break
if self.harasser is None:
print("Harasser plugin was not found")
print("This is required for all TestRun plugins")
print("cannot continue")
sys.exit(1)
# similarly, capture the highest priority defaults stage here
pri = -1
for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"):
if pri < pluginInfo.plugin_object.priority():
self.defaults = pluginInfo.plugin_object
pri = pluginInfo.plugin_object.priority()
return
def printInfo(self):
# Print the available MTT sections out, if requested
if self.options['listsections']:
print("Supported MTT stages:")
# print them in the default order of execution
for stage in self.loader.stageOrder:
print(" " + stage)
sys.exit(0)
# Print the detected plugins for a given stage
if self.options['listplugins']:
# if the list is '*', print the plugins for every stage
if self.options['listplugins'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['listplugins'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(section):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid stage name " + section)
print()
sys.exit(1)
# Print the options for a given plugin
if self.options['liststageoptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['liststageoptions'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['liststageoptions'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(section):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid stage name " + section)
print()
sys.exit(1)
# Print the available MTT tools out, if requested
if self.options['listtools']:
print("Available MTT tools:")
availTools = list(self.loader.tools.keys())
for tool in availTools:
print(" " + tool)
sys.exit(0)
# Print the detected tool plugins for a given tool type
if self.options['listtoolmodules']:
# if the list is '*', print the plugins for every type
if self.options['listtoolmodules'] == "*":
print()
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtoolmodules'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid tool type name",tool)
print()
sys.exit(1)
# Print the options for a given plugin
if self.options['listtooloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listtooloptions'] == "*":
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtooloptions'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid tool type name " + tool)
print()
sys.exit(1)
# Print the available MTT utilities out, if requested
if self.options['listutils']:
print("Available MTT utilities:")
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
print(" " + util)
sys.exit(0)
# Print the detected utility plugins for a given tool type
if self.options['listutilmodules']:
# if the list is '*', print the plugins for every type
if self.options['listutilmodules'] == "*":
print()
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutilitymodules'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid utility type name")
print()
sys.exit(1)
# Print the options for a given plugin
if self.options['listutiloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listutiloptions'] == "*":
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutiloptions'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid utility type name " + util)
print()
sys.exit(1)
# if they asked for the version info, print it and exit
if self.options['version']:
for pluginInfo in self.tools.getPluginsOfCategory("Version"):
print("MTT Base: " + pluginInfo.plugin_object.getVersion())
print("MTT Client: " + pluginInfo.plugin_object.getClientVersion())
sys.exit(0)
def openLogger(self):
# there must be a logger utility or we can't do
# anything useful
if not self.utilities.activatePluginByName("Logger", "Base"):
print("Required Logger plugin not found or could not be activated")
sys.exit(1)
# execute the provided test description
self.logger = self.utilities.getPluginByName("Logger", "Base").plugin_object
self.logger.open(self)
return
def fill_log_interpolation(self, basestr, sublog):
if isinstance(sublog, str):
self.config.set("LOG", basestr, sublog.replace("$","$$"))
elif isinstance(sublog, dict):
for k,v in sublog.items():
self.fill_log_interpolation("%s.%s" % (basestr, k), v)
elif isinstance(sublog, list):
if sum([((isinstance(t, list) or isinstance(t, tuple)) and len(t) == 2) for t in sublog]) == len(sublog):
self.fill_log_interpolation(basestr, {k:v for k,v in sublog})
else:
for i,v in enumerate(sublog):
self.fill_log_interpolation("%s.%d" % (basestr, i), v)
else:
self.fill_log_interpolation(basestr, str(sublog))
def expandWildCards(self, sections):
expsec = []
cpsections = list(sections)
for sec in cpsections:
if '*' in sec:
modsec = sec.split('*')
startswith = modsec[0]
endswith = modsec[-1]
findsec = modsec[1:-1]
allsections = self.config.sections()
for s in allsections:
if not s.startswith(startswith):
continue
if not s.endswith(endswith):
continue
found = True
s_tmp = s
for f in findsec:
if not f in s_tmp:
found = False
break
s_tmp = f.join(s_tmp.split(f)[1:])
if not found:
continue
expsec.append(s)
sections.remove(sec)
return sections + expsec
def fill_env_hidden_section(self):
"""fill ENV section with environment variables
"""
try:
self.config.add_section('ENV')
except configparser.DuplicateSectionError:
pass
for k,v in os.environ.items():
self.config.set('ENV', k, v.replace("$","$$"))
def fill_log_hidden_section(self):
"""Add LOG section filled with log results of stages
"""
try:
self.config.add_section('LOG')
except configparser.DuplicateSectionError:
pass
thefulllog = self.logger.getLog(None)
for e in thefulllog:
self.fill_log_interpolation(e['section'].replace(":","_"), e)
def check_for_nondefined_env_variables(self):
# Check for ENV input
required_env = []
all_file_contents = []
for testFile in self.log['inifiles']:
file_contents = open(testFile, "r").read()
file_contents = "\n".join(["%s %d: %s" % (testFile.split("/")[-1],i,l) for i,l in enumerate(file_contents.split("\n")) if not l.lstrip().startswith("#")])
all_file_contents.append(file_contents)
if "${ENV:" in file_contents:
required_env.extend([s.split("}")[0] for s in file_contents.split("${ENV:")[1:]])
env_not_found = set([e for e in required_env if e not in os.environ.keys()])
lines_with_env_not_found = []
for file_contents in all_file_contents:
lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \
for l in file_contents.split("\n") \
if sum(["${ENV:%s}"%e in l for e in env_not_found])])
if lines_with_env_not_found:
print("ERROR: Not all required environment variables are defined.")
print("ERROR: Still need:")
for l in lines_with_env_not_found:
print("ERROR: %s"%l)
sys.exit(1)
def configTest(self):
# setup the configuration parser
self.config = configparser.SafeConfigParser(interpolation=configparser.ExtendedInterpolation())
# Set the config parser to make option names case sensitive.
self.config.optionxform = str
# fill ENV section with environemt variables
self.fill_env_hidden_section()
# log the list of files - note that the argument parser
# puts the input files in a list, with the first member
# being the list of input files
self.log['inifiles'] = self.args.ini_files[0]
# initialize the list of active sections
self.actives = []
# if they specified a list to execute, then use it
sections = []
if self.args.section:
sections = self.args.section.split(",")
skip = False
elif self.args.skipsections:
sections = self.args.skipsections.split(",")
skip = True
else:
sections = None
# cycle thru the input files
for testFile in self.log['inifiles']:
if not os.path.isfile(testFile):
print("Test description file",testFile,"not found!")
sys.exit(1)
self.config.read(self.log['inifiles'])
# Check for ENV input
self.check_for_nondefined_env_variables()
# find all the sections that match the wild card and expand them
# this is simple wild carding, ie *text, text*, *text* and *
# should all work
if sections is not None:
sections = self.expandWildCards(sections)
#if sections is not None:
# expsec = []
# cpsections = list(sections)
# for sec in cpsections:
# if '*' in sec:
# modsec = sec.replace('*','')
# for s in self.config.sections():
# if modsec in s:
# expsec.append(s)
# sections.remove(sec)
# sections = sections + expsec
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
# if we are to filter the sections, then do so
takeus = True
if sections is not None:
found = False
for sec in sections:
if sec == section:
found = True
sections.remove(sec)
if skip:
takeus = False
break
if not found and not skip:
takeus = False
if takeus:
self.actives.append(section)
if sections is not None and 0 != len(sections) and not skip:
print("ERROR: sections were specified for execution and not found:",sections)
sys.exit(1)
# set Defaults -command line args supercede .ini args
try:
if not self.options['scratchdir']:
self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratchdir')
except:
try:
self.options['scratchdir'] = self.config.get('MTTDefaults', 'scratch')
except:
self.options['scratchdir'] = os.path.abspath('./mttscratch')
self.options['scratchdir'] = os.path.abspath(self.options['scratchdir'])
try:
if not self.options['executor']:
self.options['executor'] = self.config.get('MTTDefaults', 'executor')
except:
self.options['executor'] = 'sequential'
# if they want us to clear the scratch, then do so
if self.options['clean'] and os.path.isdir(self.options['scratchdir']) :
shutil.rmtree(self.options['scratchdir'])
# setup the scratch directory
_mkdir_recursive(self.options['scratchdir'])
return
# Used with combinatorial executor, loads next .ini file to be run with the
# sequential executor
def configNewTest(self, file):
# clear the configuration parser
for section in self.config.sections():
self.config.remove_section(section)
# read in the file
self.config.read(file)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
return
def executeTest(self, executor="sequential"):
self.logger.print_cmdline_args(self)
if not self.loaded:
print("Plugins have not been loaded - cannot execute test")
sys.exit(1)
if self.config is None:
print("No test definition file was parsed - cannot execute test")
sys.exit(1)
if not self.tools.getPluginByName(executor, "Executor"):
print("Specified executor %s not found" % executor)
sys.exit(1)
# activate the specified plugin
self.tools.activatePluginByName(executor, "Executor")
# execute the provided test description
executor = self.tools.getPluginByName(executor, "Executor")
status = executor.plugin_object.execute(self)
if status == 0 and self.options['clean_after'] and os.path.isdir(self.options['scratchdir']):
self.logger.verbose_print("Cleaning up scratchdir after successful run")
shutil.rmtree(self.options['scratchdir'])
return status
def printOptions(self, options):
# if the options are empty, report that
if not options:
lines = ["None"]
return lines
# create the list of options
opts = []
vals = list(options.keys())
for val in vals:
opts.append(val)
if options[val][0] is None:
opts.append("None")
elif isinstance(options[val][0], bool):
if options[val][0]:
opts.append("True")
else:
opts.append("False")
elif isinstance(options[val][0], list):
opts.append(" ".join(options[val][0]))
elif isinstance(options[val][0], int):
opts.append(str(options[val][0]))
else:
opts.append(options[val][0])
opts.append(options[val][1])
# print the options, their default value, and
# the help description in 3 column format
max1 = 0
max2 = 0
for i in range(0,len(opts),3):
# we want all the columns to line up
# and left-justify, so first find out
# the max len of each of the first two
# column entries
if len(opts[i]) > max1:
max1 = len(opts[i])
if type(opts[i+1]) is not str:
optout = str(opts[i+1])
else:
optout = opts[i+1]
if len(optout) > max2:
max2 = len(optout)
# provide some spacing
max1 = max1 + 4
max2 = max2 + 4
# cycle thru again, padding each entry to
# align the columns
lines = []
sp = " "
for i in range(0,len(opts),3):
line = opts[i] + (max1-len(opts[i]))*sp
if type(opts[i+1]) is not str:
optout = str(opts[i+1])
else:
optout = opts[i+1]
line = line + optout + (max2-len(optout))*sp
# to make this more readable, we will wrap the line at
# 130 characters. First, see if the line is going to be
# too long
if 130 < (len(line) + len(opts[i+2])):
# split the remaining column into individual words
words = opts[i+2].split()
first = True
for word in words:
if (len(line) + len(word)) < 130:
if first:
line = line + word
first = False
else:
line = line + " " + word
else:
lines.append(line)
line = (max1 + max2)*sp + word
if 0 < len(line):
lines.append(line)
else:
# the line is fine - so just add the last piece
line = line + opts[i+2]
# append the result
lines.append(line)
# add one blank line
lines.append("")
return lines
def selectPlugin(self, name, category):
if category == "stage":
try:
availStages = list(self.loader.stages.keys())
for stage in availStages:
for pluginInfo in self.stages.getPluginsOfCategory(stage):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "tool":
try:
availTools = list(self.loader.tools.keys())
for tool in availTools:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "utility":
try:
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
else:
print("Unrecognized category:",category)
return None
from yapsy.PluginManager import PluginManager
import logging
logging.basicConfig(level=logging.DEBUG)
# Build the manager
simplePluginManager = PluginManager()
# Tell it the default place(s) where to find plugins
simplePluginManager.setPluginPlaces(["plugins"])
# Load all plugins
simplePluginManager.setPluginLocator()
simplePluginManager.collectPlugins()
print("start")
# Activate all loaded plugins
print(simplePluginManager.getAllPlugins())
for pluginInfo in simplePluginManager.getAllPlugins():
print(pluginInfo)
simplePluginManager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.speak()
class EditorController:
def __init__(self, editor, global_state, buffer_list):
self._editor = editor
self._global_state = global_state
self._buffer_list = buffer_list
self._lexer = Lexer()
self._plugin_manager = PluginManager()
self._plugin_manager.getPluginLocator().setPluginInfoExtension("ini")
self._plugin_manager.setPluginPlaces([
paths.pluginsDir("user", "commands"),
paths.pluginsDir("system", "commands")
])
self._plugin_manager.collectPlugins()
for plugin_info in self._plugin_manager.getAllPlugins():
self._plugin_manager.activatePluginByName(plugin_info.name)
# To speed up resolution, we cache the keyword->plugin association. It is filled lazy
self._keyword_to_plugin_cache = {}
self._buffer_list.currentBufferChanged.connect(
self.registerCurrentBuffer)
def registerCurrentBuffer(self, *args):
self._editor.edit_area.buffer = self._buffer_list.current
self._editor.status_bar_controller.buffer = self._buffer_list.current
self._editor.side_ruler_controller.buffer = self._buffer_list.current
self._editor.info_hover_box.buffer = self._buffer_list.current
self._lexer.setModel(self._buffer_list.current.document)
def forceQuit(self):
for b in self._buffer_list.buffers:
if b.document.documentMetaInfo("Filename").data() is None:
continue
models.EditorState.instance().setCursorPosForPath(
os.path.abspath(
b.document.documentMetaInfo("Filename").data()),
b.cursor.pos)
models.EditorState.instance().save()
models.Configuration.save()
gui.VApplication.vApp.exit()
def doSave(self):
self._doSave()
self._doLint()
def doSaveAs(self, filename):
self._doSave(filename)
self._doLint()
def doInsertFile(self, filename):
buffer = self._buffer_list.current
command = commands.InsertFileCommand(buffer, filename)
result = command.execute()
if result.success:
buffer.command_history.add(command)
def tryQuit(self):
if any([b.isModified() for b in self._buffer_list.buffers]):
self._editor.status_bar.setMessage(
"Document has been modified. " +
"Use :q! to quit without saving " + "or :qw to save and quit.",
3000)
else:
self.forceQuit()
def searchForward(self, search_text):
if search_text == '':
if self._global_state.current_search is not None:
search_text = self._global_state.current_search[0]
if search_text != '':
self._global_state.current_search = (
search_text, Search.SearchDirection.FORWARD)
Search.find(self._buffer_list.current, search_text,
Search.SearchDirection.FORWARD)
def searchBackward(self, search_text):
if search_text == '':
if self._global_state.current_search is not None:
search_text = self._global_state.current_search[0]
if search_text != '':
self._global_state.current_search = (
search_text, Search.SearchDirection.BACKWARD)
Search.find(self._buffer_list.current, search_text,
Search.SearchDirection.BACKWARD)
def selectPrevBuffer(self):
self._buffer_list.selectPrev()
def selectNextBuffer(self):
self._buffer_list.selectNext()
def doSaveAndExit(self):
self._doSave()
self.forceQuit()
def openFile(self, filename):
buffer = self._buffer_list.bufferForFilename(filename)
if buffer is not None:
self._buffer_list.select(buffer)
return
current_buffer = self._buffer_list.current
new_buffer = models.Buffer()
status_bar = self._editor.status_bar
try:
with open(filename, 'r') as f:
new_buffer.document.read(f)
except FileNotFoundError:
status_bar.setMessage("%s [New file]" % filename, 3000)
except Exception as e:
status_bar.setMessage("%s [Error: %s]" % (filename, str(e)), 3000)
new_buffer.document.documentMetaInfo("Filename").setData(filename)
new_buffer.document.documentMetaInfo("Modified").setData(False)
initial_md5 = None
if not new_buffer.document.isEmpty():
initial_md5 = hashlib.md5(
new_buffer.document.documentText().encode("utf-8"))
new_buffer.document.documentMetaInfo("InitialMD5").setData(initial_md5)
if current_buffer.isEmpty(
) and not current_buffer.document.documentMetaInfo("Modified").data():
self._buffer_list.replaceAndSelect(current_buffer, new_buffer)
else:
self._buffer_list.addAndSelect(new_buffer)
recovered_cursor_pos = models.EditorState.instance().cursorPosForPath(
os.path.abspath(filename))
if recovered_cursor_pos is not None:
new_buffer.cursor.toPos(recovered_cursor_pos)
self._doLint()
def createEmptyBuffer(self):
self._buffer_list.addAndSelect(models.Buffer())
def setMode(self, mode):
self._global_state.edit_mode = mode
def interpretCommandLine(self, command_line):
"""
Interprets and dispatch the command line to the appropriate command execution routine,
or the plugin system.
command_line contains the full command line as specified by the user, as a string.
"""
if len(command_line.strip()) == 0:
return
command_tokens = shlex.split(command_line)
keyword = command_tokens[0]
status_bar = self._editor.status_bar
if keyword == 'q!':
self.forceQuit()
elif keyword == 'q':
self.tryQuit()
elif keyword == "w":
if len(command_tokens) == 1:
self.doSave()
elif len(command_tokens) == 2:
self.doSaveAs(command_tokens[1])
else:
status_bar.setMessage("Only one filename allowed at write",
3000)
return
elif keyword == "r":
if len(command_tokens) == 1:
status_bar.setMessage("Specify filename", 3000)
return
elif len(command_tokens) == 2:
self.doInsertFile(command_tokens[1])
else:
status_bar.setMessage("Only one filename allowed", 3000)
return
elif keyword in ("wq", "x"):
self.doSaveAndExit()
elif keyword == "e":
if len(command_tokens) == 1:
status_bar.setMessage("Specify filename", 3000)
elif len(command_tokens) == 2:
self.openFile(command_tokens[1])
else:
status_bar.setMessage("Only one filename allowed", 3000)
return
elif keyword == "bp":
self.selectPrevBuffer()
elif keyword == "bn":
self.selectNextBuffer()
else:
if keyword not in self._keyword_to_plugin_cache:
for plugin_info in self._plugin_manager.getAllPlugins():
plugin_object = plugin_info.plugin_object
if plugin_object.keyword() == keyword:
self._keyword_to_plugin_cache[keyword] = plugin_object
if keyword in self._keyword_to_plugin_cache:
plugin_object = self._keyword_to_plugin_cache[keyword]
plugin_object.execute(command_line)
else:
status_bar.setMessage("Unrecognized command", 3000)
# Private
def _doLint(self):
document = self._buffer_list.current.document
linter1 = linting.PyFlakesLinter(document)
all_info = linter1.runOnce()
meta_info = {}
for info in all_info:
meta_info[info.line] = info
line_info = document.lineMetaInfo("LinterResult")
line_info.clear()
line_info.setDataForLines(meta_info)
def _doSave(self, filename=None):
status_bar = self._editor.status_bar
document = self._buffer_list.current.document
if filename is not None and len(filename) == 0:
status_bar.setMessage("Error! Unspecified file name.", 3000)
return
status_bar.setMessage("Saving...")
gui.VApplication.vApp.processEvents()
if filename is None:
filename = document.documentMetaInfo("Filename").data()
if filename is None:
status_bar.setMessage(
"Error! Cannot save unnamed file. Please specify a filename with :w filename",
3000)
return
try:
with open(filename, 'w') as f:
document.write(f)
except Exception as e:
status_bar.setMessage("Error! Cannot save file. %s" % str(e), 3000)
return
else:
status_bar.setMessage("Saved %s" % filename, 3000)
document.documentMetaInfo("Filename").setData(filename)
document.documentMetaInfo("Modified").setData(False)
document.lineMetaInfo("Change").clear()
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
"DISABLED_PLUGINS": [],
"EXTRA_PLUGINS": [],
"EXTRA_PLUGINS_DIRS": [extra_v6_plugin_dir],
"DEFAULT_LANG": "en",
"MARKDOWN_EXTENSIONS": ["fenced_code", "codehilite"],
"TRANSLATIONS_PATTERN": "{path}.{lang}.{ext}",
"LISTINGS_FOLDERS": {},
}
self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"]
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
}
)
self.loghandlers = [nikola.utils.STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension("plugin")
extra_plugins_dirs = self.config["EXTRA_PLUGINS_DIRS"]
if sys.version_info[0] == 3:
places = [os.path.join(os.path.dirname(nikola.utils.__file__), "plugins")] + [
path for path in extra_plugins_dirs if path
]
else:
places = [os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode("plugins"))] + [
nikola.utils.sys_encode(path) for path in extra_plugins_dirs if path
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension")
self.timeline = [FakePost(title="Fake post", slug="fake-post")]
self.debug = True
self.rst_transforms = []
# This is to make plugin initialization happy
self.template_system = self
self.name = "mako"
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get("DISABLED_PLUGINS"):
self.plugin_manager.removePluginFromCategory(plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return '<img src="IMG.jpg">'
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = [
'planetoid',
'ipynb',
'local_search',
'render_mustache',
]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ANALYTICS': '',
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'CONTENT_FOOTER': '',
'CREATE_MONTHLY_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'DISQUS_FORUM': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FILE_METADATA_REGEXP': None,
'FILES_FOLDERS': {'files': ''},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GZIP_FILES': False,
'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json'),
'HIDE_UNTRANSLATED_POSTS': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEX_PATH': '',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'post_compilers': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile',),
"txt2tags": ('.t2t',),
"bbcode": ('.bb',),
"wiki": ('.wiki',),
"ipynb": ('.ipynb',),
"html": ('.html', '.htm')
},
'POST_PAGES': (
("posts/*.txt", "posts", "post.tmpl", True),
("stories/*.txt", "stories", "story.tmpl", False),
),
'PRETTY_URLS': False,
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'site',
'THEME_REVEAL_CONGIF_SUBTHEME': 'sky',
'THEME_REVEAL_CONGIF_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': None,
}
self.config.update(config)
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
print("WARNING: You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML")
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
{self.config['DEFAULT_'
'LANG']: ''})
self.THEMES = utils.get_theme_chain(self.config['THEME'])
self.MESSAGES = utils.load_messages(self.THEMES,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'])
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
print("WARNING: You should configure SITE_URL instead of BLOG_URL")
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(__file__), utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
if plugin_info.name in self.config["post_compilers"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = {
}
self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
self.GLOBAL_CONTEXT['_link'] = self.link
self.GLOBAL_CONTEXT['set_locale'] = s_l
self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
self.GLOBAL_CONTEXT['exists'] = self.file_exists
self.GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config[
'SLUG_TAG_PATH']
self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
'ADD_THIS_BUTTONS']
self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self.GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self.GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self.GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self.GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self.GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
self.GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL', self.config.get('BLOG_URL'))
self.GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self.GLOBAL_CONTEXT['analytics'] = self.config.get('ANALYTICS')
self.GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self.GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self.GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self.GLOBAL_CONTEXT['disqus_forum'] = self.config.get('DISQUS_FORUM')
self.GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self.GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONGIF_SUBTHEME')
self.GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONGIF_TRANSITION')
self.GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self.GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self.GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self.GLOBAL_CONTEXT['sidebar_links'] = utils.Functionary(list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('SIDEBAR_LINKS', {}).items():
self.GLOBAL_CONTEXT['sidebar_links'][k] = v
self.GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self.GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA')
self.GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# check if custom css exist and is not empty
for files_path in list(self.config['FILES_FOLDERS'].keys()):
custom_css_path = os.path.join(files_path, 'assets/css/custom.css')
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT['has_custom_css'] = True
break
else:
self.GLOBAL_CONTEXT['has_custom_css'] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(
template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES]
self.template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config['USE_CDN']:
bootstrap_path = utils.get_asset_path(os.path.join(
'assets', 'css', 'bootstrap.min.css'), self.THEMES)
if bootstrap_path.split(os.sep)[-4] != 'site':
warnings.warn('The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.')
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in
list(self.config['post_compilers'].items())
if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(
template_name, None, local_context)
assert output_name.startswith(
self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def current_lang(self): # FIXME: this is duplicated, turn into a mixin
"""Return the currently set locale, if it's one of the
available translations, or default_lang."""
lang = utils.LocaleBorg().current_lang
if lang:
if lang in self.translations:
return lang
lang = lang.split('_')[0]
if lang in self.translations:
return lang
# whatever
return self.default_lang
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a post_pages tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = self.current_lang()
path = []
if kind == "tag_index":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'],
self.config['INDEX_FILE']] if _f]
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".html"] if
_f]
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".xml"] if
_f]
elif kind == "index":
if name not in [None, 0]:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
'index-{0}.html'.format(name)] if _f]
else:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
self.config['INDEX_FILE']]
if _f]
elif kind == "post_path":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
os.path.dirname(name),
self.config['INDEX_FILE']] if _f]
elif kind == "rss":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['RSS_PATH'], 'rss.xml'] if _f]
elif kind == "archive":
if name:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], name,
self.config['INDEX_FILE']] if _f]
else:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'],
self.config['ARCHIVE_FILENAME']] if _f]
elif kind == "gallery":
path = [_f for _f in [self.config['GALLERY_PATH'], name,
self.config['INDEX_FILE']] if _f]
elif kind == "listing":
path = [_f for _f in [self.config['LISTINGS_FOLDER'], name +
'.html'] if _f]
if is_link:
link = '/' + ('/'.join(path))
index_len = len(self.config['INDEX_FILE'])
if self.config['STRIP_INDEXES'] and \
link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BASE_URL'], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config['BASE_URL'], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self, name, plugin_category):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
yield task
for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield task
if flag:
task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': name,
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
print("Scanning posts", end='')
tzinfo = None
if self.config['TIMEZONE'] is not None:
tzinfo = pytz.timezone(self.config['TIMEZONE'])
current_time = utils.current_time(tzinfo)
targets = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='')
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination,
os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config['TRANSLATIONS'].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
for base_path in full_list:
post = Post(
base_path,
self.config['CACHE_FOLDER'],
dest_dir,
use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BASE_URL'],
self.MESSAGES,
template_name,
self.config['FILE_METADATA_REGEXP'],
self.config['STRIP_INDEXES'],
self.config['INDEX_FILE'],
tzinfo,
current_time,
self.config['HIDE_UNTRANSLATED_POSTS'],
self.config['PRETTY_URLS'],
)
for lang, langpath in list(
self.config['TRANSLATIONS'].items()):
dest = (destination, langpath, dir_glob,
post.meta[lang]['slug'])
if dest in targets:
raise Exception('Duplicated output path {0!r} '
'in post {1!r}'.format(
post.meta[lang]['slug'],
base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[
str(post.date.year)].append(post.post_name)
self.posts_per_month[
'{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post.post_name)
for tag in post.alltags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
if self.config['OLD_THEME_SUPPORT']:
post._add_old_metadata()
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
if post:
deps_dict['post_translations'] = post.translated_to
task = {
'name': os.path.normpath(output_name),
'file_dep': deps,
'targets': [output_name],
'actions': [(self.render_template, [post.template_name,
output_name, context])],
'clean': True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in
posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name': os.path.normpath(output_name),
'targets': [output_name],
'file_dep': deps,
'actions': [(self.render_template, [template_name, output_name,
context])],
'clean': True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
# This is the default config
# TODO: fill it
self.config = {
"ARCHIVE_PATH": "",
"ARCHIVE_FILENAME": "archive.html",
"DEFAULT_LANG": "en",
"OUTPUT_FOLDER": "output",
"CACHE_FOLDER": "cache",
"FILES_FOLDERS": {"files": ""},
"LISTINGS_FOLDER": "listings",
"ADD_THIS_BUTTONS": True,
"INDEX_DISPLAY_POST_COUNT": 10,
"INDEX_TEASERS": False,
"RSS_TEASERS": True,
"MAX_IMAGE_SIZE": 1280,
"USE_FILENAME_AS_TITLE": True,
"SLUG_TAG_PATH": False,
"INDEXES_TITLE": "",
"INDEXES_PAGES": "",
"FILTERS": {},
"USE_BUNDLES": True,
"TAG_PAGES_ARE_INDEXES": False,
"THEME": "default",
"COMMENTS_IN_GALLERIES": False,
"COMMENTS_IN_STORIES": False,
"FILE_METADATA_REGEXP": None,
"post_compilers": {
"rest": [".txt", ".rst"],
"markdown": [".md", ".mdown", ".markdown"],
"html": [".html", ".htm"],
},
}
self.config.update(config)
self.config["TRANSLATIONS"] = self.config.get("TRANSLATIONS", {self.config["DEFAULT_LANG"]: ""})
self.THEMES = utils.get_theme_chain(self.config["THEME"])
self.MESSAGES = utils.load_messages(self.THEMES, self.config["TRANSLATIONS"])
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
}
)
self.plugin_manager.setPluginInfoExtension("plugin")
self.plugin_manager.setPluginPlaces(
[str(os.path.join(os.path.dirname(__file__), "plugins")), str(os.path.join(os.getcwd(), "plugins"))]
)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
pluginInfo.plugin_object.short_help = pluginInfo.description
self.commands[pluginInfo.name] = pluginInfo.plugin_object
# Activate all task plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = self.config.get("GLOBAL_CONTEXT", {})
self.GLOBAL_CONTEXT["messages"] = self.MESSAGES
self.GLOBAL_CONTEXT["_link"] = self.link
self.GLOBAL_CONTEXT["rel_link"] = self.rel_link
self.GLOBAL_CONTEXT["abs_link"] = self.abs_link
self.GLOBAL_CONTEXT["exists"] = self.file_exists
self.GLOBAL_CONTEXT["add_this_buttons"] = self.config["ADD_THIS_BUTTONS"]
self.GLOBAL_CONTEXT["index_display_post_count"] = self.config["INDEX_DISPLAY_POST_COUNT"]
self.GLOBAL_CONTEXT["use_bundles"] = self.config["USE_BUNDLES"]
if "date_format" not in self.GLOBAL_CONTEXT:
self.GLOBAL_CONTEXT["date_format"] = "%Y-%m-%d %H:%M"
# check if custom css exist and is not empty
for files_path in list(self.config["FILES_FOLDERS"].keys()):
custom_css_path = os.path.join(files_path, "assets/css/custom.css")
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT["has_custom_css"] = True
break
else:
self.GLOBAL_CONTEXT["has_custom_css"] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading %s template system plugin\n" % template_sys_name)
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES]
self.template_system.set_directories(lookup_dirs, self.config["CACHE_FOLDER"])
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for pluginInfo in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
self.compilers[pluginInfo.name] = pluginInfo.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in list(self.config["post_compilers"].items()) if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit(
"Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of %s)" % ", ".join(langs)
)
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to " "handle '%s' extensions." % ext)
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.config["GLOBAL_CONTEXT"])
local_context.update(context)
data = self.template_system.render_template(template_name, None, local_context)
assert isinstance(output_name, bytes)
assert output_name.startswith(self.config["OUTPUT_FOLDER"].encode("utf8"))
url_part = output_name.decode("utf8")[len(self.config["OUTPUT_FOLDER"]) + 1 :]
# This is to support windows paths
url_part = "/".join(url_part.split(os.sep))
src = urljoin(self.config["BLOG_URL"], url_part)
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split("/")[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == "link": # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip("/"), context["lang"])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b"<!DOCTYPE html>" + lxml.html.tostring(doc, encoding="utf8")
with open(output_name, "wb+") as post_file:
post_file.write(data)
def path(self, kind, name, lang, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
path = []
if kind == "tag_index":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], "index.html"] if _f]
elif kind == "tag":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".html"] if _f]
elif kind == "tag_rss":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".xml"] if _f]
elif kind == "index":
if name > 0:
path = [
_f
for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index-%s.html" % name]
if _f
]
else:
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index.html"] if _f]
elif kind == "rss":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["RSS_PATH"], "rss.xml"] if _f]
elif kind == "archive":
if name:
path = [
_f
for _f in [self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], name, "index.html"]
if _f
]
else:
path = [
_f
for _f in [
self.config["TRANSLATIONS"][lang],
self.config["ARCHIVE_PATH"],
self.config["ARCHIVE_FILENAME"],
]
if _f
]
elif kind == "gallery":
path = [_f for _f in [self.config["GALLERY_PATH"], name, "index.html"] if _f]
elif kind == "listing":
path = [_f for _f in [self.config["LISTINGS_FOLDER"], name + ".html"] if _f]
if is_link:
return "/" + ("/".join(path))
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config["BLOG_URL"], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config["BLOG_URL"], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split("/")[1:]
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {"name": b"all", "actions": None, "clean": True, "task_dep": task_dep}
def scan_posts(self):
"""Scan all the posts."""
if not self._scanned:
print("Scanning posts", end="")
targets = set([])
for wildcard, destination, template_name, use_in_feeds in self.config["post_pages"]:
print(".", end="")
base_len = len(destination.split(os.sep))
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.join(*([destination] + dirpath.split(os.sep)[base_len:]))
for base_path in glob.glob(dir_glob):
post = Post(
base_path,
self.config["CACHE_FOLDER"],
dest_dir,
use_in_feeds,
self.config["TRANSLATIONS"],
self.config["DEFAULT_LANG"],
self.config["BLOG_URL"],
self.MESSAGES,
template_name,
self.config["FILE_METADATA_REGEXP"],
)
for lang, langpath in list(self.config["TRANSLATIONS"].items()):
dest = (destination, langpath, dir_glob, post.pagenames[lang])
if dest in targets:
raise Exception(
"Duplicated output path %r in post %r" % (post.pagenames[lang], base_path)
)
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(post.post_name)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + self.template_system.template_deps(post.template_name)
context["post"] = post
context["lang"] = lang
context["title"] = post.title(lang)
context["description"] = post.description(lang)
context["permalink"] = post.permalink(lang)
context["page_list"] = self.pages
if post.use_in_feeds:
context["enable_comments"] = True
else:
context["enable_comments"] = self.config["COMMENTS_IN_STORIES"]
output_name = os.path.join(self.config["OUTPUT_FOLDER"], post.destination_path(lang)).encode("utf8")
deps_dict = copy(context)
deps_dict.pop("post")
if post.prev_post:
deps_dict["PREV_LINK"] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict["NEXT_LINK"] = [post.next_post.permalink(lang)]
deps_dict["OUTPUT_FOLDER"] = self.config["OUTPUT_FOLDER"]
deps_dict["TRANSLATIONS"] = self.config["TRANSLATIONS"]
deps_dict["global"] = self.config["GLOBAL_CONTEXT"]
deps_dict["comments"] = context["enable_comments"]
task = {
"name": output_name,
"file_dep": deps,
"targets": [output_name],
"actions": [(self.render_template, [post.template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context):
"""Renders pages with lists of posts."""
# This is a name on disk, has to be bytes
assert isinstance(output_name, bytes)
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config["BLOG_TITLE"]
context["description"] = self.config["BLOG_DESCRIPTION"]
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.titles[lang], p.permalink(lang)) for p in posts]
deps_context["global"] = self.config["GLOBAL_CONTEXT"]
task = {
"name": output_name,
"targets": [output_name],
"file_dep": deps,
"actions": [(self.render_template, [template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_context)],
}
return utils.apply_filters(task, filters)
class NimbusPI(object):
"""The NimbusPi Weather Station"""
# Current NimbusPI Version
VERSION = "0.1.0-rc1"
def __init__(self, config='nimbus.cfg'):
"""Initializes the NimbusPI Weather Station"""
self.sensors = dict()
self.broadcasters = dict()
self.threads = []
# Initialize a named logger
self.__logger = logging.getLogger('nimbuspi')
# Load our config defaults
self.config = configparser.ConfigParser(allow_no_value=True)
self.config.add_section('station')
self.config.set('station', 'name', 'N/A')
self.config.set('station', 'location', 'N/A')
self.config.set('station', 'longitude', '0.000000')
self.config.set('station', 'latitude', '0.000000')
self.config.set('station', 'altitude', '0')
self.config.add_section('sensors')
self.config.add_section('broadcasters')
# Load the provided config file
if not os.path.isfile(config):
self.__logger.warn('Configuration file "%s" not found!', config)
else:
self.__logger.debug('Loading configuration from "%s"', config)
self.config.read(config)
# Get our station details
self.__logger.debug(' name :: %s',
self.config.get('station', 'name'))
self.__logger.debug(' location :: %s',
self.config.get('station', 'location'))
self.__logger.debug(' latitude :: %s',
self.config.get('station', 'latitude'))
self.__logger.debug(' longitude :: %s',
self.config.get('station', 'longitude'))
self.__logger.debug(' altitude :: %s feet',
self.config.get('station', 'altitude'))
self.__logger.debug('Sensors Configured:')
for sensor in self.config.options('sensors'):
self.__logger.debug(' %s', sensor)
self.__logger.debug('Broadcasters Configured:')
for broadcaster in self.config.options('broadcasters'):
self.__logger.debug(' %s', broadcaster)
# Search for available plugins
self.__logger.debug("Searching for available plugins...")
self.__plugins = PluginManager(plugin_info_ext='info')
self.__plugins.setPluginPlaces([
'./sensors', './broadcasters', './nimbuspi/sensors',
'./nimbuspi/broadcasters'
])
self.__plugins.setCategoriesFilter({
plugins.ISensorPlugin.CATEGORY:
plugins.ISensorPlugin,
plugins.IBroadcasterPlugin.CATEGORY:
plugins.IBroadcasterPlugin
})
self.__plugins.collectPlugins()
for plugin in self.__plugins.getAllPlugins():
self.__logger.debug(" %s (%s)", plugin.name, plugin.path)
plugin.plugin_object.set_nimbus(self)
self.__logger.debug("%d plugins available",
len(self.__plugins.getAllPlugins()))
def run(self):
"""Runs the NimbusPI Weather Station loop"""
self.__logger.debug('-' * 80)
self.__logger.info('NimbusPI Weather Station v%s', self.VERSION)
self.__logger.info('-' * 80)
# Load all configured sensor plugins
self.__logger.info("Activating sensor plugins...")
for sensor in self.config.options('sensors'):
try:
self.activate_sensor(sensor)
except LookupError:
self.__logger.error("Could not load sensor '%s'", sensor)
return
if len(self.sensors) <= 0:
self.__logger.error('Cannot continue - no sensors configured')
return
# Load all configured broadcaster plugins
self.__logger.info("Activating broadcaster plugins...")
for broadcaster in self.config.options('broadcasters'):
try:
self.activate_broadcaster(broadcaster)
except LookupError:
self.__logger.error("Could not load broadcaster '%s'",
broadcaster)
return
if len(self.broadcasters) <= 0:
self.__logger.error('Cannot continue - no broadcasters configured')
return
# # Thread run loop until keyboard interrupt
self.__logger.debug("Entering thread loop")
while len(self.threads) > 0:
try:
self.threads = [
t.join(30) for t in self.threads
if t is not None and t.isAlive()
]
except (KeyboardInterrupt, SystemExit):
self.__logger.info(
"Shutting down plugins (this may take a minute)...")
for thread in self.threads:
thread.stop()
self.__logger.debug("Exiting thread loop")
# Deactivate plugins
self.__logger.debug("Deactivating sensors")
sensors = self.sensors.keys()
for sensor in sensors:
self.deactivate_sensor(sensor)
self.__logger.debug("Deactivating broadcasters")
broadcasters = self.broadcasters.keys()
for broadcaster in broadcasters:
self.deactivate_broadcaster(broadcaster)
def activate_sensor(self, sensor):
"""Activates a sensor on the service"""
if sensor in self.sensors:
self.__logger.warn(
"Cannot activate sensor '%s' - sensor already active", sensor)
return False
self.__logger.debug("Activating sensor '%s'", sensor)
self.sensors[sensor] = self.__plugins.getPluginByName(
sensor, plugins.ISensorPlugin.CATEGORY)
if not self.sensors[sensor]:
raise LookupError
self.__plugins.activatePluginByName(sensor,
plugins.ISensorPlugin.CATEGORY)
self.threads.append(self.sensors[sensor].plugin_object.thread)
return True
def deactivate_sensor(self, sensor):
"""Deactivates a sensor on the service"""
if sensor not in self.sensors:
self.__logger.warn(
"Cannot deactivate sensor '%s' - sensor not active", sensor)
return False
self.__logger.debug("Deactivating sensor '%s'", sensor)
if self.sensors[sensor].plugin_object.thread:
self.sensors[sensor].plugin_object.thread.stop()
self.__plugins.deactivatePluginByName(sensor,
plugins.ISensorPlugin.CATEGORY)
del self.sensors[sensor]
return True
def activate_broadcaster(self, broadcaster):
"""Activates a broadcaster on the service"""
if broadcaster in self.broadcasters:
self.__logger.warn(
"Cannot activate broadcaster '%s' - broadcaster already active",
broadcaster)
return False
self.__logger.debug("Activating broadcaster '%s'", broadcaster)
self.broadcasters[broadcaster] = self.__plugins.getPluginByName(
broadcaster, plugins.IBroadcasterPlugin.CATEGORY)
if not self.broadcasters[broadcaster]:
raise LookupError
self.__plugins.activatePluginByName(
broadcaster, plugins.IBroadcasterPlugin.CATEGORY)
self.threads.append(
self.broadcasters[broadcaster].plugin_object.thread)
return True
def deactivate_broadcaster(self, broadcaster):
"""Deactivates a broadcaster on the service"""
if broadcaster not in self.broadcasters:
self.__logger.warn(
"Cannot deactivate broadcaster '%s' - broadcaster not active",
broadcaster)
return False
self.__logger.debug("Deactivating broadcaster '%s'", broadcaster)
if self.broadcasters[broadcaster].plugin_object.thread:
self.broadcasters[broadcaster].plugin_object.thread.stop()
self.__plugins.deactivatePluginByName(
broadcaster, plugins.IBroadcasterPlugin.CATEGORY)
del self.broadcasters[broadcaster]
return True
def get_states(self):
"""Returns the current state of all sensors"""
states = dict()
# Add our station configuration information as well
states['config'] = dict()
for option in self.config.options('station'):
states['config'][option] = self.config.get('station', option)
# Add all current plugin states
for sensor in self.sensors:
states[sensor] = self.sensors[sensor].plugin_object.get_state()
return states
from threading import Thread
from app.models import User, Post
from app.forms import LoginForm, RegistrationForm, EditProfileForm, ResetPasswordRequestForm, PostForm, \
ResetPasswordForm
from app.email import send_password_reset_email
# Load yapsy plugins from the plugin directory specified in the ANALYSES_FOLDER configuration variable.
plugin_manager = PluginManager()
plugin_manager.setPluginPlaces([app.config['ANALYSES_FOLDER']])
plugin_manager.collectPlugins()
# Activate all loaded plugins
all_plugins = plugin_manager.getAllPlugins()
for pluginInfo in all_plugins:
plugin_manager.activatePluginByName(pluginInfo.name)
print([_.name for _ in all_plugins])
global_dict = {'analysis_names': [_.name for _ in all_plugins]}
# # note, here's how you can print the metadata
# print(pluginInfo.name, pluginInfo.author, pluginInfo.version, pluginInfo.website, pluginInfo.description)
@app.context_processor
def inject_dict_for_all_templates():
return {'analysis_names': [_.name for _ in all_plugins]}
@app.before_request
class PluginsManagerService(object):
"""
Description: main class in order to manage all the plugin operations like loading and triggering
"""
def __init__(self):
self.plugin_manager = PluginManager()
self.loaded_plugin_list = []
self.loaded_plugin_objects = None
self.plugin_db = Plugins.Plugins(mongodb)
self.plugin_location = os.path.abspath(
os.path.join(os.getcwd(), 'Plugins'))
self.deleted_plugin_location = os.path.abspath(
os.path.join(os.getcwd(), 'Plugins_deleted'))
self.load_plugin()
def load_plugin(self):
"""loads all plugins from folder and activates the plugins
Returns:
:dict: active_plugins_dict_from_db
Raises:
no_value
no_active_plugins
"""
# Build the manager
# Tell it the default place(s) where to find plugins
print "###################"
print "STARTED PLUGIN LOAD "
print "###################"
if os.path.isdir(self.plugin_location) is True:
try:
# set plugins from folder
self.plugin_manager.setPluginPlaces([self.plugin_location])
# define interfaces to load you have defined
self.plugin_manager.setCategoriesFilter({
"PreDeploymentPlugin":
IPreDeploymentPlugin,
"PostDeploymentPlugin":
IPostDeploymentPlugin
})
# Load all plugins from plugin folder
self.plugin_manager.collectPlugins()
loaded_plugin_from_folder = [
x.name for x in self.plugin_manager.getAllPlugins()
]
# validate plugin if not valid deactivate
for plugin in loaded_plugin_from_folder:
if not self.is_valid_plugin(plugin):
self.deactivate(plugin)
print(
"the following plugin is deactivate cause is not valid: "
+ plugin)
# update DB with new plugins
self.detect_new_plugins()
# load all activated plugins from DB
active_plugins_dict_from_db = self.plugin_db.get_all_plugin(
status='active')
active_plugins_in_db = [
x['name']
for x in self.plugin_db.get_all_plugin(status='active')
]
# final active_plugins list
if active_plugins_in_db is None:
print "no plugins installed"
elif loaded_plugin_from_folder is None:
print "no plugins in the plugins folder"
else:
active_plugins = [
x for x in loaded_plugin_from_folder
if x in active_plugins_in_db
]
for plugin in active_plugins:
# validate plugin if not valid deactivate
if not self.is_valid_plugin(plugin):
print "plugin is not valid: " + plugin + " deactivate"
self.deactivate(plugin)
else:
# activate
self.plugin_manager.activatePluginByName(plugin)
print "loaded plugin name: " + plugin
ValueError("loaded plugin name: " + plugin)
print "###################"
print "COMPLETED PLUGIN LOAD "
print "###################"
return active_plugins_dict_from_db
except (Exception, ValueError) as error:
print error
raise ValueError("Unable to load plugins :", error)
elif os.path.isdir(self.plugin_location) is False:
raise ValueError("plugin folder is set to " +
self.plugin_location +
" missing or not configured well")
else:
raise ValueError("unknown err during plugin loading")
def detect_new_plugins(self):
"""
updates new plugins in db
:param loaded_plugin_from_folder:
:return:
"""
# load all from db
loaded_plugin_from_folder = self.plugin_manager.getAllPlugins()
plugins_in_db = [
x.get('name') for x in self.plugin_db.get_all_plugin()
]
if not plugins_in_db:
new_plugins = loaded_plugin_from_folder
else:
new_plugins = [
x for x in loaded_plugin_from_folder
if x.name not in plugins_in_db
]
for plugin_object in new_plugins:
# get plugin type
parent_class = type(plugin_object.plugin_object).__bases__
parent_class_name = parent_class[0].__name__
plugin_dict = {
'name': plugin_object.name,
'category': parent_class_name[1:],
'author': plugin_object.author,
'version': str(plugin_object.version),
'description': plugin_object.description,
'status': 'inactive'
}
try:
self.plugin_db.add_plugin(plugin_dict)
except Exception as error:
print error
raise RuntimeError("unable to insert the plugin to DB")
def install_plugin(self, plugin_name):
'''
activates the plugin and triggers the install method of the plugin
:param plugin_name: string
:return:
:raises
RuntimeError "unable to insert the plugin to DB"
'''
# set plugins from folder
# self.plugin_location = os.path.abspath(os.path.join(os.getcwd(), 'Plugins'))
# self.load_plugin()
plugin_details = self.plugin_db.get_plugin_by_name(plugin_name)
plugin_object = self.plugin_manager.getPluginByName(
plugin_name, plugin_details.get('category'))
if plugin_object:
try:
self.activate(plugin_name)
return plugin_object.plugin_object.install()
except Exception as error:
print error
raise RuntimeError("unable to insert the plugin to DB: ",
error)
else:
raise ValueError("no active plugin with this name")
def is_valid_plugin(self, plugin):
'''
validate that plugin has all the required methods and attributes
:param plugin:
:return:
True or False
'''
# mandatory element lists
plugin_mandatory_attributes = [
'author', 'name', 'version', 'description'
]
plugin_mandatory_methods = [
'preform', 'activate', 'deactivate', 'is_activated'
]
# get plugin
plugin = self.plugin_manager.getPluginByName(plugin,
'PreDeploymentPlugin')
# check plugin manifest
try:
for attribute in plugin_mandatory_attributes:
if str(getattr(plugin, attribute, None)) == "":
raise ValueError("plugin is missing value in " + attribute)
except (ValueError, AttributeError) as error:
print error
return False
# check plugin methods
try:
methods = [
x for x in plugin_mandatory_methods
if x not in dir(plugin.plugin_object)
]
if len(methods) > 0:
raise ValueError("folowwing methods are missing " +
str(methods))
except (ValueError, AttributeError) as error:
print error
return False
return True
def uninstall_plugin(self, plugin):
'''
triggers the uninstall method of the plugin by name
remove the db details
rename the plugin folder
:param plugin_name:
'''
try:
# get category form DB
plugin_details = self.plugin_db.get_plugin_by_name(plugin)
if plugin_details:
plugin_object = self.plugin_manager.getPluginByName(
plugin, plugin_details.get('category'))
# uninstal
plugin_object.plugin_object.uninstall()
# delete from db
self.plugin_db.delete_plugin_by_name(plugin)
# check if deleted folder exists
if os.path.isdir(self.deleted_plugin_location) is not True:
os.mkdir(self.deleted_plugin_location)
# delete folder
os.rename(
self.plugin_location + "/" + plugin,
self.deleted_plugin_location + "/" + plugin + ".deleted")
else:
raise ValueError('couldnt find plugin')
except (ValueError, Exception) as error:
print error
raise RuntimeError(" could not uninstall plugin " + plugin + " " +
error)
def activate(self, plugin_name):
'''
activates plugin by name
:param plugin_name:
:raise
RuntimeError "could not activate plugin"
'''
try:
# get category form DB
plugin_details = self.plugin_db.get_plugin_by_name(plugin_name)
# activate
self.plugin_manager.activatePluginByName(
plugin_name, plugin_details.get('category'))
# updateDB
self.plugin_db.update_plugin_status(plugin_name, 'active')
except Exception as error:
print error
raise RuntimeError(" could not activate plugin " + plugin_name)
def deactivate(self, plugin_name):
'''
deactivates plugin by name
:param plugin_name: string
:raise
RuntimeError "could not deactivate plugin"
'''
try:
# get category form DB
plugin_details = self.plugin_db.get_plugin_by_name(plugin_name)
# deactivate
self.plugin_manager.deactivatePluginByName(
plugin_name, plugin_details.get('category'))
# updateDB
self.plugin_db.update_plugin_status(plugin_name, 'inactive')
except Exception as error:
print error
raise RuntimeError(" could not deactivate plugin " + plugin_name)
def preform(self, category, plugin_name):
"""
execute the preform method of plugin in category
:param plugin_name: string
:param category: string
:raises
RuntimeError unable to trigger operation of plugin:
"""
try:
plugin = self.plugin_manager.getPluginByName(plugin_name, category)
plugin.plugin_object.preform()
except Exception as error:
print error
raise RuntimeError(" unable to trigger operation of plugin:" +
plugin_name)
def preform_all_in_category(self, category, **keyargs):
'''
triggers all preform methods of plugin in the provided category
:param category:
:return:
:raises
RuntimeError "unable to execute the plugin logic"
'''
for (key, value) in keyargs.iteritems():
print(key, value)
try:
for plugin in self.plugin_manager.getPluginsOfCategory(category):
print("preforming action of plugin " + plugin.name)
plugin.plugin_object.preform()
except (ValueError, Exception) as error:
print error
raise RuntimeError("unable to execute the plugin logic: " + error)
# ## test
def test(self):
# just for test
for p in self.plugin_manager.getPluginsOfCategory(
'PreDeploymentPlugin'):
self.install_plugin('TestPlugin')
class SimpleTestsCase(unittest.TestCase):
"""
Test the correct loading of a simple plugin as well as basic
commands.
"""
def setUp(self):
"""
init
"""
# create the plugin manager
self.simplePluginManager = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# load the plugins that may be found
self.simplePluginManager.collectPlugins()
# Will be used later
self.plugin_info = None
def plugin_loading_check(self):
"""
Test if the correct plugin has been loaded.
"""
if self.plugin_info is None:
# check nb of categories
self.assertEqual(len(self.simplePluginManager.getCategories()), 1)
sole_category = self.simplePluginManager.getCategories()[0]
# check the number of plugins
self.assertEqual(
len(
self.simplePluginManager.getPluginsOfCategory(
sole_category)), 1)
self.plugin_info = self.simplePluginManager.getPluginsOfCategory(
sole_category)[0]
# test that the name of the plugin has been correctly defined
self.assertEqual(self.plugin_info.name, "Simple Plugin")
self.assertEqual(sole_category, self.plugin_info.category)
else:
self.assert_(True)
def testLoaded(self):
"""
Test if the correct plugin has been loaded.
"""
self.plugin_loading_check()
def testGetAll(self):
"""
Test if the correct plugin has been loaded.
"""
self.plugin_loading_check()
self.assertEqual(len(self.simplePluginManager.getAllPlugins()), 1)
self.assertEqual(self.simplePluginManager.getAllPlugins()[0],
self.plugin_info)
def testActivationAndDeactivation(self):
"""
Test if the activation procedure works.
"""
self.plugin_loading_check()
self.assert_(not self.plugin_info.plugin_object.is_activated)
self.simplePluginManager.activatePluginByName(
self.plugin_info.name, self.plugin_info.category)
self.assert_(self.plugin_info.plugin_object.is_activated)
self.simplePluginManager.deactivatePluginByName(
self.plugin_info.name, self.plugin_info.category)
self.assert_(not self.plugin_info.plugin_object.is_activated)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = [
'planetoid',
'ipynb',
'local_search',
'render_mustache',
]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.posts_per_category = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
self._template_system = None
self._THEMES = None
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'BODY_END': "",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENT_SYSTEM': 'disqus',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'COMPILERS': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile', ),
"txt2tags": ('.t2t', ),
"bbcode": ('.bb', ),
"wiki": ('.wiki', ),
"ipynb": ('.ipynb', ),
"html": ('.html', '.htm')
},
'CONTENT_FOOTER': '',
'COPY_SOURCES': True,
'CREATE_MONTHLY_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'COMMENT_SYSTEM_ID': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FEED_LENGTH': 10,
'FILE_METADATA_REGEXP': None,
'ADDITIONAL_METADATA': {},
'FILES_FOLDERS': {
'files': ''
},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GZIP_FILES': False,
'GZIP_EXTENSIONS':
('.txt', '.htm', '.html', '.css', '.js', '.json'),
'HIDE_SOURCELINK': False,
'HIDE_UNTRANSLATED_POSTS': False,
'HYPHENATE': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEX_PATH': '',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'NAVIGATION_LINKS': None,
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'POSTS': (("posts/*.txt", "posts", "post.tmpl"), ),
'PAGES': (("stories/*.txt", "stories", "story.tmpl"), ),
'PRETTY_URLS': False,
'FUTURE_IS_NOW': False,
'READ_MORE_LINK':
'<p class="more"><a href="{link}">{read_more}…</a></p>',
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'SOCIAL_BUTTONS_CODE': SOCIAL_BUTTONS_CODE,
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'bootstrap',
'THEME_REVEAL_CONFIG_SUBTHEME': 'sky',
'THEME_REVEAL_CONFIG_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': None,
'DEPLOY_DRAFTS': True,
'DEPLOY_FUTURE': False,
'SCHEDULE_ALL': False,
'SCHEDULE_RULE': '',
'SCHEDULE_FORCE_TODAY': False
}
self.config.update(config)
# Make sure we have pyphen installed if we are using it
if self.config.get('HYPHENATE') and pyphen is None:
print('WARNING: To use the hyphenation, you have to install '
'the "pyphen" package.')
print('WARNING: Setting HYPHENATE to False.')
self.config['HYPHENATE'] = False
# Deprecating post_compilers
# TODO: remove on v7
if 'post_compilers' in config:
print(
"WARNING: The post_compilers option is deprecated, use COMPILERS instead."
)
if 'COMPILERS' in config:
print(
"WARNING: COMPILERS conflicts with post_compilers, ignoring post_compilers."
)
else:
self.config['COMPILERS'] = config['post_compilers']
# Deprecating post_pages
# TODO: remove on v7
if 'post_pages' in config:
print(
"WARNING: The post_pages option is deprecated, use POSTS and PAGES instead."
)
if 'POSTS' in config or 'PAGES' in config:
print(
"WARNING: POSTS and PAGES conflict with post_pages, ignoring post_pages."
)
else:
self.config['POSTS'] = [
item[:3] for item in config['post_pages'] if item[-1]
]
self.config['PAGES'] = [
item[:3] for item in config['post_pages'] if not item[-1]
]
# FIXME: Internally, we still use post_pages because it's a pain to change it
self.config['post_pages'] = []
for i1, i2, i3 in self.config['POSTS']:
self.config['post_pages'].append([i1, i2, i3, True])
for i1, i2, i3 in self.config['PAGES']:
self.config['post_pages'].append([i1, i2, i3, False])
# Deprecating DISQUS_FORUM
# TODO: remove on v7
if 'DISQUS_FORUM' in config:
print(
"WARNING: The DISQUS_FORUM option is deprecated, use COMMENT_SYSTEM_ID instead."
)
if 'COMMENT_SYSTEM_ID' in config:
print(
"WARNING: DISQUS_FORUM conflicts with COMMENT_SYSTEM_ID, ignoring DISQUS_FORUM."
)
else:
self.config['COMMENT_SYSTEM_ID'] = config['DISQUS_FORUM']
# Deprecating the ANALYTICS option
# TODO: remove on v7
if 'ANALYTICS' in config:
print(
"WARNING: The ANALYTICS option is deprecated, use BODY_END instead."
)
if 'BODY_END' in config:
print(
"WARNING: ANALYTICS conflicts with BODY_END, ignoring ANALYTICS."
)
else:
self.config['BODY_END'] = config['ANALYTICS']
# Deprecating the SIDEBAR_LINKS option
# TODO: remove on v7
if 'SIDEBAR_LINKS' in config:
print(
"WARNING: The SIDEBAR_LINKS option is deprecated, use NAVIGATION_LINKS instead."
)
if 'NAVIGATION_LINKS' in config:
print(
"WARNING: The SIDEBAR_LINKS conflicts with NAVIGATION_LINKS, ignoring SIDEBAR_LINKS."
)
else:
self.config['NAVIGATION_LINKS'] = config['SIDEBAR_LINKS']
# Compatibility alias
self.config['SIDEBAR_LINKS'] = self.config['NAVIGATION_LINKS']
if self.config['NAVIGATION_LINKS'] in (None, {}):
self.config['NAVIGATION_LINKS'] = {self.config['DEFAULT_LANG']: ()}
# Deprecating the ADD_THIS_BUTTONS option
# TODO: remove on v7
if 'ADD_THIS_BUTTONS' in config:
print(
"WARNING: The ADD_THIS_BUTTONS option is deprecated, use SOCIAL_BUTTONS_CODE instead."
)
if not config['ADD_THIS_BUTTONS']:
print(
"WARNING: Setting SOCIAL_BUTTONS_CODE to empty because ADD_THIS_BUTTONS is False."
)
self.config['SOCIAL_BUTTONS_CODE'] = ''
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
# TODO: remove on v7
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
print(
"WARNING: You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML"
)
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
if config.get('COPY_SOURCES') and not self.config['HIDE_SOURCELINK']:
self.config['HIDE_SOURCELINK'] = True
self.config['TRANSLATIONS'] = self.config.get(
'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''})
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
# TODO: remove on v7
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
print(
"WARNING: You should configure SITE_URL instead of BLOG_URL"
)
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
# BASE_URL should *always* end in /
if self.config['BASE_URL'] and self.config['BASE_URL'][-1] != '/':
print("WARNING: Your BASE_URL doesn't end in / -- adding it.")
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(__file__),
utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS
and plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(
task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS
and plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
if plugin_info.name in self.config["COMPILERS"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self._GLOBAL_CONTEXT = {}
self._GLOBAL_CONTEXT['_link'] = self.link
self._GLOBAL_CONTEXT['set_locale'] = s_l
self._GLOBAL_CONTEXT['rel_link'] = self.rel_link
self._GLOBAL_CONTEXT['abs_link'] = self.abs_link
self._GLOBAL_CONTEXT['exists'] = self.file_exists
self._GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config['SLUG_TAG_PATH']
self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self._GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self._GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
# TODO: remove fallback in v7
self._GLOBAL_CONTEXT['blog_url'] = self.config.get(
'SITE_URL', self.config.get('BLOG_URL'))
self._GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self._GLOBAL_CONTEXT['body_end'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['analytics'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['add_this_buttons'] = self.config.get(
'SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['social_buttons_code'] = self.config.get(
'SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self._GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self._GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self._GLOBAL_CONTEXT['comment_system'] = self.config.get(
'COMMENT_SYSTEM')
self._GLOBAL_CONTEXT['comment_system_id'] = self.config.get(
'COMMENT_SYSTEM_ID')
# TODO: remove in v7
self._GLOBAL_CONTEXT['disqus_forum'] = self.config.get(
'COMMENT_SYSTEM_ID')
self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self._GLOBAL_CONTEXT['subtheme'] = self.config.get(
'THEME_REVEAL_CONFIG_SUBTHEME')
self._GLOBAL_CONTEXT['transition'] = self.config.get(
'THEME_REVEAL_CONFIG_TRANSITION')
self._GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self._GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self._GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self._GLOBAL_CONTEXT['navigation_links'] = utils.Functionary(
list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('NAVIGATION_LINKS', {}).items():
self._GLOBAL_CONTEXT['navigation_links'][k] = v
# TODO: remove on v7
# Compatibility alias
self._GLOBAL_CONTEXT['sidebar_links'] = self._GLOBAL_CONTEXT[
'navigation_links']
self._GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self._GLOBAL_CONTEXT['hide_sourcelink'] = self.config.get(
'HIDE_SOURCELINK')
self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get(
'EXTRA_HEAD_DATA')
self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
def _get_themes(self):
if self._THEMES is None:
# Check for old theme names (Issue #650) TODO: remove in v7
theme_replacements = {
'site': 'bootstrap',
'orphan': 'base',
'default': 'oldfashioned',
}
if self.config['THEME'] in theme_replacements:
warnings.warn(
'You are using the old theme "{0}", using "{1}" instead.'.
format(self.config['THEME'],
theme_replacements[self.config['THEME']]))
self.config['THEME'] = theme_replacements[self.config['THEME']]
if self.config['THEME'] == 'oldfashioned':
warnings.warn(
'''You may need to install the "oldfashioned" theme '''
'''from themes.nikola.ralsina.com.ar because it's not '''
'''shipped by default anymore.''')
warnings.warn('Please change your THEME setting.')
try:
self._THEMES = utils.get_theme_chain(self.config['THEME'])
except Exception:
warnings.warn(
'''Can't load theme "{0}", using 'bootstrap' instead.'''.
format(self.config['THEME']))
self.config['THEME'] = 'bootstrap'
return self._get_themes()
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config['USE_CDN']:
bootstrap_path = utils.get_asset_path(
os.path.join('assets', 'css', 'bootstrap.min.css'),
self._THEMES)
if bootstrap_path and bootstrap_path.split(
os.sep)[-4] not in ['bootstrap', 'bootstrap3']:
warnings.warn(
'The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.'
)
return self._THEMES
THEMES = property(_get_themes)
def _get_messages(self):
return utils.load_messages(self.THEMES, self.translations,
self.default_lang)
MESSAGES = property(_get_messages)
def _get_global_context(self):
"""Initialize some parts of GLOBAL_CONTEXT only when it's queried."""
if 'messages' not in self._GLOBAL_CONTEXT:
self._GLOBAL_CONTEXT['messages'] = self.MESSAGES
if 'has_custom_css' not in self._GLOBAL_CONTEXT:
# check if custom css exist and is not empty
custom_css_path = utils.get_asset_path(
'assets/css/custom.css', self.THEMES,
self.config['FILES_FOLDERS'])
if custom_css_path and self.file_exists(custom_css_path,
not_empty=True):
self._GLOBAL_CONTEXT['has_custom_css'] = True
else:
self._GLOBAL_CONTEXT['has_custom_css'] = False
return self._GLOBAL_CONTEXT
GLOBAL_CONTEXT = property(_get_global_context)
def _get_template_system(self):
if self._template_system is None:
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name,
"TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self._template_system = pi.plugin_object
lookup_dirs = ['templates'] + [
os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES
]
self._template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
return self._template_system
template_system = property(_get_template_system)
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.COMPILERS`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [
lang for lang, exts in list(self.config['COMPILERS'].items())
if ext in exts
]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'COMPILERS' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("COMPILERS in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(template_name, None,
local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def current_lang(self): # FIXME: this is duplicated, turn into a mixin
"""Return the currently set locale, if it's one of the
available translations, or default_lang."""
lang = utils.LocaleBorg().current_lang
if lang:
if lang in self.translations:
return lang
lang = lang.split('_')[0]
if lang in self.translations:
return lang
# whatever
return self.default_lang
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* category (and name is the category name)
* category_rss (and name is the category name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a POSTS/PAGES tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = self.current_lang()
path = []
if kind == "tag_index":
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
self.config['INDEX_FILE']
] if _f
]
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".html"
] if _f
]
elif kind == "category":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
"cat_" + name + ".html"
] if _f
]
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".xml"
] if _f
]
elif kind == "category_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
"cat_" + name + ".xml"
] if _f
]
elif kind == "index":
if name not in [None, 0]:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.
config['INDEX_PATH'], 'index-{0}.html'.format(name)
] if _f
]
else:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], self.config['INDEX_FILE']
] if _f
]
elif kind == "post_path":
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang],
os.path.dirname(name), self.config['INDEX_FILE']
] if _f
]
elif kind == "rss":
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'],
'rss.xml'
] if _f
]
elif kind == "archive":
if name:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.
config['ARCHIVE_PATH'], name, self.config['INDEX_FILE']
] if _f
]
else:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.
config['ARCHIVE_PATH'], self.config['ARCHIVE_FILENAME']
] if _f
]
elif kind == "gallery":
path = [
_f for _f in
[self.config['GALLERY_PATH'], name, self.config['INDEX_FILE']]
if _f
]
elif kind == "listing":
path = [
_f for _f in [self.config['LISTINGS_FOLDER'], name + '.html']
if _f
]
if is_link:
link = '/' + ('/'.join(path))
index_len = len(self.config['INDEX_FILE'])
if self.config['STRIP_INDEXES'] and \
link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BASE_URL'], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config['BASE_URL'], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self, name, plugin_category):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(
plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
yield task
for multi in self.plugin_manager.getPluginsOfCategory(
"TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield task
if flag:
task_dep.append('{0}_{1}'.format(
name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': name,
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
seen = set([])
print("Scanning posts", end='')
tzinfo = None
if self.config['TIMEZONE'] is not None:
tzinfo = pytz.timezone(self.config['TIMEZONE'])
if self.config['FUTURE_IS_NOW']:
current_time = None
else:
current_time = utils.current_time(tzinfo)
targets = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='')
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(
os.path.join(destination,
os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config['TRANSLATIONS'].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
# We eliminate from the list the files inside any .ipynb folder
full_list = [
p for p in full_list
if not any([x.startswith('.') for x in p.split(os.sep)])
]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.config['CACHE_FOLDER'],
dest_dir,
use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BASE_URL'],
self.MESSAGES,
template_name,
self.config['FILE_METADATA_REGEXP'],
self.config['STRIP_INDEXES'],
self.config['INDEX_FILE'],
tzinfo,
current_time,
self.config['HIDE_UNTRANSLATED_POSTS'],
self.config['PRETTY_URLS'],
self.config['HYPHENATE'],
)
for lang, langpath in list(
self.config['TRANSLATIONS'].items()):
dest = (destination, langpath, dir_glob,
post.meta[lang]['slug'])
if dest in targets:
raise Exception('Duplicated output path {0!r} '
'in post {1!r}'.format(
post.meta[lang]['slug'],
base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(
post.post_name)
self.posts_per_month['{0}/{1:02d}'.format(
post.date.year,
post.date.month)].append(post.post_name)
for tag in post.alltags:
self.posts_per_tag[tag].append(post.post_name)
self.posts_per_category[post.meta('category')].append(
post.post_name)
else:
self.pages.append(post)
if self.config['OLD_THEME_SUPPORT']:
post._add_old_metadata()
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(
lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
if post:
deps_dict['post_translations'] = post.translated_to
task = {
'name':
os.path.normpath(output_name),
'file_dep':
deps,
'targets': [output_name],
'actions':
[(self.render_template, [post.template_name, output_name,
context])],
'clean':
True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang))
for p in posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name':
os.path.normpath(output_name),
'targets': [output_name],
'file_dep':
deps,
'actions':
[(self.render_template, [template_name, output_name, context])],
'clean':
True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {
'listings': 'listings'
},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension
})
self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
nikola.utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category(
"CompilerExtension")
self.timeline = [FakePost(title='Fake post', slug='fake-post')]
self.debug = True
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(
plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return ('<img src="IMG.jpg">')
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data, *a, **kw):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data,
self.shortcode_registry,
**kw)
class WeaponSystem(rpyc.Service):
'''
RPC Services: This is the code that does the actual password cracking
and returns the results to orbital control. Currently only supports
cracking using rainbow tables (RCrackPy)
'''
is_initialized = False
mutex = Lock()
is_busy = False
job_id = None
def initialize(self):
''' Initializes variables, this should only be called once '''
logging.info("Weapon system initializing ...")
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["plugins/"])
self.plugin_manager.setCategoriesFilter(FILTERS)
self.plugin_manager.collectPlugins()
self.plugins = {}
logging.info("Loaded %d plugin(s)" %
len(self.plugin_manager.getAllPlugins()))
self.__cpu__()
logging.info("Weapon system online, good hunting.")
@atomic
def on_connect(self):
''' Called when successfully connected '''
if not self.is_initialized:
self.initialize()
self.is_initialized = True
logging.info("Uplink to orbital control active")
def on_disconnect(self):
''' Called if the connection is lost/disconnected '''
logging.info("Disconnected from orbital command server.")
def __cpu__(self):
''' Detects the number of CPU cores on a system (including virtual cores) '''
if cpu_count is not None:
try:
self.cpu_cores = cpu_count()
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except NotImplementedError:
logging.error(
"Could not detect number of processors; assuming 1")
self.cpu_cores = 1
else:
try:
self.cpu_cores = int(sysconf("SC_NPROCESSORS_CONF"))
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except ValueError:
logging.error(
"Could not detect number of processors; assuming 1")
self.cpu_cores = 1
############################ [ EXPOSED METHODS ] ############################
@atomic
def exposed_crack(self, plugin_name, job_id, hashes, **kwargs):
''' Exposes plugins calls '''
assert plugin_name in self.plugins
self.is_busy = True
self.job_id = job_id
self.plugin_manager.activatePluginByName(plugin_name)
plugin = self.plugin_manager.getPluginByName(plugin_name)
results = plugin.execute(hashes, **kwargs)
self.plugin_manager.deactivatePluginByName(plugin_name)
self.job_id = None
self.is_busy = False
return results
def exposed_get_plugin_names(self):
''' Returns what algorithms can be cracked '''
logging.info("Method called: exposed_get_capabilities")
plugins = self.plugin_manager.getAllPlugins()
return [plugin.name for plugin in plugins]
def exposed_get_categories(self):
''' Return categories for which we have plugins '''
categories = []
for category in self.plugin_manager.getCategories():
if 0 < len(self.plugin_manager.getPluginsOfCategory(category)):
categories.append(category)
return categories
def exposed_get_category_plugins(self, category):
''' Get plugin names for a category '''
plugins = self.plugin_manager.getPluginsOfCategory(category)
return [plugin.name for plugin in plugins]
def exposed_get_plugin_details(self, category, plugin_name):
''' Get plugin based on name details '''
plugin = self.plugin_manager.getPluginByName(plugin_name, category)
info = {'name': plugin.name}
info['author'] = plugin.details.get('Documentation', 'author')
info['website'] = plugin.details.get('Documentation', 'website')
info['version'] = plugin.details.get('Documentation', 'version')
info['description'] = plugin.details.get('Documentation',
'description')
info['copyright'] = plugin.details.get('Documentation', 'copyright')
info['precomputation'] = plugin.details.getboolean(
'Core', 'precomputation')
return info
def exposed_ping(self):
''' Returns a pong message '''
return "PONG"
def exposed_is_busy(self):
''' Returns True/False if the current system is busy (thread safe) '''
return self.is_busy
def exposed_current_job_id(self):
''' Returns the current job id (thread safe) '''
return self.job_id
def exposed_cpu_count(self):
''' Returns the number of detected cpu cores '''
return self.cpu_cores
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {'listings': 'listings'},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension
})
self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension")
self.timeline = [
FakePost(title='Fake post',
slug='fake-post')
]
self.debug = True
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return('<img src="IMG.jpg">')
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry)
from yapsy.PluginManager import PluginManager
from . import db
allPlugins = PluginManager()
# Tell it the default place(s) where to find plugins
allPlugins.setPluginPlaces(["precollapse/plugins"])
# Load all plugins
allPlugins.collectPlugins()
# Activate all loaded plugins
for pluginInfo in allPlugins.getAllPlugins():
print(pluginInfo)
print(pluginInfo.name)
allPlugins.activatePluginByName(pluginInfo.name)
print(pluginInfo.plugin_object)
__all__ = [allPlugins]
class SimpleTestsCase(unittest.TestCase): """ Test the correct loading of a simple plugin as well as basic commands. """ def setUp(self): """ init """ # create the plugin manager self.simplePluginManager = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # load the plugins that may be found self.simplePluginManager.collectPlugins() # Will be used later self.plugin_info = None def plugin_loading_check(self): """ Test if the correct plugin has been loaded. """ if self.plugin_info is None: # check nb of categories self.assertEqual(len(self.simplePluginManager.getCategories()),1) sole_category = self.simplePluginManager.getCategories()[0] # check the number of plugins self.assertEqual(len(self.simplePluginManager.getPluginsOfCategory(sole_category)),1) self.plugin_info = self.simplePluginManager.getPluginsOfCategory(sole_category)[0] # test that the name of the plugin has been correctly defined self.assertEqual(self.plugin_info.name,"Simple Plugin") self.assertEqual(sole_category,self.plugin_info.category) else: self.assert_(True) def testLoaded(self): """ Test if the correct plugin has been loaded. """ self.plugin_loading_check() def testGetAll(self): """ Test if the correct plugin has been loaded. """ self.plugin_loading_check() self.assertEqual(len(self.simplePluginManager.getAllPlugins()),1) self.assertEqual(self.simplePluginManager.getAllPlugins()[0],self.plugin_info) def testActivationAndDeactivation(self): """ Test if the activation procedure works. """ self.plugin_loading_check() self.assert_(not self.plugin_info.plugin_object.is_activated) self.simplePluginManager.activatePluginByName(self.plugin_info.name, self.plugin_info.category) self.assert_(self.plugin_info.plugin_object.is_activated) self.simplePluginManager.deactivatePluginByName(self.plugin_info.name, self.plugin_info.category) self.assert_(not self.plugin_info.plugin_object.is_activated)
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'EXTRA_PLUGINS_DIRS': [extra_v6_plugin_dir],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
})
self.loghandlers = [nikola.utils.STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), 'plugins'),
] + [path for path in extra_plugins_dirs if path]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode('plugins')),
] + [nikola.utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension")
self.timeline = [
FakePost(title='Fake post',
slug='fake-post')
]
self.debug = True
self.rst_transforms = []
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return('<img src="IMG.jpg">')
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
# This is the default config
# TODO: fill it
self.config = {
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'DEFAULT_LANG': "en",
'OUTPUT_FOLDER': 'output',
'FILES_FOLDERS': {
'files': ''
},
'LISTINGS_FOLDER': 'listings',
'ADD_THIS_BUTTONS': True,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_TEASERS': False,
'MAX_IMAGE_SIZE': 1280,
'USE_FILENAME_AS_TITLE': True,
'SLUG_TAG_PATH': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'FILTERS': {},
'USE_BUNDLES': True,
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'default',
'post_compilers': {
"rest": ['.txt', '.rst'],
"markdown": ['.md', '.mdown', '.markdown'],
"html": ['.html', '.htm'],
},
}
self.config.update(config)
self.config['TRANSLATIONS'] = self.config.get(
'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''})
self.GLOBAL_CONTEXT = self.config.get('GLOBAL_CONTEXT', {})
self.THEMES = utils.get_theme_chain(self.config['THEME'])
self.MESSAGES = utils.load_messages(self.THEMES,
self.config['TRANSLATIONS'])
self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
self.GLOBAL_CONTEXT['_link'] = self.link
self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
self.GLOBAL_CONTEXT['exists'] = self.file_exists
self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
'ADD_THIS_BUTTONS']
self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
self.plugin_manager.setPluginPlaces([
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
])
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
pluginInfo.plugin_object.short_help = pluginInfo.description
self.commands[pluginInfo.name] = pluginInfo.plugin_object
# Activate all task plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name,
"TemplateSystem")
if pi is None:
sys.stderr.write("Error loading %s template system plugin\n" %
template_sys_name)
sys.exit(1)
self.template_system = pi.plugin_object
self.template_system.set_directories([
os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES
])
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for pluginInfo in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[pluginInfo.name] = \
pluginInfo.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [
lang for lang, exts in self.config['post_compilers'].items()
if ext in exts
]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of %s)" % ', '.join(langs))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to "
"handle '%s' extensions." % ext)
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.config['GLOBAL_CONTEXT'])
local_context.update(context)
data = self.template_system.render_template(template_name, None,
local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# This is to support windows paths
url_part = "/".join(url_part.split(os.sep))
src = urlparse.urljoin(self.config["BLOG_URL"], url_part)
parsed_src = urlparse.urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse.urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = '<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "w+") as post_file:
post_file.write(data)
def path(self, kind, name, lang, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
path = []
if kind == "tag_index":
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
'index.html'
])
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".html"
])
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".xml"
])
elif kind == "index":
if name > 0:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
'index-%s.html' % name
])
else:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], 'index.html'
])
elif kind == "rss":
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'],
'rss.xml'
])
elif kind == "archive":
if name:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], name, 'index.html'
])
else:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'],
self.config['ARCHIVE_FILENAME']
])
elif kind == "gallery":
path = filter(None,
[self.config['GALLERY_PATH'], name, 'index.html'])
elif kind == "listing":
path = filter(None,
[self.config['LISTINGS_FOLDER'], name + '.html'])
if is_link:
return '/' + ('/'.join(path))
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urlparse.urljoin(self.config['BLOG_URL'], dst)
return urlparse.urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urlparse.urljoin(self.config['BLOG_URL'], src)
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlparse.urlsplit(src)
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': 'all',
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if not self._scanned:
print "Scanning posts ",
targets = set([])
for wildcard, destination, _, use_in_feeds in \
self.config['post_pages']:
print ".",
for base_path in glob.glob(wildcard):
post = Post(base_path, destination, use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BLOG_URL'], self.MESSAGES)
for lang, langpath in self.config['TRANSLATIONS'].items():
dest = (destination, langpath, post.pagenames[lang])
if dest in targets:
raise Exception(
'Duplicated output path %r in post %r' %
(post.pagenames[lang], base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(
post.post_name)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
for name, post in self.global_data.items():
self.timeline.append(post)
self.timeline.sort(cmp=lambda a, b: cmp(a.date, b.date))
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print "done!"
def generic_page_renderer(self, lang, wildcard, template_name, destination,
filters):
"""Render post fragments to final HTML pages."""
for post in glob.glob(wildcard):
post_name = os.path.splitext(post)[0]
context = {}
post = self.global_data[post_name]
deps = post.deps(lang) + \
self.template_system.template_deps(template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
self.config['TRANSLATIONS'][lang],
destination,
post.pagenames[lang] + ".html")
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.config['GLOBAL_CONTEXT']
task = {
'name':
output_name.encode('utf-8'),
'file_dep':
deps,
'targets': [output_name],
'actions':
[(self.render_template, [template_name, output_name,
context])],
'clean':
True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.titles[lang], p.permalink(lang))
for p in posts]
deps_context["global"] = self.config['GLOBAL_CONTEXT']
task = {
'name':
output_name.encode('utf8'),
'targets': [output_name],
'file_dep':
deps,
'actions':
[(self.render_template, [template_name, output_name, context])],
'clean':
True,
'uptodate': [config_changed(deps_context)]
}
yield utils.apply_filters(task, filters)
class EfetchPluginManager(object):
"""This class manages and creates plugin objects"""
def __init__(self, plugins_file, curr_directory):
# Plugin Manager Setup
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces([curr_directory + u'/plugins/'])
self.plugins_file = plugins_file
self.reload_plugins()
def reload_plugins_file(self):
"""Reloads all plugins from the YAML file"""
self.config_file_plugins = self.load_plugin_config(self.plugins_file)
def reload_plugins(self):
"""Reloads all Yapsy and YAML file plugins"""
self.plugin_manager.collectPlugins()
for plugin in self.plugin_manager.getAllPlugins():
self.plugin_manager.activatePluginByName(plugin.name)
self.reload_plugins_file()
def load_plugin_config(self, plugins_file):
"""Loads the plugin config file"""
if not os.path.isfile(plugins_file):
logging.warn(u'Could not find Plugin Configuration File "' +
plugins_file + u'"')
return {}
with open(plugins_file, 'r') as stream:
try:
return yaml.load(stream)
except yaml.YAMLError as exc:
logging.error(u'Failed to parse Plugin Configuration File')
logging.error(exc)
return {}
def get_all_plugins(self):
"""Gets a list of all the plugins"""
plugins = []
for plugin in self.plugin_manager.getAllPlugins():
plugins.append(plugin.name)
for key in self.config_file_plugins:
plugins.append(key)
return plugins
def get_plugin_by_name(self, name):
"""Gets an Efetch plugin by name"""
plugin = self.plugin_manager.getPluginByName(str(name).lower())
if not plugin and name not in self.config_file_plugins:
logging.warn(u'Request made for unknown plugin "' + name + u'"')
abort(404, u'Could not find plugin "' + name + u'"')
elif not plugin:
plugin = self.config_file_plugins[name]
return Plugin(plugin.get('name', 'None'),
plugin.get('description', 'None'),
plugin.get('cache',
True), plugin.get('popularity', 5),
plugin.get('fast', False),
plugin.get('store', False),
map(str.lower, plugin.get('mimetypes', [])),
map(str.lower, plugin.get('extensions', [])),
map(str.lower, plugin.get('os', [])),
plugin.get('command', False),
plugin.get('format', 'Text'),
plugin.get('file', False),
plugin.get('openwith', False),
plugin.get('icon', 'fa-file-o'))
else:
return plugin.plugin_object
# -*- coding: utf-8 -*-
from yapsy.PluginManager import PluginManager
# cargamos los plugins
simplePluginManager = PluginManager()
simplePluginManager.setPluginPlaces(["plugins"])
simplePluginManager.collectPlugins()
# activate all loaded plugins
for pluginInfo in simplePluginManager.getAllPlugins():
simplePluginManager.activatePluginByName(pluginInfo.name)
# creamos el nucleo
beruby = Beruby()
# login
beruby.login()
def main():
#Find and load plugins
pm = PluginManager()
libpath = '%s/OpenMesher/plugins' % (get_python_lib())
pm.setPluginPlaces([
"/usr/share/openmesher/plugins", "~/.openmesher/plugins",
"./OpenMesher/plugins", "./plugins", libpath
])
pm.setPluginInfoExtension('plugin')
pm.setCategoriesFilter({
'config': IOpenMesherConfigPlugin,
'package': IOpenMesherPackagePlugin,
'deploy': IOpenMesherDeployPlugin,
})
pm.collectPlugins()
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Configure, package, and deploy an OpenVPN mesh")
parser.add_argument('-r',
'--router',
action='append',
help='Adds a router that can be a client and server')
parser.add_argument(
'-s',
'--server',
action='append',
help='Adds a router that can only act as a server, not a client.')
parser.add_argument(
'-c',
'--client',
action='append',
help=
'Adds a router than can only act as a client. For example, a router that is behind NAT and not accessible by a public IP'
)
#BUG: Stupid argparse appends your switches to the default.
#parser.add_argument('-n', '--network', action='append', default=['10.99.99.0/24'])
parser.add_argument('-n', '--network', action='append', required=True)
portgroup = parser.add_mutually_exclusive_group()
portgroup.add_argument('-p',
'--ports',
action='append',
default=['7000-7999'])
portgroup.add_argument('-a', '--random', action='store_true')
parser.add_argument(
'-v',
'--verbose',
action='append_const',
const='verbose',
help='Specify multiple times to make things more verbose')
parser.add_argument('--version', action='version', version='v0.6.4')
pluginargsgroup = parser.add_argument_group('plugins')
for plugin in pm.getAllPlugins():
plugin.plugin_object.setupargs(pluginargsgroup)
arg = parser.parse_args()
for plugin in pm.getAllPlugins():
if plugin.plugin_object.__class__.__name__.lower() in arg:
if eval('arg.%s' %
(plugin.plugin_object.__class__.__name__.lower())):
logging.debug('Plugin enabled: %s' % (plugin.name))
pm.activatePluginByName(plugin.name)
else:
logging.debug('Plugin disabled: %s' % (plugin.name))
pm.deactivatePluginByName(plugin.name)
else:
logging.debug('Plugin disabled: %s' % (plugin.name))
pm.deactivatePluginByName(plugin.name)
l = logging.getLogger()
if arg.verbose:
if len(arg.verbose) == 1:
l.setLevel(logging.INFO)
if len(arg.verbose) >= 2:
l.setLevel(logging.DEBUG)
# Call activate() on all plugins so they prep themselves for use
for plugin in pm.getAllPlugins():
if eval('arg.%s' % (plugin.plugin_object.__class__.__name__.lower())):
logging.info('Enabled plugin: %s' % (plugin.name))
pm.activatePluginByName(plugin.name)
plugin.plugin_object.activate()
plugin.plugin_object._enabled = True
if len(arg.ports) > 1:
arg.ports.reverse()
arg.ports.pop()
port_list = []
if arg.random:
numdevs = 0
if arg.router:
numdevs += len(arg.router)
if arg.server:
numdevs += len(arg.server)
if arg.client:
numdevs += len(arg.client)
ports_needed = numdevs * (numdevs - 1) / 2
for i in range(0, ports_needed):
port_list.append(random.randrange(1025, 32767))
try:
if not arg.random:
# If we're not using random ports, pull whatever is in arg.ports
for portrange in arg.ports:
portstart, portstop = portrange.split('-')
port_list += range(int(portstart), int(portstop))
except ValueError as e:
print 'Invalid port range: %s' % (portrange)
raise ValueError('Invalid port range: %s' % (portrange))
linkmesh = create_link_mesh(routers=arg.router,
servers=arg.server,
clients=arg.client)
m = Mesh(linkmesh, port_list, arg.network)
files = None
# Run through config plugins
configPlugins = []
for plugin in pm.getPluginsOfCategory('config'):
if plugin.plugin_object._enabled:
plugin.plugin_object.process(m, arg)
configPlugins.append(plugin.plugin_object)
if files:
files = nested_dict_merge(files, plugin.plugin_object.files())
else:
files = plugin.plugin_object.files()
#Grab list of folders that need to be in the package root
includedirs = []
for f in files:
for fldr in files[f]:
rootfldr = fldr.split('/')[1]
if rootfldr not in includedirs:
includedirs.append(rootfldr)
logging.debug('The following folders will be packaged: %s' % (includedirs))
# Run through packaging plugins
packagePlugins = []
for plugin in pm.getPluginsOfCategory('package'):
if plugin.plugin_object._enabled:
#BUG: Services to restart may not necessarily be the same name as their config dir...
plugin.plugin_object.process(m,
include_dirs=includedirs,
restart_services=includedirs,
configPlugins=configPlugins,
cliargs=arg)
packagePlugins.append(plugin.plugin_object)
# Run through deployment plugins
for plugin in pm.getPluginsOfCategory('deploy'):
if plugin.plugin_object._enabled:
try:
plugin.plugin_object.deploy(packagePlugins=packagePlugins,
cliargs=arg,
stoponfailure=False)
except Exception as e:
print "Unable to deploy due to error: %s" % (e)
logging.info('OpenMesher run complete')
