def test_plugins_found(self):
pm = PluginManager()
libpath = '%s/OpenMesher/plugins' % (get_python_lib())
pm.setPluginPlaces([
'/usr/share/openmesher/plugins',
'~/.openmesher/plugins',
'./OpenMesher/plugins',
'./plugins',
libpath,
])
pm.setPluginInfoExtension('plugin')
pm.setCategoriesFilter({
'config': IOpenMesherConfigPlugin,
'package': IOpenMesherPackagePlugin,
'deploy': IOpenMesherDeployPlugin,
})
pm.collectPlugins()
for plugin in pm.getAllPlugins():
print('Author: %s' % (plugin.author))
print('Categories: %s' % (plugin.categories))
print('Copyright: %s' % (plugin.copyright))
print('Descr: %s' % (plugin.description))
print('Error: %s' % (plugin.error))
print('Name: %s' % (plugin.name))
print('Path: %s' % (plugin.path))
print('Version: %s' % (plugin.version))
print('Website: %s' % (plugin.website))
print('')
return True
class FakeSite(object):
def __init__(self):
self.template_system = self
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(utils.__file__), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(utils.__file__), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
def render_template(self, name, _, context):
return('<img src="IMG.jpg">')
class FakeSite(object):
def __init__(self):
self.template_system = self
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(utils.__file__), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(utils.__file__),
utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
def render_template(self, name, _, context):
return ('<img src="IMG.jpg">')
def main():
# Yapsy uses Python’s standard logging module to record most important events and especially plugin loading failures.
logging.basicConfig(level=logging.INFO)
logging.getLogger('yapsy').setLevel(logging.DEBUG) # Enable DEBUG logging during development
# Build the manager
simplePluginManager = PluginManager()
# Tell it the directories it should look in to find plugins
simplePluginManager.setPluginPlaces(['plugins'])
# Tell it the file extension to use for PluginInfo files (INI format)
simplePluginManager.setPluginInfoExtension('plug')
# Load all plugins
simplePluginManager.collectPlugins()
# Activate all loaded plugins (calls activate method)
for pluginInfo in simplePluginManager.getAllPlugins():
# pluginInfo is a plugin_info object, which is typically an instance of IPlugin
simplePluginManager.activatePluginByName(pluginInfo.name)
# Loop through the plugins and call a custom method to print their names
for plugin in simplePluginManager.getAllPlugins():
plugin.plugin_object.print_name()
class FakeSite(object):
def __init__(self):
self.template_system = self
self.config = {"DISABLED_PLUGINS": [], "EXTRA_PLUGINS": [], "DEFAULT_LANG": "en"}
self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"]
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
}
)
self.loghandlers = [STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension("plugin")
if sys.version_info[0] == 3:
places = [os.path.join(os.path.dirname(utils.__file__), "plugins")]
else:
places = [os.path.join(os.path.dirname(utils.__file__), utils.sys_encode("plugins"))]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.timeline = [FakePost(title="Fake post", slug="fake-post")]
def render_template(self, name, _, context):
return '<img src="IMG.jpg">'
def pane_data(self):
list = [] # Holds the filter tuples (name, activated=True)
# Get the default plugin directory, using XML
path = os.path.expanduser("~")
xml = xml_controller.Controller(path + "\.cxvrc.xml")
xml.load_file()
if os.path.exists(os.path.expanduser("~") + os.sep + "plugins"):
default_dir = os.path.expanduser("~") + os.sep + "plugins"
else:
default_dir = self.dicom_view.get_main_dir() + os.sep + "plugins"
if xml.get_plugin_directory() == "" or xml.get_plugin_directory() is None:
directory = [default_dir]
else:
directory = [default_dir, xml.get_plugin_directory()]
# Load the plugins from the default plugin directory.
manager = PluginManager()
manager.setPluginPlaces(directory)
manager.setPluginInfoExtension("plugin")
manager.collectPlugins()
# Append tuple with plugin name and enabled=True to the list
for plugin in manager.getAllPlugins():
list.append((plugin.name, True))
return list
def init_plugin_manager():
global simplePluginManager
if not holder.plugin_manager:
logging.debug('init plugin manager')
simplePluginManager = PluginManager(categories_filter={"bots": BotPlugin})
simplePluginManager.setPluginInfoExtension('plug')
holder.plugin_manager = simplePluginManager
else:
simplePluginManager = holder.plugin_manager
def init_plugin_manager():
global simplePluginManager
if not holder.plugin_manager:
logging.info("init plugin manager")
simplePluginManager = PluginManager(categories_filter={"bots": BotPlugin})
simplePluginManager.setPluginInfoExtension("plug")
holder.plugin_manager = simplePluginManager
else:
simplePluginManager = holder.plugin_manager
def init_plugin_manager(directory, created_app):
global plugin_manager
global plugin_directory
global app
plugin_directory = directory
app = created_app
plugin_manager = PluginManager()
plugin_manager.setPluginPlaces([plugin_directory])
plugin_manager.setPluginInfoExtension('plugin')
plugin_manager.collectPlugins()
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'EXTRA_PLUGINS_DIRS': [extra_v6_plugin_dir],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
})
self.loghandlers = [nikola.utils.STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
'plugins'),
] + [path for path in extra_plugins_dirs if path]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
nikola.utils.sys_encode('plugins')),
] + [
nikola.utils.sys_encode(path)
for path in extra_plugins_dirs if path
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.timeline = [FakePost(title='Fake post', slug='fake-post')]
self.debug = True
self.rst_transforms = []
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def render_template(self, name, _, context):
return ('<img src="IMG.jpg">')
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'EXTRA_PLUGINS_DIRS': [extra_v6_plugin_dir],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
})
self.loghandlers = [nikola.utils.STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), 'plugins'),
] + [path for path in extra_plugins_dirs if path]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode('plugins')),
] + [nikola.utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.timeline = [
FakePost(title='Fake post',
slug='fake-post')
]
self.debug = True
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def render_template(self, name, _, context):
return('<img src="IMG.jpg">')
def plugin_submenu(self):
""" Creates the plugin submenu in the menubar which displays all plugins
and allows the user to specify a secondary plugin directory.
"""
menu = wx.Menu()
"""
# Add a plugin from another directory to the default directory
addPlugin = wx.MenuItem(menu, wx.ID_ANY, 'Add Plugin')
menu.AppendItem(addPlugin)
self.Bind(wx.EVT_MENU, self.controller.on_add_plugin, addPlugin)
"""
# Set directory where extra plugins are held
props = wx.MenuItem(menu, wx.ID_ANY, 'Set Directory')
menu.AppendItem(props)
self.Bind(wx.EVT_MENU, self.controller.on_plugin_properties, props)
menu.AppendSeparator()
# Get the default plugin directory, using XML
path = os.path.expanduser('~')
xml = xml_controller.Controller(path + '\.cxvrc.xml')
xml.load_file()
if os.path.exists(os.path.expanduser('~') + os.sep + "plugins"):
default_dir = os.path.expanduser('~') + os.sep + "plugins"
else:
default_dir = self.get_main_dir() + os.sep + "plugins"
if xml.get_plugin_directory() == "" or xml.get_plugin_directory() is None:
directory = [default_dir]
else:
directory = [default_dir, xml.get_plugin_directory()]
# Load the plugins from the specified plugin directory/s.
manager = PluginManager()
manager.setPluginPlaces(directory)
manager.setPluginInfoExtension('plugin')
manager.collectPlugins()
for plugin in manager.getAllPlugins():
item = wx.MenuItem(menu, wx.ID_ANY, plugin.name)
menu.AppendItem(item)
self.better_bind(wx.EVT_MENU, item, self.controller.on_about_filter, plugin)
return menu
class FakeNikola(object):
def __init__(self):
self.config = {'DISABLED_PLUGINS': []}
self.debug = False
self.loghandlers = []
self.timeline = []
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
"SignalHandler": SignalHandler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = ''
places = [
resource_filename('nikola', utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
os.path.expanduser('~/.nikola/plugins'),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
#self.pug = None
#for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
# if plugin_info.name == 'rest':
# self.plugin_manager.activatePluginByName(plugin_info.name)
# plugin_info.plugin_object.set_site(self)
# self.pug = plugin_info
# Now we have our pug
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"RestExtension"):
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
def init_plugin_manager():
global tom_plugin_manager
from engine import BuiltinEngine, AnsibleEngine, Engine
if not holder.plugin_manager:
tom_plugin_manager = PluginManager()
tom_plugin_manager.setPluginPlaces(config.plugin_dirs)
tom_plugin_manager.setPluginInfoExtension('plug')
# 3 types of plugins, Built-in for plugin come with Tom,
# Ansible for ansible simplerunner or playbook running on tom
# User for user-defined plugin
tom_plugin_manager.setCategoriesFilter({
'Built-in': BuiltinEngine,
'Ansible': AnsibleEngine,
'User': Engine
})
holder.plugin_manager = tom_plugin_manager
else:
tom_plugin_manager = holder.plugin_manager
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.loghandlers = [STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(utils.__file__), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(utils.__file__), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.timeline = [
FakePost(title='Fake post',
slug='fake-post')
]
self.debug = True
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def render_template(self, name, _, context):
return('<img src="IMG.jpg">')
def init_plugin_manager():
global tom_plugin_manager
from engine import BuiltinEngine, AnsibleEngine, Engine
if not holder.plugin_manager:
tom_plugin_manager = PluginManager()
tom_plugin_manager.setPluginPlaces(config.plugin_dirs)
tom_plugin_manager.setPluginInfoExtension('plug')
# 3 types of plugins, Built-in for plugin come with Tom,
# Ansible for ansible simplerunner or playbook running on tom
# User for user-defined plugin
tom_plugin_manager.setCategoriesFilter({
'Built-in': BuiltinEngine,
'Ansible': AnsibleEngine,
'User': Engine
})
holder.plugin_manager = tom_plugin_manager
else:
tom_plugin_manager = holder.plugin_manager
class ExtensionLoader(object):
def __init__(self, core_config):
default_dir = os.path.join(os.path.expanduser("~"), "epops")
self.plugin_path = core_config.get('plugin_dir', [default_dir])
self.simple_plugin_manager = PluginManager()
self.simple_plugin_manager.setPluginPlaces(self.plugin_path)
self.simple_plugin_manager.setPluginInfoExtension('plugin-meta')
self.simple_plugin_manager.setCategoriesFilter({"Extension": IExtension})
self.pluginManager = AutoInstallPluginManager(self.plugin_path[0], self.simple_plugin_manager)
def ext_init(self):
self.simple_plugin_manager.collectPlugins()
return self.simple_plugin_manager.getAllPlugins()
def ext_load_new(self, filename):
zip_path = os.path.join(self.plugin_path[0], filename)
self.pluginManager.installFromZIP(zip_path)
self.simple_plugin_manager.collectPlugins()
return self.simple_plugin_manager.getAllPlugins()
class FakeNikola(object):
def __init__(self):
self.config = {'DISABLED_PLUGINS': []}
self.debug = False
self.loghandlers = []
self.timeline = []
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
"SignalHandler": SignalHandler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = ''
places = [
resource_filename('nikola', utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
os.path.expanduser('~/.nikola/plugins'),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
#self.pug = None
#for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
# if plugin_info.name == 'rest':
# self.plugin_manager.activatePluginByName(plugin_info.name)
# plugin_info.plugin_object.set_site(self)
# self.pug = plugin_info
# Now we have our pug
for plugin_info in self.plugin_manager.getPluginsOfCategory("RestExtension"):
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
class FakeSite(object):
def __init__(self):
self.template_system = self
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.loghandlers = [STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(utils.__file__), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(utils.__file__),
utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.timeline = [FakePost(title='Fake post', slug='fake-post')]
self.debug = True
def render_template(self, name, _, context):
return ('<img src="IMG.jpg">')
def getPluginCount(self):
# Get the default plugin directory, using XML
path = os.path.expanduser('~')
xml = xml_controller.Controller(path + '\.cxvrc.xml')
xml.load_file()
if os.path.exists(os.path.expanduser('~') + os.sep + "plugins"):
default_dir = os.path.expanduser('~') + os.sep + "plugins"
else:
default_dir = self.dicom_view.get_main_dir() + os.sep + "plugins"
if xml.get_plugin_directory() == "" or xml.get_plugin_directory() is None:
directory = [default_dir]
else:
directory = [default_dir, xml.get_plugin_directory()]
# Load the plugins from the plugin directory.
manager = PluginManager()
manager.setPluginPlaces(directory)
manager.setPluginInfoExtension('plugin')
manager.collectPlugins()
return len(manager.getAllPlugins())
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = [
'planetoid',
'ipynb',
'local_search',
'render_mustache',
]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
# Register our own path handlers
self.path_handlers = {
'slug': self.slug_path,
'post_path': self.post_path,
}
self.strict = False
self.global_data = {}
self.posts = []
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.posts_per_category = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
self._template_system = None
self._THEMES = None
self.debug = DEBUG
self.loghandlers = []
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ANNOTATIONS': False,
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'BLOG_TITLE': 'Default Title',
'BLOG_DESCRIPTION': 'Default Description',
'BODY_END': "",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENT_SYSTEM': 'disqus',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'COMPILERS': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile',),
"txt2tags": ('.t2t',),
"bbcode": ('.bb',),
"wiki": ('.wiki',),
"ipynb": ('.ipynb',),
"html": ('.html', '.htm')
},
'CONTENT_FOOTER': '',
'COPY_SOURCES': True,
'CREATE_MONTHLY_ARCHIVE': False,
'CREATE_SINGLE_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'EXTRA_PLUGINS_DIRS': [],
'COMMENT_SYSTEM_ID': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FEED_LENGTH': 10,
'FILE_METADATA_REGEXP': None,
'ADDITIONAL_METADATA': {},
'FILES_FOLDERS': {'files': ''},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GALLERY_SORT_BY_DATE': True,
'GZIP_COMMAND': None,
'GZIP_FILES': False,
'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json', '.xml'),
'HIDE_SOURCELINK': False,
'HIDE_UNTRANSLATED_POSTS': False,
'HYPHENATE': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEXES_PAGES_MAIN': False,
'INDEX_PATH': '',
'IPYNB_CONFIG': {},
'LESS_COMPILER': 'lessc',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'NAVIGATION_LINKS': None,
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'POSTS': (("posts/*.txt", "posts", "post.tmpl"),),
'PAGES': (("stories/*.txt", "stories", "story.tmpl"),),
'PRETTY_URLS': False,
'FUTURE_IS_NOW': False,
'READ_MORE_LINK': '<p class="more"><a href="{link}">{read_more}…</a></p>',
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SASS_COMPILER': 'sass',
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'SOCIAL_BUTTONS_CODE': SOCIAL_BUTTONS_CODE,
'SITE_URL': 'http://getnikola.com/',
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'bootstrap',
'THEME_REVEAL_CONFIG_SUBTHEME': 'sky',
'THEME_REVEAL_CONFIG_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'URL_TYPE': 'rel_path',
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': 'UTC',
'DEPLOY_DRAFTS': True,
'DEPLOY_FUTURE': False,
'SCHEDULE_ALL': False,
'SCHEDULE_RULE': '',
'SCHEDULE_FORCE_TODAY': False,
'LOGGING_HANDLERS': {'stderr': {'loglevel': 'WARNING', 'bubble': True}},
'DEMOTE_HEADERS': 1,
}
self.config.update(config)
# Make sure we have pyphen installed if we are using it
if self.config.get('HYPHENATE') and pyphen is None:
utils.LOGGER.warn('To use the hyphenation, you have to install '
'the "pyphen" package.')
utils.LOGGER.warn('Setting HYPHENATE to False.')
self.config['HYPHENATE'] = False
# Deprecating post_compilers
# TODO: remove on v7
if 'post_compilers' in config:
utils.LOGGER.warn('The post_compilers option is deprecated, use COMPILERS instead.')
if 'COMPILERS' in config:
utils.LOGGER.warn('COMPILERS conflicts with post_compilers, ignoring post_compilers.')
else:
self.config['COMPILERS'] = config['post_compilers']
# Deprecating post_pages
# TODO: remove on v7
if 'post_pages' in config:
utils.LOGGER.warn('The post_pages option is deprecated, use POSTS and PAGES instead.')
if 'POSTS' in config or 'PAGES' in config:
utils.LOGGER.warn('POSTS and PAGES conflict with post_pages, ignoring post_pages.')
else:
self.config['POSTS'] = [item[:3] for item in config['post_pages'] if item[-1]]
self.config['PAGES'] = [item[:3] for item in config['post_pages'] if not item[-1]]
# FIXME: Internally, we still use post_pages because it's a pain to change it
self.config['post_pages'] = []
for i1, i2, i3 in self.config['POSTS']:
self.config['post_pages'].append([i1, i2, i3, True])
for i1, i2, i3 in self.config['PAGES']:
self.config['post_pages'].append([i1, i2, i3, False])
# Deprecating DISQUS_FORUM
# TODO: remove on v7
if 'DISQUS_FORUM' in config:
utils.LOGGER.warn('The DISQUS_FORUM option is deprecated, use COMMENT_SYSTEM_ID instead.')
if 'COMMENT_SYSTEM_ID' in config:
utils.LOGGER.warn('DISQUS_FORUM conflicts with COMMENT_SYSTEM_ID, ignoring DISQUS_FORUM.')
else:
self.config['COMMENT_SYSTEM_ID'] = config['DISQUS_FORUM']
# Deprecating the ANALYTICS option
# TODO: remove on v7
if 'ANALYTICS' in config:
utils.LOGGER.warn('The ANALYTICS option is deprecated, use BODY_END instead.')
if 'BODY_END' in config:
utils.LOGGER.warn('ANALYTICS conflicts with BODY_END, ignoring ANALYTICS.')
else:
self.config['BODY_END'] = config['ANALYTICS']
# Deprecating the SIDEBAR_LINKS option
# TODO: remove on v7
if 'SIDEBAR_LINKS' in config:
utils.LOGGER.warn('The SIDEBAR_LINKS option is deprecated, use NAVIGATION_LINKS instead.')
if 'NAVIGATION_LINKS' in config:
utils.LOGGER.warn('The SIDEBAR_LINKS conflicts with NAVIGATION_LINKS, ignoring SIDEBAR_LINKS.')
else:
self.config['NAVIGATION_LINKS'] = config['SIDEBAR_LINKS']
# Compatibility alias
self.config['SIDEBAR_LINKS'] = self.config['NAVIGATION_LINKS']
if self.config['NAVIGATION_LINKS'] in (None, {}):
self.config['NAVIGATION_LINKS'] = {self.config['DEFAULT_LANG']: ()}
# Deprecating the ADD_THIS_BUTTONS option
# TODO: remove on v7
if 'ADD_THIS_BUTTONS' in config:
utils.LOGGER.warn('The ADD_THIS_BUTTONS option is deprecated, use SOCIAL_BUTTONS_CODE instead.')
if not config['ADD_THIS_BUTTONS']:
utils.LOGGER.warn('Setting SOCIAL_BUTTONS_CODE to empty because ADD_THIS_BUTTONS is False.')
self.config['SOCIAL_BUTTONS_CODE'] = ''
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
# TODO: remove on v7
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
utils.LOGGER.warn('You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML')
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
if config.get('COPY_SOURCES') and not self.config['HIDE_SOURCELINK']:
self.config['HIDE_SOURCELINK'] = True
self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
{self.config['DEFAULT_LANG']: ''})
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
# TODO: remove on v7
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
utils.LOGGER.warn('You should configure SITE_URL instead of BLOG_URL')
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
locale_fallback, locale_default, locales = sanitized_locales(
self.config.get('LOCALE_FALLBACK', None),
self.config.get('LOCALE_DEFAULT', None),
self.config.get('LOCALES', {}),
self.translations) # NOQA
utils.LocaleBorg.initialize(locales, self.default_lang)
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
# BASE_URL should *always* end in /
if self.config['BASE_URL'] and self.config['BASE_URL'][-1] != '/':
utils.LOGGER.warn("Your BASE_URL doesn't end in / -- adding it.")
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"SignalHandler": SignalHandler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
] + [path for path in extra_plugins_dirs if path]
else:
places = [
os.path.join(os.path.dirname(__file__), utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
] + [utils.sys_encode(path) for path in extra_plugins_dirs if path]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
# Activate all required SignalHandler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("SignalHandler"):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(plugin_info, "SignalHandler")
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Emit signal for SignalHandlers which need to start running immediately.
signal('sighandlers_loaded').send(self)
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
if plugin_info.name in self.config["COMPILERS"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self._GLOBAL_CONTEXT = {}
self._GLOBAL_CONTEXT['_link'] = self.link
self._GLOBAL_CONTEXT['set_locale'] = utils.LocaleBorg().set_locale
self._GLOBAL_CONTEXT['rel_link'] = self.rel_link
self._GLOBAL_CONTEXT['abs_link'] = self.abs_link
self._GLOBAL_CONTEXT['exists'] = self.file_exists
self._GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config['SLUG_TAG_PATH']
self._GLOBAL_CONTEXT['annotations'] = self.config['ANNOTATIONS']
self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self._GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self._GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
# TODO: remove fallback in v7
self._GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL', self.config.get('BLOG_URL'))
self._GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self._GLOBAL_CONTEXT['body_end'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['analytics'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['add_this_buttons'] = self.config.get('SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['social_buttons_code'] = self.config.get('SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self._GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self._GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self._GLOBAL_CONTEXT['comment_system'] = self.config.get('COMMENT_SYSTEM')
self._GLOBAL_CONTEXT['comment_system_id'] = self.config.get('COMMENT_SYSTEM_ID')
# TODO: remove in v7
self._GLOBAL_CONTEXT['disqus_forum'] = self.config.get('COMMENT_SYSTEM_ID')
self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self._GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONFIG_SUBTHEME')
self._GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONFIG_TRANSITION')
self._GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self._GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self._GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self._GLOBAL_CONTEXT['navigation_links'] = utils.Functionary(list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('NAVIGATION_LINKS', {}).items():
self._GLOBAL_CONTEXT['navigation_links'][k] = v
# TODO: remove on v7
# Compatibility alias
self._GLOBAL_CONTEXT['sidebar_links'] = self._GLOBAL_CONTEXT['navigation_links']
self._GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self._GLOBAL_CONTEXT['hide_sourcelink'] = self.config.get(
'HIDE_SOURCELINK')
self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA')
self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
signal('configured').send(self)
def _get_themes(self):
if self._THEMES is None:
# Check for old theme names (Issue #650) TODO: remove in v7
theme_replacements = {
'site': 'bootstrap',
'orphan': 'base',
'default': 'oldfashioned',
}
if self.config['THEME'] in theme_replacements:
utils.LOGGER.warn('You are using the old theme "{0}", using "{1}" instead.'.format(
self.config['THEME'], theme_replacements[self.config['THEME']]))
self.config['THEME'] = theme_replacements[self.config['THEME']]
if self.config['THEME'] == 'oldfashioned':
utils.LOGGER.warn('''You may need to install the "oldfashioned" theme '''
'''from themes.nikola.ralsina.com.ar because it's not '''
'''shipped by default anymore.''')
utils.LOGGER.warn('Please change your THEME setting.')
try:
self._THEMES = utils.get_theme_chain(self.config['THEME'])
except Exception:
utils.LOGGER.warn('''Can't load theme "{0}", using 'bootstrap' instead.'''.format(self.config['THEME']))
self.config['THEME'] = 'bootstrap'
return self._get_themes()
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config['USE_CDN']:
bootstrap_path = utils.get_asset_path(os.path.join(
'assets', 'css', 'bootstrap.min.css'), self._THEMES)
if bootstrap_path and bootstrap_path.split(os.sep)[-4] not in ['bootstrap', 'bootstrap3']:
utils.LOGGER.warn('The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.')
return self._THEMES
THEMES = property(_get_themes)
def _get_messages(self):
return utils.load_messages(self.THEMES,
self.translations,
self.default_lang)
MESSAGES = property(_get_messages)
def _get_global_context(self):
"""Initialize some parts of GLOBAL_CONTEXT only when it's queried."""
if 'messages' not in self._GLOBAL_CONTEXT:
self._GLOBAL_CONTEXT['messages'] = self.MESSAGES
if 'has_custom_css' not in self._GLOBAL_CONTEXT:
# check if custom css exist and is not empty
custom_css_path = utils.get_asset_path(
'assets/css/custom.css',
self.THEMES,
self.config['FILES_FOLDERS']
)
if custom_css_path and self.file_exists(custom_css_path, not_empty=True):
self._GLOBAL_CONTEXT['has_custom_css'] = True
else:
self._GLOBAL_CONTEXT['has_custom_css'] = False
return self._GLOBAL_CONTEXT
GLOBAL_CONTEXT = property(_get_global_context)
def _get_template_system(self):
if self._template_system is None:
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(
template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self._template_system = pi.plugin_object
lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES]
self._template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
return self._template_system
template_system = property(_get_template_system)
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.COMPILERS`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in
list(self.config['COMPILERS'].items())
if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'COMPILERS' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("COMPILERS in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
# string, arguments
local_context["formatmsg"] = lambda s, *a: s % a
data = self.template_system.render_template(
template_name, None, local_context)
assert output_name.startswith(
self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Refuse to replace links that consist of a fragment only
if ((not dst_url.scheme) and (not dst_url.netloc) and
(not dst_url.path) and (not dst_url.params) and
(not dst_url.query) and dst_url.fragment):
return dst
# Normalize
dst = urljoin(src, dst.lstrip('/'))
# Avoid empty links.
if src == dst:
if self.config.get('URL_TYPE') == 'absolute':
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return dst
elif self.config.get('URL_TYPE') == 'full_path':
return dst
else:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
if self.config.get('URL_TYPE') == 'absolute':
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return dst
if self.config.get('URL_TYPE') in ('full_path', 'absolute'):
if self.config.get('URL_TYPE') == 'absolute':
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
utils.makedirs(os.path.dirname(output_name))
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
These are mostly defined by plugins by registering via
the register_path_handler method, except for slug and
post_path which are defined in this class' init method.
Here's some of the others, for historical reasons:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* category (and name is the category name)
* category_rss (and name is the category name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a POSTS/PAGES tuple)
* slug (name is the slug of a post or story)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = utils.LocaleBorg().current_lang
path = self.path_handlers[kind](name, lang)
if is_link:
link = '/' + ('/'.join(path))
index_len = len(self.config['INDEX_FILE'])
if self.config['STRIP_INDEXES'] and \
link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def post_path(self, name, lang):
"""post_path path handler"""
return [_f for _f in [self.config['TRANSLATIONS'][lang],
os.path.dirname(name),
self.config['INDEX_FILE']] if _f]
def slug_path(self, name, lang):
"""slug path handler"""
results = [p for p in self.timeline if p.meta('slug') == name]
if not results:
utils.LOGGER.warning("Can't resolve path request for slug: {0}".format(name))
else:
if len(results) > 1:
utils.LOGGER.warning('Ambiguous path request for slug: {0}'.format(name))
return [_f for _f in results[0].permalink(lang).split('/') if _f]
def register_path_handler(self, kind, f):
if kind in self.path_handlers:
utils.LOGGER.warning('Conflicting path handlers for kind: {0}'.format(kind))
else:
self.path_handlers[kind] = f
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
return urlparse(dst).geturl()
def rel_link(self, src, dst):
# Normalize
try:
src = urljoin(self.config['BASE_URL'], src.lstrip('/'))
except AttributeError:
# sometimes, it’s an Undefined object.
src = urljoin(self.config['BASE_URL'], src)
try:
dst = urljoin(src, dst.lstrip('/'))
except AttributeError:
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def clean_task_paths(self, task):
"""Normalize target paths in the task."""
targets = task.get('targets', None)
if targets is not None:
task['targets'] = [os.path.normpath(t) for t in targets]
return task
def gen_tasks(self, name, plugin_category, doc=''):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
assert 'basename' in task
task = self.clean_task_paths(task)
yield task
for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield self.clean_task_paths(task)
if flag:
task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'basename': name,
'doc': doc,
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
seen = set([])
print("Scanning posts", end='', file=sys.stderr)
lower_case_tags = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='', file=sys.stderr)
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination,
os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config['TRANSLATIONS'].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
# We eliminate from the list the files inside any .ipynb folder
full_list = [p for p in full_list
if not any([x.startswith('.')
for x in p.split(os.sep)])]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.config,
dest_dir,
use_in_feeds,
self.MESSAGES,
template_name,
self.get_compiler(base_path)
)
self.global_data[post.source_path] = post
if post.use_in_feeds:
self.posts.append(post.source_path)
self.posts_per_year[
str(post.date.year)].append(post.source_path)
self.posts_per_month[
'{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post.source_path)
for tag in post.alltags:
if tag.lower() in lower_case_tags:
if tag not in self.posts_per_tag:
# Tags that differ only in case
other_tag = [k for k in self.posts_per_tag.keys() if k.lower() == tag.lower()][0]
utils.LOGGER.error('You have cases that differ only in upper/lower case: {0} and {1}'.format(tag, other_tag))
utils.LOGGER.error('Tag {0} is used in: {1}'.format(tag, post.source_path))
utils.LOGGER.error('Tag {0} is used in: {1}'.format(other_tag, ', '.join(self.posts_per_tag[other_tag])))
sys.exit(1)
else:
lower_case_tags.add(tag.lower())
self.posts_per_tag[tag].append(post.source_path)
self.posts_per_category[post.meta('category')].append(post.source_path)
else:
self.pages.append(post)
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!", file=sys.stderr)
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine'))
deps = list(filter(None, deps))
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
if post:
deps_dict['post_translations'] = post.translated_to
task = {
'name': os.path.normpath(output_name),
'file_dep': deps,
'targets': [output_name],
'actions': [(self.render_template, [post.template_name,
output_name, context])],
'clean': True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in
posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name': os.path.normpath(output_name),
'targets': [output_name],
'file_dep': deps,
'actions': [(self.render_template, [template_name, output_name,
context])],
'clean': True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
def run(self):
""" Runs the given algorithms """
self.controller.overlays = []
if self.alphas is None:
self.controller.alphas = []
if self.pb is not None:
wx.CallAfter(self.pb.update, 'Retrieving coral region')
# Load in original dicom pixel data and normalize
coral_slab = self.model.ds.pixel_array.astype(np.double)
coral_slab = np.rot90(coral_slab, self.rotations)
coral_slab = self.model.normalize_intensity(coral_slab)
# Cut down dicom pixel data to user defined coral slab region
x, y, dx, dy = self.dicom_controller.coral_slab
coral_slab = coral_slab[y:dy, x:dx]
# Get the default plugin directory, using XML
path = os.path.expanduser('~')
xml = xml_controller.Controller(path + '\.cxvrc.xml')
xml.load_file()
if os.path.exists(os.path.expanduser('~') + os.sep + "plugins"):
default_dir = os.path.expanduser('~') + os.sep + "plugins"
else:
default_dir = self.dicom_controller.view.get_main_dir() + os.sep + "plugins"
if xml.get_plugin_directory() == "" or xml.get_plugin_directory() is None:
directory = [default_dir]
else:
directory = [default_dir, xml.get_plugin_directory()]
# Load the plugins from the default plugin directory.
manager = PluginManager()
manager.setPluginPlaces(directory)
manager.setPluginInfoExtension('plugin')
manager.collectPlugins()
# Loop over all plugins that have been found
count = 0;
for plugin in manager.getAllPlugins():
# Initialize the plugin so that we can call it's methods
plugin.plugin_object.initPlugin(self.controller, coral_slab, self.model, self.pb, count, self.alphas)
# Run the plugin's algorithm
plugin.plugin_object.calc_filter()
# Update count so that the next filter will be added to the next overlay
count += 1
# Add the original coral_slab to the overlay
self.controller.overlays.append(coral_slab)
if self.alphas is None:
self.controller.alphas.append(100)
else:
self.controller.alphas = []
self.controller.alphas = self.alphas
wx.CallAfter(self.controller.add_overlay)
# Update the progress bar to let the user know we've finished
# running the algorithms
if self.pb is not None:
wx.CallAfter(self.pb.finish, 'Finishing up')
# Display the filters
if self.controller.show and self.alphas is None:
# Get the size of alphas (number of filters) minus one.
# The last alpha should be 0. Set all the alphas to the same
# value (except for the last one), so divide 100 by the number
# of alphas minus one.
# value = 100 / (len(self.controller.alphas) - 1)
value = 0
for i in range(len(self.controller.alphas) - 1):
self.controller.alphas[i] = int(value)
# Set the last one to 0 ourselves just to make sure
self.controller.alphas[len(self.controller.alphas) - 1] = 100
wx.CallAfter(self.controller.display)
wx.CallAfter(self.controller.view.Show)
else:
wx.CallAfter(self.controller.display)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
# This is the default config
# TODO: fill it
self.config = {
"ARCHIVE_PATH": "",
"ARCHIVE_FILENAME": "archive.html",
"DEFAULT_LANG": "en",
"OUTPUT_FOLDER": "output",
"CACHE_FOLDER": "cache",
"FILES_FOLDERS": {"files": ""},
"LISTINGS_FOLDER": "listings",
"ADD_THIS_BUTTONS": True,
"INDEX_DISPLAY_POST_COUNT": 10,
"INDEX_TEASERS": False,
"RSS_TEASERS": True,
"MAX_IMAGE_SIZE": 1280,
"USE_FILENAME_AS_TITLE": True,
"SLUG_TAG_PATH": False,
"INDEXES_TITLE": "",
"INDEXES_PAGES": "",
"FILTERS": {},
"USE_BUNDLES": True,
"TAG_PAGES_ARE_INDEXES": False,
"THEME": "default",
"COMMENTS_IN_GALLERIES": False,
"COMMENTS_IN_STORIES": False,
"FILE_METADATA_REGEXP": None,
"post_compilers": {
"rest": [".txt", ".rst"],
"markdown": [".md", ".mdown", ".markdown"],
"html": [".html", ".htm"],
},
}
self.config.update(config)
self.config["TRANSLATIONS"] = self.config.get("TRANSLATIONS", {self.config["DEFAULT_LANG"]: ""})
self.THEMES = utils.get_theme_chain(self.config["THEME"])
self.MESSAGES = utils.load_messages(self.THEMES, self.config["TRANSLATIONS"])
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
}
)
self.plugin_manager.setPluginInfoExtension("plugin")
self.plugin_manager.setPluginPlaces(
[str(os.path.join(os.path.dirname(__file__), "plugins")), str(os.path.join(os.getcwd(), "plugins"))]
)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
pluginInfo.plugin_object.short_help = pluginInfo.description
self.commands[pluginInfo.name] = pluginInfo.plugin_object
# Activate all task plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = self.config.get("GLOBAL_CONTEXT", {})
self.GLOBAL_CONTEXT["messages"] = self.MESSAGES
self.GLOBAL_CONTEXT["_link"] = self.link
self.GLOBAL_CONTEXT["rel_link"] = self.rel_link
self.GLOBAL_CONTEXT["abs_link"] = self.abs_link
self.GLOBAL_CONTEXT["exists"] = self.file_exists
self.GLOBAL_CONTEXT["add_this_buttons"] = self.config["ADD_THIS_BUTTONS"]
self.GLOBAL_CONTEXT["index_display_post_count"] = self.config["INDEX_DISPLAY_POST_COUNT"]
self.GLOBAL_CONTEXT["use_bundles"] = self.config["USE_BUNDLES"]
if "date_format" not in self.GLOBAL_CONTEXT:
self.GLOBAL_CONTEXT["date_format"] = "%Y-%m-%d %H:%M"
# check if custom css exist and is not empty
for files_path in list(self.config["FILES_FOLDERS"].keys()):
custom_css_path = os.path.join(files_path, "assets/css/custom.css")
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT["has_custom_css"] = True
break
else:
self.GLOBAL_CONTEXT["has_custom_css"] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading %s template system plugin\n" % template_sys_name)
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES]
self.template_system.set_directories(lookup_dirs, self.config["CACHE_FOLDER"])
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for pluginInfo in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
self.compilers[pluginInfo.name] = pluginInfo.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in list(self.config["post_compilers"].items()) if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit(
"Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of %s)" % ", ".join(langs)
)
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to " "handle '%s' extensions." % ext)
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.config["GLOBAL_CONTEXT"])
local_context.update(context)
data = self.template_system.render_template(template_name, None, local_context)
assert isinstance(output_name, bytes)
assert output_name.startswith(self.config["OUTPUT_FOLDER"].encode("utf8"))
url_part = output_name.decode("utf8")[len(self.config["OUTPUT_FOLDER"]) + 1 :]
# This is to support windows paths
url_part = "/".join(url_part.split(os.sep))
src = urljoin(self.config["BLOG_URL"], url_part)
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split("/")[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == "link": # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip("/"), context["lang"])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b"<!DOCTYPE html>" + lxml.html.tostring(doc, encoding="utf8")
with open(output_name, "wb+") as post_file:
post_file.write(data)
def path(self, kind, name, lang, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
path = []
if kind == "tag_index":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], "index.html"] if _f]
elif kind == "tag":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".html"] if _f]
elif kind == "tag_rss":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".xml"] if _f]
elif kind == "index":
if name > 0:
path = [
_f
for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index-%s.html" % name]
if _f
]
else:
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index.html"] if _f]
elif kind == "rss":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["RSS_PATH"], "rss.xml"] if _f]
elif kind == "archive":
if name:
path = [
_f
for _f in [self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], name, "index.html"]
if _f
]
else:
path = [
_f
for _f in [
self.config["TRANSLATIONS"][lang],
self.config["ARCHIVE_PATH"],
self.config["ARCHIVE_FILENAME"],
]
if _f
]
elif kind == "gallery":
path = [_f for _f in [self.config["GALLERY_PATH"], name, "index.html"] if _f]
elif kind == "listing":
path = [_f for _f in [self.config["LISTINGS_FOLDER"], name + ".html"] if _f]
if is_link:
return "/" + ("/".join(path))
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config["BLOG_URL"], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config["BLOG_URL"], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split("/")[1:]
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {"name": b"all", "actions": None, "clean": True, "task_dep": task_dep}
def scan_posts(self):
"""Scan all the posts."""
if not self._scanned:
print("Scanning posts", end="")
targets = set([])
for wildcard, destination, template_name, use_in_feeds in self.config["post_pages"]:
print(".", end="")
base_len = len(destination.split(os.sep))
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.join(*([destination] + dirpath.split(os.sep)[base_len:]))
for base_path in glob.glob(dir_glob):
post = Post(
base_path,
self.config["CACHE_FOLDER"],
dest_dir,
use_in_feeds,
self.config["TRANSLATIONS"],
self.config["DEFAULT_LANG"],
self.config["BLOG_URL"],
self.MESSAGES,
template_name,
self.config["FILE_METADATA_REGEXP"],
)
for lang, langpath in list(self.config["TRANSLATIONS"].items()):
dest = (destination, langpath, dir_glob, post.pagenames[lang])
if dest in targets:
raise Exception(
"Duplicated output path %r in post %r" % (post.pagenames[lang], base_path)
)
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(post.post_name)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + self.template_system.template_deps(post.template_name)
context["post"] = post
context["lang"] = lang
context["title"] = post.title(lang)
context["description"] = post.description(lang)
context["permalink"] = post.permalink(lang)
context["page_list"] = self.pages
if post.use_in_feeds:
context["enable_comments"] = True
else:
context["enable_comments"] = self.config["COMMENTS_IN_STORIES"]
output_name = os.path.join(self.config["OUTPUT_FOLDER"], post.destination_path(lang)).encode("utf8")
deps_dict = copy(context)
deps_dict.pop("post")
if post.prev_post:
deps_dict["PREV_LINK"] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict["NEXT_LINK"] = [post.next_post.permalink(lang)]
deps_dict["OUTPUT_FOLDER"] = self.config["OUTPUT_FOLDER"]
deps_dict["TRANSLATIONS"] = self.config["TRANSLATIONS"]
deps_dict["global"] = self.config["GLOBAL_CONTEXT"]
deps_dict["comments"] = context["enable_comments"]
task = {
"name": output_name,
"file_dep": deps,
"targets": [output_name],
"actions": [(self.render_template, [post.template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context):
"""Renders pages with lists of posts."""
# This is a name on disk, has to be bytes
assert isinstance(output_name, bytes)
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config["BLOG_TITLE"]
context["description"] = self.config["BLOG_DESCRIPTION"]
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.titles[lang], p.permalink(lang)) for p in posts]
deps_context["global"] = self.config["GLOBAL_CONTEXT"]
task = {
"name": output_name,
"targets": [output_name],
"file_dep": deps,
"actions": [(self.render_template, [template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_context)],
}
return utils.apply_filters(task, filters)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = ["planetoid", "ipynb", "local_search", "render_mustache"]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
"ADD_THIS_BUTTONS": True,
"ANALYTICS": "",
"ARCHIVE_PATH": "",
"ARCHIVE_FILENAME": "archive.html",
"CACHE_FOLDER": "cache",
"CODE_COLOR_SCHEME": "default",
"COMMENTS_IN_GALLERIES": False,
"COMMENTS_IN_STORIES": False,
"CONTENT_FOOTER": "",
"CREATE_MONTHLY_ARCHIVE": False,
"DATE_FORMAT": "%Y-%m-%d %H:%M",
"DEFAULT_LANG": "en",
"DEPLOY_COMMANDS": [],
"DISABLED_PLUGINS": (),
"DISQUS_FORUM": "nikolademo",
"ENABLED_EXTRAS": (),
"EXTRA_HEAD_DATA": "",
"FAVICONS": {},
"FILE_METADATA_REGEXP": None,
"FILES_FOLDERS": {"files": ""},
"FILTERS": {},
"GALLERY_PATH": "galleries",
"GZIP_FILES": False,
"GZIP_EXTENSIONS": (".txt", ".htm", ".html", ".css", ".js", ".json"),
"HIDE_UNTRANSLATED_POSTS": False,
"INDEX_DISPLAY_POST_COUNT": 10,
"INDEX_TEASERS": False,
"INDEXES_TITLE": "",
"INDEXES_PAGES": "",
"INDEX_PATH": "",
"LICENSE": "",
"LISTINGS_FOLDER": "listings",
"MAX_IMAGE_SIZE": 1280,
"MATHJAX_CONFIG": "",
"OLD_THEME_SUPPORT": True,
"OUTPUT_FOLDER": "output",
"post_compilers": {
"rest": (".txt", ".rst"),
"markdown": (".md", ".mdown", ".markdown"),
"textile": (".textile",),
"txt2tags": (".t2t",),
"bbcode": (".bb",),
"wiki": (".wiki",),
"ipynb": (".ipynb",),
"html": (".html", ".htm"),
},
"POST_PAGES": (
("posts/*.txt", "posts", "post.tmpl", True),
("stories/*.txt", "stories", "story.tmpl", False),
),
"REDIRECTIONS": [],
"RSS_LINK": None,
"RSS_PATH": "",
"RSS_TEASERS": True,
"SEARCH_FORM": "",
"SLUG_TAG_PATH": True,
"STORY_INDEX": False,
"STRIP_INDEX_HTML": False,
"TAG_PATH": "categories",
"TAG_PAGES_ARE_INDEXES": False,
"THEME": "site",
"THEME_REVEAL_CONGIF_SUBTHEME": "sky",
"THEME_REVEAL_CONGIF_TRANSITION": "cube",
"THUMBNAIL_SIZE": 180,
"USE_BUNDLES": True,
"USE_CDN": False,
"USE_FILENAME_AS_TITLE": True,
"TIMEZONE": None,
}
self.config.update(config)
self.config["TRANSLATIONS"] = self.config.get("TRANSLATIONS", {self.config["DEFAULT_" "LANG"]: ""})
self.THEMES = utils.get_theme_chain(self.config["THEME"])
self.MESSAGES = utils.load_messages(self.THEMES, self.config["TRANSLATIONS"], self.config["DEFAULT_LANG"])
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
if "SITE_URL" not in self.config:
if "BLOG_URL" in self.config:
print("WARNING: You should configure SITE_URL instead of BLOG_URL")
self.config["SITE_URL"] = self.config["BLOG_URL"]
self.default_lang = self.config["DEFAULT_LANG"]
self.translations = self.config["TRANSLATIONS"]
# BASE_URL defaults to SITE_URL
if "BASE_URL" not in self.config:
self.config["BASE_URL"] = self.config.get("SITE_URL")
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
}
)
self.plugin_manager.setPluginInfoExtension("plugin")
self.plugin_manager.setPluginPlaces(
[str(os.path.join(os.path.dirname(__file__), "plugins")), str(os.path.join(os.getcwd(), "plugins"))]
)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if plugin_info.name in self.config["DISABLED_PLUGINS"] or (
plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"]
):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if plugin_info.name in self.config["DISABLED_PLUGINS"] or (
plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"]
):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = {}
self.GLOBAL_CONTEXT["messages"] = self.MESSAGES
self.GLOBAL_CONTEXT["_link"] = self.link
self.GLOBAL_CONTEXT["set_locale"] = s_l
self.GLOBAL_CONTEXT["rel_link"] = self.rel_link
self.GLOBAL_CONTEXT["abs_link"] = self.abs_link
self.GLOBAL_CONTEXT["exists"] = self.file_exists
self.GLOBAL_CONTEXT["SLUG_TAG_PATH"] = self.config["SLUG_TAG_PATH"]
self.GLOBAL_CONTEXT["add_this_buttons"] = self.config["ADD_THIS_BUTTONS"]
self.GLOBAL_CONTEXT["index_display_post_count"] = self.config["INDEX_DISPLAY_POST_COUNT"]
self.GLOBAL_CONTEXT["use_bundles"] = self.config["USE_BUNDLES"]
self.GLOBAL_CONTEXT["use_cdn"] = self.config.get("USE_CDN")
self.GLOBAL_CONTEXT["favicons"] = self.config["FAVICONS"]
self.GLOBAL_CONTEXT["date_format"] = self.config.get("DATE_FORMAT", "%Y-%m-%d %H:%M")
self.GLOBAL_CONTEXT["blog_author"] = self.config.get("BLOG_AUTHOR")
self.GLOBAL_CONTEXT["blog_title"] = self.config.get("BLOG_TITLE")
self.GLOBAL_CONTEXT["blog_url"] = self.config.get("SITE_URL", self.config.get("BLOG_URL"))
self.GLOBAL_CONTEXT["blog_desc"] = self.config.get("BLOG_DESCRIPTION")
self.GLOBAL_CONTEXT["analytics"] = self.config.get("ANALYTICS")
self.GLOBAL_CONTEXT["translations"] = self.config.get("TRANSLATIONS")
self.GLOBAL_CONTEXT["license"] = self.config.get("LICENSE")
self.GLOBAL_CONTEXT["search_form"] = self.config.get("SEARCH_FORM")
self.GLOBAL_CONTEXT["disqus_forum"] = self.config.get("DISQUS_FORUM")
self.GLOBAL_CONTEXT["mathjax_config"] = self.config.get("MATHJAX_CONFIG")
self.GLOBAL_CONTEXT["subtheme"] = self.config.get("THEME_REVEAL_CONGIF_SUBTHEME")
self.GLOBAL_CONTEXT["transition"] = self.config.get("THEME_REVEAL_CONGIF_TRANSITION")
self.GLOBAL_CONTEXT["content_footer"] = self.config.get("CONTENT_FOOTER")
self.GLOBAL_CONTEXT["rss_path"] = self.config.get("RSS_PATH")
self.GLOBAL_CONTEXT["rss_link"] = self.config.get("RSS_LINK")
self.GLOBAL_CONTEXT["sidebar_links"] = utils.Functionary(list, self.config["DEFAULT_LANG"])
for k, v in self.config.get("SIDEBAR_LINKS", {}).items():
self.GLOBAL_CONTEXT["sidebar_links"][k] = v
self.GLOBAL_CONTEXT["twitter_card"] = self.config.get("TWITTER_CARD", {})
self.GLOBAL_CONTEXT["extra_head_data"] = self.config.get("EXTRA_HEAD_DATA")
self.GLOBAL_CONTEXT.update(self.config.get("GLOBAL_CONTEXT", {}))
# check if custom css exist and is not empty
for files_path in list(self.config["FILES_FOLDERS"].keys()):
custom_css_path = os.path.join(files_path, "assets/css/custom.css")
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT["has_custom_css"] = True
break
else:
self.GLOBAL_CONTEXT["has_custom_css"] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system " "plugin\n".format(template_sys_name))
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES]
self.template_system.set_directories(lookup_dirs, self.config["CACHE_FOLDER"])
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config["USE_CDN"]:
bootstrap_path = utils.get_asset_path(os.path.join("assets", "css", "bootstrap.min.css"), self.THEMES)
if bootstrap_path.split(os.sep)[-4] != "site":
warnings.warn(
"The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap."
)
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
self.compilers[plugin_info.name] = plugin_info.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in list(self.config["post_compilers"].items()) if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit(
"Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of {0})".format(", ".join(langs))
)
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to " "handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(template_name, None, local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1 :]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split("/")[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == "link": # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip("/"), context["lang"])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b"<!DOCTYPE html>" + lxml.html.tostring(doc, encoding="utf8")
with open(output_name, "wb+") as post_file:
post_file.write(data)
def current_lang(self): # FIXME: this is duplicated, turn into a mixin
"""Return the currently set locale, if it's one of the
available translations, or default_lang."""
lang = utils.LocaleBorg().current_lang
if lang:
if lang in self.translations:
return lang
lang = lang.split("_")[0]
if lang in self.translations:
return lang
# whatever
return self.default_lang
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a post_pages tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = self.current_lang()
path = []
if kind == "tag_index":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], "index.html"] if _f]
elif kind == "tag":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".html"] if _f]
elif kind == "tag_rss":
if self.config["SLUG_TAG_PATH"]:
name = utils.slugify(name)
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["TAG_PATH"], name + ".xml"] if _f]
elif kind == "index":
if name not in [None, 0]:
path = [
_f
for _f in [
self.config["TRANSLATIONS"][lang],
self.config["INDEX_PATH"],
"index-{0}.html".format(name),
]
if _f
]
else:
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["INDEX_PATH"], "index.html"] if _f]
elif kind == "post_path":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], os.path.dirname(name), "index.html"] if _f]
elif kind == "rss":
path = [_f for _f in [self.config["TRANSLATIONS"][lang], self.config["RSS_PATH"], "rss.xml"] if _f]
elif kind == "archive":
if name:
path = [
_f
for _f in [self.config["TRANSLATIONS"][lang], self.config["ARCHIVE_PATH"], name, "index.html"]
if _f
]
else:
path = [
_f
for _f in [
self.config["TRANSLATIONS"][lang],
self.config["ARCHIVE_PATH"],
self.config["ARCHIVE_FILENAME"],
]
if _f
]
elif kind == "gallery":
path = [_f for _f in [self.config["GALLERY_PATH"], name, "index.html"] if _f]
elif kind == "listing":
path = [_f for _f in [self.config["LISTINGS_FOLDER"], name + ".html"] if _f]
if is_link:
link = "/" + ("/".join(path))
if self.config["STRIP_INDEX_HTML"] and link.endswith("/index.html"):
return link[:-10]
else:
return link
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config["BASE_URL"], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config["BASE_URL"], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split("/")[1:]
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
def create_gzipped_copy(in_path, out_path):
with gzip.GzipFile(out_path, "wb+") as outf:
with open(in_path, "rb") as inf:
outf.write(inf.read())
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
def add_gzipped_copies(task):
if not self.config["GZIP_FILES"]:
return None
if task.get("name") is None:
return None
gzip_task = {
"file_dep": [],
"targets": [],
"actions": [],
"basename": "gzip",
"name": task.get("name") + ".gz",
"clean": True,
}
targets = task.get("targets", [])
flag = False
for target in targets:
ext = os.path.splitext(target)[1]
if ext.lower() in self.config["GZIP_EXTENSIONS"] and target.startswith(self.config["OUTPUT_FOLDER"]):
flag = True
gzipped = target + ".gz"
gzip_task["file_dep"].append(target)
gzip_task["targets"].append(gzipped)
gzip_task["actions"].append((create_gzipped_copy, (target, gzipped)))
if not flag:
return None
return gzip_task
if self.config["GZIP_FILES"]:
task_dep = ["gzip"]
else:
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
gztask = add_gzipped_copies(task)
if gztask:
yield gztask
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
gztask = add_gzipped_copies(task)
if gztask:
yield gztask
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {"name": b"all", "actions": None, "clean": True, "task_dep": task_dep}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
print("Scanning posts", end="")
tzinfo = None
if self.config["TIMEZONE"] is not None:
tzinfo = pytz.timezone(self.config["TIMEZONE"])
targets = set([])
for wildcard, destination, template_name, use_in_feeds in self.config["post_pages"]:
print(".", end="")
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination, os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config["TRANSLATIONS"].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
for base_path in full_list:
post = Post(
base_path,
self.config["CACHE_FOLDER"],
dest_dir,
use_in_feeds,
self.config["TRANSLATIONS"],
self.config["DEFAULT_LANG"],
self.config["BASE_URL"],
self.MESSAGES,
template_name,
self.config["FILE_METADATA_REGEXP"],
self.config["STRIP_INDEX_HTML"],
tzinfo,
self.config["HIDE_UNTRANSLATED_POSTS"],
)
for lang, langpath in list(self.config["TRANSLATIONS"].items()):
dest = (destination, langpath, dir_glob, post.meta[lang]["slug"])
if dest in targets:
raise Exception(
"Duplicated output path {0!r} "
"in post {1!r}".format(post.meta[lang]["slug"], base_path)
)
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(post.post_name)
self.posts_per_month["{0}/{1:02d}".format(post.date.year, post.date.month)].append(
post.post_name
)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
if self.config["OLD_THEME_SUPPORT"]:
post._add_old_metadata()
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + self.template_system.template_deps(post.template_name)
context["post"] = post
context["lang"] = lang
context["title"] = post.title(lang)
context["description"] = post.description(lang)
context["permalink"] = post.permalink(lang)
context["page_list"] = self.pages
if post.use_in_feeds:
context["enable_comments"] = True
else:
context["enable_comments"] = self.config["COMMENTS_IN_STORIES"]
output_name = os.path.join(self.config["OUTPUT_FOLDER"], post.destination_path(lang))
deps_dict = copy(context)
deps_dict.pop("post")
if post.prev_post:
deps_dict["PREV_LINK"] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict["NEXT_LINK"] = [post.next_post.permalink(lang)]
deps_dict["OUTPUT_FOLDER"] = self.config["OUTPUT_FOLDER"]
deps_dict["TRANSLATIONS"] = self.config["TRANSLATIONS"]
deps_dict["global"] = self.GLOBAL_CONTEXT
deps_dict["comments"] = context["enable_comments"]
task = {
"name": os.path.normpath(output_name),
"file_dep": deps,
"targets": [output_name],
"actions": [(self.render_template, [post.template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config["BLOG_TITLE"]
context["description"] = self.config["BLOG_DESCRIPTION"]
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]["title"], p.permalink(lang)) for p in posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
"name": os.path.normpath(output_name),
"targets": [output_name],
"file_dep": deps,
"actions": [(self.render_template, [template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_context)],
}
return utils.apply_filters(task, filters)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = ["planetoid", "ipynb", "local_search", "render_mustache"]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
# Register our own path handlers
self.path_handlers = {"slug": self.slug_path, "post_path": self.post_path}
self.strict = False
self.global_data = {}
self.posts = []
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.posts_per_category = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
self._template_system = None
self._THEMES = None
self.loghandlers = []
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
"ADD_THIS_BUTTONS": True,
"ANNOTATIONS": False,
"ARCHIVE_PATH": "",
"ARCHIVE_FILENAME": "archive.html",
"BLOG_TITLE": "Default Title",
"BLOG_DESCRIPTION": "Default Description",
"BODY_END": "",
"CACHE_FOLDER": "cache",
"CODE_COLOR_SCHEME": "default",
"COMMENT_SYSTEM": "disqus",
"COMMENTS_IN_GALLERIES": False,
"COMMENTS_IN_STORIES": False,
"COMPILERS": {
"rest": (".txt", ".rst"),
"markdown": (".md", ".mdown", ".markdown"),
"textile": (".textile",),
"txt2tags": (".t2t",),
"bbcode": (".bb",),
"wiki": (".wiki",),
"ipynb": (".ipynb",),
"html": (".html", ".htm"),
},
"CONTENT_FOOTER": "",
"COPY_SOURCES": True,
"CREATE_MONTHLY_ARCHIVE": False,
"CREATE_SINGLE_ARCHIVE": False,
"DATE_FORMAT": "%Y-%m-%d %H:%M",
"DEFAULT_LANG": "en",
"DEPLOY_COMMANDS": [],
"DISABLED_PLUGINS": (),
"COMMENT_SYSTEM_ID": "nikolademo",
"ENABLED_EXTRAS": (),
"EXTRA_HEAD_DATA": "",
"FAVICONS": {},
"FEED_LENGTH": 10,
"FILE_METADATA_REGEXP": None,
"ADDITIONAL_METADATA": {},
"FILES_FOLDERS": {"files": ""},
"FILTERS": {},
"GALLERY_PATH": "galleries",
"GALLERY_SORT_BY_DATE": True,
"GZIP_COMMAND": None,
"GZIP_FILES": False,
"GZIP_EXTENSIONS": (".txt", ".htm", ".html", ".css", ".js", ".json", ".xml"),
"HIDE_SOURCELINK": False,
"HIDE_UNTRANSLATED_POSTS": False,
"HYPHENATE": False,
"INDEX_DISPLAY_POST_COUNT": 10,
"INDEX_FILE": "index.html",
"INDEX_TEASERS": False,
"INDEXES_TITLE": "",
"INDEXES_PAGES": "",
"INDEX_PATH": "",
"IPYNB_CONFIG": {},
"LICENSE": "",
"LINK_CHECK_WHITELIST": [],
"LISTINGS_FOLDER": "listings",
"NAVIGATION_LINKS": None,
"MARKDOWN_EXTENSIONS": ["fenced_code", "codehilite"],
"MAX_IMAGE_SIZE": 1280,
"MATHJAX_CONFIG": "",
"OLD_THEME_SUPPORT": True,
"OUTPUT_FOLDER": "output",
"POSTS": (("posts/*.txt", "posts", "post.tmpl"),),
"PAGES": (("stories/*.txt", "stories", "story.tmpl"),),
"PRETTY_URLS": False,
"FUTURE_IS_NOW": False,
"READ_MORE_LINK": '<p class="more"><a href="{link}">{read_more}…</a></p>',
"REDIRECTIONS": [],
"RSS_LINK": None,
"RSS_PATH": "",
"RSS_TEASERS": True,
"SEARCH_FORM": "",
"SLUG_TAG_PATH": True,
"SOCIAL_BUTTONS_CODE": SOCIAL_BUTTONS_CODE,
"SITE_URL": "http://getnikola.com/",
"STORY_INDEX": False,
"STRIP_INDEXES": False,
"SITEMAP_INCLUDE_FILELESS_DIRS": True,
"TAG_PATH": "categories",
"TAG_PAGES_ARE_INDEXES": False,
"THEME": "bootstrap",
"THEME_REVEAL_CONFIG_SUBTHEME": "sky",
"THEME_REVEAL_CONFIG_TRANSITION": "cube",
"THUMBNAIL_SIZE": 180,
"URL_TYPE": "rel_path",
"USE_BUNDLES": True,
"USE_CDN": False,
"USE_FILENAME_AS_TITLE": True,
"TIMEZONE": "UTC",
"DEPLOY_DRAFTS": True,
"DEPLOY_FUTURE": False,
"SCHEDULE_ALL": False,
"SCHEDULE_RULE": "",
"SCHEDULE_FORCE_TODAY": False,
"LOGGING_HANDLERS": {"stderr": {"loglevel": "WARNING", "bubble": True}},
"DEMOTE_HEADERS": 1,
}
self.config.update(config)
# Make sure we have pyphen installed if we are using it
if self.config.get("HYPHENATE") and pyphen is None:
utils.LOGGER.warn("To use the hyphenation, you have to install " 'the "pyphen" package.')
utils.LOGGER.warn("Setting HYPHENATE to False.")
self.config["HYPHENATE"] = False
# Deprecating post_compilers
# TODO: remove on v7
if "post_compilers" in config:
utils.LOGGER.warn("The post_compilers option is deprecated, use COMPILERS instead.")
if "COMPILERS" in config:
utils.LOGGER.warn("COMPILERS conflicts with post_compilers, ignoring post_compilers.")
else:
self.config["COMPILERS"] = config["post_compilers"]
# Deprecating post_pages
# TODO: remove on v7
if "post_pages" in config:
utils.LOGGER.warn("The post_pages option is deprecated, use POSTS and PAGES instead.")
if "POSTS" in config or "PAGES" in config:
utils.LOGGER.warn("POSTS and PAGES conflict with post_pages, ignoring post_pages.")
else:
self.config["POSTS"] = [item[:3] for item in config["post_pages"] if item[-1]]
self.config["PAGES"] = [item[:3] for item in config["post_pages"] if not item[-1]]
# FIXME: Internally, we still use post_pages because it's a pain to change it
self.config["post_pages"] = []
for i1, i2, i3 in self.config["POSTS"]:
self.config["post_pages"].append([i1, i2, i3, True])
for i1, i2, i3 in self.config["PAGES"]:
self.config["post_pages"].append([i1, i2, i3, False])
# Deprecating DISQUS_FORUM
# TODO: remove on v7
if "DISQUS_FORUM" in config:
utils.LOGGER.warn("The DISQUS_FORUM option is deprecated, use COMMENT_SYSTEM_ID instead.")
if "COMMENT_SYSTEM_ID" in config:
utils.LOGGER.warn("DISQUS_FORUM conflicts with COMMENT_SYSTEM_ID, ignoring DISQUS_FORUM.")
else:
self.config["COMMENT_SYSTEM_ID"] = config["DISQUS_FORUM"]
# Deprecating the ANALYTICS option
# TODO: remove on v7
if "ANALYTICS" in config:
utils.LOGGER.warn("The ANALYTICS option is deprecated, use BODY_END instead.")
if "BODY_END" in config:
utils.LOGGER.warn("ANALYTICS conflicts with BODY_END, ignoring ANALYTICS.")
else:
self.config["BODY_END"] = config["ANALYTICS"]
# Deprecating the SIDEBAR_LINKS option
# TODO: remove on v7
if "SIDEBAR_LINKS" in config:
utils.LOGGER.warn("The SIDEBAR_LINKS option is deprecated, use NAVIGATION_LINKS instead.")
if "NAVIGATION_LINKS" in config:
utils.LOGGER.warn("The SIDEBAR_LINKS conflicts with NAVIGATION_LINKS, ignoring SIDEBAR_LINKS.")
else:
self.config["NAVIGATION_LINKS"] = config["SIDEBAR_LINKS"]
# Compatibility alias
self.config["SIDEBAR_LINKS"] = self.config["NAVIGATION_LINKS"]
if self.config["NAVIGATION_LINKS"] in (None, {}):
self.config["NAVIGATION_LINKS"] = {self.config["DEFAULT_LANG"]: ()}
# Deprecating the ADD_THIS_BUTTONS option
# TODO: remove on v7
if "ADD_THIS_BUTTONS" in config:
utils.LOGGER.warn("The ADD_THIS_BUTTONS option is deprecated, use SOCIAL_BUTTONS_CODE instead.")
if not config["ADD_THIS_BUTTONS"]:
utils.LOGGER.warn("Setting SOCIAL_BUTTONS_CODE to empty because ADD_THIS_BUTTONS is False.")
self.config["SOCIAL_BUTTONS_CODE"] = ""
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
# TODO: remove on v7
if "STRIP_INDEX_HTML" in config and "STRIP_INDEXES" not in config:
utils.LOGGER.warn("You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML")
self.config["STRIP_INDEXES"] = config["STRIP_INDEX_HTML"]
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get("PRETTY_URLS", False) and "STRIP_INDEXES" not in config:
self.config["STRIP_INDEXES"] = True
if config.get("COPY_SOURCES") and not self.config["HIDE_SOURCELINK"]:
self.config["HIDE_SOURCELINK"] = True
self.config["TRANSLATIONS"] = self.config.get("TRANSLATIONS", {self.config["DEFAULT_LANG"]: ""})
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
# TODO: remove on v7
if "SITE_URL" not in self.config:
if "BLOG_URL" in self.config:
utils.LOGGER.warn("You should configure SITE_URL instead of BLOG_URL")
self.config["SITE_URL"] = self.config["BLOG_URL"]
self.default_lang = self.config["DEFAULT_LANG"]
self.translations = self.config["TRANSLATIONS"]
locale_fallback, locale_default, locales = sanitized_locales(
self.config.get("LOCALE_FALLBACK", None),
self.config.get("LOCALE_DEFAULT", None),
self.config.get("LOCALES", {}),
self.translations,
) # NOQA
utils.LocaleBorg.initialize(locales, self.default_lang)
# BASE_URL defaults to SITE_URL
if "BASE_URL" not in self.config:
self.config["BASE_URL"] = self.config.get("SITE_URL")
# BASE_URL should *always* end in /
if self.config["BASE_URL"] and self.config["BASE_URL"][-1] != "/":
utils.LOGGER.warn("Your BASE_URL doesn't end in / -- adding it.")
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
"SignalHandler": SignalHandler,
}
)
self.plugin_manager.setPluginInfoExtension("plugin")
if sys.version_info[0] == 3:
places = [os.path.join(os.path.dirname(__file__), "plugins"), os.path.join(os.getcwd(), "plugins")]
else:
places = [
os.path.join(os.path.dirname(__file__), utils.sys_encode("plugins")),
os.path.join(os.getcwd(), utils.sys_encode("plugins")),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
# Activate all required SignalHandler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("SignalHandler"):
if plugin_info.name in self.config.get("DISABLED_PLUGINS"):
self.plugin_manager.removePluginFromCategory(plugin_info, "SignalHandler")
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Emit signal for SignalHandlers which need to start running immediately.
signal("sighandlers_loaded").send(self)
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if plugin_info.name in self.config["DISABLED_PLUGINS"] or (
plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"]
):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if plugin_info.name in self.config["DISABLED_PLUGINS"] or (
plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"]
):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
if plugin_info.name in self.config["DISABLED_PLUGINS"] or (
plugin_info.name in self.EXTRA_PLUGINS and plugin_info.name not in self.config["ENABLED_EXTRAS"]
):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
if plugin_info.name in self.config["COMPILERS"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self._GLOBAL_CONTEXT = {}
self._GLOBAL_CONTEXT["_link"] = self.link
self._GLOBAL_CONTEXT["set_locale"] = utils.LocaleBorg().set_locale
self._GLOBAL_CONTEXT["rel_link"] = self.rel_link
self._GLOBAL_CONTEXT["abs_link"] = self.abs_link
self._GLOBAL_CONTEXT["exists"] = self.file_exists
self._GLOBAL_CONTEXT["SLUG_TAG_PATH"] = self.config["SLUG_TAG_PATH"]
self._GLOBAL_CONTEXT["annotations"] = self.config["ANNOTATIONS"]
self._GLOBAL_CONTEXT["index_display_post_count"] = self.config["INDEX_DISPLAY_POST_COUNT"]
self._GLOBAL_CONTEXT["use_bundles"] = self.config["USE_BUNDLES"]
self._GLOBAL_CONTEXT["use_cdn"] = self.config.get("USE_CDN")
self._GLOBAL_CONTEXT["favicons"] = self.config["FAVICONS"]
self._GLOBAL_CONTEXT["date_format"] = self.config.get("DATE_FORMAT", "%Y-%m-%d %H:%M")
self._GLOBAL_CONTEXT["blog_author"] = self.config.get("BLOG_AUTHOR")
self._GLOBAL_CONTEXT["blog_title"] = self.config.get("BLOG_TITLE")
# TODO: remove fallback in v7
self._GLOBAL_CONTEXT["blog_url"] = self.config.get("SITE_URL", self.config.get("BLOG_URL"))
self._GLOBAL_CONTEXT["blog_desc"] = self.config.get("BLOG_DESCRIPTION")
self._GLOBAL_CONTEXT["body_end"] = self.config.get("BODY_END")
# TODO: remove in v7
self._GLOBAL_CONTEXT["analytics"] = self.config.get("BODY_END")
# TODO: remove in v7
self._GLOBAL_CONTEXT["add_this_buttons"] = self.config.get("SOCIAL_BUTTONS_CODE")
self._GLOBAL_CONTEXT["social_buttons_code"] = self.config.get("SOCIAL_BUTTONS_CODE")
self._GLOBAL_CONTEXT["translations"] = self.config.get("TRANSLATIONS")
self._GLOBAL_CONTEXT["license"] = self.config.get("LICENSE")
self._GLOBAL_CONTEXT["search_form"] = self.config.get("SEARCH_FORM")
self._GLOBAL_CONTEXT["comment_system"] = self.config.get("COMMENT_SYSTEM")
self._GLOBAL_CONTEXT["comment_system_id"] = self.config.get("COMMENT_SYSTEM_ID")
# TODO: remove in v7
self._GLOBAL_CONTEXT["disqus_forum"] = self.config.get("COMMENT_SYSTEM_ID")
self._GLOBAL_CONTEXT["mathjax_config"] = self.config.get("MATHJAX_CONFIG")
self._GLOBAL_CONTEXT["subtheme"] = self.config.get("THEME_REVEAL_CONFIG_SUBTHEME")
self._GLOBAL_CONTEXT["transition"] = self.config.get("THEME_REVEAL_CONFIG_TRANSITION")
self._GLOBAL_CONTEXT["content_footer"] = self.config.get("CONTENT_FOOTER")
self._GLOBAL_CONTEXT["rss_path"] = self.config.get("RSS_PATH")
self._GLOBAL_CONTEXT["rss_link"] = self.config.get("RSS_LINK")
self._GLOBAL_CONTEXT["navigation_links"] = utils.Functionary(list, self.config["DEFAULT_LANG"])
for k, v in self.config.get("NAVIGATION_LINKS", {}).items():
self._GLOBAL_CONTEXT["navigation_links"][k] = v
# TODO: remove on v7
# Compatibility alias
self._GLOBAL_CONTEXT["sidebar_links"] = self._GLOBAL_CONTEXT["navigation_links"]
self._GLOBAL_CONTEXT["twitter_card"] = self.config.get("TWITTER_CARD", {})
self._GLOBAL_CONTEXT["hide_sourcelink"] = self.config.get("HIDE_SOURCELINK")
self._GLOBAL_CONTEXT["extra_head_data"] = self.config.get("EXTRA_HEAD_DATA")
self._GLOBAL_CONTEXT.update(self.config.get("GLOBAL_CONTEXT", {}))
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
self.compilers[plugin_info.name] = plugin_info.plugin_object
signal("configured").send(self)
def _get_themes(self):
if self._THEMES is None:
# Check for old theme names (Issue #650) TODO: remove in v7
theme_replacements = {"site": "bootstrap", "orphan": "base", "default": "oldfashioned"}
if self.config["THEME"] in theme_replacements:
utils.LOGGER.warn(
'You are using the old theme "{0}", using "{1}" instead.'.format(
self.config["THEME"], theme_replacements[self.config["THEME"]]
)
)
self.config["THEME"] = theme_replacements[self.config["THEME"]]
if self.config["THEME"] == "oldfashioned":
utils.LOGGER.warn(
"""You may need to install the "oldfashioned" theme """
"""from themes.nikola.ralsina.com.ar because it's not """
"""shipped by default anymore."""
)
utils.LOGGER.warn("Please change your THEME setting.")
try:
self._THEMES = utils.get_theme_chain(self.config["THEME"])
except Exception:
utils.LOGGER.warn("""Can't load theme "{0}", using 'bootstrap' instead.""".format(self.config["THEME"]))
self.config["THEME"] = "bootstrap"
return self._get_themes()
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config["USE_CDN"]:
bootstrap_path = utils.get_asset_path(os.path.join("assets", "css", "bootstrap.min.css"), self._THEMES)
if bootstrap_path and bootstrap_path.split(os.sep)[-4] not in ["bootstrap", "bootstrap3"]:
utils.LOGGER.warn(
"The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap."
)
return self._THEMES
THEMES = property(_get_themes)
def _get_messages(self):
return utils.load_messages(self.THEMES, self.translations, self.default_lang)
MESSAGES = property(_get_messages)
def _get_global_context(self):
"""Initialize some parts of GLOBAL_CONTEXT only when it's queried."""
if "messages" not in self._GLOBAL_CONTEXT:
self._GLOBAL_CONTEXT["messages"] = self.MESSAGES
if "has_custom_css" not in self._GLOBAL_CONTEXT:
# check if custom css exist and is not empty
custom_css_path = utils.get_asset_path("assets/css/custom.css", self.THEMES, self.config["FILES_FOLDERS"])
if custom_css_path and self.file_exists(custom_css_path, not_empty=True):
self._GLOBAL_CONTEXT["has_custom_css"] = True
else:
self._GLOBAL_CONTEXT["has_custom_css"] = False
return self._GLOBAL_CONTEXT
GLOBAL_CONTEXT = property(_get_global_context)
def _get_template_system(self):
if self._template_system is None:
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system " "plugin\n".format(template_sys_name))
sys.exit(1)
self._template_system = pi.plugin_object
lookup_dirs = ["templates"] + [
os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES
]
self._template_system.set_directories(lookup_dirs, self.config["CACHE_FOLDER"])
return self._template_system
template_system = property(_get_template_system)
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.COMPILERS`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in list(self.config["COMPILERS"].items()) if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit(
"Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'COMPILERS' in conf.py\n(The error is in"
"one of {0})".format(", ".join(langs))
)
elif len(langs) > 1:
langs = langs[:1]
else:
exit("COMPILERS in conf.py does not tell me how to " "handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
# string, arguments
local_context["formatmsg"] = lambda s, *a: s % a
data = self.template_system.render_template(template_name, None, local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1 :]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split("/")[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == "link": # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip("/"), context["lang"])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
if self.config.get("URL_TYPE") == "absolute":
dst = urljoin(self.config["BASE_URL"], dst)
return dst
elif self.config.get("URL_TYPE") == "full_path":
return dst
else:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
if self.config.get("URL_TYPE") == "absolute":
dst = urljoin(self.config["BASE_URL"], dst)
return dst
if self.config.get("URL_TYPE") in ("full_path", "absolute"):
if self.config.get("URL_TYPE") == "absolute":
dst = urljoin(self.config["BASE_URL"], dst)
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
utils.makedirs(os.path.dirname(output_name))
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b"<!DOCTYPE html>" + lxml.html.tostring(doc, encoding="utf8")
with open(output_name, "wb+") as post_file:
post_file.write(data)
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
These are mostly defined by plugins by registering via
the register_path_handler method, except for slug and
post_path which are defined in this class' init method.
Here's some of the others, for historical reasons:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* category (and name is the category name)
* category_rss (and name is the category name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a POSTS/PAGES tuple)
* slug (name is the slug of a post or story)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = utils.LocaleBorg().current_lang
path = self.path_handlers[kind](name, lang)
if is_link:
link = "/" + ("/".join(path))
index_len = len(self.config["INDEX_FILE"])
if self.config["STRIP_INDEXES"] and link[-(1 + index_len) :] == "/" + self.config["INDEX_FILE"]:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def post_path(self, name, lang):
"""post_path path handler"""
return [
_f for _f in [self.config["TRANSLATIONS"][lang], os.path.dirname(name), self.config["INDEX_FILE"]] if _f
]
def slug_path(self, name, lang):
"""slug path handler"""
results = [p for p in self.timeline if p.meta("slug") == name]
if not results:
utils.LOGGER.warning("Can't resolve path request for slug: {0}".format(name))
else:
if len(results) > 1:
utils.LOGGER.warning("Ambiguous path request for slug: {0}".format(name))
return [_f for _f in results[0].permalink(lang).split("/") if _f]
def register_path_handler(self, kind, f):
if kind in self.path_handlers:
utils.LOGGER.warning("Conflicting path handlers for kind: {0}".format(kind))
else:
self.path_handlers[kind] = f
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config["BASE_URL"], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config["BASE_URL"], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split("/")[1:]
dst_elems = parsed_dst.path.split("/")[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return "/".join([".."] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def clean_task_paths(self, task):
"""Normalize target paths in the task."""
targets = task.get("targets", None)
if targets is not None:
task["targets"] = [os.path.normpath(t) for t in targets]
return task
def gen_tasks(self, name, plugin_category, doc=""):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
assert "basename" in task
task = self.clean_task_paths(task)
yield task
for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield self.clean_task_paths(task)
if flag:
task_dep.append("{0}_{1}".format(name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {"basename": name, "doc": doc, "actions": None, "clean": True, "task_dep": task_dep}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
seen = set([])
print("Scanning posts", end="", file=sys.stderr)
lower_case_tags = set([])
for wildcard, destination, template_name, use_in_feeds in self.config["post_pages"]:
print(".", end="", file=sys.stderr)
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination, os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config["TRANSLATIONS"].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
# We eliminate from the list the files inside any .ipynb folder
full_list = [p for p in full_list if not any([x.startswith(".") for x in p.split(os.sep)])]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.config,
dest_dir,
use_in_feeds,
self.MESSAGES,
template_name,
self.get_compiler(base_path),
)
self.global_data[post.source_path] = post
if post.use_in_feeds:
self.posts.append(post.source_path)
self.posts_per_year[str(post.date.year)].append(post.source_path)
self.posts_per_month["{0}/{1:02d}".format(post.date.year, post.date.month)].append(
post.source_path
)
for tag in post.alltags:
if tag.lower() in lower_case_tags:
if tag not in self.posts_per_tag:
# Tags that differ only in case
other_tag = [k for k in self.posts_per_tag.keys() if k.lower() == tag.lower()][0]
utils.LOGGER.error(
"You have cases that differ only in upper/lower case: {0} and {1}".format(
tag, other_tag
)
)
utils.LOGGER.error("Tag {0} is used in: {1}".format(tag, post.source_path))
utils.LOGGER.error(
"Tag {0} is used in: {1}".format(
other_tag, ", ".join(self.posts_per_tag[other_tag])
)
)
sys.exit(1)
else:
lower_case_tags.add(tag.lower())
self.posts_per_tag[tag].append(post.source_path)
self.posts_per_category[post.meta("category")].append(post.source_path)
else:
self.pages.append(post)
if self.config["OLD_THEME_SUPPORT"]:
post._add_old_metadata()
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!", file=sys.stderr)
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + self.template_system.template_deps(post.template_name)
deps.extend(utils.get_asset_path(x, self.THEMES) for x in ("bundles", "parent", "engine"))
deps = list(filter(None, deps))
context["post"] = post
context["lang"] = lang
context["title"] = post.title(lang)
context["description"] = post.description(lang)
context["permalink"] = post.permalink(lang)
context["page_list"] = self.pages
if post.use_in_feeds:
context["enable_comments"] = True
else:
context["enable_comments"] = self.config["COMMENTS_IN_STORIES"]
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config["OUTPUT_FOLDER"], post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop("post")
if post.prev_post:
deps_dict["PREV_LINK"] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict["NEXT_LINK"] = [post.next_post.permalink(lang)]
deps_dict["OUTPUT_FOLDER"] = self.config["OUTPUT_FOLDER"]
deps_dict["TRANSLATIONS"] = self.config["TRANSLATIONS"]
deps_dict["global"] = self.GLOBAL_CONTEXT
deps_dict["comments"] = context["enable_comments"]
if post:
deps_dict["post_translations"] = post.translated_to
task = {
"name": os.path.normpath(output_name),
"file_dep": deps,
"targets": [output_name],
"actions": [(self.render_template, [post.template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config["BLOG_TITLE"]
context["description"] = self.config["BLOG_DESCRIPTION"]
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]["title"], p.permalink(lang)) for p in posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
"name": os.path.normpath(output_name),
"targets": [output_name],
"file_dep": deps,
"actions": [(self.render_template, [template_name, output_name, context])],
"clean": True,
"uptodate": [config_changed(deps_context)],
}
return utils.apply_filters(task, filters)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = [
'planetoid',
'ipynb',
'local_search',
'render_mustache',
]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ANALYTICS': '',
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'CONTENT_FOOTER': '',
'CREATE_MONTHLY_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'DISQUS_FORUM': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FILE_METADATA_REGEXP': None,
'FILES_FOLDERS': {'files': ''},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GZIP_FILES': False,
'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json'),
'HIDE_UNTRANSLATED_POSTS': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEX_PATH': '',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'post_compilers': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile',),
"txt2tags": ('.t2t',),
"bbcode": ('.bb',),
"wiki": ('.wiki',),
"ipynb": ('.ipynb',),
"html": ('.html', '.htm')
},
'POST_PAGES': (
("posts/*.txt", "posts", "post.tmpl", True),
("stories/*.txt", "stories", "story.tmpl", False),
),
'PRETTY_URLS': False,
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'site',
'THEME_REVEAL_CONGIF_SUBTHEME': 'sky',
'THEME_REVEAL_CONGIF_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': None,
}
self.config.update(config)
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
print("WARNING: You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML")
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
{self.config['DEFAULT_'
'LANG']: ''})
self.THEMES = utils.get_theme_chain(self.config['THEME'])
self.MESSAGES = utils.load_messages(self.THEMES,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'])
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
print("WARNING: You should configure SITE_URL instead of BLOG_URL")
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(__file__), utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS']
or (plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
if plugin_info.name in self.config["post_compilers"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = {
}
self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
self.GLOBAL_CONTEXT['_link'] = self.link
self.GLOBAL_CONTEXT['set_locale'] = s_l
self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
self.GLOBAL_CONTEXT['exists'] = self.file_exists
self.GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config[
'SLUG_TAG_PATH']
self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
'ADD_THIS_BUTTONS']
self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self.GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self.GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self.GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self.GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self.GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
self.GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL', self.config.get('BLOG_URL'))
self.GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self.GLOBAL_CONTEXT['analytics'] = self.config.get('ANALYTICS')
self.GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self.GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self.GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self.GLOBAL_CONTEXT['disqus_forum'] = self.config.get('DISQUS_FORUM')
self.GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self.GLOBAL_CONTEXT['subtheme'] = self.config.get('THEME_REVEAL_CONGIF_SUBTHEME')
self.GLOBAL_CONTEXT['transition'] = self.config.get('THEME_REVEAL_CONGIF_TRANSITION')
self.GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self.GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self.GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self.GLOBAL_CONTEXT['sidebar_links'] = utils.Functionary(list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('SIDEBAR_LINKS', {}).items():
self.GLOBAL_CONTEXT['sidebar_links'][k] = v
self.GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self.GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA')
self.GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# check if custom css exist and is not empty
for files_path in list(self.config['FILES_FOLDERS'].keys()):
custom_css_path = os.path.join(files_path, 'assets/css/custom.css')
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT['has_custom_css'] = True
break
else:
self.GLOBAL_CONTEXT['has_custom_css'] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(
template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES]
self.template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config['USE_CDN']:
bootstrap_path = utils.get_asset_path(os.path.join(
'assets', 'css', 'bootstrap.min.css'), self.THEMES)
if bootstrap_path.split(os.sep)[-4] != 'site':
warnings.warn('The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.')
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in
list(self.config['post_compilers'].items())
if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(
template_name, None, local_context)
assert output_name.startswith(
self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def current_lang(self): # FIXME: this is duplicated, turn into a mixin
"""Return the currently set locale, if it's one of the
available translations, or default_lang."""
lang = utils.LocaleBorg().current_lang
if lang:
if lang in self.translations:
return lang
lang = lang.split('_')[0]
if lang in self.translations:
return lang
# whatever
return self.default_lang
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a post_pages tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = self.current_lang()
path = []
if kind == "tag_index":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'],
self.config['INDEX_FILE']] if _f]
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".html"] if
_f]
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".xml"] if
_f]
elif kind == "index":
if name not in [None, 0]:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
'index-{0}.html'.format(name)] if _f]
else:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
self.config['INDEX_FILE']]
if _f]
elif kind == "post_path":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
os.path.dirname(name),
self.config['INDEX_FILE']] if _f]
elif kind == "rss":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['RSS_PATH'], 'rss.xml'] if _f]
elif kind == "archive":
if name:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], name,
self.config['INDEX_FILE']] if _f]
else:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'],
self.config['ARCHIVE_FILENAME']] if _f]
elif kind == "gallery":
path = [_f for _f in [self.config['GALLERY_PATH'], name,
self.config['INDEX_FILE']] if _f]
elif kind == "listing":
path = [_f for _f in [self.config['LISTINGS_FOLDER'], name +
'.html'] if _f]
if is_link:
link = '/' + ('/'.join(path))
index_len = len(self.config['INDEX_FILE'])
if self.config['STRIP_INDEXES'] and \
link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BASE_URL'], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config['BASE_URL'], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self, name, plugin_category):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
yield task
for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield task
if flag:
task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': name,
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
print("Scanning posts", end='')
tzinfo = None
if self.config['TIMEZONE'] is not None:
tzinfo = pytz.timezone(self.config['TIMEZONE'])
current_time = utils.current_time(tzinfo)
targets = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='')
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(os.path.join(destination,
os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config['TRANSLATIONS'].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
for base_path in full_list:
post = Post(
base_path,
self.config['CACHE_FOLDER'],
dest_dir,
use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BASE_URL'],
self.MESSAGES,
template_name,
self.config['FILE_METADATA_REGEXP'],
self.config['STRIP_INDEXES'],
self.config['INDEX_FILE'],
tzinfo,
current_time,
self.config['HIDE_UNTRANSLATED_POSTS'],
self.config['PRETTY_URLS'],
)
for lang, langpath in list(
self.config['TRANSLATIONS'].items()):
dest = (destination, langpath, dir_glob,
post.meta[lang]['slug'])
if dest in targets:
raise Exception('Duplicated output path {0!r} '
'in post {1!r}'.format(
post.meta[lang]['slug'],
base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[
str(post.date.year)].append(post.post_name)
self.posts_per_month[
'{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post.post_name)
for tag in post.alltags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
if self.config['OLD_THEME_SUPPORT']:
post._add_old_metadata()
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
if post:
deps_dict['post_translations'] = post.translated_to
task = {
'name': os.path.normpath(output_name),
'file_dep': deps,
'targets': [output_name],
'actions': [(self.render_template, [post.template_name,
output_name, context])],
'clean': True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in
posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name': os.path.normpath(output_name),
'targets': [output_name],
'file_dep': deps,
'actions': [(self.render_template, [template_name, output_name,
context])],
'clean': True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {
'listings': 'listings'
},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension
})
self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
nikola.utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category(
"CompilerExtension")
self.timeline = [FakePost(title='Fake post', slug='fake-post')]
self.debug = True
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(
plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return ('<img src="IMG.jpg">')
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data, *a, **kw):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data,
self.shortcode_registry,
**kw)
class Parser(object):
def __init__(
self,
arquivo,
provider_conf = {},
xsd_path=None,
plugins_path=None,
providers_path=None,
skeep=False
):
self.provider_conf = provider_conf
self.arquivo = arquivo
self.version = None
self.skeep_validation = skeep
self.plugins_path = plugins_path
self.providers_path = providers_path
self.xsd_path = xsd_path
self.xsd_valido = False
self.xsd_erros = []
self.valido = False
self.erros = {
'lote': {},
'guias': {},
}
self.providers = {}
self.arquivo_xsd = None
# analisa o arquivo
if not self.skeep_validation:
self.parse()
if self.arquivo_xsd:
# # descobre versao
self.get_version()
if self.version:
# valida o xsd
self.xsd_validate()
if self.xsd_valido:
# validação estrutural
self.valida_estrutura()
if self.valido:
# executa plugins de providers
self.executa_providers()
# validação do negócios em modelo de plugins
self.executa_plugins()
# se existir erros de guia, marca lote invalido
guias_invalidas = len(self.erros['guias'].items())
inconsistencias = 0
if guias_invalidas:
for guia in self.erros['guias'].items():
inconsistencias += len(guia[1])
self.valido = False
self.erros['lote']['_guias'] = u"%s Guias apresentaram %s inconsistências" % (
guias_invalidas,
inconsistencias
)
else:
self.erros['lote']['_estrutura'] = "Estrutura não é válida"
xml_errors = None
def parse(self):
#
# tenta identar o XML para apontar erro em linha
try:
conteudo = open(self.arquivo, encoding='iso-8859-1').read()
except TypeError:
# hack para python 2.7
conteudo = io.open(self.arquivo, encoding='iso-8859-1').read()
# remove o encoding do xml para deixar o lxml trabalhar
conteudo_sem = conteudo.replace('encoding="iso-8859-1"', '')
conteudo_sem = conteudo.replace('encoding="ISO-8859-1"', '')
root = etree.fromstring(conteudo_sem)
# conteudo bonito, identado
cb = etree.tostring(root, pretty_print=True).decode()
self.root = etree.fromstring(cb)
# checa se namespace valido.
# namespace errado da erro de interpretação.
try:
# se nao possuir o esquema da ans no namespace
# forçar definir hardcoded.
self.nsmap = self.root.nsmap
self.schema_ans_url = self.nsmap['ans']
except KeyError:
self.nsmap = {
'ans': 'http://www.ans.gov.br/padroes/tiss/schemas',
'ds': 'http://www.w3.org/2000/09/xmldsig#',
'xsi': 'http://www.w3.org/2001/XMLSchema-instance'
}
# devoler pro root o nsmap forçado
#self.root.nsmap = self.nsmap
# define o tipo de trasacao
self.tipo_transacao = self.get_xpath(
'//ans:tipoTransacao'
)[0].text.replace("\n", '').replace("\t", '')
# no momento so suporta envio de lote
if self.tipo_transacao == 'ENVIO_LOTE_GUIAS':
# neste caso, o arquivo xsd é o tiss, dentro da pasta xsd
self.arquivo_xsd = 'tiss'
# tambem existe um numero do lote
self.numero_lote = self.get_xpath("//ans:numeroLote")[0].text
# numero sequencial
self.numero_sequencial = self.get_xpath("//ans:sequencialTransacao")[0].text
#
# aqui precisa fazer testes em outros tipos de transação
# deve ser possivel/necessario filtrar isso nos plugins e providers
#
# arquivo xsd não encontrado.
if not self.arquivo_xsd:
self.valido = False
self.erros['lote']['_transacao'] = 'Tipo de Transação não identificado ou não suportado: %s' % self.tipo_transacao
# extrai as guias para utilizacao nos plugins
self.guias = self.root.xpath('//ans:guiasTISS', namespaces=self.nsmap)[0].getchildren()
# extrai o prestador do lote
#self.codigo_prestador = self.get_xpath(
# "//ans:cabecalho//ans:codigoPrestadorNaOperadora"
#)[0].text
def get_version(self):
'''retorna versao no formato N.NN.NN, conforme xml
'''
try:
# versao 3.0.1
versao301 = self.get_xpath(
'//ans:Padrao'
)
if versao301:
self.version = versao301[0].text
else:
self.version = self.get_xpath(
'//ans:versaoPadrao'
)[0].text.replace("\n", '').replace("\t", '')
except:
self.valid = False
self.erros['lote']['_versao'] = u"Erro ao detectar a versão do padrão TISS"
def xsd_validate(self):
if not self.xsd_path:
xsd_file = 'xsd/%sV%s.xsd' % (self.arquivo_xsd, self.version.replace('.', '_'))
self.xsd_path = os.path.join(os.path.dirname(__file__), xsd_file)
#f = open(self.xsd_path)
self.root_xsd = etree.parse(self.xsd_path)
try:
self.xsd_schema = etree.XMLSchema(self.root_xsd)
self.xsd_valido = True
except etree.XMLSchemaParseError as xsd_erros:
self.xsd_valido = False
self.xsd_erros = xsd_erros
i = 1
for erro in self.xsd_erros.error_log:
self.erros['lote']['_xsd_invalido%i' % i] = "Arquivo: %s, Linha %s, Coluna: %s: %s" % (
erro.filename, erro.line, erro.column, erro.message
)
i += 1
def valida_estrutura(self):
try:
self.xsd_schema.assertValid(self.root)
self.valido = True
self.schema_valido = True
self.calcula_hash()
self.valida_hash()
except etree.DocumentInvalid as xml_errors:
self.valid = False
self.schema_valido = False
self.schema_erros = xml_errors
i = 1
for erro in self.schema_erros.error_log:
self.erros['lote']['_schema_invalido%i' % i] = "Linha %s, Coluna: %s: %s" % (erro.line, erro.column, erro.message)
i += 1
def calcula_hash(self):
# remove epilogo do calculo
self.root_no_epilogo = self.root
self.epilogo = self.root_no_epilogo.xpath(
'//ans:epilogo', namespaces=self.nsmap)[0]
self.root_no_epilogo.remove(self.epilogo)
self.hash_fornecido = self.epilogo.getchildren()[0].text
reslist = list(self.root_no_epilogo.iter())
conteudos = []
for i in reslist:
if not i.getchildren():
conteudos.append(i.text)
self.conteudo = ''.join(conteudos)
cod = hashlib.md5()
cod.update(self.conteudo.encode('iso-8859-1'))
self.hash = cod.hexdigest()
self.parse()
def valida_hash(self):
if self.hash != self.hash_fornecido:
self.valido = False
self.erros['lote']['_hash'] = "Hash Inválido! Fornecido: %s, Calculado: %s" % (
self.hash_fornecido, self.hash)
def executa_plugins(self, plugin_path=None):
print("Executando plugins")
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginInfoExtension("tiss-plugin")
if not self.plugins_path:
self.plugins_path = os.path.join(os.path.dirname(__file__), 'extensoes/plugins')
self.plugin_manager.setPluginPlaces([self.plugins_path])
self.plugin_manager.collectPlugins()
for plugin in self.plugin_manager.getAllPlugins():
plugin.plugin_object.executa(objeto=self)
def executa_providers(self, providers_path=None):
print("Executando Providers")
self.provider_manager = PluginManager()
self.provider_manager.setPluginInfoExtension("tiss-provider")
if not self.providers_path:
self.providers_path = os.path.join(os.path.dirname(__file__), 'extensoes/providers')
self.provider_manager.setPluginPlaces([self.providers_path])
self.provider_manager.collectPlugins()
for plugin in self.provider_manager.getAllPlugins():
plugin.plugin_object.executa(objeto=self)
def registra_erro_guia(self, erro):
try:
self.erros['guias'][erro['numero']]
except KeyError:
self.erros['guias'][erro['numero']] = []
self.erros['guias'][erro['numero']].append(erro)
def registra_provider(self, provider_name, provider):
'''
registra os dados de uma provider
'''
try:
self.providers[provider_name]
except:
self.providers[provider_name] = {}
self.providers[provider_name] = [provider]
def get_xpath(self, xpath):
return self.root.xpath(xpath, namespaces=self.nsmap)
def init_plugin_manager():
global simplePluginManager
simplePluginManager = PluginManager(categories_filter={"bots": BotPlugin})
simplePluginManager.setPluginInfoExtension('plug')
def main():
#Find and load plugins
pm = PluginManager()
libpath = '%s/OpenMesher/plugins' %(get_python_lib())
pm.setPluginPlaces(["/usr/share/openmesher/plugins", "~/.openmesher/plugins", "./OpenMesher/plugins", "./plugins", libpath])
pm.setPluginInfoExtension('plugin')
pm.setCategoriesFilter({
'config': IOpenMesherConfigPlugin,
'package': IOpenMesherPackagePlugin,
'deploy': IOpenMesherDeployPlugin,
})
pm.collectPlugins()
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="Configure, package, and deploy an OpenVPN mesh")
parser.add_argument('-r', '--router', action='append', help='Adds a router that can be a client and server')
parser.add_argument('-s', '--server', action='append', help='Adds a router that can only act as a server, not a client.')
parser.add_argument('-c', '--client', action='append', help='Adds a router than can only act as a client. For example, a router that is behind NAT and not accessible by a public IP')
#BUG: Stupid argparse appends your switches to the default.
#parser.add_argument('-n', '--network', action='append', default=['10.99.99.0/24'])
parser.add_argument('-n', '--network', action='append', required=True)
portgroup = parser.add_mutually_exclusive_group()
portgroup.add_argument('-p', '--ports', action='append', default=['7000-7999'])
portgroup.add_argument('-a', '--random', action='store_true')
parser.add_argument('-v', '--verbose', action='append_const', const='verbose', help='Specify multiple times to make things more verbose')
parser.add_argument('--version', action='version', version='v0.6.4')
pluginargsgroup = parser.add_argument_group('plugins')
for plugin in pm.getAllPlugins():
plugin.plugin_object.setupargs(pluginargsgroup)
arg = parser.parse_args()
for plugin in pm.getAllPlugins():
if plugin.plugin_object.__class__.__name__.lower() in arg:
if eval('arg.%s' %(plugin.plugin_object.__class__.__name__.lower())):
logging.debug('Plugin enabled: %s' %(plugin.name))
pm.activatePluginByName(plugin.name)
else:
logging.debug('Plugin disabled: %s' %(plugin.name))
pm.deactivatePluginByName(plugin.name)
else:
logging.debug('Plugin disabled: %s' %(plugin.name))
pm.deactivatePluginByName(plugin.name)
l = logging.getLogger()
if arg.verbose:
if len(arg.verbose) == 1:
l.setLevel(logging.INFO)
if len(arg.verbose) >= 2:
l.setLevel(logging.DEBUG)
# Call activate() on all plugins so they prep themselves for use
for plugin in pm.getAllPlugins():
if eval('arg.%s' %(plugin.plugin_object.__class__.__name__.lower())):
logging.info('Enabled plugin: %s' %(plugin.name))
pm.activatePluginByName(plugin.name)
plugin.plugin_object.activate()
plugin.plugin_object._enabled = True
if len(arg.ports) > 1:
arg.ports.reverse()
arg.ports.pop()
port_list = []
if arg.random:
numdevs = 0
if arg.router:
numdevs += len(arg.router)
if arg.server:
numdevs += len(arg.server)
if arg.client:
numdevs += len(arg.client)
ports_needed = numdevs * (numdevs - 1) / 2
for i in range(0, ports_needed):
port_list.append(random.randrange(1025,32767))
try:
if not arg.random:
# If we're not using random ports, pull whatever is in arg.ports
for portrange in arg.ports:
portstart, portstop = portrange.split('-')
port_list += range(int(portstart),int(portstop))
except ValueError as e:
print 'Invalid port range: %s' %(portrange)
raise ValueError('Invalid port range: %s' %(portrange))
linkmesh = create_link_mesh(routers=arg.router, servers=arg.server, clients=arg.client)
m = Mesh(linkmesh, port_list, arg.network)
files = None
# Run through config plugins
configPlugins = []
for plugin in pm.getPluginsOfCategory('config'):
if plugin.plugin_object._enabled:
plugin.plugin_object.process(m, arg)
configPlugins.append(plugin.plugin_object)
if files:
files = nested_dict_merge(files, plugin.plugin_object.files())
else:
files = plugin.plugin_object.files()
#Grab list of folders that need to be in the package root
includedirs = []
for f in files:
for fldr in files[f]:
rootfldr = fldr.split('/')[1]
if rootfldr not in includedirs:
includedirs.append(rootfldr)
logging.debug('The following folders will be packaged: %s' %(includedirs))
# Run through packaging plugins
packagePlugins = []
for plugin in pm.getPluginsOfCategory('package'):
if plugin.plugin_object._enabled:
#BUG: Services to restart may not necessarily be the same name as their config dir...
plugin.plugin_object.process(m, include_dirs=includedirs, restart_services=includedirs, configPlugins=configPlugins, cliargs=arg)
packagePlugins.append(plugin.plugin_object)
# Run through deployment plugins
for plugin in pm.getPluginsOfCategory('deploy'):
if plugin.plugin_object._enabled:
try:
plugin.plugin_object.deploy(packagePlugins=packagePlugins, cliargs=arg, stoponfailure=False)
except Exception as e:
print "Unable to deploy due to error: %s" %(e)
logging.info('OpenMesher run complete')
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
# This is the default config
# TODO: fill it
self.config = {
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'DEFAULT_LANG': "en",
'OUTPUT_FOLDER': 'output',
'FILES_FOLDERS': {
'files': ''
},
'LISTINGS_FOLDER': 'listings',
'ADD_THIS_BUTTONS': True,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_TEASERS': False,
'MAX_IMAGE_SIZE': 1280,
'USE_FILENAME_AS_TITLE': True,
'SLUG_TAG_PATH': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'FILTERS': {},
'USE_BUNDLES': True,
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'default',
'post_compilers': {
"rest": ['.txt', '.rst'],
"markdown": ['.md', '.mdown', '.markdown'],
"html": ['.html', '.htm'],
},
}
self.config.update(config)
self.config['TRANSLATIONS'] = self.config.get(
'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''})
self.GLOBAL_CONTEXT = self.config.get('GLOBAL_CONTEXT', {})
self.THEMES = utils.get_theme_chain(self.config['THEME'])
self.MESSAGES = utils.load_messages(self.THEMES,
self.config['TRANSLATIONS'])
self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
self.GLOBAL_CONTEXT['_link'] = self.link
self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
self.GLOBAL_CONTEXT['exists'] = self.file_exists
self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
'ADD_THIS_BUTTONS']
self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
self.plugin_manager.setPluginPlaces([
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
])
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
pluginInfo.plugin_object.short_help = pluginInfo.description
self.commands[pluginInfo.name] = pluginInfo.plugin_object
# Activate all task plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name,
"TemplateSystem")
if pi is None:
sys.stderr.write("Error loading %s template system plugin\n" %
template_sys_name)
sys.exit(1)
self.template_system = pi.plugin_object
self.template_system.set_directories([
os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES
])
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for pluginInfo in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[pluginInfo.name] = \
pluginInfo.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [
lang for lang, exts in self.config['post_compilers'].items()
if ext in exts
]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of %s)" % ', '.join(langs))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to "
"handle '%s' extensions." % ext)
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.config['GLOBAL_CONTEXT'])
local_context.update(context)
data = self.template_system.render_template(template_name, None,
local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# This is to support windows paths
url_part = "/".join(url_part.split(os.sep))
src = urlparse.urljoin(self.config["BLOG_URL"], url_part)
parsed_src = urlparse.urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse.urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = '<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "w+") as post_file:
post_file.write(data)
def path(self, kind, name, lang, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
path = []
if kind == "tag_index":
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
'index.html'
])
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".html"
])
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".xml"
])
elif kind == "index":
if name > 0:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
'index-%s.html' % name
])
else:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], 'index.html'
])
elif kind == "rss":
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'],
'rss.xml'
])
elif kind == "archive":
if name:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], name, 'index.html'
])
else:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'],
self.config['ARCHIVE_FILENAME']
])
elif kind == "gallery":
path = filter(None,
[self.config['GALLERY_PATH'], name, 'index.html'])
elif kind == "listing":
path = filter(None,
[self.config['LISTINGS_FOLDER'], name + '.html'])
if is_link:
return '/' + ('/'.join(path))
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urlparse.urljoin(self.config['BLOG_URL'], dst)
return urlparse.urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urlparse.urljoin(self.config['BLOG_URL'], src)
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlparse.urlsplit(src)
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': 'all',
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if not self._scanned:
print "Scanning posts ",
targets = set([])
for wildcard, destination, _, use_in_feeds in \
self.config['post_pages']:
print ".",
for base_path in glob.glob(wildcard):
post = Post(base_path, destination, use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BLOG_URL'], self.MESSAGES)
for lang, langpath in self.config['TRANSLATIONS'].items():
dest = (destination, langpath, post.pagenames[lang])
if dest in targets:
raise Exception(
'Duplicated output path %r in post %r' %
(post.pagenames[lang], base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(
post.post_name)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
for name, post in self.global_data.items():
self.timeline.append(post)
self.timeline.sort(cmp=lambda a, b: cmp(a.date, b.date))
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print "done!"
def generic_page_renderer(self, lang, wildcard, template_name, destination,
filters):
"""Render post fragments to final HTML pages."""
for post in glob.glob(wildcard):
post_name = os.path.splitext(post)[0]
context = {}
post = self.global_data[post_name]
deps = post.deps(lang) + \
self.template_system.template_deps(template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
self.config['TRANSLATIONS'][lang],
destination,
post.pagenames[lang] + ".html")
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.config['GLOBAL_CONTEXT']
task = {
'name':
output_name.encode('utf-8'),
'file_dep':
deps,
'targets': [output_name],
'actions':
[(self.render_template, [template_name, output_name,
context])],
'clean':
True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.titles[lang], p.permalink(lang))
for p in posts]
deps_context["global"] = self.config['GLOBAL_CONTEXT']
task = {
'name':
output_name.encode('utf8'),
'targets': [output_name],
'file_dep':
deps,
'actions':
[(self.render_template, [template_name, output_name, context])],
'clean':
True,
'uptodate': [config_changed(deps_context)]
}
yield utils.apply_filters(task, filters)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
# This is the default config
# TODO: fill it
self.config = {
'ADD_THIS_BUTTONS': True,
'ANALYTICS': '',
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'CACHE_FOLDER': 'cache',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'CONTENT_FOOTER': '',
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISQUS_FORUM': 'nikolademo',
'FAVICONS': {},
'FILE_METADATA_REGEXP': None,
'FILES_FOLDERS': {'files': ''},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEX_PATH': '',
'LICENSE': '',
'LISTINGS_FOLDER': 'listings',
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OUTPUT_FOLDER': 'output',
'post_compilers': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile',),
"txt2tags": ('.t2t',),
"bbcode": ('.bb',),
"wiki": ('.wiki',),
"ipynb": ('.ipynb',),
"html": ('.html', '.htm')
},
'POST_PAGES': (
("posts/*.txt", "posts", "post.tmpl", True),
("stories/*.txt", "stories", "story.tmpl", False),
),
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'STORY_INDEX': False,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'site',
'THUMBNAIL_SIZE': 180,
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': None,
}
self.config.update(config)
self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
{self.config['DEFAULT_'
'LANG']: ''})
self.THEMES = utils.get_theme_chain(self.config['THEME'])
self.MESSAGES = utils.load_messages(self.THEMES,
self.config['TRANSLATIONS'])
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
self.plugin_manager.setPluginPlaces([
str(os.path.join(os.path.dirname(__file__), 'plugins')),
str(os.path.join(os.getcwd(), 'plugins')),
])
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
pluginInfo.plugin_object.short_help = pluginInfo.description
self.commands[pluginInfo.name] = pluginInfo.plugin_object
# Activate all task plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
# set global_context for template rendering
self.GLOBAL_CONTEXT = {
}
self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
self.GLOBAL_CONTEXT['_link'] = self.link
self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
self.GLOBAL_CONTEXT['exists'] = self.file_exists
self.GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config[
'SLUG_TAG_PATH']
self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
'ADD_THIS_BUTTONS']
self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self.GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self.GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self.GLOBAL_CONTEXT['date_format'] = self.config.get('DATE_FORMAT', '%Y-%m-%d %H:%M')
self.GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self.GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
self.GLOBAL_CONTEXT['blog_url'] = self.config.get('BLOG_URL')
self.GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self.GLOBAL_CONTEXT['analytics'] = self.config.get('ANALYTICS')
self.GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self.GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self.GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self.GLOBAL_CONTEXT['disqus_forum'] = self.config.get('DISQUS_FORUM')
self.GLOBAL_CONTEXT['mathjax_config'] = self.config.get('MATHJAX_CONFIG')
self.GLOBAL_CONTEXT['content_footer'] = self.config.get('CONTENT_FOOTER')
self.GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self.GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self.GLOBAL_CONTEXT['sidebar_links'] = self.config.get('SIDEBAR_LINKS')
self.GLOBAL_CONTEXT['twitter_card'] = self.config.get('TWITTER_CARD', {})
self.GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# check if custom css exist and is not empty
for files_path in list(self.config['FILES_FOLDERS'].keys()):
custom_css_path = os.path.join(files_path, 'assets/css/custom.css')
if self.file_exists(custom_css_path, not_empty=True):
self.GLOBAL_CONTEXT['has_custom_css'] = True
break
else:
self.GLOBAL_CONTEXT['has_custom_css'] = False
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(
template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self.template_system = pi.plugin_object
lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES]
self.template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for pluginInfo in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[pluginInfo.name] = \
pluginInfo.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in
list(self.config['post_compilers'].items())
if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(
template_name, None, local_context)
assert isinstance(output_name, bytes)
assert output_name.startswith(
self.config["OUTPUT_FOLDER"].encode('utf8'))
url_part = output_name.decode('utf8')[len(self.config["OUTPUT_FOLDER"])
+ 1:]
# This is to support windows paths
url_part = "/".join(url_part.split(os.sep))
src = urljoin(self.config["BLOG_URL"], url_part)
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def path(self, kind, name, lang, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a post_pages tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
path = []
if kind == "tag_index":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], 'index.html'] if _f]
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".html"] if
_f]
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".xml"] if
_f]
elif kind == "index":
if name not in [None, 0]:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
'index-{0}.html'.format(name)] if _f]
else:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], 'index.html']
if _f]
elif kind == "post_path":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
os.path.dirname(name), "index.html"] if _f]
elif kind == "rss":
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['RSS_PATH'], 'rss.xml'] if _f]
elif kind == "archive":
if name:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], name,
'index.html'] if _f]
else:
path = [_f for _f in [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'],
self.config['ARCHIVE_FILENAME']] if _f]
elif kind == "gallery":
path = [_f for _f in [self.config['GALLERY_PATH'], name,
'index.html'] if _f]
elif kind == "listing":
path = [_f for _f in [self.config['LISTINGS_FOLDER'], name +
'.html'] if _f]
if is_link:
return '/' + ('/'.join(path))
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BLOG_URL'], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config['BLOG_URL'], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': b'all',
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if not self._scanned:
print("Scanning posts", end='')
tzinfo = None
if self.config['TIMEZONE'] is not None:
tzinfo = pytz.timezone(self.config['TIMEZONE'])
targets = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='')
base_len = len(destination.split(os.sep))
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath,
os.path.basename(wildcard))
dest_dir = os.path.join(*([destination] +
dirpath.split(
os.sep)[base_len:]))
for base_path in glob.glob(dir_glob):
post = Post(
base_path,
self.config['CACHE_FOLDER'],
dest_dir,
use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BLOG_URL'],
self.MESSAGES,
template_name,
self.config['FILE_METADATA_REGEXP'],
tzinfo,
)
for lang, langpath in list(
self.config['TRANSLATIONS'].items()):
dest = (destination, langpath, dir_glob,
post.pagenames[lang])
if dest in targets:
raise Exception('Duplicated output path {0!r} '
'in post {1!r}'.format(
post.pagenames[lang],
base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[
str(post.date.year)].append(post.post_name)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang)).encode('utf8')
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
task = {
'name': output_name,
'file_dep': deps,
'targets': [output_name],
'actions': [(self.render_template, [post.template_name,
output_name, context])],
'clean': True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
# This is a name on disk, has to be bytes
assert isinstance(output_name, bytes)
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.titles[lang], p.permalink(lang)) for p in
posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name': output_name,
'targets': [output_name],
'file_dep': deps,
'actions': [(self.render_template, [template_name, output_name,
context])],
'clean': True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
class Paratest(object):
def __init__(self, workspace_num, scripts, source_path, workspace_path,
output_path, test_pattern, persistence):
self.workspace_num = workspace_num
self.workspace_path = workspace_path
self.scripts = scripts
self.source_path = source_path
self.output_path = output_path
self.test_pattern = test_pattern
self._workers = []
self.pluginmgr = PluginManager()
self.pluginmgr.setPluginInfoExtension('paratest')
self.pluginmgr.setPluginPlaces(["plugins", ""])
self.pluginmgr.collectPlugins()
self.persistence = persistence
if not os.path.exists(self.source_path):
os.makedirs(self.source_path)
if not os.path.exists(self.output_path):
os.makedirs(self.output_path)
def list_plugins(self):
msg = "Available plugins are:\n"
for plugin in self.pluginmgr.getAllPlugins():
msg += " %s" % plugin.name
print(msg)
def run(self, plugin):
plugin = self.pluginmgr.getPluginByName(plugin)
pluginobj = plugin.plugin_object
self.run_script_setup()
test_number = self.queue_tests(pluginobj)
self.create_workers(pluginobj, self.num_of_workers(test_number))
self.start_workers()
self.wait_workers()
self.run_script_teardown()
self.assert_all_messages_were_processed()
def run_script_setup(self):
if run_script(self.scripts.setup, path=self.workspace_path):
raise Abort('The setup script failed. aborting.')
def run_script_teardown(self):
if run_script(self.scripts.teardown, path=self.workspace_path):
raise Abort('The teardown script failed, but nothing can be done.')
def queue_tests(self, pluginobj):
tids = 0
for tid in pluginobj.find(self.source_path, self.test_pattern):
shared_queue.put((self.persistence.get_priority(tid), tid))
tids += 1
return tids
def create_workers(self, pluginobj, workers):
for i in range(workers):
t = Worker(
pluginobj,
scripts=self.scripts,
workspace_path=self.workspace_path,
source_path=self.source_path,
output_path=self.output_path,
persistence=self.persistence,
name=str(i),
)
self._workers.append(t)
def num_of_workers(self, test_number):
return min(self.workspace_num, test_number)
def start_workers(self):
logger.debug("start workers")
for t in self._workers:
t.start()
shared_queue.put((INFINITE, FINISH))
def wait_workers(self):
logger.debug("wait for all workers to finish")
for t in self._workers:
t.join()
def assert_all_messages_were_processed(self):
if not shared_queue.empty():
raise Abort(
'There were unprocessed tests, but all workers are dead. Aborting.'
)
class EventDispatcher(object):
failed_event_threshold = 10
failed_events = {}
def __init__(self):
logging.basicConfig(format='%(asctime)s %(levelname)s %(module)s: %(message)s',
datefmt='%d.%m.%Y %H:%M:%S',
level=logging.INFO)
self.logger = logging.getLogger(__name__)
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["plugins"])
self.plugin_manager.setPluginInfoExtension('plugin')
self._init_plugins()
self.sqs_queue = SqsQueue()
def _init_plugins(self):
self.plugin_manager.collectPlugins()
for pluginInfo in self.plugin_manager.getAllPlugins():
self.plugin_manager.activatePluginByName(pluginInfo.name)
self.logger.info("Loaded plugin: {0}".format(pluginInfo.name))
def get_plugin_by_name(self, name):
plugin = self.plugin_manager.getPluginByName(name)
if not plugin:
self.logger.debug("No plugin found, using Default!")
plugin = self.plugin_manager.getPluginByName("Default")
assert plugin, "No handler plugin loaded for resource name {0}".format(name)
self.logger.info("Choose {0} handler for resource name {1}".format(plugin.name, name))
return plugin
def increment_failure_counter(self, event):
if event.id in self.failed_events:
self.failed_events[event.id] += 1
else:
self.failed_events[event.id] = 1
return self.failed_events[event.id]
def get_resource_type_name(self, event):
return event.get_property("ResourceType").lstrip("Custom::")
def dispatch_event(self, sqs_message):
event = CustomResourceEvent(sqs_message)
self.logger.info("Handling event: {0}".format(event.id))
self.logger.debug(event)
resource_type = self.get_resource_type_name(event)
try:
event_handler = self.get_plugin_by_name(resource_type)
event_handler.plugin_object.handle_event(event)
except Exception as e:
failure_count = self.increment_failure_counter(event)
self.logger.error("Couldn't handle event: "
"{0}, error was {1} (try {2}/{3})".format(event, str(e), failure_count,
self.failed_event_threshold))
if failure_count < self.failed_event_threshold:
sys.exit(1)
self.sqs_queue.delete_message(sqs_message)
self.logger.info("Deleted event from queue: {0}".format(event.id))
def event_loop(self):
while True:
# TODO: handle errors here
sqs_messages = self.sqs_queue.get_messages()
if sqs_messages:
self.logger.info("Found {0} events in the queue".format(len(sqs_messages)))
for sqs_message in sqs_messages:
thread = Thread(target=self.dispatch_event, args=(sqs_message,))
thread.start()
def load_plugins(self):
pass
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {'listings': 'listings'},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension
})
self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension")
self.timeline = [
FakePost(title='Fake post',
slug='fake-post')
]
self.debug = True
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return('<img src="IMG.jpg">')
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry)
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
"DISABLED_PLUGINS": [],
"EXTRA_PLUGINS": [],
"EXTRA_PLUGINS_DIRS": [extra_v6_plugin_dir],
"DEFAULT_LANG": "en",
"MARKDOWN_EXTENSIONS": ["fenced_code", "codehilite"],
"TRANSLATIONS_PATTERN": "{path}.{lang}.{ext}",
"LISTINGS_FOLDERS": {},
}
self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"]
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"RestExtension": RestExtension,
"MarkdownExtension": MarkdownExtension,
}
)
self.loghandlers = [nikola.utils.STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension("plugin")
extra_plugins_dirs = self.config["EXTRA_PLUGINS_DIRS"]
if sys.version_info[0] == 3:
places = [os.path.join(os.path.dirname(nikola.utils.__file__), "plugins")] + [
path for path in extra_plugins_dirs if path
]
else:
places = [os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode("plugins"))] + [
nikola.utils.sys_encode(path) for path in extra_plugins_dirs if path
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension")
self.timeline = [FakePost(title="Fake post", slug="fake-post")]
self.debug = True
self.rst_transforms = []
# This is to make plugin initialization happy
self.template_system = self
self.name = "mako"
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get("DISABLED_PLUGINS"):
self.plugin_manager.removePluginFromCategory(plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return '<img src="IMG.jpg">'
class Paratest(object):
def __init__(self, workspace_num, scripts, source_path, workspace_path, output_path, test_pattern, persistence):
self.workspace_num = workspace_num
self.workspace_path = workspace_path
self.scripts = scripts
self.source_path = source_path
self.output_path = output_path
self.test_pattern = test_pattern
self._workers = []
self.pluginmgr = PluginManager()
self.pluginmgr.setPluginInfoExtension('paratest')
self.pluginmgr.setPluginPlaces(["plugins", ""])
self.pluginmgr.collectPlugins()
self.persistence = persistence
if not os.path.exists(self.source_path):
os.makedirs(self.source_path)
if not os.path.exists(self.output_path):
os.makedirs(self.output_path)
def list_plugins(self):
msg = "Available plugins are:\n"
for plugin in self.pluginmgr.getAllPlugins():
msg += " %s" % plugin.name
print(msg)
def run(self, plugin):
plugin = self.pluginmgr.getPluginByName(plugin)
pluginobj = plugin.plugin_object
self.run_script_setup()
test_number = self.queue_tests(pluginobj)
self.create_workers(pluginobj, self.num_of_workers(test_number))
self.start_workers()
self.wait_workers()
self.run_script_teardown()
self.assert_all_messages_were_processed()
def run_script_setup(self):
if run_script(self.scripts.setup, path=self.workspace_path):
raise Abort('The setup script failed. aborting.')
def run_script_teardown(self):
if run_script(self.scripts.teardown, path=self.workspace_path):
raise Abort('The teardown script failed, but nothing can be done.')
def queue_tests(self, pluginobj):
tids = 0
for tid in pluginobj.find(self.source_path, self.test_pattern):
shared_queue.put((self.persistence.get_priority(tid), tid))
tids += 1
return tids
def create_workers(self, pluginobj, workers):
for i in range(workers):
t = Worker(
pluginobj,
scripts=self.scripts,
workspace_path=self.workspace_path,
source_path=self.source_path,
output_path=self.output_path,
persistence=self.persistence,
name=str(i),
)
self._workers.append(t)
def num_of_workers(self, test_number):
return min(self.workspace_num, test_number)
def start_workers(self):
logger.debug("start workers")
for t in self._workers:
t.start()
shared_queue.put((INFINITE, FINISH))
def wait_workers(self):
logger.debug("wait for all workers to finish")
for t in self._workers:
t.join()
def assert_all_messages_were_processed(self):
if not shared_queue.empty():
raise Abort('There were unprocessed tests, but all workers are dead. Aborting.')
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
# This is the default config
# TODO: fill it
self.config = {
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'DEFAULT_LANG': "en",
'OUTPUT_FOLDER': 'output',
'FILES_FOLDERS': {'files': ''},
'LISTINGS_FOLDER': 'listings',
'ADD_THIS_BUTTONS': True,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_TEASERS': False,
'MAX_IMAGE_SIZE': 1280,
'USE_FILENAME_AS_TITLE': True,
'SLUG_TAG_PATH': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'FILTERS': {},
'USE_BUNDLES': True,
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'default',
'post_compilers': {
"rest": ['.txt', '.rst'],
"markdown": ['.md', '.mdown', '.markdown'],
"html": ['.html', '.htm'],
},
}
self.config.update(config)
self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
{self.config['DEFAULT_LANG']: ''})
# FIXME: find way to achieve this with the plugins
#if self.config['USE_BUNDLES'] and not webassets:
#self.config['USE_BUNDLES'] = False
self.GLOBAL_CONTEXT = self.config.get('GLOBAL_CONTEXT', {})
self.THEMES = utils.get_theme_chain(self.config['THEME'])
self.MESSAGES = utils.load_messages(self.THEMES,
self.config['TRANSLATIONS'])
self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
self.GLOBAL_CONTEXT['_link'] = self.link
self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
self.GLOBAL_CONTEXT['exists'] = self.file_exists
self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
'ADD_THIS_BUTTONS']
self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
self.plugin_manager.setPluginPlaces([
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
])
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
pluginInfo.plugin_object.short_help = pluginInfo.description
self.commands[pluginInfo.name] = pluginInfo.plugin_object
# Activate all task plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(
template_sys_name, "TemplateSystem")
if pi is None:
sys.stderr.write("Error loading %s template system plugin\n"
% template_sys_name)
sys.exit(1)
self.template_system = pi.plugin_object
self.template_system.set_directories(
[os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES])
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for pluginInfo in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[pluginInfo.name] = \
pluginInfo.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [lang for lang, exts in
self.config['post_compilers'].items()
if ext in exts]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of %s)" % ', '.join(langs))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to "
"handle '%s' extensions." % ext)
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
data = self.template_system.render_template(
template_name, None, context, self.GLOBAL_CONTEXT)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# This is to support windows paths
url_part = "/".join(url_part.split(os.sep))
src = urlparse.urljoin(self.config["BLOG_URL"], url_part)
parsed_src = urlparse.urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse.urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = '<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "w+") as post_file:
post_file.write(data)
def path(self, kind, name, lang, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
path = []
if kind == "tag_index":
path = filter(None, [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], 'index.html'])
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".html"])
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [self.config['TRANSLATIONS'][lang],
self.config['TAG_PATH'], name + ".xml"])
elif kind == "index":
if name > 0:
path = filter(None, [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], 'index-%s.html' % name])
else:
path = filter(None, [self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], 'index.html'])
elif kind == "rss":
path = filter(None, [self.config['TRANSLATIONS'][lang],
self.config['RSS_PATH'], 'rss.xml'])
elif kind == "archive":
if name:
path = filter(None, [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], name, 'index.html'])
else:
path = filter(None, [self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], self.config['ARCHIVE_FILENAME']])
elif kind == "gallery":
path = filter(None,
[self.config['GALLERY_PATH'], name, 'index.html'])
elif kind == "listing":
path = filter(None,
[self.config['LISTINGS_FOLDER'], name + '.html'])
if is_link:
return '/' + ('/'.join(path))
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urlparse.urljoin(self.config['BLOG_URL'], dst)
return urlparse.urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urlparse.urljoin(self.config['BLOG_URL'], src)
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlparse.urlsplit(src)
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': 'all',
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if not self._scanned:
print "Scanning posts ",
targets = set([])
for wildcard, destination, _, use_in_feeds in \
self.config['post_pages']:
print ".",
for base_path in glob.glob(wildcard):
post = Post(base_path, destination, use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BLOG_URL'],
self.MESSAGES)
for lang, langpath in self.config['TRANSLATIONS'].items():
dest = (destination, langpath, post.pagenames[lang])
if dest in targets:
raise Exception(
'Duplicated output path %r in post %r' %
(post.pagenames[lang], base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[
str(post.date.year)].append(post.post_name)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
for name, post in self.global_data.items():
self.timeline.append(post)
self.timeline.sort(cmp=lambda a, b: cmp(a.date, b.date))
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print "done!"
def generic_page_renderer(self, lang, wildcard,
template_name, destination, filters):
"""Render post fragments to final HTML pages."""
for post in glob.glob(wildcard):
post_name = os.path.splitext(post)[0]
context = {}
post = self.global_data[post_name]
deps = post.deps(lang) + \
self.template_system.template_deps(template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
output_name = os.path.join(
self.config['OUTPUT_FOLDER'],
self.config['TRANSLATIONS'][lang],
destination,
post.pagenames[lang] + ".html")
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
task = {
'name': output_name.encode('utf-8'),
'file_dep': deps,
'targets': [output_name],
'actions': [(self.render_template,
[template_name, output_name, context])],
'clean': True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts,
output_name, template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.titles[lang], p.permalink(lang))
for p in posts]
task = {
'name': output_name.encode('utf8'),
'targets': [output_name],
'file_dep': deps,
'actions': [(self.render_template,
[template_name, output_name, context])],
'clean': True,
'uptodate': [config_changed(deps_context)]
}
yield utils.apply_filters(task, filters)
def init_plugin_manager():
global simplePluginManager
simplePluginManager = PluginManager(categories_filter={"bots": BotPlugin})
simplePluginManager.setPluginInfoExtension('plug')
def main():
#Find and load plugins
pm = PluginManager()
libpath = '%s/OpenMesher/plugins' % (get_python_lib())
pm.setPluginPlaces([
"/usr/share/openmesher/plugins", "~/.openmesher/plugins",
"./OpenMesher/plugins", "./plugins", libpath
])
pm.setPluginInfoExtension('plugin')
pm.setCategoriesFilter({
'config': IOpenMesherConfigPlugin,
'package': IOpenMesherPackagePlugin,
'deploy': IOpenMesherDeployPlugin,
})
pm.collectPlugins()
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Configure, package, and deploy an OpenVPN mesh")
parser.add_argument('-r',
'--router',
action='append',
help='Adds a router that can be a client and server')
parser.add_argument(
'-s',
'--server',
action='append',
help='Adds a router that can only act as a server, not a client.')
parser.add_argument(
'-c',
'--client',
action='append',
help=
'Adds a router than can only act as a client. For example, a router that is behind NAT and not accessible by a public IP'
)
#BUG: Stupid argparse appends your switches to the default.
#parser.add_argument('-n', '--network', action='append', default=['10.99.99.0/24'])
parser.add_argument('-n', '--network', action='append', required=True)
portgroup = parser.add_mutually_exclusive_group()
portgroup.add_argument('-p',
'--ports',
action='append',
default=['7000-7999'])
portgroup.add_argument('-a', '--random', action='store_true')
parser.add_argument(
'-v',
'--verbose',
action='append_const',
const='verbose',
help='Specify multiple times to make things more verbose')
parser.add_argument('--version', action='version', version='v0.6.4')
pluginargsgroup = parser.add_argument_group('plugins')
for plugin in pm.getAllPlugins():
plugin.plugin_object.setupargs(pluginargsgroup)
arg = parser.parse_args()
for plugin in pm.getAllPlugins():
if plugin.plugin_object.__class__.__name__.lower() in arg:
if eval('arg.%s' %
(plugin.plugin_object.__class__.__name__.lower())):
logging.debug('Plugin enabled: %s' % (plugin.name))
pm.activatePluginByName(plugin.name)
else:
logging.debug('Plugin disabled: %s' % (plugin.name))
pm.deactivatePluginByName(plugin.name)
else:
logging.debug('Plugin disabled: %s' % (plugin.name))
pm.deactivatePluginByName(plugin.name)
l = logging.getLogger()
if arg.verbose:
if len(arg.verbose) == 1:
l.setLevel(logging.INFO)
if len(arg.verbose) >= 2:
l.setLevel(logging.DEBUG)
# Call activate() on all plugins so they prep themselves for use
for plugin in pm.getAllPlugins():
if eval('arg.%s' % (plugin.plugin_object.__class__.__name__.lower())):
logging.info('Enabled plugin: %s' % (plugin.name))
pm.activatePluginByName(plugin.name)
plugin.plugin_object.activate()
plugin.plugin_object._enabled = True
if len(arg.ports) > 1:
arg.ports.reverse()
arg.ports.pop()
port_list = []
if arg.random:
numdevs = 0
if arg.router:
numdevs += len(arg.router)
if arg.server:
numdevs += len(arg.server)
if arg.client:
numdevs += len(arg.client)
ports_needed = numdevs * (numdevs - 1) / 2
for i in range(0, ports_needed):
port_list.append(random.randrange(1025, 32767))
try:
if not arg.random:
# If we're not using random ports, pull whatever is in arg.ports
for portrange in arg.ports:
portstart, portstop = portrange.split('-')
port_list += range(int(portstart), int(portstop))
except ValueError as e:
print 'Invalid port range: %s' % (portrange)
raise ValueError('Invalid port range: %s' % (portrange))
linkmesh = create_link_mesh(routers=arg.router,
servers=arg.server,
clients=arg.client)
m = Mesh(linkmesh, port_list, arg.network)
files = None
# Run through config plugins
configPlugins = []
for plugin in pm.getPluginsOfCategory('config'):
if plugin.plugin_object._enabled:
plugin.plugin_object.process(m, arg)
configPlugins.append(plugin.plugin_object)
if files:
files = nested_dict_merge(files, plugin.plugin_object.files())
else:
files = plugin.plugin_object.files()
#Grab list of folders that need to be in the package root
includedirs = []
for f in files:
for fldr in files[f]:
rootfldr = fldr.split('/')[1]
if rootfldr not in includedirs:
includedirs.append(rootfldr)
logging.debug('The following folders will be packaged: %s' % (includedirs))
# Run through packaging plugins
packagePlugins = []
for plugin in pm.getPluginsOfCategory('package'):
if plugin.plugin_object._enabled:
#BUG: Services to restart may not necessarily be the same name as their config dir...
plugin.plugin_object.process(m,
include_dirs=includedirs,
restart_services=includedirs,
configPlugins=configPlugins,
cliargs=arg)
packagePlugins.append(plugin.plugin_object)
# Run through deployment plugins
for plugin in pm.getPluginsOfCategory('deploy'):
if plugin.plugin_object._enabled:
try:
plugin.plugin_object.deploy(packagePlugins=packagePlugins,
cliargs=arg,
stoponfailure=False)
except Exception as e:
print "Unable to deploy due to error: %s" % (e)
logging.info('OpenMesher run complete')
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = [
'planetoid',
'ipynb',
'local_search',
'render_mustache',
]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.posts_per_category = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
self._template_system = None
self._THEMES = None
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'BODY_END': "",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENT_SYSTEM': 'disqus',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'COMPILERS': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile', ),
"txt2tags": ('.t2t', ),
"bbcode": ('.bb', ),
"wiki": ('.wiki', ),
"ipynb": ('.ipynb', ),
"html": ('.html', '.htm')
},
'CONTENT_FOOTER': '',
'COPY_SOURCES': True,
'CREATE_MONTHLY_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'COMMENT_SYSTEM_ID': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FEED_LENGTH': 10,
'FILE_METADATA_REGEXP': None,
'ADDITIONAL_METADATA': {},
'FILES_FOLDERS': {
'files': ''
},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GZIP_FILES': False,
'GZIP_EXTENSIONS':
('.txt', '.htm', '.html', '.css', '.js', '.json'),
'HIDE_SOURCELINK': False,
'HIDE_UNTRANSLATED_POSTS': False,
'HYPHENATE': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEX_PATH': '',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'NAVIGATION_LINKS': None,
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'POSTS': (("posts/*.txt", "posts", "post.tmpl"), ),
'PAGES': (("stories/*.txt", "stories", "story.tmpl"), ),
'PRETTY_URLS': False,
'FUTURE_IS_NOW': False,
'READ_MORE_LINK':
'<p class="more"><a href="{link}">{read_more}…</a></p>',
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'SOCIAL_BUTTONS_CODE': SOCIAL_BUTTONS_CODE,
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'bootstrap',
'THEME_REVEAL_CONFIG_SUBTHEME': 'sky',
'THEME_REVEAL_CONFIG_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': None,
'DEPLOY_DRAFTS': True,
'DEPLOY_FUTURE': False,
'SCHEDULE_ALL': False,
'SCHEDULE_RULE': '',
'SCHEDULE_FORCE_TODAY': False
}
self.config.update(config)
# Make sure we have pyphen installed if we are using it
if self.config.get('HYPHENATE') and pyphen is None:
print('WARNING: To use the hyphenation, you have to install '
'the "pyphen" package.')
print('WARNING: Setting HYPHENATE to False.')
self.config['HYPHENATE'] = False
# Deprecating post_compilers
# TODO: remove on v7
if 'post_compilers' in config:
print(
"WARNING: The post_compilers option is deprecated, use COMPILERS instead."
)
if 'COMPILERS' in config:
print(
"WARNING: COMPILERS conflicts with post_compilers, ignoring post_compilers."
)
else:
self.config['COMPILERS'] = config['post_compilers']
# Deprecating post_pages
# TODO: remove on v7
if 'post_pages' in config:
print(
"WARNING: The post_pages option is deprecated, use POSTS and PAGES instead."
)
if 'POSTS' in config or 'PAGES' in config:
print(
"WARNING: POSTS and PAGES conflict with post_pages, ignoring post_pages."
)
else:
self.config['POSTS'] = [
item[:3] for item in config['post_pages'] if item[-1]
]
self.config['PAGES'] = [
item[:3] for item in config['post_pages'] if not item[-1]
]
# FIXME: Internally, we still use post_pages because it's a pain to change it
self.config['post_pages'] = []
for i1, i2, i3 in self.config['POSTS']:
self.config['post_pages'].append([i1, i2, i3, True])
for i1, i2, i3 in self.config['PAGES']:
self.config['post_pages'].append([i1, i2, i3, False])
# Deprecating DISQUS_FORUM
# TODO: remove on v7
if 'DISQUS_FORUM' in config:
print(
"WARNING: The DISQUS_FORUM option is deprecated, use COMMENT_SYSTEM_ID instead."
)
if 'COMMENT_SYSTEM_ID' in config:
print(
"WARNING: DISQUS_FORUM conflicts with COMMENT_SYSTEM_ID, ignoring DISQUS_FORUM."
)
else:
self.config['COMMENT_SYSTEM_ID'] = config['DISQUS_FORUM']
# Deprecating the ANALYTICS option
# TODO: remove on v7
if 'ANALYTICS' in config:
print(
"WARNING: The ANALYTICS option is deprecated, use BODY_END instead."
)
if 'BODY_END' in config:
print(
"WARNING: ANALYTICS conflicts with BODY_END, ignoring ANALYTICS."
)
else:
self.config['BODY_END'] = config['ANALYTICS']
# Deprecating the SIDEBAR_LINKS option
# TODO: remove on v7
if 'SIDEBAR_LINKS' in config:
print(
"WARNING: The SIDEBAR_LINKS option is deprecated, use NAVIGATION_LINKS instead."
)
if 'NAVIGATION_LINKS' in config:
print(
"WARNING: The SIDEBAR_LINKS conflicts with NAVIGATION_LINKS, ignoring SIDEBAR_LINKS."
)
else:
self.config['NAVIGATION_LINKS'] = config['SIDEBAR_LINKS']
# Compatibility alias
self.config['SIDEBAR_LINKS'] = self.config['NAVIGATION_LINKS']
if self.config['NAVIGATION_LINKS'] in (None, {}):
self.config['NAVIGATION_LINKS'] = {self.config['DEFAULT_LANG']: ()}
# Deprecating the ADD_THIS_BUTTONS option
# TODO: remove on v7
if 'ADD_THIS_BUTTONS' in config:
print(
"WARNING: The ADD_THIS_BUTTONS option is deprecated, use SOCIAL_BUTTONS_CODE instead."
)
if not config['ADD_THIS_BUTTONS']:
print(
"WARNING: Setting SOCIAL_BUTTONS_CODE to empty because ADD_THIS_BUTTONS is False."
)
self.config['SOCIAL_BUTTONS_CODE'] = ''
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
# TODO: remove on v7
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
print(
"WARNING: You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML"
)
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
if config.get('COPY_SOURCES') and not self.config['HIDE_SOURCELINK']:
self.config['HIDE_SOURCELINK'] = True
self.config['TRANSLATIONS'] = self.config.get(
'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''})
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
# TODO: remove on v7
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
print(
"WARNING: You should configure SITE_URL instead of BLOG_URL"
)
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
# BASE_URL should *always* end in /
if self.config['BASE_URL'] and self.config['BASE_URL'][-1] != '/':
print("WARNING: Your BASE_URL doesn't end in / -- adding it.")
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(__file__),
utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS
and plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(
task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS
and plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
if plugin_info.name in self.config["COMPILERS"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self._GLOBAL_CONTEXT = {}
self._GLOBAL_CONTEXT['_link'] = self.link
self._GLOBAL_CONTEXT['set_locale'] = s_l
self._GLOBAL_CONTEXT['rel_link'] = self.rel_link
self._GLOBAL_CONTEXT['abs_link'] = self.abs_link
self._GLOBAL_CONTEXT['exists'] = self.file_exists
self._GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config['SLUG_TAG_PATH']
self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self._GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self._GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
# TODO: remove fallback in v7
self._GLOBAL_CONTEXT['blog_url'] = self.config.get(
'SITE_URL', self.config.get('BLOG_URL'))
self._GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self._GLOBAL_CONTEXT['body_end'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['analytics'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['add_this_buttons'] = self.config.get(
'SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['social_buttons_code'] = self.config.get(
'SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self._GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self._GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self._GLOBAL_CONTEXT['comment_system'] = self.config.get(
'COMMENT_SYSTEM')
self._GLOBAL_CONTEXT['comment_system_id'] = self.config.get(
'COMMENT_SYSTEM_ID')
# TODO: remove in v7
self._GLOBAL_CONTEXT['disqus_forum'] = self.config.get(
'COMMENT_SYSTEM_ID')
self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self._GLOBAL_CONTEXT['subtheme'] = self.config.get(
'THEME_REVEAL_CONFIG_SUBTHEME')
self._GLOBAL_CONTEXT['transition'] = self.config.get(
'THEME_REVEAL_CONFIG_TRANSITION')
self._GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self._GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self._GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self._GLOBAL_CONTEXT['navigation_links'] = utils.Functionary(
list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('NAVIGATION_LINKS', {}).items():
self._GLOBAL_CONTEXT['navigation_links'][k] = v
# TODO: remove on v7
# Compatibility alias
self._GLOBAL_CONTEXT['sidebar_links'] = self._GLOBAL_CONTEXT[
'navigation_links']
self._GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self._GLOBAL_CONTEXT['hide_sourcelink'] = self.config.get(
'HIDE_SOURCELINK')
self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get(
'EXTRA_HEAD_DATA')
self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
def _get_themes(self):
if self._THEMES is None:
# Check for old theme names (Issue #650) TODO: remove in v7
theme_replacements = {
'site': 'bootstrap',
'orphan': 'base',
'default': 'oldfashioned',
}
if self.config['THEME'] in theme_replacements:
warnings.warn(
'You are using the old theme "{0}", using "{1}" instead.'.
format(self.config['THEME'],
theme_replacements[self.config['THEME']]))
self.config['THEME'] = theme_replacements[self.config['THEME']]
if self.config['THEME'] == 'oldfashioned':
warnings.warn(
'''You may need to install the "oldfashioned" theme '''
'''from themes.nikola.ralsina.com.ar because it's not '''
'''shipped by default anymore.''')
warnings.warn('Please change your THEME setting.')
try:
self._THEMES = utils.get_theme_chain(self.config['THEME'])
except Exception:
warnings.warn(
'''Can't load theme "{0}", using 'bootstrap' instead.'''.
format(self.config['THEME']))
self.config['THEME'] = 'bootstrap'
return self._get_themes()
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config['USE_CDN']:
bootstrap_path = utils.get_asset_path(
os.path.join('assets', 'css', 'bootstrap.min.css'),
self._THEMES)
if bootstrap_path and bootstrap_path.split(
os.sep)[-4] not in ['bootstrap', 'bootstrap3']:
warnings.warn(
'The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.'
)
return self._THEMES
THEMES = property(_get_themes)
def _get_messages(self):
return utils.load_messages(self.THEMES, self.translations,
self.default_lang)
MESSAGES = property(_get_messages)
def _get_global_context(self):
"""Initialize some parts of GLOBAL_CONTEXT only when it's queried."""
if 'messages' not in self._GLOBAL_CONTEXT:
self._GLOBAL_CONTEXT['messages'] = self.MESSAGES
if 'has_custom_css' not in self._GLOBAL_CONTEXT:
# check if custom css exist and is not empty
custom_css_path = utils.get_asset_path(
'assets/css/custom.css', self.THEMES,
self.config['FILES_FOLDERS'])
if custom_css_path and self.file_exists(custom_css_path,
not_empty=True):
self._GLOBAL_CONTEXT['has_custom_css'] = True
else:
self._GLOBAL_CONTEXT['has_custom_css'] = False
return self._GLOBAL_CONTEXT
GLOBAL_CONTEXT = property(_get_global_context)
def _get_template_system(self):
if self._template_system is None:
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name,
"TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self._template_system = pi.plugin_object
lookup_dirs = ['templates'] + [
os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES
]
self._template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
return self._template_system
template_system = property(_get_template_system)
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.COMPILERS`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [
lang for lang, exts in list(self.config['COMPILERS'].items())
if ext in exts
]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'COMPILERS' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("COMPILERS in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(template_name, None,
local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def current_lang(self): # FIXME: this is duplicated, turn into a mixin
"""Return the currently set locale, if it's one of the
available translations, or default_lang."""
lang = utils.LocaleBorg().current_lang
if lang:
if lang in self.translations:
return lang
lang = lang.split('_')[0]
if lang in self.translations:
return lang
# whatever
return self.default_lang
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* category (and name is the category name)
* category_rss (and name is the category name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a POSTS/PAGES tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = self.current_lang()
path = []
if kind == "tag_index":
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
self.config['INDEX_FILE']
] if _f
]
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".html"
] if _f
]
elif kind == "category":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
"cat_" + name + ".html"
] if _f
]
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".xml"
] if _f
]
elif kind == "category_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
"cat_" + name + ".xml"
] if _f
]
elif kind == "index":
if name not in [None, 0]:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.
config['INDEX_PATH'], 'index-{0}.html'.format(name)
] if _f
]
else:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], self.config['INDEX_FILE']
] if _f
]
elif kind == "post_path":
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang],
os.path.dirname(name), self.config['INDEX_FILE']
] if _f
]
elif kind == "rss":
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'],
'rss.xml'
] if _f
]
elif kind == "archive":
if name:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.
config['ARCHIVE_PATH'], name, self.config['INDEX_FILE']
] if _f
]
else:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.
config['ARCHIVE_PATH'], self.config['ARCHIVE_FILENAME']
] if _f
]
elif kind == "gallery":
path = [
_f for _f in
[self.config['GALLERY_PATH'], name, self.config['INDEX_FILE']]
if _f
]
elif kind == "listing":
path = [
_f for _f in [self.config['LISTINGS_FOLDER'], name + '.html']
if _f
]
if is_link:
link = '/' + ('/'.join(path))
index_len = len(self.config['INDEX_FILE'])
if self.config['STRIP_INDEXES'] and \
link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BASE_URL'], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config['BASE_URL'], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self, name, plugin_category):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(
plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
yield task
for multi in self.plugin_manager.getPluginsOfCategory(
"TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield task
if flag:
task_dep.append('{0}_{1}'.format(
name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': name,
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
seen = set([])
print("Scanning posts", end='')
tzinfo = None
if self.config['TIMEZONE'] is not None:
tzinfo = pytz.timezone(self.config['TIMEZONE'])
if self.config['FUTURE_IS_NOW']:
current_time = None
else:
current_time = utils.current_time(tzinfo)
targets = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='')
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(
os.path.join(destination,
os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config['TRANSLATIONS'].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
# We eliminate from the list the files inside any .ipynb folder
full_list = [
p for p in full_list
if not any([x.startswith('.') for x in p.split(os.sep)])
]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.config['CACHE_FOLDER'],
dest_dir,
use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BASE_URL'],
self.MESSAGES,
template_name,
self.config['FILE_METADATA_REGEXP'],
self.config['STRIP_INDEXES'],
self.config['INDEX_FILE'],
tzinfo,
current_time,
self.config['HIDE_UNTRANSLATED_POSTS'],
self.config['PRETTY_URLS'],
self.config['HYPHENATE'],
)
for lang, langpath in list(
self.config['TRANSLATIONS'].items()):
dest = (destination, langpath, dir_glob,
post.meta[lang]['slug'])
if dest in targets:
raise Exception('Duplicated output path {0!r} '
'in post {1!r}'.format(
post.meta[lang]['slug'],
base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(
post.post_name)
self.posts_per_month['{0}/{1:02d}'.format(
post.date.year,
post.date.month)].append(post.post_name)
for tag in post.alltags:
self.posts_per_tag[tag].append(post.post_name)
self.posts_per_category[post.meta('category')].append(
post.post_name)
else:
self.pages.append(post)
if self.config['OLD_THEME_SUPPORT']:
post._add_old_metadata()
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(
lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
if post:
deps_dict['post_translations'] = post.translated_to
task = {
'name':
os.path.normpath(output_name),
'file_dep':
deps,
'targets': [output_name],
'actions':
[(self.render_template, [post.template_name, output_name,
context])],
'clean':
True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang))
for p in posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name':
os.path.normpath(output_name),
'targets': [output_name],
'file_dep':
deps,
'actions':
[(self.render_template, [template_name, output_name, context])],
'clean':
True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
class FakeSite:
def __init__(self):
self.template_system = self
self.invariant = False
self.debug = True
self.config = {
"DISABLED_PLUGINS": [],
"EXTRA_PLUGINS": [],
"DEFAULT_LANG":
"en",
"MARKDOWN_EXTENSIONS": [
"markdown.extensions.fenced_code",
"markdown.extensions.codehilite",
],
"TRANSLATIONS_PATTERN":
"{path}.{lang}.{ext}",
"LISTINGS_FOLDERS": {
"listings": "listings"
},
"TRANSLATIONS": {
"en": ""
},
}
self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"]
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension,
})
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension("plugin")
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), "plugins")
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category(
"CompilerExtension")
self.timeline = [FakePost(title="Fake post", slug="fake-post")]
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = "mako"
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get("DISABLED_PLUGINS"):
self.plugin_manager.removePluginFromCategory(
plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return '<img src="IMG.jpg">'
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warning('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data, *a, **kw):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data,
self.shortcode_registry,
**kw)
def apply_shortcodes_uuid(self, data, shortcodes, *a, **kw):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data,
self.shortcode_registry,
**kw)
