def assert_rst_equal(self, rstfile, expectations):
reader = Readers(DEFAULT_CONFIG)
content = reader.read_file(base_path=RESOURCES_PATH, path=rstfile).content
extracted_parts = RE_EXTRACT.findall(content)
self.assertEqual(len(extracted_parts), len(expectations))
for expected, extracted in zip(expectations, extracted_parts):
self.assertEqual(extracted, expected)
def add_static_comments(gen, content):
if gen.settings["PELICAN_COMMENT_SYSTEM"] is not True:
return
global _all_comments
content.comments_count = 0
content.comments = []
# Modify the local context, so we get proper values for the feed
context = copy.copy(gen.context)
context["SITEURL"] += "/" + content.url
context["SITENAME"] += " - Comments: " + content.title
context["SITESUBTITLE"] = ""
folder = os.path.join(gen.settings["PATH"], gen.settings["PELICAN_COMMENT_SYSTEM_DIR"], content.slug)
if not os.path.isdir(folder):
logger.debug("No comments found for: %s", content.slug)
write_feed(gen, [], context, content.slug)
return
reader = Readers(gen.settings)
comments = []
replies = []
for file in os.listdir(folder):
name, extension = os.path.splitext(file)
if extension[1:].lower() in reader.extensions:
com = reader.read_file(base_path=folder, path=file, content_class=Comment, context=context)
com.article = content
_all_comments.append(com)
if hasattr(com, "replyto"):
replies.append(com)
else:
comments.append(com)
feed_items = sorted(comments + replies)
feed_items.reverse()
warn_on_slug_collision(feed_items)
write_feed(gen, feed_items, context, content.slug)
# TODO: Fix this O(n²) loop
for reply in replies:
for comment in chain(comments, replies):
if comment.slug == reply.replyto:
comment.addReply(reply)
count = 0
for comment in comments:
comment.sortReplies()
count += comment.countReplies()
comments = sorted(comments)
content.comments_count = len(comments) + count
content.comments = comments
def add_static_comments(gen, content):
if gen.settings['PELICAN_COMMENT_SYSTEM'] is not True:
return
content.comments_count = 0
content.comments = []
# Modify the local context, so we get proper values for the feed
context = copy.copy(gen.context)
context['SITEURL'] += "/" + content.url
context['SITENAME'] += " - Comments: " + content.title
context['SITESUBTITLE'] = ""
folder = os.path.join(
gen.settings['PATH'],
gen.settings['PELICAN_COMMENT_SYSTEM_DIR'],
content.slug
)
if not os.path.isdir(folder):
logger.debug("No comments found for: " + content.slug)
write_feed(gen, [], context, content.slug)
return
reader = Readers(gen.settings)
comments = []
replies = []
for file in os.listdir(folder):
name, extension = os.path.splitext(file)
if extension[1:].lower() in reader.extensions:
com = reader.read_file(
base_path=folder, path=file,
content_class=Comment, context=context)
if hasattr(com, 'replyto'):
replies.append(com)
else:
comments.append(com)
warn_on_slug_collision(comments + replies)
write_feed(gen, comments + replies, context, content.slug)
# TODO: Fix this O(n²) loop
for reply in replies:
for comment in chain(comments, replies):
if comment.slug == reply.replyto:
comment.addReply(reply)
count = 0
for comment in comments:
comment.sortReplies()
count += comment.countReplies()
comments = sorted(comments)
content.comments_count = len(comments) + count
content.comments = comments
def __init__(self, output_path, settings=None):
super(FeedAlterSettingsWriter, self).__init__(
output_path, settings=settings)
# copy and alter settings
new_settings = copy.deepcopy(self.settings)
self.settings[CALLBACK_NAME](new_settings)
# put the new settings into readers
self.readers = Readers(new_settings)
def get_posts(pelican_root_directory: str, ):
"""Given a directory containing pelican draft files, return a generator of all pelican post files."""
# certain keys are required by the reader, even if empty.
reader_settings = {"READERS": None, "CACHE_PATH": "/tmp"}
pelican_readers = Readers(reader_settings)
for (dirpath, _, filename_list) in os.walk(target_directory):
for filename in filename_list:
content = pelican_readers.read_file(dirpath, filename)
yield content
def sync_file(self, settings, filename, content_type):
"""syncs a single file to the database."""
context = settings.copy()
from pelican.readers import Readers
from pelican.contents import Article, Page
readers = Readers(context)
content_class = Article if content_type == "article" else Page
content = readers.read_file(base_path=settings['PATH'], path=filename, content_class=content_class, context=context)
self.__sync_content(content)
def __init__(self, context, settings, path, theme, output_path,
readers_cache_name='', **kwargs):
self.context = context
self.settings = settings
self.path = path
self.theme = theme
self.output_path = output_path
for arg, value in kwargs.items():
setattr(self, arg, value)
self.readers = Readers(self.settings, readers_cache_name)
# templates cache
self._templates = {}
self._templates_path = list(self.settings['THEME_TEMPLATES_OVERRIDES'])
theme_templates_path = os.path.expanduser(
os.path.join(self.theme, 'templates'))
self._templates_path.append(theme_templates_path)
theme_loader = FileSystemLoader(theme_templates_path)
simple_theme_path = os.path.dirname(os.path.abspath(__file__))
simple_loader = FileSystemLoader(
os.path.join(simple_theme_path, "themes", "simple", "templates"))
self.env = Environment(
loader=ChoiceLoader([
FileSystemLoader(self._templates_path),
simple_loader, # implicit inheritance
PrefixLoader({
'!simple': simple_loader,
'!theme': theme_loader
}) # explicit ones
]),
**self.settings['JINJA_ENVIRONMENT']
)
logger.debug('Template list: %s', self.env.list_templates())
# provide utils.strftime as a jinja filter
self.env.filters.update({'strftime': DateFormatter()})
# get custom Jinja filters from user settings
custom_filters = self.settings['JINJA_FILTERS']
self.env.filters.update(custom_filters)
# get custom Jinja globals from user settings
custom_globals = self.settings['JINJA_GLOBALS']
self.env.globals.update(custom_globals)
# get custom Jinja tests from user settings
custom_tests = self.settings['JINJA_TESTS']
self.env.tests.update(custom_tests)
signals.generator_init.send(self)
def __init__(self,
context,
settings,
path,
theme,
output_path,
readers_cache_name='',
**kwargs):
self.context = context
self.settings = settings
self.path = path
self.theme = theme
self.output_path = output_path
for arg, value in kwargs.items():
setattr(self, arg, value)
self.readers = Readers(self.settings, readers_cache_name)
# templates cache
self._templates = {}
self._templates_path = []
self._templates_path.append(
os.path.expanduser(os.path.join(self.theme, 'templates')))
self._templates_path += self.settings['EXTRA_TEMPLATES_PATHS']
theme_path = os.path.dirname(os.path.abspath(__file__))
simple_loader = FileSystemLoader(
os.path.join(theme_path, "themes", "simple", "templates"))
self.env = Environment(
trim_blocks=True,
lstrip_blocks=True,
loader=ChoiceLoader([
FileSystemLoader(self._templates_path),
simple_loader, # implicit inheritance
PrefixLoader({'!simple': simple_loader}) # explicit one
]),
extensions=self.settings['JINJA_EXTENSIONS'],
)
logger.debug('template list: {0}'.format(self.env.list_templates()))
# provide utils.strftime as a jinja filter
self.env.filters.update({'strftime': DateFormatter()})
# get custom Jinja filters from user settings
custom_filters = self.settings['JINJA_FILTERS']
self.env.filters.update(custom_filters)
signals.generator_init.send(self)
class FeedAlterSettingsWriter(Writer):
def __init__(self, output_path, settings=None):
super(FeedAlterSettingsWriter, self).__init__(
output_path, settings=settings)
# copy and alter settings
new_settings = copy.deepcopy(self.settings)
self.settings[CALLBACK_NAME](new_settings)
# put the new settings into readers
self.readers = Readers(new_settings)
def alter_item(self, item):
# split path to dir and filename
abspath = os.path.abspath(item.source_path)
base_path = os.path.dirname(abspath)
path = os.path.basename(abspath)
# read content with the new settings
return self.readers.read_file(base_path=base_path, path=path,
content_class=item.__class__)
def _add_item_to_the_feed(self, feed, item):
# sunstitute the item with new one created with the new settings
item = self.alter_item(item)
super(FeedAlterSettingsWriter, self)._add_item_to_the_feed(feed, item)
def add_static_comments(gen, content):
if gen.settings['PELICAN_COMMENT_SYSTEM'] is not True:
return
global _all_comments
content.comments_count = 0
content.comments = []
mirror_to_translations(content)
# Modify the local context, so we get proper values for the feed
context = copy.copy(gen.context)
context['SITEURL'] += "/" + content.url
context['SITENAME'] += " - Comments: " + content.title
context['SITESUBTITLE'] = ""
folder = os.path.join(gen.settings['PATH'],
gen.settings['PELICAN_COMMENT_SYSTEM_DIR'],
content.slug)
if not os.path.isdir(folder):
logger.debug("No comments found for: %s", content.slug)
write_feed(gen, [], context, content.slug)
return
reader = Readers(gen.settings)
comments = []
replies = []
for file in os.listdir(folder):
name, extension = os.path.splitext(file)
if extension[1:].lower() in reader.extensions:
com = reader.read_file(base_path=folder,
path=file,
content_class=Comment,
context=context)
com.article = content
_all_comments.append(com)
if hasattr(com, 'replyto'):
replies.append(com)
else:
comments.append(com)
feed_items = sorted(comments + replies)
feed_items.reverse()
warn_on_slug_collision(feed_items)
write_feed(gen, feed_items, context, content.slug)
# TODO: Fix this O(n²) loop
for reply in replies:
found_parent = False
for comment in chain(comments, replies):
if comment.slug == reply.replyto:
comment.addReply(reply)
found_parent = True
break
if not found_parent:
logger.warning(
'Comment "%s/%s" is a reply to non-existent comment "%s". '
'Make sure the replyto attribute is set correctly.',
content.slug, reply.slug, reply.replyto)
count = 0
for comment in comments:
comment.sortReplies()
count += comment.countReplies()
comments = sorted(comments)
content.comments_count = len(comments) + count
content.comments = comments
mirror_to_translations(content)
def add_static_comments(gen, content):
if gen.settings['PELICAN_COMMENT_SYSTEM'] is not True:
return
global _all_comments
content.comments_count = 0
content.comments = []
mirror_to_translations(content)
# Modify the local context, so we get proper values for the feed
context = copy.copy(gen.context)
context['SITEURL'] += "/" + content.url
context['SITENAME'] += " - Comments: " + content.title
context['SITESUBTITLE'] = ""
folder = os.path.join(
gen.settings['PATH'],
gen.settings['PELICAN_COMMENT_SYSTEM_DIR'],
content.slug
)
if not os.path.isdir(folder):
logger.debug("No comments found for: %s", content.slug)
write_feed(gen, [], context, content.slug)
return
reader = Readers(gen.settings)
comments = []
replies = []
for file in os.listdir(folder):
name, extension = os.path.splitext(file)
if extension[1:].lower() in reader.extensions:
com = reader.read_file(
base_path=folder, path=file,
content_class=Comment, context=context)
com.article = content
_all_comments.append(com)
if hasattr(com, 'replyto'):
replies.append(com)
else:
comments.append(com)
feed_items = sorted(comments + replies)
feed_items.reverse()
warn_on_slug_collision(feed_items)
write_feed(gen, feed_items, context, content.slug)
# TODO: Fix this O(n²) loop
for reply in replies:
found_parent = False
for comment in chain(comments, replies):
if comment.slug == reply.replyto:
comment.addReply(reply)
found_parent = True
break
if not found_parent:
logger.warning('Comment "%s/%s" is a reply to non-existent comment "%s". '
'Make sure the replyto attribute is set correctly.',
content.slug, reply.slug, reply.replyto)
count = 0
for comment in comments:
comment.sortReplies()
count += comment.countReplies()
comments = sorted(comments)
content.comments_count = len(comments) + count
content.comments = comments
mirror_to_translations(content)
def main():
args = parse_arguments()
logs_dedup_min_level = getattr(logging, args.logs_dedup_min_level)
init_logging(args.verbosity, args.fatal,
logs_dedup_min_level=logs_dedup_min_level)
logger.debug('Pelican version: %s', __version__)
logger.debug('Python version: %s', sys.version.split()[0])
try:
pelican, settings = get_instance(args)
readers = Readers(settings)
reader_descs = sorted(set(['%s (%s)' %
(type(r).__name__,
', '.join(r.file_extensions))
for r in readers.readers.values()
if r.enabled]))
watchers = {'content': folder_watcher(pelican.path,
readers.extensions,
pelican.ignore_files),
'theme': folder_watcher(pelican.theme,
[''],
pelican.ignore_files),
'settings': file_watcher(args.settings)}
old_static = settings.get("STATIC_PATHS", [])
for static_path in old_static:
# use a prefix to avoid possible overriding of standard watchers
# above
watchers['[static]%s' % static_path] = folder_watcher(
os.path.join(pelican.path, static_path),
[''],
pelican.ignore_files)
if args.autoreload and args.listen:
excqueue = multiprocessing.Queue()
p1 = multiprocessing.Process(
target=autoreload,
args=(watchers, args, old_static, reader_descs, excqueue))
p2 = multiprocessing.Process(
target=listen,
args=(settings.get('BIND'), settings.get('PORT'),
settings.get("OUTPUT_PATH"), excqueue))
p1.start()
p2.start()
exc = excqueue.get()
p1.terminate()
p2.terminate()
logger.critical(exc)
elif args.autoreload:
print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
' `settings` for changes. ---')
autoreload(watchers, args, old_static, reader_descs)
elif args.listen:
listen(settings.get('BIND'), settings.get('PORT'),
settings.get("OUTPUT_PATH"))
else:
if next(watchers['content']) is None:
logger.warning(
'No valid files found in content for '
+ 'the active readers:\n'
+ '\n'.join(reader_descs))
if next(watchers['theme']) is None:
logger.warning('Empty theme folder. Using `basic` theme.')
pelican.run()
except Exception as e:
logger.critical('%s', e)
if args.verbosity == logging.DEBUG:
raise
else:
sys.exit(getattr(e, 'exitcode', 1))
def main():
args = parse_arguments()
logs_dedup_min_level = getattr(logging, args.logs_dedup_min_level)
init_logging(args.verbosity, args.fatal,
logs_dedup_min_level=logs_dedup_min_level)
logger.debug('Pelican version: %s', __version__)
logger.debug('Python version: %s', sys.version.split()[0])
try:
pelican, settings = get_instance(args)
readers = Readers(settings)
reader_descs = sorted(set(['%s (%s)' %
(type(r).__name__,
', '.join(r.file_extensions))
for r in readers.readers.values()
if r.enabled]))
watchers = {'content': folder_watcher(pelican.path,
readers.extensions,
pelican.ignore_files),
'theme': folder_watcher(pelican.theme,
[''],
pelican.ignore_files),
'settings': file_watcher(args.settings)}
old_static = settings.get("STATIC_PATHS", [])
for static_path in old_static:
# use a prefix to avoid possible overriding of standard watchers
# above
watchers['[static]%s' % static_path] = folder_watcher(
os.path.join(pelican.path, static_path),
[''],
pelican.ignore_files)
if args.autoreload:
print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
' `settings` for changes. ---')
while True:
try:
# Check source dir for changed files ending with the given
# extension in the settings. In the theme dir is no such
# restriction; all files are recursively checked if they
# have changed, no matter what extension the filenames
# have.
modified = {k: next(v) for k, v in watchers.items()}
if modified['settings']:
pelican, settings = get_instance(args)
# Adjust static watchers if there are any changes
new_static = settings.get("STATIC_PATHS", [])
# Added static paths
# Add new watchers and set them as modified
new_watchers = set(new_static).difference(old_static)
for static_path in new_watchers:
static_key = '[static]%s' % static_path
watchers[static_key] = folder_watcher(
os.path.join(pelican.path, static_path),
[''],
pelican.ignore_files)
modified[static_key] = next(watchers[static_key])
# Removed static paths
# Remove watchers and modified values
old_watchers = set(old_static).difference(new_static)
for static_path in old_watchers:
static_key = '[static]%s' % static_path
watchers.pop(static_key)
modified.pop(static_key)
# Replace old_static with the new one
old_static = new_static
if any(modified.values()):
print('\n-> Modified: {}. re-generating...'.format(
', '.join(k for k, v in modified.items() if v)))
if modified['content'] is None:
logger.warning(
'No valid files found in content for '
+ 'the active readers:\n'
+ '\n'.join(reader_descs))
if modified['theme'] is None:
logger.warning('Empty theme folder. Using `basic` '
'theme.')
pelican.run()
except KeyboardInterrupt:
logger.warning("Keyboard interrupt, quitting.")
break
except Exception as e:
if (args.verbosity == logging.DEBUG):
raise
logger.warning(
'Caught exception "%s". Reloading.', e)
finally:
time.sleep(.5) # sleep to avoid cpu load
else:
if next(watchers['content']) is None:
logger.warning(
'No valid files found in content for '
+ 'the active readers:\n'
+ '\n'.join(reader_descs))
if next(watchers['theme']) is None:
logger.warning('Empty theme folder. Using `basic` theme.')
pelican.run()
except Exception as e:
logger.critical('%s', e)
if args.verbosity == logging.DEBUG:
raise
else:
sys.exit(getattr(e, 'exitcode', 1))
def read_file(self, path, **kwargs):
# Isolate from future API changes to readers.read_file
r = Readers(settings=get_settings(**kwargs))
return r.read_file(base_path=CONTENT_PATH, path=path)
def assert_rst_equal(self, rstfile, expected, **kwargs):
pelican = Pelican(settings=get_settings(**kwargs))
reader = Readers(pelican.settings)
content = reader.read_file(base_path=RESOURCES_PATH, path=rstfile).content
self.assertEqual(normalize(content), normalize(expected))
def main():
args = parse_arguments()
init(args.verbosity)
pelican, settings = get_instance(args)
readers = Readers(settings)
watchers = {
'content':
folder_watcher(pelican.path, readers.extensions, pelican.ignore_files),
'theme':
folder_watcher(pelican.theme, [''], pelican.ignore_files),
'settings':
file_watcher(args.settings)
}
old_static = settings.get("STATIC_PATHS", [])
for static_path in old_static:
# use a prefix to avoid possible overriding of standard watchers above
watchers['[static]%s' % static_path] = folder_watcher(
os.path.join(pelican.path, static_path), [''],
pelican.ignore_files)
try:
if args.autoreload:
print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
' `settings` for changes. ---')
def _ignore_cache(pelican_obj):
if pelican_obj.settings['AUTORELOAD_IGNORE_CACHE']:
pelican_obj.settings['LOAD_CONTENT_CACHE'] = False
while True:
try:
# Check source dir for changed files ending with the given
# extension in the settings. In the theme dir is no such
# restriction; all files are recursively checked if they
# have changed, no matter what extension the filenames
# have.
modified = {k: next(v) for k, v in watchers.items()}
original_load_cache = settings['LOAD_CONTENT_CACHE']
if modified['settings']:
pelican, settings = get_instance(args)
original_load_cache = settings['LOAD_CONTENT_CACHE']
_ignore_cache(pelican)
# Adjust static watchers if there are any changes
new_static = settings.get("STATIC_PATHS", [])
# Added static paths
# Add new watchers and set them as modified
for static_path in set(new_static).difference(
old_static):
static_key = '[static]%s' % static_path
watchers[static_key] = folder_watcher(
os.path.join(pelican.path, static_path), [''],
pelican.ignore_files)
modified[static_key] = next(watchers[static_key])
# Removed static paths
# Remove watchers and modified values
for static_path in set(old_static).difference(
new_static):
static_key = '[static]%s' % static_path
watchers.pop(static_key)
modified.pop(static_key)
# Replace old_static with the new one
old_static = new_static
if any(modified.values()):
print('\n-> Modified: {}. re-generating...'.format(
', '.join(k for k, v in modified.items() if v)))
if modified['content'] is None:
logger.warning('No valid files found in content.')
if modified['theme'] is None:
logger.warning('Empty theme folder. Using `basic` '
'theme.')
pelican.run()
# restore original caching policy
pelican.settings[
'LOAD_CONTENT_CACHE'] = original_load_cache
except KeyboardInterrupt:
logger.warning("Keyboard interrupt, quitting.")
break
except Exception as e:
if (args.verbosity == logging.DEBUG):
logger.critical(e.args)
raise
logger.warning('Caught exception "%s". Reloading.', e)
finally:
time.sleep(.5) # sleep to avoid cpu load
else:
if next(watchers['content']) is None:
logger.warning('No valid files found in content.')
if next(watchers['theme']) is None:
logger.warning('Empty theme folder. Using `basic` theme.')
pelican.run()
except Exception as e:
logger.critical('%s', e)
if args.verbosity == logging.DEBUG:
raise
else:
sys.exit(getattr(e, 'exitcode', 1))
def main():
args = parse_arguments()
init(args.verbosity)
pelican, settings = get_instance(args)
readers = Readers(settings)
watchers = {
'content':
folder_watcher(pelican.path, readers.extensions, pelican.ignore_files),
'theme':
folder_watcher(pelican.theme, [''], pelican.ignore_files),
'settings':
file_watcher(args.settings)
}
try:
if args.autoreload:
print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
' `settings` for changes. ---')
while True:
try:
# Check source dir for changed files ending with the given
# extension in the settings. In the theme dir is no such
# restriction; all files are recursively checked if they
# have changed, no matter what extension the filenames
# have.
modified = {k: next(v) for k, v in watchers.items()}
if modified['settings']:
pelican, settings = get_instance(args)
if any(modified.values()):
print('\n-> Modified: {}. re-generating...'.format(
', '.join(k for k, v in modified.items() if v)))
if modified['content'] is None:
logger.warning('No valid files found in content.')
if modified['theme'] is None:
logger.warning(
'Empty theme folder. Using `basic` theme.')
pelican.run()
except KeyboardInterrupt:
logger.warning("Keyboard interrupt, quitting.")
break
except Exception as e:
if (args.verbosity == logging.DEBUG):
logger.critical(e.args)
raise
logger.warning(
'Caught exception "{0}". Reloading.'.format(e))
finally:
time.sleep(.5) # sleep to avoid cpu load
else:
if next(watchers['content']) is None:
logger.warning('No valid files found in content.')
if next(watchers['theme']) is None:
logger.warning('Empty theme folder. Using `basic` theme.')
pelican.run()
except Exception as e:
# localized systems have errors in native language if locale is set
# so convert the message to unicode with the correct encoding
msg = str(e)
if not six.PY3:
msg = msg.decode(locale.getpreferredencoding())
logger.critical(msg)
if (args.verbosity == logging.DEBUG):
raise
else:
sys.exit(getattr(e, 'exitcode', 1))
def read_file(self, path, **kwargs):
# Isolate from future API changes to readers.read_file
r = Readers(settings=get_settings(**kwargs))
return r.read_file(base_path=CONTENT_PATH, path=path)