def generate_context(self):
all_pages = []
hidden_pages = []
for f in self.get_files(
os.path.join(self.path, self.settings['PAGE_DIR']),
exclude=self.settings['PAGE_EXCLUDES']):
try:
content, metadata = read_file(f, settings=self.settings)
except Exception as e:
logger.warning(u'Could not process %s\n%s' % (f, str(e)))
continue
signals.pages_generate_context.send(self, metadata=metadata)
page = Page(content, metadata, settings=self.settings,
filename=f, context=self.context)
if not is_valid_content(page, f):
continue
self.add_filename(page)
if page.status == "published":
all_pages.append(page)
elif page.status == "hidden":
hidden_pages.append(page)
else:
logger.warning(u"Unknown status %s for file %s, skipping it." %
(repr(unicode.encode(page.status, 'utf-8')),
repr(f)))
self.pages, self.translations = process_translations(all_pages)
self.hidden_pages, self.hidden_translations = process_translations(hidden_pages)
self._update_context(('pages', ))
self.context['PAGES'] = self.pages
def generate_context(self):
all_pages = []
hidden_pages = []
for f in self.get_files(
os.path.join(self.path, self.settings["PAGE_DIR"]), exclude=self.settings["PAGE_EXCLUDES"]
):
try:
content, metadata = read_file(f, settings=self.settings)
except Exception as e:
logger.warning("Could not process %s\n%s" % (f, str(e)))
continue
signals.pages_generate_context.send(self, metadata=metadata)
page = Page(content, metadata, settings=self.settings, source_path=f, context=self.context)
if not is_valid_content(page, f):
continue
self.add_source_path(page)
if page.status == "published":
all_pages.append(page)
elif page.status == "hidden":
hidden_pages.append(page)
else:
logger.warning("Unknown status %s for file %s, skipping it." % (repr(page.status), repr(f)))
self.pages, self.translations = process_translations(all_pages)
self.hidden_pages, self.hidden_translations = process_translations(hidden_pages)
self._update_context(("pages",))
self.context["PAGES"] = self.pages
signals.pages_generator_finalized.send(self)
def generate_context(self):
"""
Here is the meat of the class - where the heavy lifting occurs. It
generates a list of tutorials and places them in the context object so we
can access them in templates.
Some of this is leftover from the stock Article class. Ideally those aspect
will be removed as it is shown they can be safely done away with.
However, this works.
"""
all_tutorials = []
hidden_tutorials = []
for f in self.get_files(
self.settings['TUTORIAL_DIR'],
exclude=self.settings['TUTORIAL_EXCLUDES']):
tutorial = self.get_cached_data(f, None)
if tutorial is None:
try:
tutorial = self.readers.read_file(
base_path=self.path, path=f, content_class=Tutorial,
context=self.context,
preread_signal=tutorial_generator_preread,
preread_sender=self,
context_signal=tutorial_generator_context,
context_sender=self)
except Exception as e:
logger.warning('Could not process {}\n{}'.format(f, e))
continue
if not is_valid_content(tutorial, f):
continue
self.cache_data(f, tutorial)
self.add_source_path(tutorial)
if tutorial.status == "published":
all_tutorials.append(tutorial)
for author_data in self.context['site_authors']:
if author_data.name == tutorial.author.name:
tutorial.author.data = author_data
elif tutorial.status == "hidden":
hidden_tutorials.append(tutorial)
else:
logger.warning("Unknown status %s for file %s, skipping it." %
(repr(tutorial.status),
repr(f)))
self.tutorials, self.translations = process_translations(all_tutorials)
self.hidden_tutorials, self.hidden_translations = (
process_translations(hidden_tutorials))
self._update_context(('tutorials', ))
self.context['TUTORIALS'] = self.tutorials
self.save_cache()
self.readers.save_cache()
tutorial_generator_finalized.send(self)
def generate_context(self):
all_pages = []
hidden_pages = []
for f in self.get_files(self.settings['PAGE_PATHS'],
exclude=self.settings['PAGE_EXCLUDES']):
page = self.get_cached_data(f, None)
if page is None:
try:
page = self.readers.read_file(
base_path=self.path,
path=f,
content_class=Page,
context=self.context,
preread_signal=signals.page_generator_preread,
preread_sender=self,
context_signal=signals.page_generator_context,
context_sender=self)
except Exception as e:
logger.error('Could not process %s\n%s',
f,
e,
exc_info=self.settings.get('DEBUG', False))
self._add_failed_source_path(f)
continue
if not is_valid_content(page, f):
self._add_failed_source_path(f)
continue
if page.status.lower() == "published":
all_pages.append(page)
elif page.status.lower() == "hidden":
hidden_pages.append(page)
else:
logger.error(
"Unknown status '%s' for file %s, skipping it.",
page.status, f)
self._add_failed_source_path(f)
continue
self.cache_data(f, page)
self.add_source_path(page)
self.pages, self.translations = process_translations(
all_pages, order_by=self.settings['PAGE_ORDER_BY'])
self.hidden_pages, self.hidden_translations = (
process_translations(hidden_pages))
self._update_context(('pages', ))
self.context['PAGES'] = self.pages
self.save_cache()
self.readers.save_cache()
signals.page_generator_finalized.send(self)
def generate_context(self):
"""
Here is the meat of the class - where the heavy lifting occurs. It
generates a list of events and places them in the context object so we
can access them in templates.
Some of this is leftover from the stock Article class. Ideally those aspect
will be removed as it is shown they can be safely done away with.
However, this works.
"""
all_events = []
hidden_events = []
for f in self.get_files(self.settings["EVENT_DIR"], exclude=self.settings["EVENT_EXCLUDES"]):
event = self.get_cached_data(f, None)
if event is None:
try:
event = self.readers.read_file(
base_path=self.path,
path=f,
content_class=Event,
context=self.context,
preread_signal=event_generator_preread,
preread_sender=self,
context_signal=event_generator_context,
context_sender=self,
)
except Exception as e:
logger.warning("Could not process {}\n{}".format(f, e))
continue
if not is_valid_content(event, f):
continue
self.cache_data(f, event)
self.add_source_path(event)
if event.status == "published":
all_events.append(event)
elif event.status == "hidden":
hidden_events.append(event)
else:
logger.warning("Unknown status %s for file %s, skipping it." % (repr(event.status), repr(f)))
self.events, self.translations = process_translations(all_events)
self.hidden_events, self.hidden_translations = process_translations(hidden_events)
self._update_context(("events",))
self.context["EVENTS"] = self.events
self.save_cache()
self.readers.save_cache()
event_generator_finalized.send(self)
def generate_context(self):
all_pages = []
hidden_pages = []
for f in self.get_files(
self.settings['PAGE_PATHS'],
exclude=self.settings['PAGE_EXCLUDES']):
page = self.get_cached_data(f, None)
if page is None:
try:
page = self.readers.read_file(
base_path=self.path, path=f, content_class=Page,
context=self.context,
preread_signal=signals.page_generator_preread,
preread_sender=self,
context_signal=signals.page_generator_context,
context_sender=self)
except Exception as e:
logger.error(
'Could not process %s\n%s', f, e,
exc_info=self.settings.get('DEBUG', False))
self._add_failed_source_path(f)
continue
if not is_valid_content(page, f):
self._add_failed_source_path(f)
continue
if page.status.lower() == "published":
all_pages.append(page)
elif page.status.lower() == "hidden":
hidden_pages.append(page)
else:
logger.error(
"Unknown status '%s' for file %s, skipping it.",
page.status, f)
self._add_failed_source_path(f)
continue
self.cache_data(f, page)
self.add_source_path(page)
self.pages, self.translations = process_translations(
all_pages,
order_by=self.settings['PAGE_ORDER_BY'])
self.hidden_pages, self.hidden_translations = \
process_translations(hidden_pages)
self._update_context(('pages', 'hidden_pages'))
self.save_cache()
self.readers.save_cache()
signals.page_generator_finalized.send(self)
def generate_output(self, writer):
start_time = time.time()
page_count = 0
article_count = 0
write = partial(writer.write_file,
relative_urls=self.settings.get('RELATIVE_URLS'))
for cat in self.config:
category_name = cat.get('category_name')
if not category_name:
warning("Improperly configured Standalone Category: "
"missing required `category_name` parameter.")
continue
category_name_lc = category_name.lower()
cat_page_name = "{}.html".format(
slugify(category_name_lc,
self.settings.get('SLUG_SUBSTITUTIONS', ())))
articles = self.context['standalone_articles'][category_name_lc]
write(cat_page_name,
self.get_template(cat['page_template']),
self.context,
articles=articles,
category=cat['category_name'])
page_count += 1
the_articles, translations = process_translations(articles)
for article in chain(the_articles, translations):
write(article.save_as,
self.get_template(cat['article_template']),
self.context,
article=article,
category=article.category,
override_output=hasattr(article, 'override_save_as'))
article_count += 1
print("Standalone categories done: Processed {} articles and {}"
" pages in {:.2f} seconds.".format(article_count, page_count,
time.time() - start_time))
class PagesGenerator(Generator):
"""Generate pages"""
def __init__(self, *args, **kwargs):
self.pages = []
super(PagesGenerator, self).__init__(*args, **kwargs)
def generate_context(self):
all_pages = []
for f in self.get_files(os.path.join(self.path,
self.settings['PAGE_DIR']),
exclude=self.settings['PAGE_EXCLUDES']):
try:
content, metadata = read_file(f)
except Exception, e:
logger.error(u'Could not process %s\n%s' % (f, str(e)))
continue
page = Page(content, metadata, settings=self.settings, filename=f)
if not is_valid_content(page, f):
continue
all_pages.append(page)
self.pages, self.translations = process_translations(all_pages)
self._update_context(('pages', ))
self.context['PAGES'] = self.pages
def generate_context(self):
all_pages = []
hidden_pages = []
for f in self.get_files(
self.settings['PAGE_DIR'],
exclude=self.settings['PAGE_EXCLUDES']):
page = self.get_cached_data(f, None)
if page is None:
try:
page = self.readers.read_file(
base_path=self.path, path=f, content_class=Page,
context=self.context,
preread_signal=signals.page_generator_preread,
preread_sender=self,
context_signal=signals.page_generator_context,
context_sender=self)
except Exception as e:
logger.warning('Could not process {}\n{}'.format(f, e))
continue
if not is_valid_content(page, f):
continue
self.cache_data(f, page)
self.add_source_path(page)
if page.status == "published":
all_pages.append(page)
elif page.status == "hidden":
hidden_pages.append(page)
else:
logger.warning("Unknown status %s for file %s, skipping it." %
(repr(page.status),
repr(f)))
self.pages, self.translations = process_translations(all_pages)
self.hidden_pages, self.hidden_translations = (
process_translations(hidden_pages))
self._update_context(('pages', ))
self.context['PAGES'] = self.pages
self.save_cache()
self.readers.save_cache()
signals.page_generator_finalized.send(self)
def generate_context(self):
all_pages = []
hidden_pages = []
for f in self.get_files(self.settings['PAGE_DIR'],
exclude=self.settings['PAGE_EXCLUDES']):
page = self.get_cached_data(f, None)
if page is None:
try:
page = self.readers.read_file(
base_path=self.path,
path=f,
content_class=Page,
context=self.context,
preread_signal=signals.page_generator_preread,
preread_sender=self,
context_signal=signals.page_generator_context,
context_sender=self)
except Exception as e:
logger.warning('Could not process {}\n{}'.format(f, e))
continue
if not is_valid_content(page, f):
continue
self.cache_data(f, page)
self.add_source_path(page)
if page.status == "published":
all_pages.append(page)
elif page.status == "hidden":
hidden_pages.append(page)
else:
logger.warning("Unknown status %s for file %s, skipping it." %
(repr(page.status), repr(f)))
self.pages, self.translations = process_translations(all_pages)
self.hidden_pages, self.hidden_translations = (
process_translations(hidden_pages))
self._update_context(('pages', ))
self.context['PAGES'] = self.pages
self.save_cache()
self.readers.save_cache()
signals.page_generator_finalized.send(self)
def test_process_translations(self):
fr_articles = []
en_articles = []
# create a bunch of articles
# 0: no translation metadata
fr_articles.append(get_article(lang='fr', slug='yay0', title='Titre',
content='en français'))
en_articles.append(get_article(lang='en', slug='yay0', title='Title',
content='in english'))
# 1: translation metadata on default lang
fr_articles.append(get_article(lang='fr', slug='yay1', title='Titre',
content='en français'))
en_articles.append(get_article(lang='en', slug='yay1', title='Title',
content='in english',
extra_metadata={'translation': 'true'}))
# 2: translation metadata not on default lang
fr_articles.append(get_article(lang='fr', slug='yay2', title='Titre',
content='en français',
extra_metadata={'translation': 'true'}))
en_articles.append(get_article(lang='en', slug='yay2', title='Title',
content='in english'))
# 3: back to default language detection if all items have the
# translation metadata
fr_articles.append(get_article(lang='fr', slug='yay3', title='Titre',
content='en français',
extra_metadata={'translation': 'yep'}))
en_articles.append(get_article(lang='en', slug='yay3', title='Title',
content='in english',
extra_metadata={'translation': 'yes'}))
# try adding articles in both orders
for lang0_articles, lang1_articles in ((fr_articles, en_articles),
(en_articles, fr_articles)):
articles = lang0_articles + lang1_articles
index, trans = utils.process_translations(articles)
self.assertIn(en_articles[0], index)
self.assertIn(fr_articles[0], trans)
self.assertNotIn(en_articles[0], trans)
self.assertNotIn(fr_articles[0], index)
self.assertIn(fr_articles[1], index)
self.assertIn(en_articles[1], trans)
self.assertNotIn(fr_articles[1], trans)
self.assertNotIn(en_articles[1], index)
self.assertIn(en_articles[2], index)
self.assertIn(fr_articles[2], trans)
self.assertNotIn(en_articles[2], trans)
self.assertNotIn(fr_articles[2], index)
self.assertIn(en_articles[3], index)
self.assertIn(fr_articles[3], trans)
self.assertNotIn(en_articles[3], trans)
self.assertNotIn(fr_articles[3], index)
def test_process_translations(self):
fr_articles = []
en_articles = []
# create a bunch of articles
# 0: no translation metadata
fr_articles.append(get_article(lang='fr', slug='yay0', title='Titre',
content='en français'))
en_articles.append(get_article(lang='en', slug='yay0', title='Title',
content='in english'))
# 1: translation metadata on default lang
fr_articles.append(get_article(lang='fr', slug='yay1', title='Titre',
content='en français'))
en_articles.append(get_article(lang='en', slug='yay1', title='Title',
content='in english',
extra_metadata={'translation': 'true'}))
# 2: translation metadata not on default lang
fr_articles.append(get_article(lang='fr', slug='yay2', title='Titre',
content='en français',
extra_metadata={'translation': 'true'}))
en_articles.append(get_article(lang='en', slug='yay2', title='Title',
content='in english'))
# 3: back to default language detection if all items have the
# translation metadata
fr_articles.append(get_article(lang='fr', slug='yay3', title='Titre',
content='en français',
extra_metadata={'translation': 'yep'}))
en_articles.append(get_article(lang='en', slug='yay3', title='Title',
content='in english',
extra_metadata={'translation': 'yes'}))
# try adding articles in both orders
for lang0_articles, lang1_articles in ((fr_articles, en_articles),
(en_articles, fr_articles)):
articles = lang0_articles + lang1_articles
index, trans = utils.process_translations(articles)
self.assertIn(en_articles[0], index)
self.assertIn(fr_articles[0], trans)
self.assertNotIn(en_articles[0], trans)
self.assertNotIn(fr_articles[0], index)
self.assertIn(fr_articles[1], index)
self.assertIn(en_articles[1], trans)
self.assertNotIn(fr_articles[1], trans)
self.assertNotIn(en_articles[1], index)
self.assertIn(en_articles[2], index)
self.assertIn(fr_articles[2], trans)
self.assertNotIn(en_articles[2], trans)
self.assertNotIn(fr_articles[2], index)
self.assertIn(en_articles[3], index)
self.assertIn(fr_articles[3], trans)
self.assertNotIn(en_articles[3], trans)
self.assertNotIn(fr_articles[3], index)
def test_process_translations(self):
# create a bunch of articles
# 1: no translation metadata
fr_article1 = get_article(lang='fr',
slug='yay',
title='Un titre',
content='en français')
en_article1 = get_article(lang='en',
slug='yay',
title='A title',
content='in english')
# 2: reverse which one is the translation thanks to metadata
fr_article2 = get_article(lang='fr',
slug='yay2',
title='Un titre',
content='en français')
en_article2 = get_article(lang='en',
slug='yay2',
title='A title',
content='in english',
extra_metadata={'translation': 'true'})
# 3: back to default language detection if all items have the
# translation metadata
fr_article3 = get_article(lang='fr',
slug='yay3',
title='Un titre',
content='en français',
extra_metadata={'translation': 'yep'})
en_article3 = get_article(lang='en',
slug='yay3',
title='A title',
content='in english',
extra_metadata={'translation': 'yes'})
articles = [
fr_article1, en_article1, fr_article2, en_article2, fr_article3,
en_article3
]
index, trans = utils.process_translations(articles)
self.assertIn(en_article1, index)
self.assertIn(fr_article1, trans)
self.assertNotIn(en_article1, trans)
self.assertNotIn(fr_article1, index)
self.assertIn(fr_article2, index)
self.assertIn(en_article2, trans)
self.assertNotIn(fr_article2, trans)
self.assertNotIn(en_article2, index)
self.assertIn(en_article3, index)
self.assertIn(fr_article3, trans)
self.assertNotIn(en_article3, trans)
self.assertNotIn(fr_article3, index)
def generate_context(self):
"""change the context"""
# return the list of files to use
files = self.get_files(self.path, exclude=[
'pages',
])
all_articles = []
for f in files:
content, metadatas = read_file(f)
# if no category is set, use the name of the path as a category
if 'category' not in metadatas.keys():
category = os.path.basename(os.path.dirname(f))
if category == self.path:
category = self.settings['DEFAULT_CATEGORY']
if category != '':
metadatas['category'] = unicode(category)
if 'date' not in metadatas.keys()\
and self.settings['FALLBACK_ON_FS_DATE']:
metadatas['date'] = datetime.fromtimestamp(os.stat(f).st_ctime)
article = Article(content,
metadatas,
settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['REVERSE_ARCHIVE_ORDER'])
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings.get('REVERSE_CATEGORY_ORDER'))
self._update_context(('articles', 'dates', 'tags', 'categories'))
def generate_context(self):
# Update the context (only articles in default language)
self.articles = self.context['articles']
all_articles = []
new_articles = self._generate_butter_articles()
all_articles.extend(new_articles)
# Continue with the rest of ArticleGenerator, code adapted from:
# https://github.com/getpelican/pelican/blob/master/pelican/generators.py#L548
# ARTICLE_ORDER_BY doesn't exist in 3.3, which was in Fedora 21.
# (I wanted to be able to build this on F21 at the time).
articles, translations = process_translations(all_articles)
# , order_by=self.settings['ARTICLE_ORDER_BY'])
self.articles.extend(articles)
self.translations.extend(translations)
# Disabled for 3.3 compatibility, great.
# signals.article_generator_pretaxonomy.send(self)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
for author in getattr(article, 'authors', []):
self.authors[author].append(article)
# This may not technically be right, but...
# Sort the articles by date too.
self.articles = list(self.articles)
self.dates = self.articles
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
logger.info('++++++++++++++++++++++++++++++++++++')
logger.info('GOT categories %s' % str(self.categories))
logger.info('++++++++++++++++++++++++++++++++++++')
self._update_context(('articles', 'dates', 'categories', 'authors'))
def add_coming_soon(article_generator):
for draft in article_generator.drafts:
if hasattr(draft, "visible_draft") and draft.visible_draft.strip().lower() == "true":
soon_article = Article(content=None, metadata=draft.metadata)
soon_article.author = draft.author
soon_article.slug = draft.slug
soon_article.source_path = draft.source_path
article_generator.articles.append(soon_article)
article_generator.articles, article_generator.translations = process_translations(
article_generator.articles, order_by=article_generator.settings["ARTICLE_ORDER_BY"]
)
def generate_context(self):
"""
Overwrites `PagesGenerator.generate_context` and builds up `HiPage`s.
"""
root = HiPage("", slug='../index', name="index", title='Home',
parent=None, settings=self.settings)
all_pages = []
hidden_pages = []
for page_path in self.settings['PAGE_PATHS']:
ps, hs = self._scan_dir_r(
os.path.join(self.path, page_path),
exclude=self.settings['PAGE_EXCLUDES'],
parent=root)
all_pages.extend(ps)
hidden_pages.extend(hs)
self.pages, self.translations = utils.process_translations(all_pages)
self.hidden_pages, self.hidden_translations = (
utils.process_translations(hidden_pages))
self.PAGES_TREE = root
# Fix (hack): Go through all translations and add the missing sub-pages
for page in self.translations:
# find corresponding page
orig_page = None
for p in self.pages:
if p.name == page.name:
orig_page = p
break
page.sub_pages.update(orig_page.sub_pages)
self._update_context(['pages', 'hidden_pages', 'PAGES_TREE'])
# self.context['PAGES'] = self.pages
# self.context['PAGES_TREE'] = root
logger.info("\n" + ascii_tree(self.context['PAGES_TREE'],
print_item=lambda x: x.title))
signals.page_generator_finalized.send(self)
def generate_context(self):
all_pages = []
for f in self.get_files(os.sep.join((self.path, 'pages'))):
content, metadatas = read_file(f)
page = Page(content, metadatas, settings=self.settings, filename=f)
if not is_valid_content(page, f):
continue
all_pages.append(page)
self.pages, self.translations = process_translations(all_pages)
self._update_context(('pages', ))
self.context['PAGES'] = self.pages
def generate_context(self):
"""change the context"""
# return the list of files to use
files = self.get_files(self.path, exclude=['pages',])
all_articles = []
for f in files:
content, metadatas = read_file(f)
# if no category is set, use the name of the path as a category
if 'category' not in metadatas.keys():
category = os.path.basename(os.path.dirname(f))
if category == self.path:
category = self.settings['DEFAULT_CATEGORY']
if category != '':
metadatas['category'] = unicode(category)
if 'date' not in metadatas.keys()\
and self.settings['FALLBACK_ON_FS_DATE']:
metadatas['date'] = datetime.fromtimestamp(os.stat(f).st_ctime)
article = Article(content, metadatas, settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['REVERSE_ARCHIVE_ORDER'])
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings.get('REVERSE_CATEGORY_ORDER'))
self._update_context(('articles', 'dates', 'tags', 'categories'))
def generate_context(self):
all_pages = []
for f in self.get_files(os.sep.join((self.path, 'pages'))):
content, metadata = read_file(f)
page = Page(content, metadata, settings=self.settings,
filename=f)
if not is_valid_content(page, f):
continue
all_pages.append(page)
self.pages, self.translations = process_translations(all_pages)
self._update_context(('pages', ))
self.context['PAGES'] = self.pages
def test_process_translations(self):
# create a bunch of articles
fr_article1 = get_article(lang='fr', slug='yay', title='Un titre',
content='en français')
en_article1 = get_article(lang='en', slug='yay', title='A title',
content='in english')
articles = [fr_article1, en_article1]
index, trans = utils.process_translations(articles)
self.assertIn(en_article1, index)
self.assertIn(fr_article1, trans)
self.assertNotIn(en_article1, trans)
self.assertNotIn(fr_article1, index)
def test_process_translations(self):
# create a bunch of articles
fr_article1 = get_article(lang='fr', slug='yay', title='Un titre',
content='en français')
en_article1 = get_article(lang='en', slug='yay', title='A title',
content='in english')
articles = [fr_article1, en_article1]
index, trans = utils.process_translations(articles)
self.assertIn(en_article1, index)
self.assertIn(fr_article1, trans)
self.assertNotIn(en_article1, trans)
self.assertNotIn(fr_article1, index)
def generate_context(self):
all_pages = []
for f in self.get_files(os.sep.join((self.path, 'pages'))):
content, metadata = read_file(f)
page = Page(content, metadata, settings=self.settings,
filename=f)
if not is_valid_content(page, f):
continue
if self.settings.get('CLEAN_URLS_NO_PROXY'):
# cleaning page url
page.save_as = os.path.join(page.slug, 'index.html')
page.url = os.path.dirname(page.save_as) + '/'
all_pages.append(page)
self.pages, self.translations = process_translations(all_pages)
self._update_context(('pages', ))
self.context['PAGES'] = self.pages
def test_process_translations(self):
# create a bunch of articles
# 1: no translation metadata
fr_article1 = get_article(lang='fr', slug='yay', title='Un titre',
content='en français')
en_article1 = get_article(lang='en', slug='yay', title='A title',
content='in english')
# 2: reverse which one is the translation thanks to metadata
fr_article2 = get_article(lang='fr', slug='yay2', title='Un titre',
content='en français')
en_article2 = get_article(lang='en', slug='yay2', title='A title',
content='in english',
extra_metadata={'translation': 'true'})
# 3: back to default language detection if all items have the
# translation metadata
fr_article3 = get_article(lang='fr', slug='yay3', title='Un titre',
content='en français',
extra_metadata={'translation': 'yep'})
en_article3 = get_article(lang='en', slug='yay3', title='A title',
content='in english',
extra_metadata={'translation': 'yes'})
articles = [fr_article1, en_article1, fr_article2, en_article2,
fr_article3, en_article3]
index, trans = utils.process_translations(articles)
self.assertIn(en_article1, index)
self.assertIn(fr_article1, trans)
self.assertNotIn(en_article1, trans)
self.assertNotIn(fr_article1, index)
self.assertIn(fr_article2, index)
self.assertIn(en_article2, trans)
self.assertNotIn(fr_article2, trans)
self.assertNotIn(en_article2, index)
self.assertIn(en_article3, index)
self.assertIn(fr_article3, trans)
self.assertNotIn(en_article3, trans)
self.assertNotIn(fr_article3, index)
def test_process_translations(self):
# create a bunch of articles
# 1: no translation metadata
fr_article1 = get_article(lang="fr", slug="yay", title="Un titre", content="en français")
en_article1 = get_article(lang="en", slug="yay", title="A title", content="in english")
# 2: reverse which one is the translation thanks to metadata
fr_article2 = get_article(lang="fr", slug="yay2", title="Un titre", content="en français")
en_article2 = get_article(
lang="en", slug="yay2", title="A title", content="in english", extra_metadata={"translation": "true"}
)
# 3: back to default language detection if all items have the
# translation metadata
fr_article3 = get_article(
lang="fr", slug="yay3", title="Un titre", content="en français", extra_metadata={"translation": "yep"}
)
en_article3 = get_article(
lang="en", slug="yay3", title="A title", content="in english", extra_metadata={"translation": "yes"}
)
articles = [fr_article1, en_article1, fr_article2, en_article2, fr_article3, en_article3]
index, trans = utils.process_translations(articles)
self.assertIn(en_article1, index)
self.assertIn(fr_article1, trans)
self.assertNotIn(en_article1, trans)
self.assertNotIn(fr_article1, index)
self.assertIn(fr_article2, index)
self.assertIn(en_article2, trans)
self.assertNotIn(fr_article2, trans)
self.assertNotIn(en_article2, index)
self.assertIn(en_article3, index)
self.assertIn(fr_article3, trans)
self.assertNotIn(en_article3, trans)
self.assertNotIn(fr_article3, index)
class ArticlesGenerator(Generator):
"""Generate blog articles"""
def __init__(self, *args, **kwargs):
"""initialize properties"""
self.articles = [] # only articles in default language
self.translations = []
self.dates = {}
self.tags = defaultdict(list)
self.categories = defaultdict(list)
self.authors = defaultdict(list)
super(ArticlesGenerator, self).__init__(*args, **kwargs)
self.drafts = []
signals.article_generator_init.send(self)
def generate_feeds(self, writer):
"""Generate the feeds from the current context, and output files."""
if self.settings.get('FEED'):
writer.write_feed(self.articles, self.context,
self.settings['FEED'])
if self.settings.get('FEED_RSS'):
writer.write_feed(self.articles,
self.context,
self.settings['FEED_RSS'],
feed_type='rss')
for cat, arts in self.categories:
arts.sort(key=attrgetter('date'), reverse=True)
if self.settings.get('CATEGORY_FEED'):
writer.write_feed(arts, self.context,
self.settings['CATEGORY_FEED'] % cat)
if self.settings.get('CATEGORY_FEED_RSS'):
writer.write_feed(arts,
self.context,
self.settings['CATEGORY_FEED_RSS'] % cat,
feed_type='rss')
if self.settings.get('TAG_FEED') or self.settings.get('TAG_FEED_RSS'):
for tag, arts in self.tags.items():
arts.sort(key=attrgetter('date'), reverse=True)
if self.settings.get('TAG_FEED'):
writer.write_feed(arts, self.context,
self.settings['TAG_FEED'] % tag)
if self.settings.get('TAG_FEED_RSS'):
writer.write_feed(arts,
self.context,
self.settings['TAG_FEED_RSS'] % tag,
feed_type='rss')
if self.settings.get('TRANSLATION_FEED'):
translations_feeds = defaultdict(list)
for article in chain(self.articles, self.translations):
translations_feeds[article.lang].append(article)
for lang, items in translations_feeds.items():
items.sort(key=attrgetter('date'), reverse=True)
writer.write_feed(items, self.context,
self.settings['TRANSLATION_FEED'] % lang)
def generate_articles(self, write):
"""Generate the articles."""
article_template = self.get_template('article')
for article in chain(self.translations, self.articles):
write(article.save_as,
article_template,
self.context,
article=article,
category=article.category)
def generate_direct_templates(self, write):
"""Generate direct templates pages"""
PAGINATED_TEMPLATES = self.settings.get('PAGINATED_DIRECT_TEMPLATES')
for template in self.settings.get('DIRECT_TEMPLATES'):
paginated = {}
if template in PAGINATED_TEMPLATES:
paginated = {'articles': self.articles, 'dates': self.dates}
save_as = self.settings.get("%s_SAVE_AS" % template.upper(),
'%s.html' % template)
if not save_as:
continue
write(save_as,
self.get_template(template),
self.context,
blog=True,
paginated=paginated,
page_name=template)
def generate_tags(self, write):
"""Generate Tags pages."""
tag_template = self.get_template('tag')
for tag, articles in self.tags.items():
articles.sort(key=attrgetter('date'), reverse=True)
dates = [article for article in self.dates if article in articles]
write(tag.save_as,
tag_template,
self.context,
tag=tag,
articles=articles,
dates=dates,
paginated={
'articles': articles,
'dates': dates
},
page_name=u'tag/%s' % tag)
def generate_categories(self, write):
"""Generate category pages."""
category_template = self.get_template('category')
for cat, articles in self.categories:
dates = [article for article in self.dates if article in articles]
write(cat.save_as,
category_template,
self.context,
category=cat,
articles=articles,
dates=dates,
paginated={
'articles': articles,
'dates': dates
},
page_name=u'category/%s' % cat)
def generate_authors(self, write):
"""Generate Author pages."""
author_template = self.get_template('author')
for aut, articles in self.authors:
dates = [article for article in self.dates if article in articles]
write(aut.save_as,
author_template,
self.context,
author=aut,
articles=articles,
dates=dates,
paginated={
'articles': articles,
'dates': dates
},
page_name=u'author/%s' % aut)
def generate_drafts(self, write):
"""Generate drafts pages."""
article_template = self.get_template('article')
for article in self.drafts:
write('drafts/%s.html' % article.slug,
article_template,
self.context,
article=article,
category=article.category)
def generate_pages(self, writer):
"""Generate the pages on the disk"""
write = partial(writer.write_file,
relative_urls=self.settings.get('RELATIVE_URLS'))
# to minimize the number of relative path stuff modification
# in writer, articles pass first
self.generate_articles(write)
self.generate_direct_templates(write)
# and subfolders after that
self.generate_tags(write)
self.generate_categories(write)
self.generate_authors(write)
self.generate_drafts(write)
def generate_context(self):
"""change the context"""
article_path = os.path.normpath( # we have to remove trailing slashes
os.path.join(self.path, self.settings['ARTICLE_DIR']))
all_articles = []
for f in self.get_files(article_path,
exclude=self.settings['ARTICLE_EXCLUDES']):
try:
content, metadata = read_file(f, settings=self.settings)
except Exception, e:
logger.warning(u'Could not process %s\n%s' % (f, str(e)))
continue
# if no category is set, use the name of the path as a category
if 'category' not in metadata:
if os.path.dirname(
f
) == article_path: # if the article is not in a subdirectory
category = self.settings['DEFAULT_CATEGORY']
else:
category = os.path.basename(os.path.dirname(f))\
.decode('utf-8')
if category != '':
metadata['category'] = Category(category, self.settings)
if 'date' not in metadata and self.settings['FALLBACK_ON_FS_DATE']:
metadata['date'] = datetime.datetime.fromtimestamp(
os.stat(f).st_ctime)
signals.article_generate_context.send(self, metadata=metadata)
article = Article(content,
metadata,
settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
if article.status == "published":
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
else:
logger.warning(
u"Unknown status %s for file %s, skipping it." %
(repr(unicode.encode(article.status, 'utf-8')), repr(f)))
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['REVERSE_ARCHIVE_ORDER'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = map(itemgetter(1), tag_cloud)
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(tag,
int(
math.floor(steps - (steps - 1) * math.log(count) /
(math.log(max_count) or 1))))
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(key=lambda item: item[0].name,
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort(key=lambda item: item[0].name)
self._update_context(('articles', 'dates', 'tags', 'categories',
'tag_cloud', 'authors'))
def generate_context(self):
"""Add the articles into the shared context"""
all_articles = []
all_drafts = []
for f in self.get_files(
self.settings['ARTICLE_PATHS'],
exclude=self.settings['ARTICLE_EXCLUDES']):
article_or_draft = self.get_cached_data(f, None)
if article_or_draft is None:
#TODO needs overhaul, maybe nomad for read_file solution, unified behaviour
try:
article_or_draft = self.readers.read_file(
base_path=self.path, path=f, content_class=Article,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self)
except Exception as e:
logger.error('Could not process %s\n%s', f, e,
exc_info=self.settings.get('DEBUG', False))
self._add_failed_source_path(f)
continue
if not is_valid_content(article_or_draft, f):
self._add_failed_source_path(f)
continue
if article_or_draft.status.lower() == "published":
all_articles.append(article_or_draft)
elif article_or_draft.status.lower() == "draft":
article_or_draft = self.readers.read_file(
base_path=self.path, path=f, content_class=Draft,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self)
self.add_source_path(article_or_draft)
all_drafts.append(article_or_draft)
else:
logger.error("Unknown status '%s' for file %s, skipping it.",
article_or_draft.status, f)
self._add_failed_source_path(f)
continue
self.cache_data(f, article_or_draft)
self.add_source_path(article_or_draft)
self.articles, self.translations = process_translations(all_articles,
order_by=self.settings['ARTICLE_ORDER_BY'])
self.drafts, self.drafts_translations = \
process_translations(all_drafts)
signals.article_generator_pretaxonomy.send(self)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
for author in getattr(article, 'authors', []):
self.authors[author].append(article)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories',
'authors', 'related_posts', 'drafts'))
self.save_cache()
self.readers.save_cache()
signals.article_generator_finalized.send(self)
def generate_context(self):
all_pages = []
limit = 20
urls = []
num = 0
news = [i for i in who_news.get_feeds() if i['date'] and i['title']]
news.sort(key=lambda i: i['date'], reverse=True)
page_num = 0
pages = math.ceil(len(news) / limit)
for article in news:
article['date_str'] = ''
if article['date']:
article['date_str'] = datetime.date.strftime(
article['date'], '%d %b %Y')
try:
fpath = os.path.join(
'news',
slugify('{}-{}-{}'.format(article['date_str'][:20],
article['title'][:50], num)) +
'.html')
except UnicodeEncodeError:
logger.info('Could not encode airline')
continue
num += 1
metadata = {
'title': article['title'],
'article': article,
'save_as': fpath
}
page = contents.Page(deepcopy(article), metadata=metadata)
article['url'] = fpath
all_pages.append(page)
urls.append(article)
if len(urls) >= limit:
page_num += 1
url = 'newsroom.html' if page_num == 1 else f'newsroom-{page_num}.html'
all_pages.append(
contents.Page({'urls': urls},
metadata={
'title': 'News list',
'save_as': url,
'template': 'news_list',
'urls': urls,
'pages_num': page_num,
'pages': pages
}))
urls = []
if urls:
page_num += 1
url = 'newsroom.html' if page_num == 1 else f'newsroom-{page_num}.html'
all_pages.append(
contents.Page({'urls': urls},
metadata={
'title': 'News list',
'save_as': url,
'template': 'news_list',
'urls': urls,
'pages_num': page_num,
'pages': pages
}))
self.pages, self.translations = process_translations(all_pages, )
signals.page_generator_finalized.send(self)
def generate_context(self):
"""Add the articles into the shared context"""
all_articles = []
all_drafts = []
for f in self.get_files(
self.settings['ARTICLE_PATHS'],
exclude=self.settings['ARTICLE_EXCLUDES']):
article = self.get_cached_data(f, None)
if article is None:
try:
article = self.readers.read_file(
base_path=self.path, path=f, content_class=Article,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self)
except Exception as e:
logger.error(
'Could not process %s\n%s', f, e,
exc_info=self.settings.get('DEBUG', False))
self._add_failed_source_path(f)
continue
if not article.is_valid():
self._add_failed_source_path(f)
continue
self.cache_data(f, article)
if article.status == "published":
all_articles.append(article)
elif article.status == "draft":
all_drafts.append(article)
self.add_source_path(article)
self.articles, self.translations = process_translations(all_articles)
self.articles = order_content(
self.articles,
order_by=self.settings['ARTICLE_ORDER_BY'])
self.drafts, self.drafts_translations = \
process_translations(all_drafts)
signals.article_generator_pretaxonomy.send(self)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
for author in getattr(article, 'authors', []):
self.authors[author].append(article)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories',
'authors', 'related_posts', 'drafts'))
self.save_cache()
self.readers.save_cache()
signals.article_generator_finalized.send(self)
def generate_context(self):
"""Add the articles into the shared context"""
all_articles = []
all_drafts = []
for f in self.get_files(self.settings["ARTICLE_PATHS"], exclude=self.settings["ARTICLE_EXCLUDES"]):
article = self.get_cached_data(f, None)
if article is None:
try:
article = self.readers.read_file(
base_path=self.path,
path=f,
content_class=Article,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self,
)
except Exception as e:
logger.error("Could not process %s\n%s", f, e, exc_info=self.settings.get("DEBUG", False))
continue
if not is_valid_content(article, f):
continue
self.cache_data(f, article)
self.add_source_path(article)
if article.status.lower() == "published":
all_articles.append(article)
elif article.status.lower() == "draft":
draft = self.readers.read_file(
base_path=self.path,
path=f,
content_class=Draft,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self,
)
all_drafts.append(draft)
else:
logger.error("Unknown status '%s' for file %s, skipping it.", article.status, f)
self.articles, self.translations = process_translations(all_articles)
self.drafts, self.drafts_translations = process_translations(all_drafts)
signals.article_generator_pretaxonomy.send(self)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, "tags"):
for tag in article.tags:
self.tags[tag].append(article)
# ignore blank authors as well as undefined
for author in getattr(article, "authors", []):
if author.name != "":
self.authors[author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter("date"), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter("date"), reverse=self.context["NEWEST_FIRST_ARCHIVES"])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, "tags", []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[: self.settings.get("TAG_CLOUD_MAX_ITEMS")]
tags = list(map(itemgetter(1), tag_cloud))
if tags:
max_count = max(tags)
steps = self.settings.get("TAG_CLOUD_STEPS")
# calculate word sizes
self.tag_cloud = [
(tag, int(math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count) or 1))))
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings["REVERSE_CATEGORY_ORDER"])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(("articles", "dates", "tags", "categories", "tag_cloud", "authors", "related_posts"))
self.save_cache()
self.readers.save_cache()
signals.article_generator_finalized.send(self)
def generate_context(self):
"""Add the articles into the shared context"""
all_articles = []
all_drafts = []
for f in self.get_files(
self.settings['ARTICLE_PATHS'],
exclude=self.settings['ARTICLE_EXCLUDES']):
article = self.get_cached_data(f, None)
if article is None:
try:
article = self.readers.read_file(
base_path=self.path, path=f, content_class=Article,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self)
except Exception as e:
logger.error('Could not process %s\n%s', f, e,
exc_info=self.settings.get('DEBUG', False))
self._add_failed_source_path(f)
continue
if not is_valid_content(article, f):
self._add_failed_source_path(f)
continue
self.cache_data(f, article)
self.add_source_path(article)
if article.status.lower() == "published":
all_articles.append(article)
elif article.status.lower() == "draft":
draft = self.readers.read_file(
base_path=self.path, path=f, content_class=Draft,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self)
self.add_source_path(draft)
all_drafts.append(draft)
else:
logger.error("Unknown status '%s' for file %s, skipping it.",
article.status, f)
self.articles, self.translations = process_translations(all_articles,
order_by=self.settings['ARTICLE_ORDER_BY'])
self.drafts, self.drafts_translations = \
process_translations(all_drafts)
signals.article_generator_pretaxonomy.send(self)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
for author in getattr(article, 'authors', []):
self.authors[author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = list(map(itemgetter(1), tag_cloud))
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(
tag,
int(math.floor(steps - (steps - 1) * math.log(count)
/ (math.log(max_count)or 1)))
)
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories',
'tag_cloud', 'authors', 'related_posts'))
self.save_cache()
self.readers.save_cache()
signals.article_generator_finalized.send(self)
def generate_context(self):
"""Add the articles into the shared context"""
all_articles = []
for f in self.get_files(self.settings['ARTICLE_DIR'],
exclude=self.settings['ARTICLE_EXCLUDES']):
try:
article = read_file(
base_path=self.path,
path=f,
content_class=Article,
settings=self.settings,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self)
except Exception as e:
logger.warning('Could not process {}\n{}'.format(f, e))
continue
if not is_valid_content(article, f):
continue
self.add_source_path(article)
if article.status == "published":
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
else:
logger.warning("Unknown status %s for file %s, skipping it." %
(repr(article.status), repr(f)))
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
# ignore blank authors as well as undefined
if hasattr(article, 'author') and article.author.name != '':
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = list(map(itemgetter(1), tag_cloud))
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(tag,
int(
math.floor(steps - (steps - 1) * math.log(count) /
(math.log(max_count) or 1))))
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories',
'tag_cloud', 'authors', 'related_posts'))
signals.article_generator_finalized.send(self)
def generate_context(self):
"""change the context"""
# return the list of files to use
files = self.get_files(self.path, exclude=['pages',])
all_articles = []
for f in files:
content, metadata = read_file(f)
# if no category is set, use the name of the path as a category
if 'category' not in metadata.keys():
if os.path.dirname(f) == self.path:
category = self.settings['DEFAULT_CATEGORY']
else:
category = os.path.basename(os.path.dirname(f))
if category != '':
metadata['category'] = unicode(category)
if 'date' not in metadata.keys()\
and self.settings['FALLBACK_ON_FS_DATE']:
metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime)
article = Article(content, metadata, settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
if article.status == "published":
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['REVERSE_ARCHIVE_ORDER'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key = itemgetter(1), reverse = True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = map(itemgetter(1), tag_cloud)
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(
tag,
int(
math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count)or 1))
)
)
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings.get('REVERSE_CATEGORY_ORDER'))
self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud'))
def generate_context(self):
"""change the context"""
# return the list of files to use
files = self.get_files(self.path, exclude=[
'pages',
])
all_articles = []
for f in files:
content, metadatas = read_file(f)
# if no category is set, use the name of the path as a category
if 'category' not in metadatas.keys():
if os.path.dirname(f) == self.path:
category = self.settings['DEFAULT_CATEGORY']
else:
category = os.path.basename(os.path.dirname(f))
if category != '':
metadatas['category'] = unicode(category)
if 'date' not in metadatas.keys()\
and self.settings['FALLBACK_ON_FS_DATE']:
metadatas['date'] = datetime.fromtimestamp(os.stat(f).st_ctime)
article = Article(content,
metadatas,
settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['REVERSE_ARCHIVE_ORDER'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = map(itemgetter(1), tag_cloud)
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(tag,
int(
math.floor(steps - (steps - 1) * math.log(count) /
math.log(max_count)))) for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
self._update_context(
('articles', 'dates', 'tags', 'categories', 'tag_cloud'))
def generate_context(self):
"""Add the entities into the shared context"""
all_entities = []
all_drafts = []
for f in self.get_files(
self.settings['PATHS'],
exclude=self.settings['EXCLUDES']):
entity = self.get_cached_data(f, None)
if entity is None:
entity_class = EntityFactory(
self.entity_type,
self.settings['MANDATORY_PROPERTIES'],
self.settings['DEFAULT_TEMPLATE'])
try:
entity = self.readers.read_file(
base_path=self.path, path=f, content_class=entity_class,
context=self.context,
preread_signal=entity_subgenerator_preread,
preread_sender=self,
context_signal=entity_subgenerator_context,
context_sender=self)
except Exception as e:
logger.error('Could not process %s\n%s', f, e,
exc_info=self.settings.get('DEBUG', False))
continue
if not contents.is_valid_content(entity, f):
continue
self.cache_data(f, entity)
self.add_source_path(entity)
if entity.status.lower() == "published":
all_entities.append(entity)
elif entity.status.lower() == "draft":
all_drafts.append(entity)
else:
logger.warning("Unknown status '%s' for file %s, skipping it.",
entity.status, f)
self.entities, self.translations = process_translations(all_entities)
self.drafts, self.drafts_translations = \
process_translations(all_drafts)
custom_sort_attr = self.settings.get("SORT_ATTRIBUTES", [])
self.sort_attrs = ["date"]
if custom_sort_attr:
self.sort_attrs = custom_sort_attr + self.sort_attrs
self.entities.sort(key=attrgetter(*self.sort_attrs), reverse=True)
entity_subgenerator_pretaxonomy.send(self)
for entity in self.entities:
# only main entities are listed in categories and tags
# not translations
if hasattr(entity, 'category'):
self.categories[entity.category].append(entity)
if hasattr(entity, 'tags'):
for tag in entity.tags:
self.tags[tag].append(entity)
# ignore blank authors as well as undefined
for author in getattr(entity, 'authors', []):
if author.name != '':
self.authors[author].append(entity)
# create tag cloud
tag_cloud = defaultdict(int)
for entity in self.entities:
for tag in getattr(entity, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = list(map(itemgetter(1), tag_cloud))
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(
tag,
int(math.floor(steps - (steps - 1) * math.log(count)
/ (math.log(max_count)or 1)))
)
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self.save_cache()
self.readers.save_cache()
entity_subgenerator_finalized.send(self)
def generate_context(self):
"""Add the articles into the shared context"""
article_path = os.path.normpath( # we have to remove trailing slashes
os.path.join(self.path, self.settings["ARTICLE_DIR"])
)
all_articles = []
for f in self.get_files(article_path, exclude=self.settings["ARTICLE_EXCLUDES"]):
try:
signals.article_generate_preread.send(self)
content, metadata = read_file(f, settings=self.settings)
except Exception as e:
logger.warning("Could not process %s\n%s" % (f, str(e)))
continue
# if no category is set, use the name of the path as a category
if "category" not in metadata:
if self.settings["USE_FOLDER_AS_CATEGORY"] and os.path.dirname(f) != article_path:
# if the article is in a subdirectory
category = os.path.basename(os.path.dirname(f))
else:
# if the article is not in a subdirectory
category = self.settings["DEFAULT_CATEGORY"]
if category != "":
metadata["category"] = Category(category, self.settings)
if "date" not in metadata and self.settings.get("DEFAULT_DATE"):
if self.settings["DEFAULT_DATE"] == "fs":
metadata["date"] = datetime.datetime.fromtimestamp(os.stat(f).st_ctime)
else:
metadata["date"] = datetime.datetime(*self.settings["DEFAULT_DATE"])
signals.article_generate_context.send(self, metadata=metadata)
article = Article(content, metadata, settings=self.settings, source_path=f, context=self.context)
if not is_valid_content(article, f):
continue
self.add_source_path(article)
if article.status == "published":
if hasattr(article, "tags"):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
else:
logger.warning("Unknown status %s for file %s, skipping it." % (repr(article.status), repr(f)))
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
# ignore blank authors as well as undefined
if hasattr(article, "author") and article.author.name != "":
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter("date"), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter("date"), reverse=self.context["NEWEST_FIRST_ARCHIVES"])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, "tags", []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[: self.settings.get("TAG_CLOUD_MAX_ITEMS")]
tags = list(map(itemgetter(1), tag_cloud))
if tags:
max_count = max(tags)
steps = self.settings.get("TAG_CLOUD_STEPS")
# calculate word sizes
self.tag_cloud = [
(tag, int(math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count) or 1))))
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings["REVERSE_CATEGORY_ORDER"])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(("articles", "dates", "tags", "categories", "tag_cloud", "authors", "related_posts"))
signals.article_generator_finalized.send(self)
def generate_context(self):
"""Add the articles into the shared context"""
article_path = os.path.normpath( # we have to remove trailing slashes
os.path.join(self.path, self.settings['ARTICLE_DIR'])
)
all_articles = []
for f in self.get_files(
article_path,
exclude=self.settings['ARTICLE_EXCLUDES']):
try:
signals.article_generate_preread.send(self)
content, metadata = read_file(f, settings=self.settings)
except Exception as e:
logger.warning(u'Could not process %s\n%s' % (f, str(e)))
continue
# if no category is set, use the name of the path as a category
if 'category' not in metadata:
if (self.settings['USE_FOLDER_AS_CATEGORY']
and os.path.dirname(f) != article_path):
# if the article is in a subdirectory
category = os.path.basename(os.path.dirname(f))\
.decode('utf-8')
else:
# if the article is not in a subdirectory
category = self.settings['DEFAULT_CATEGORY']
if category != '':
metadata['category'] = Category(category, self.settings)
if 'date' not in metadata and self.settings.get('DEFAULT_DATE'):
if self.settings['DEFAULT_DATE'] == 'fs':
metadata['date'] = datetime.datetime.fromtimestamp(
os.stat(f).st_ctime)
else:
metadata['date'] = datetime.datetime(
*self.settings['DEFAULT_DATE'])
signals.article_generate_context.send(self, metadata=metadata)
article = Article(content, metadata, settings=self.settings,
filename=f, context=self.context)
if not is_valid_content(article, f):
continue
self.add_filename(article)
if article.status == "published":
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
else:
logger.warning(u"Unknown status %s for file %s, skipping it." %
(repr(unicode.encode(article.status, 'utf-8')),
repr(f)))
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
# ignore blank authors as well as undefined
if hasattr(article,'author') and article.author.name != '':
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = map(itemgetter(1), tag_cloud)
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(
tag,
int(math.floor(steps - (steps - 1) * math.log(count)
/ (math.log(max_count)or 1)))
)
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
key=lambda item: item[0].name,
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort(key=lambda item: item[0].name)
self._update_context(('articles', 'dates', 'tags', 'categories',
'tag_cloud', 'authors', 'related_posts'))
signals.article_generator_finalized.send(self)
def generate_context(self):
# Update the context (only articles in default language)
self.articles = self.context['articles']
# Complain if MBOX_PATH and MBOX_CATEGORY are not of the same length.
mboxPaths = self.settings.get('MBOX_PATH')
mboxCategories = self.settings.get('MBOX_CATEGORY')
errMsg = 'MBOX_PATH, MBOX_CATEGORY not of equal length or non-empty.'
if len(mboxPaths) != len(mboxCategories) or len(mboxPaths) <= 0:
logger.error(errMsg)
return
all_articles = []
for i in xrange(len(mboxPaths)):
mboxPath = mboxPaths[i]
mboxCategory = mboxCategories[i]
new_articles = self._generate_mbox_articles(mboxPath, mboxCategory)
all_articles.extend(new_articles)
# Log that we did stuff.
print(('Read in %d messages from %s and converted to articles in ' +
'category %s.') % (len(new_articles), mboxPath, mboxCategory))
print('Read in %d messages from all mailboxes.' % (len(all_articles)))
# Continue with the rest of ArticleGenerator, code adapted from:
# https://github.com/getpelican/pelican/blob/master/pelican/generators.py#L548
# ARTICLE_ORDER_BY doesn't exist in 3.3, which was in Fedora 21.
# (I wanted to be able to build this on F21 at the time).
articles, translations = process_translations(all_articles)
# , order_by=self.settings['ARTICLE_ORDER_BY'])
self.articles.extend(articles)
self.translations.extend(translations)
# Disabled for 3.3 compatibility, great.
# signals.article_generator_pretaxonomy.send(self)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
# We have to use django for this, unfortunately.
if article.date.tzinfo is None:
article.date = pytz.UTC.localize(article.date)
self.categories[article.category].append(article)
# Support for Author and Authors.
if hasattr(article, 'author') and article.author.name != '':
self.authors[article.author].append(article)
else:
for author in getattr(article, 'authors', []):
self.authors[author].append(article)
# This may not technically be right, but...
# Sort the articles by date too.
self.articles = list(self.articles)
self.dates = self.articles
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'categories', 'authors'))
class ArticlesGenerator(Generator):
"""Generate blog articles"""
def __init__(self, *args, **kwargs):
"""initialize properties"""
self.articles = [] # only articles in default language
self.translations = []
self.dates = {}
self.tags = defaultdict(list)
self.categories = defaultdict(list)
self.authors = defaultdict(list)
super(ArticlesGenerator, self).__init__(*args, **kwargs)
self.drafts = []
def generate_feeds(self, writer):
"""Generate the feeds from the current context, and output files."""
writer.write_feed(self.articles, self.context, self.settings['FEED'])
if 'FEED_RSS' in self.settings:
writer.write_feed(self.articles,
self.context,
self.settings['FEED_RSS'],
feed_type='rss')
for cat, arts in self.categories:
arts.sort(key=attrgetter('date'), reverse=True)
writer.write_feed(arts, self.context,
self.settings['CATEGORY_FEED'] % cat)
if 'CATEGORY_FEED_RSS' in self.settings:
writer.write_feed(arts,
self.context,
self.settings['CATEGORY_FEED_RSS'] % cat,
feed_type='rss')
if 'TAG_FEED' in self.settings:
for tag, arts in self.tags.items():
arts.sort(key=attrgetter('date'), reverse=True)
writer.write_feed(arts, self.context,
self.settings['TAG_FEED'] % tag)
if 'TAG_FEED_RSS' in self.settings:
writer.write_feed(arts,
self.context,
self.settings['TAG_FEED_RSS'] % tag,
feed_type='rss')
translations_feeds = defaultdict(list)
for article in chain(self.articles, self.translations):
translations_feeds[article.lang].append(article)
for lang, items in translations_feeds.items():
items.sort(key=attrgetter('date'), reverse=True)
writer.write_feed(items, self.context,
self.settings['TRANSLATION_FEED'] % lang)
def generate_pages(self, writer):
"""Generate the pages on the disk"""
write = partial(writer.write_file,
relative_urls=self.settings.get('RELATIVE_URLS'))
# to minimize the number of relative path stuff modification
# in writer, articles pass first
article_template = self.get_template('article')
for article in chain(self.translations, self.articles):
write(article.save_as,
article_template,
self.context,
article=article,
category=article.category)
PAGINATED_TEMPLATES = self.settings.get('PAGINATED_DIRECT_TEMPLATES')
for template in self.settings.get('DIRECT_TEMPLATES'):
paginated = {}
if template in PAGINATED_TEMPLATES:
paginated = {'articles': self.articles, 'dates': self.dates}
write('%s.html' % template,
self.get_template(template),
self.context,
blog=True,
paginated=paginated,
page_name=template)
# and subfolders after that
tag_template = self.get_template('tag')
for tag, articles in self.tags.items():
articles.sort(key=attrgetter('date'), reverse=True)
dates = [article for article in self.dates if article in articles]
write('tag/%s.html' % tag,
tag_template,
self.context,
tag=tag,
articles=articles,
dates=dates,
paginated={
'articles': articles,
'dates': dates
},
page_name='tag/%s' % tag)
category_template = self.get_template('category')
for cat, articles in self.categories:
dates = [article for article in self.dates if article in articles]
write('category/%s.html' % cat,
category_template,
self.context,
category=cat,
articles=articles,
dates=dates,
paginated={
'articles': articles,
'dates': dates
},
page_name='category/%s' % cat)
author_template = self.get_template('author')
for aut, articles in self.authors:
dates = [article for article in self.dates if article in articles]
write('author/%s.html' % aut,
author_template,
self.context,
author=aut,
articles=articles,
dates=dates,
paginated={
'articles': articles,
'dates': dates
},
page_name='author/%s' % aut)
for article in self.drafts:
write('drafts/%s.html' % article.slug,
article_template,
self.context,
article=article,
category=article.category)
def generate_context(self):
"""change the context"""
# return the list of files to use
files = self.get_files(self.path, exclude=[
'pages',
])
all_articles = []
for f in files:
try:
content, metadata = read_file(f, settings=self.settings)
except Exception, e:
warning(u'Could not process %s\n%s' % (f, str(e)))
continue
# if no category is set, use the name of the path as a category
if 'category' not in metadata.keys():
if os.path.dirname(f) == self.path:
category = self.settings['DEFAULT_CATEGORY']
else:
category = os.path.basename(
os.path.dirname(f)).decode('utf-8')
if category != '':
metadata['category'] = unicode(category)
if 'date' not in metadata.keys()\
and self.settings['FALLBACK_ON_FS_DATE']:
metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime)
article = Article(content,
metadata,
settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
add_to_url = u''
if 'ARTICLE_PERMALINK_STRUCTURE' in self.settings:
article_permalink_structure = self.settings[
'ARTICLE_PERMALINK_STRUCTURE']
article_permalink_structure = article_permalink_structure.lstrip(
'/').replace('%(', "%%(")
# try to substitute any python datetime directive
add_to_url = article.date.strftime(article_permalink_structure)
# try to substitute any article metadata in rest file
add_to_url = add_to_url % article.__dict__
add_to_url = [slugify(i) for i in add_to_url.split('/')]
add_to_url = os.path.join(*add_to_url)
article.url = urlparse.urljoin(add_to_url, article.url)
article.save_as = urlparse.urljoin(add_to_url, article.save_as)
if article.status == "published":
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['REVERSE_ARCHIVE_ORDER'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = map(itemgetter(1), tag_cloud)
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(tag,
int(
math.floor(steps - (steps - 1) * math.log(count) /
(math.log(max_count) or 1))))
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings.get('REVERSE_CATEGORY_ORDER'))
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories',
'tag_cloud', 'authors'))
def generate_context(self):
"""Add the articles into the shared context"""
restrict_all_articles_to_tags = set(
self.settings.get("ARTICLES_RESTRICT_TO_TAGS", []))
all_articles = []
all_drafts = []
for f in self.get_files(
self.settings['ARTICLE_PATHS'],
exclude=self.settings['ARTICLE_EXCLUDES']):
article = self.get_cached_data(f, None)
if article is None:
try:
article = self.readers.read_file(
base_path=self.path, path=f, content_class=Article,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self)
except Exception as e:
logger.error(
'Could not process %s\n%s', f, e,
exc_info=self.settings.get('DEBUG', False))
self._add_failed_source_path(f)
continue
if not article.is_valid():
self._add_failed_source_path(f)
continue
self.cache_data(f, article)
if not restrict_all_articles_to_tags <= set(
getattr(article, 'tags', [])):
continue
if article.status == "published":
all_articles.append(article)
elif article.status == "draft":
all_drafts.append(article)
self.add_source_path(article)
self.articles, self.translations = process_translations(all_articles)
self.articles = order_content(
self.articles,
order_by=self.settings['ARTICLE_ORDER_BY'])
self.drafts, self.drafts_translations = \
process_translations(all_drafts)
signals.article_generator_pretaxonomy.send(self)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
for author in getattr(article, 'authors', []):
self.authors[author].append(article)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories',
'authors', 'related_posts', 'drafts'))
self.save_cache()
self.readers.save_cache()
signals.article_generator_finalized.send(self)
def generate_context(self):
"""Add the entities into the shared context"""
all_entities = []
all_drafts = []
for f in self.get_files(
self.settings['PATHS'],
exclude=self.settings['EXCLUDES']):
entity_or_draft = self.get_cached_data(f, None)
if entity_or_draft is None:
entity_class = EntityFactory(
self.entity_type,
self.settings['MANDATORY_PROPERTIES'],
self.settings['DEFAULT_TEMPLATE'])
try:
entity_or_draft = self.readers.read_file(
base_path=self.path, path=f, content_class=entity_class,
context=self.context,
preread_signal=entity_subgenerator_preread,
preread_sender=self,
context_signal=entity_subgenerator_context,
context_sender=self)
except Exception as e:
logger.error('Could not process %s\n%s', f, e,
exc_info=self.settings.get('DEBUG', False))
self._add_failed_source_path(f)
continue
if not contents.is_valid_content(entity_or_draft, f):
self._add_failed_source_path(f)
continue
known_statuses = ("published", "draft")
if entity_or_draft.status.lower() not in known_statuses:
logger.warning("Unknown status '%s' for file %s, skipping it.",
entity_or_draft.status, f)
self._add_failed_source_path(f)
continue
self.cache_data(f, entity_or_draft)
if entity_or_draft.status.lower() == "published":
all_entities.append(entity_or_draft)
else:
all_drafts.append(entity_or_draft)
self.add_source_path(entity_or_draft)
self.entities, self.translations = process_translations(
all_entities)
self.drafts, self.drafts_translations = \
process_translations(all_drafts)
sorter = self.settings["SORTER"]
sorter(self.entities)
entity_subgenerator_pretaxonomy.send(self)
for entity_or_draft in self.entities:
# only main entities are listed in categories and tags
# not translations
if hasattr(entity_or_draft, 'category'):
self.categories[entity_or_draft.category].append(entity_or_draft)
if hasattr(entity_or_draft, 'tags'):
for tag in entity_or_draft.tags:
self.tags[tag].append(entity_or_draft)
for author in getattr(entity_or_draft, 'authors', []):
self.authors[author].append(entity_or_draft)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self.save_cache()
self.readers.save_cache()
entity_subgenerator_finalized.send(self)
def test_process_translations(self):
fr_articles = []
en_articles = []
# create a bunch of articles
# 0: no translation metadata
fr_articles.append(
get_article(lang='fr',
slug='yay0',
title='Titre',
content='en français'))
en_articles.append(
get_article(lang='en',
slug='yay0',
title='Title',
content='in english'))
# 1: translation metadata on default lang
fr_articles.append(
get_article(lang='fr',
slug='yay1',
title='Titre',
content='en français'))
en_articles.append(
get_article(lang='en',
slug='yay1',
title='Title',
content='in english',
translation='true'))
# 2: translation metadata not on default lang
fr_articles.append(
get_article(lang='fr',
slug='yay2',
title='Titre',
content='en français',
translation='true'))
en_articles.append(
get_article(lang='en',
slug='yay2',
title='Title',
content='in english'))
# 3: back to default language detection if all items have the
# translation metadata
fr_articles.append(
get_article(lang='fr',
slug='yay3',
title='Titre',
content='en français',
translation='yep'))
en_articles.append(
get_article(lang='en',
slug='yay3',
title='Title',
content='in english',
translation='yes'))
# 4-5: translation pairs with the same slug but different category
fr_articles.append(
get_article(lang='fr',
slug='yay4',
title='Titre',
content='en français',
category='foo'))
en_articles.append(
get_article(lang='en',
slug='yay4',
title='Title',
content='in english',
category='foo'))
fr_articles.append(
get_article(lang='fr',
slug='yay4',
title='Titre',
content='en français',
category='bar'))
en_articles.append(
get_article(lang='en',
slug='yay4',
title='Title',
content='in english',
category='bar'))
# try adding articles in both orders
for lang0_articles, lang1_articles in ((fr_articles, en_articles),
(en_articles, fr_articles)):
articles = lang0_articles + lang1_articles
# test process_translations with falsy translation_id
index, trans = utils.process_translations(articles,
translation_id=None)
for i in range(6):
for lang_articles in [en_articles, fr_articles]:
self.assertIn(lang_articles[i], index)
self.assertNotIn(lang_articles[i], trans)
# test process_translations with simple and complex translation_id
for translation_id in ['slug', {'slug', 'category'}]:
index, trans = utils.process_translations(
articles, translation_id=translation_id)
for a in [
en_articles[0], fr_articles[1], en_articles[2],
en_articles[3], en_articles[4], en_articles[5]
]:
self.assertIn(a, index)
self.assertNotIn(a, trans)
for a in [
fr_articles[0], en_articles[1], fr_articles[2],
fr_articles[3], fr_articles[4], fr_articles[5]
]:
self.assertIn(a, trans)
self.assertNotIn(a, index)
for i in range(6):
self.assertIn(en_articles[i], fr_articles[i].translations)
self.assertIn(fr_articles[i], en_articles[i].translations)
for a_arts in [en_articles, fr_articles]:
for b_arts in [en_articles, fr_articles]:
if translation_id == 'slug':
self.assertIn(a_arts[4], b_arts[5].translations)
self.assertIn(a_arts[5], b_arts[4].translations)
elif translation_id == {'slug', 'category'}:
self.assertNotIn(a_arts[4], b_arts[5].translations)
self.assertNotIn(a_arts[5], b_arts[4].translations)
def generate_context(self):
"""Add the entities into the shared context"""
all_entities = []
all_drafts = []
for f in self.get_files(
self.settings['PATHS'],
exclude=self.settings['EXCLUDES']):
entity_or_draft = self.get_cached_data(f, None)
if entity_or_draft is None:
entity_class = EntityFactory(
self.entity_type,
self.settings['MANDATORY_PROPERTIES'],
self.settings['DEFAULT_TEMPLATE'])
try:
entity_or_draft = self.readers.read_file(
base_path=self.path, path=f, content_class=entity_class,
context=self.context,
preread_signal=entity_subgenerator_preread,
preread_sender=self,
context_signal=entity_subgenerator_context,
context_sender=self)
except Exception as e:
logger.error('Could not process %s\n%s', f, e,
exc_info=self.settings.get('DEBUG', False))
self._add_failed_source_path(f)
continue
if not contents.is_valid_content(entity_or_draft, f):
self._add_failed_source_path(f)
continue
if entity_or_draft.status.lower() == "published":
all_entities.append(entity_or_draft)
elif entity_or_draft.status.lower() == "draft":
all_drafts.append(entity_or_draft)
self.add_source_path(entity_or_draft)
continue
else:
logger.warning("Unknown status '%s' for file %s, skipping it.",
entity_or_draft.status, f)
self._add_failed_source_path(f)
continue
self.cache_data(f, entity_or_draft)
self.add_source_path(entity_or_draft)
self.entities, self.translations = process_translations(
all_entities)
self.drafts, self.drafts_translations = \
process_translations(all_drafts)
sorter = self.settings["SORTER"]
sorter(self.entities)
entity_subgenerator_pretaxonomy.send(self)
for entity_or_draft in self.entities:
# only main entities are listed in categories and tags
# not translations
if hasattr(entity_or_draft, 'category'):
self.categories[entity_or_draft.category].append(entity_or_draft)
if hasattr(entity_or_draft, 'tags'):
for tag in entity_or_draft.tags:
self.tags[tag].append(entity_or_draft)
for author in getattr(entity_or_draft, 'authors', []):
self.authors[author].append(entity_or_draft)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self.save_cache()
self.readers.save_cache()
entity_subgenerator_finalized.send(self)
def generate_context(self):
"""change the context"""
# return the list of files to use
files = self.get_files(self.path, exclude=[
'pages',
])
all_articles = []
for f in files:
content, metadata = read_file(f)
# if no category is set, use the name of the path as a category
if 'category' not in metadata.keys():
if os.path.dirname(f) == self.path:
category = self.settings['DEFAULT_CATEGORY']
else:
category = os.path.basename(os.path.dirname(f))
if category != '':
metadata['category'] = unicode(category)
if 'date' not in metadata.keys()\
and self.settings['FALLBACK_ON_FS_DATE']:
metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime)
article = Article(content,
metadata,
settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
add_to_url = u''
if 'ARTICLE_PERMALINK_STRUCTURE' in self.settings:
article_permalink_structure = self.settings[
'ARTICLE_PERMALINK_STRUCTURE']
article_permalink_structure = article_permalink_structure.lstrip(
'/')
# try to substitute any python datetime directive
add_to_url = article.date.strftime(article_permalink_structure)
# try to substitute any article metadata in rest file
add_to_url = add_to_url % article.__dict__
add_to_url = [slugify(i) for i in add_to_url.split('/')]
add_to_url = os.path.join(*add_to_url)
article.url = urlparse.urljoin(add_to_url, article.url)
article.save_as = urlparse.urljoin(add_to_url, article.save_as)
if article.status == "published":
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['REVERSE_ARCHIVE_ORDER'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = map(itemgetter(1), tag_cloud)
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(tag,
int(
math.floor(steps - (steps - 1) * math.log(count) /
(math.log(max_count) or 1))))
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings.get('REVERSE_CATEGORY_ORDER'))
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories',
'tag_cloud', 'authors'))
def _process(pages):
origs, translations = process_translations(
pages, translation_id=self.settings['PAGE_TRANSLATION_ID'])
origs = order_content(origs, self.settings['PAGE_ORDER_BY'])
return origs, translations
def generate_context(self):
"""change the context"""
# return the list of files to use
files = self.get_files(self.path, exclude=['pages',])
all_articles = []
for f in files:
content, metadata = read_file(f)
# if no category is set, use the name of the path as a category
if 'category' not in metadata.keys():
if os.path.dirname(f) == self.path:
category = self.settings['DEFAULT_CATEGORY']
else:
category = os.path.basename(os.path.dirname(f))
if category != '':
metadata['category'] = unicode(category)
if 'date' not in metadata.keys()\
and self.settings['FALLBACK_ON_FS_DATE']:
metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime)
article = Article(content, metadata, settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
add_to_url = u''
if 'ARTICLE_PERMALINK_STRUCTURE' in self.settings:
article_permalink_structure = self.settings['ARTICLE_PERMALINK_STRUCTURE']
article_permalink_structure = article_permalink_structure.lstrip('/')
# try to substitute any python datetime directive
add_to_url = article.date.strftime(article_permalink_structure)
# try to substitute any article metadata in rest file
add_to_url = add_to_url % article.__dict__
add_to_url = [slugify(i) for i in add_to_url.split('/')]
add_to_url = os.path.join(*add_to_url)
article.url = urlparse.urljoin(add_to_url, article.url)
article.save_as = urlparse.urljoin(add_to_url, article.save_as)
if article.status == "published":
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['REVERSE_ARCHIVE_ORDER'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key = itemgetter(1), reverse = True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = map(itemgetter(1), tag_cloud)
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(
tag,
int(
math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count)or 1))
)
)
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings.get('REVERSE_CATEGORY_ORDER'))
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors'))
def generate_context(self):
# Update the context (only articles in default language)
self.articles = self.context['articles']
# Complain if MBOX_PATH and MBOX_CATEGORY are not of the same length.
mboxPaths = self.settings.get('MBOX_PATH')
mboxCategories = self.settings.get('MBOX_CATEGORY')
errMsg = 'MBOX_PATH, MBOX_CATEGORY not of equal length or non-empty.'
if len(mboxPaths) != len(mboxCategories) or len(mboxPaths) <= 0:
logger.error(errMsg)
return
all_articles = []
# To avoid pulling in a dependency, define this convenient lambda.
future_range = lambda x: range(
x) if not sys.version_info.major <= 2 else xrange(x)
for i in future_range(len(mboxPaths)):
mboxPath = mboxPaths[i]
mboxCategory = mboxCategories[i]
new_articles = self._generate_mbox_articles(mboxPath, mboxCategory)
all_articles.extend(new_articles)
# Log that we did stuff.
print(
('Read in %d messages from %s and converted to articles in ' +
'category %s.') % (len(new_articles), mboxPath, mboxCategory))
print('Read in %d messages from all mailboxes.' % (len(all_articles)))
# Continue with the rest of ArticleGenerator, code adapted from:
# https://github.com/getpelican/pelican/blob/master/pelican/generators.py#L548
# ARTICLE_ORDER_BY doesn't exist in 3.3, which was in Fedora 21.
# (I wanted to be able to build this on F21 at the time).
articles, translations = process_translations(all_articles)
# , order_by=self.settings['ARTICLE_ORDER_BY'])
self.articles.extend(articles)
self.translations.extend(translations)
# Disabled for 3.3 compatibility, great.
# signals.article_generator_pretaxonomy.send(self)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
# We have to use django for this, unfortunately.
if article.date.tzinfo is None:
article.date = pytz.UTC.localize(article.date)
self.categories[article.category].append(article)
# Support for Author and Authors.
if hasattr(article, 'author') and article.author.name != '':
self.authors[article.author].append(article)
else:
for author in getattr(article, 'authors', []):
self.authors[author].append(article)
# This may not technically be right, but...
# Sort the articles by date too.
self.articles = list(self.articles)
self.dates = self.articles
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'categories', 'authors'))
def generate_context(self):
"""Add the articles into the shared context"""
all_articles = []
for f in self.get_files(
self.settings['ARTICLE_DIR'],
exclude=self.settings['ARTICLE_EXCLUDES']):
try:
article = self.readers.read_file(
base_path=self.path, path=f, content_class=Article,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self)
except Exception as e:
logger.warning('Could not process {}\n{}'.format(f, e))
continue
if not is_valid_content(article, f):
continue
self.add_source_path(article)
if article.status == "published":
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
else:
logger.warning("Unknown status %s for file %s, skipping it." %
(repr(article.status),
repr(f)))
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
# ignore blank authors as well as undefined
if hasattr(article, 'author') and article.author.name != '':
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = list(map(itemgetter(1), tag_cloud))
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(
tag,
int(math.floor(steps - (steps - 1) * math.log(count)
/ (math.log(max_count)or 1)))
)
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(
reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories',
'tag_cloud', 'authors', 'related_posts'))
signals.article_generator_finalized.send(self)
def generate_context(self):
"""Add the articles into the shared context"""
all_articles = []
all_drafts = []
for f in self.get_files(self.settings["ARTICLE_PATHS"], exclude=self.settings["ARTICLE_EXCLUDES"]):
article_or_draft = self.get_cached_data(f, None)
if article_or_draft is None:
# TODO needs overhaul, maybe nomad for read_file
# solution, unified behaviour
try:
article_or_draft = self.readers.read_file(
base_path=self.path,
path=f,
content_class=Article,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self,
)
except Exception as e:
logger.error("Could not process %s\n%s", f, e, exc_info=self.settings.get("DEBUG", False))
self._add_failed_source_path(f)
continue
if not is_valid_content(article_or_draft, f):
self._add_failed_source_path(f)
continue
if article_or_draft.status.lower() == "published":
all_articles.append(article_or_draft)
elif article_or_draft.status.lower() == "draft":
article_or_draft = self.readers.read_file(
base_path=self.path,
path=f,
content_class=Draft,
context=self.context,
preread_signal=signals.article_generator_preread,
preread_sender=self,
context_signal=signals.article_generator_context,
context_sender=self,
)
self.add_source_path(article_or_draft)
all_drafts.append(article_or_draft)
else:
logger.error("Unknown status '%s' for file %s, skipping it.", article_or_draft.status, f)
self._add_failed_source_path(f)
continue
self.cache_data(f, article_or_draft)
self.add_source_path(article_or_draft)
self.articles, self.translations = process_translations(
all_articles, order_by=self.settings["ARTICLE_ORDER_BY"]
)
self.drafts, self.drafts_translations = process_translations(all_drafts)
signals.article_generator_pretaxonomy.send(self)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, "tags"):
for tag in article.tags:
self.tags[tag].append(article)
for author in getattr(article, "authors", []):
self.authors[author].append(article)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter("date"), reverse=self.context["NEWEST_FIRST_ARCHIVES"])
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings["REVERSE_CATEGORY_ORDER"])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(("articles", "dates", "tags", "categories", "authors", "related_posts", "drafts"))
self.save_cache()
self.readers.save_cache()
signals.article_generator_finalized.send(self)
def _process(pages):
origs, translations = process_translations(
pages, translation_id=self.settings['PAGE_TRANSLATION_ID'])
origs = order_content(origs, self.settings['PAGE_ORDER_BY'])
return origs, translations
def test_process_translations(self):
fr_articles = []
en_articles = []
# create a bunch of articles
# 0: no translation metadata
fr_articles.append(get_article(lang='fr', slug='yay0', title='Titre',
content='en français'))
en_articles.append(get_article(lang='en', slug='yay0', title='Title',
content='in english'))
# 1: translation metadata on default lang
fr_articles.append(get_article(lang='fr', slug='yay1', title='Titre',
content='en français'))
en_articles.append(get_article(lang='en', slug='yay1', title='Title',
content='in english',
translation='true'))
# 2: translation metadata not on default lang
fr_articles.append(get_article(lang='fr', slug='yay2', title='Titre',
content='en français',
translation='true'))
en_articles.append(get_article(lang='en', slug='yay2', title='Title',
content='in english'))
# 3: back to default language detection if all items have the
# translation metadata
fr_articles.append(get_article(lang='fr', slug='yay3', title='Titre',
content='en français',
translation='yep'))
en_articles.append(get_article(lang='en', slug='yay3', title='Title',
content='in english',
translation='yes'))
# 4-5: translation pairs with the same slug but different category
fr_articles.append(get_article(lang='fr', slug='yay4', title='Titre',
content='en français', category='foo'))
en_articles.append(get_article(lang='en', slug='yay4', title='Title',
content='in english', category='foo'))
fr_articles.append(get_article(lang='fr', slug='yay4', title='Titre',
content='en français', category='bar'))
en_articles.append(get_article(lang='en', slug='yay4', title='Title',
content='in english', category='bar'))
# try adding articles in both orders
for lang0_articles, lang1_articles in ((fr_articles, en_articles),
(en_articles, fr_articles)):
articles = lang0_articles + lang1_articles
# test process_translations with falsy translation_id
index, trans = utils.process_translations(
articles, translation_id=None)
for i in range(6):
for lang_articles in [en_articles, fr_articles]:
self.assertIn(lang_articles[i], index)
self.assertNotIn(lang_articles[i], trans)
# test process_translations with simple and complex translation_id
for translation_id in ['slug', {'slug', 'category'}]:
index, trans = utils.process_translations(
articles, translation_id=translation_id)
for a in [en_articles[0], fr_articles[1], en_articles[2],
en_articles[3], en_articles[4], en_articles[5]]:
self.assertIn(a, index)
self.assertNotIn(a, trans)
for a in [fr_articles[0], en_articles[1], fr_articles[2],
fr_articles[3], fr_articles[4], fr_articles[5]]:
self.assertIn(a, trans)
self.assertNotIn(a, index)
for i in range(6):
self.assertIn(en_articles[i], fr_articles[i].translations)
self.assertIn(fr_articles[i], en_articles[i].translations)
for a_arts in [en_articles, fr_articles]:
for b_arts in [en_articles, fr_articles]:
if translation_id == 'slug':
self.assertIn(a_arts[4], b_arts[5].translations)
self.assertIn(a_arts[5], b_arts[4].translations)
elif translation_id == {'slug', 'category'}:
self.assertNotIn(a_arts[4], b_arts[5].translations)
self.assertNotIn(a_arts[5], b_arts[4].translations)
