def generate_context(self): all_pages = [] hidden_pages = [] for f in self.get_files( os.path.join(self.path, self.settings['PAGE_DIR']), exclude=self.settings['PAGE_EXCLUDES']): try: content, metadata = read_file(f, settings=self.settings) except Exception as e: logger.warning(u'Could not process %s\n%s' % (f, str(e))) continue signals.pages_generate_context.send(self, metadata=metadata) page = Page(content, metadata, settings=self.settings, filename=f, context=self.context) if not is_valid_content(page, f): continue self.add_filename(page) if page.status == "published": all_pages.append(page) elif page.status == "hidden": hidden_pages.append(page) else: logger.warning(u"Unknown status %s for file %s, skipping it." % (repr(unicode.encode(page.status, 'utf-8')), repr(f))) self.pages, self.translations = process_translations(all_pages) self.hidden_pages, self.hidden_translations = process_translations(hidden_pages) self._update_context(('pages', )) self.context['PAGES'] = self.pages
def generate_context(self): all_pages = [] hidden_pages = [] for f in self.get_files( os.path.join(self.path, self.settings["PAGE_DIR"]), exclude=self.settings["PAGE_EXCLUDES"] ): try: content, metadata = read_file(f, settings=self.settings) except Exception as e: logger.warning("Could not process %s\n%s" % (f, str(e))) continue signals.pages_generate_context.send(self, metadata=metadata) page = Page(content, metadata, settings=self.settings, source_path=f, context=self.context) if not is_valid_content(page, f): continue self.add_source_path(page) if page.status == "published": all_pages.append(page) elif page.status == "hidden": hidden_pages.append(page) else: logger.warning("Unknown status %s for file %s, skipping it." % (repr(page.status), repr(f))) self.pages, self.translations = process_translations(all_pages) self.hidden_pages, self.hidden_translations = process_translations(hidden_pages) self._update_context(("pages",)) self.context["PAGES"] = self.pages signals.pages_generator_finalized.send(self)
def generate_context(self): """ Here is the meat of the class - where the heavy lifting occurs. It generates a list of tutorials and places them in the context object so we can access them in templates. Some of this is leftover from the stock Article class. Ideally those aspect will be removed as it is shown they can be safely done away with. However, this works. """ all_tutorials = [] hidden_tutorials = [] for f in self.get_files( self.settings['TUTORIAL_DIR'], exclude=self.settings['TUTORIAL_EXCLUDES']): tutorial = self.get_cached_data(f, None) if tutorial is None: try: tutorial = self.readers.read_file( base_path=self.path, path=f, content_class=Tutorial, context=self.context, preread_signal=tutorial_generator_preread, preread_sender=self, context_signal=tutorial_generator_context, context_sender=self) except Exception as e: logger.warning('Could not process {}\n{}'.format(f, e)) continue if not is_valid_content(tutorial, f): continue self.cache_data(f, tutorial) self.add_source_path(tutorial) if tutorial.status == "published": all_tutorials.append(tutorial) for author_data in self.context['site_authors']: if author_data.name == tutorial.author.name: tutorial.author.data = author_data elif tutorial.status == "hidden": hidden_tutorials.append(tutorial) else: logger.warning("Unknown status %s for file %s, skipping it." % (repr(tutorial.status), repr(f))) self.tutorials, self.translations = process_translations(all_tutorials) self.hidden_tutorials, self.hidden_translations = ( process_translations(hidden_tutorials)) self._update_context(('tutorials', )) self.context['TUTORIALS'] = self.tutorials self.save_cache() self.readers.save_cache() tutorial_generator_finalized.send(self)
def generate_context(self): all_pages = [] hidden_pages = [] for f in self.get_files(self.settings['PAGE_PATHS'], exclude=self.settings['PAGE_EXCLUDES']): page = self.get_cached_data(f, None) if page is None: try: page = self.readers.read_file( base_path=self.path, path=f, content_class=Page, context=self.context, preread_signal=signals.page_generator_preread, preread_sender=self, context_signal=signals.page_generator_context, context_sender=self) except Exception as e: logger.error('Could not process %s\n%s', f, e, exc_info=self.settings.get('DEBUG', False)) self._add_failed_source_path(f) continue if not is_valid_content(page, f): self._add_failed_source_path(f) continue if page.status.lower() == "published": all_pages.append(page) elif page.status.lower() == "hidden": hidden_pages.append(page) else: logger.error( "Unknown status '%s' for file %s, skipping it.", page.status, f) self._add_failed_source_path(f) continue self.cache_data(f, page) self.add_source_path(page) self.pages, self.translations = process_translations( all_pages, order_by=self.settings['PAGE_ORDER_BY']) self.hidden_pages, self.hidden_translations = ( process_translations(hidden_pages)) self._update_context(('pages', )) self.context['PAGES'] = self.pages self.save_cache() self.readers.save_cache() signals.page_generator_finalized.send(self)
def generate_context(self): """ Here is the meat of the class - where the heavy lifting occurs. It generates a list of events and places them in the context object so we can access them in templates. Some of this is leftover from the stock Article class. Ideally those aspect will be removed as it is shown they can be safely done away with. However, this works. """ all_events = [] hidden_events = [] for f in self.get_files(self.settings["EVENT_DIR"], exclude=self.settings["EVENT_EXCLUDES"]): event = self.get_cached_data(f, None) if event is None: try: event = self.readers.read_file( base_path=self.path, path=f, content_class=Event, context=self.context, preread_signal=event_generator_preread, preread_sender=self, context_signal=event_generator_context, context_sender=self, ) except Exception as e: logger.warning("Could not process {}\n{}".format(f, e)) continue if not is_valid_content(event, f): continue self.cache_data(f, event) self.add_source_path(event) if event.status == "published": all_events.append(event) elif event.status == "hidden": hidden_events.append(event) else: logger.warning("Unknown status %s for file %s, skipping it." % (repr(event.status), repr(f))) self.events, self.translations = process_translations(all_events) self.hidden_events, self.hidden_translations = process_translations(hidden_events) self._update_context(("events",)) self.context["EVENTS"] = self.events self.save_cache() self.readers.save_cache() event_generator_finalized.send(self)
def generate_context(self): all_pages = [] hidden_pages = [] for f in self.get_files( self.settings['PAGE_PATHS'], exclude=self.settings['PAGE_EXCLUDES']): page = self.get_cached_data(f, None) if page is None: try: page = self.readers.read_file( base_path=self.path, path=f, content_class=Page, context=self.context, preread_signal=signals.page_generator_preread, preread_sender=self, context_signal=signals.page_generator_context, context_sender=self) except Exception as e: logger.error( 'Could not process %s\n%s', f, e, exc_info=self.settings.get('DEBUG', False)) self._add_failed_source_path(f) continue if not is_valid_content(page, f): self._add_failed_source_path(f) continue if page.status.lower() == "published": all_pages.append(page) elif page.status.lower() == "hidden": hidden_pages.append(page) else: logger.error( "Unknown status '%s' for file %s, skipping it.", page.status, f) self._add_failed_source_path(f) continue self.cache_data(f, page) self.add_source_path(page) self.pages, self.translations = process_translations( all_pages, order_by=self.settings['PAGE_ORDER_BY']) self.hidden_pages, self.hidden_translations = \ process_translations(hidden_pages) self._update_context(('pages', 'hidden_pages')) self.save_cache() self.readers.save_cache() signals.page_generator_finalized.send(self)
def generate_output(self, writer): start_time = time.time() page_count = 0 article_count = 0 write = partial(writer.write_file, relative_urls=self.settings.get('RELATIVE_URLS')) for cat in self.config: category_name = cat.get('category_name') if not category_name: warning("Improperly configured Standalone Category: " "missing required `category_name` parameter.") continue category_name_lc = category_name.lower() cat_page_name = "{}.html".format( slugify(category_name_lc, self.settings.get('SLUG_SUBSTITUTIONS', ()))) articles = self.context['standalone_articles'][category_name_lc] write(cat_page_name, self.get_template(cat['page_template']), self.context, articles=articles, category=cat['category_name']) page_count += 1 the_articles, translations = process_translations(articles) for article in chain(the_articles, translations): write(article.save_as, self.get_template(cat['article_template']), self.context, article=article, category=article.category, override_output=hasattr(article, 'override_save_as')) article_count += 1 print("Standalone categories done: Processed {} articles and {}" " pages in {:.2f} seconds.".format(article_count, page_count, time.time() - start_time))
class PagesGenerator(Generator): """Generate pages""" def __init__(self, *args, **kwargs): self.pages = [] super(PagesGenerator, self).__init__(*args, **kwargs) def generate_context(self): all_pages = [] for f in self.get_files(os.path.join(self.path, self.settings['PAGE_DIR']), exclude=self.settings['PAGE_EXCLUDES']): try: content, metadata = read_file(f) except Exception, e: logger.error(u'Could not process %s\n%s' % (f, str(e))) continue page = Page(content, metadata, settings=self.settings, filename=f) if not is_valid_content(page, f): continue all_pages.append(page) self.pages, self.translations = process_translations(all_pages) self._update_context(('pages', )) self.context['PAGES'] = self.pages
def generate_context(self): all_pages = [] hidden_pages = [] for f in self.get_files( self.settings['PAGE_DIR'], exclude=self.settings['PAGE_EXCLUDES']): page = self.get_cached_data(f, None) if page is None: try: page = self.readers.read_file( base_path=self.path, path=f, content_class=Page, context=self.context, preread_signal=signals.page_generator_preread, preread_sender=self, context_signal=signals.page_generator_context, context_sender=self) except Exception as e: logger.warning('Could not process {}\n{}'.format(f, e)) continue if not is_valid_content(page, f): continue self.cache_data(f, page) self.add_source_path(page) if page.status == "published": all_pages.append(page) elif page.status == "hidden": hidden_pages.append(page) else: logger.warning("Unknown status %s for file %s, skipping it." % (repr(page.status), repr(f))) self.pages, self.translations = process_translations(all_pages) self.hidden_pages, self.hidden_translations = ( process_translations(hidden_pages)) self._update_context(('pages', )) self.context['PAGES'] = self.pages self.save_cache() self.readers.save_cache() signals.page_generator_finalized.send(self)
def generate_context(self): all_pages = [] hidden_pages = [] for f in self.get_files(self.settings['PAGE_DIR'], exclude=self.settings['PAGE_EXCLUDES']): page = self.get_cached_data(f, None) if page is None: try: page = self.readers.read_file( base_path=self.path, path=f, content_class=Page, context=self.context, preread_signal=signals.page_generator_preread, preread_sender=self, context_signal=signals.page_generator_context, context_sender=self) except Exception as e: logger.warning('Could not process {}\n{}'.format(f, e)) continue if not is_valid_content(page, f): continue self.cache_data(f, page) self.add_source_path(page) if page.status == "published": all_pages.append(page) elif page.status == "hidden": hidden_pages.append(page) else: logger.warning("Unknown status %s for file %s, skipping it." % (repr(page.status), repr(f))) self.pages, self.translations = process_translations(all_pages) self.hidden_pages, self.hidden_translations = ( process_translations(hidden_pages)) self._update_context(('pages', )) self.context['PAGES'] = self.pages self.save_cache() self.readers.save_cache() signals.page_generator_finalized.send(self)
def test_process_translations(self): fr_articles = [] en_articles = [] # create a bunch of articles # 0: no translation metadata fr_articles.append(get_article(lang='fr', slug='yay0', title='Titre', content='en français')) en_articles.append(get_article(lang='en', slug='yay0', title='Title', content='in english')) # 1: translation metadata on default lang fr_articles.append(get_article(lang='fr', slug='yay1', title='Titre', content='en français')) en_articles.append(get_article(lang='en', slug='yay1', title='Title', content='in english', extra_metadata={'translation': 'true'})) # 2: translation metadata not on default lang fr_articles.append(get_article(lang='fr', slug='yay2', title='Titre', content='en français', extra_metadata={'translation': 'true'})) en_articles.append(get_article(lang='en', slug='yay2', title='Title', content='in english')) # 3: back to default language detection if all items have the # translation metadata fr_articles.append(get_article(lang='fr', slug='yay3', title='Titre', content='en français', extra_metadata={'translation': 'yep'})) en_articles.append(get_article(lang='en', slug='yay3', title='Title', content='in english', extra_metadata={'translation': 'yes'})) # try adding articles in both orders for lang0_articles, lang1_articles in ((fr_articles, en_articles), (en_articles, fr_articles)): articles = lang0_articles + lang1_articles index, trans = utils.process_translations(articles) self.assertIn(en_articles[0], index) self.assertIn(fr_articles[0], trans) self.assertNotIn(en_articles[0], trans) self.assertNotIn(fr_articles[0], index) self.assertIn(fr_articles[1], index) self.assertIn(en_articles[1], trans) self.assertNotIn(fr_articles[1], trans) self.assertNotIn(en_articles[1], index) self.assertIn(en_articles[2], index) self.assertIn(fr_articles[2], trans) self.assertNotIn(en_articles[2], trans) self.assertNotIn(fr_articles[2], index) self.assertIn(en_articles[3], index) self.assertIn(fr_articles[3], trans) self.assertNotIn(en_articles[3], trans) self.assertNotIn(fr_articles[3], index)
def test_process_translations(self): # create a bunch of articles # 1: no translation metadata fr_article1 = get_article(lang='fr', slug='yay', title='Un titre', content='en français') en_article1 = get_article(lang='en', slug='yay', title='A title', content='in english') # 2: reverse which one is the translation thanks to metadata fr_article2 = get_article(lang='fr', slug='yay2', title='Un titre', content='en français') en_article2 = get_article(lang='en', slug='yay2', title='A title', content='in english', extra_metadata={'translation': 'true'}) # 3: back to default language detection if all items have the # translation metadata fr_article3 = get_article(lang='fr', slug='yay3', title='Un titre', content='en français', extra_metadata={'translation': 'yep'}) en_article3 = get_article(lang='en', slug='yay3', title='A title', content='in english', extra_metadata={'translation': 'yes'}) articles = [ fr_article1, en_article1, fr_article2, en_article2, fr_article3, en_article3 ] index, trans = utils.process_translations(articles) self.assertIn(en_article1, index) self.assertIn(fr_article1, trans) self.assertNotIn(en_article1, trans) self.assertNotIn(fr_article1, index) self.assertIn(fr_article2, index) self.assertIn(en_article2, trans) self.assertNotIn(fr_article2, trans) self.assertNotIn(en_article2, index) self.assertIn(en_article3, index) self.assertIn(fr_article3, trans) self.assertNotIn(en_article3, trans) self.assertNotIn(fr_article3, index)
def generate_context(self): """change the context""" # return the list of files to use files = self.get_files(self.path, exclude=[ 'pages', ]) all_articles = [] for f in files: content, metadatas = read_file(f) # if no category is set, use the name of the path as a category if 'category' not in metadatas.keys(): category = os.path.basename(os.path.dirname(f)) if category == self.path: category = self.settings['DEFAULT_CATEGORY'] if category != '': metadatas['category'] = unicode(category) if 'date' not in metadatas.keys()\ and self.settings['FALLBACK_ON_FS_DATE']: metadatas['date'] = datetime.fromtimestamp(os.stat(f).st_ctime) article = Article(content, metadatas, settings=self.settings, filename=f) if not is_valid_content(article, f): continue if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['REVERSE_ARCHIVE_ORDER']) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings.get('REVERSE_CATEGORY_ORDER')) self._update_context(('articles', 'dates', 'tags', 'categories'))
def generate_context(self): # Update the context (only articles in default language) self.articles = self.context['articles'] all_articles = [] new_articles = self._generate_butter_articles() all_articles.extend(new_articles) # Continue with the rest of ArticleGenerator, code adapted from: # https://github.com/getpelican/pelican/blob/master/pelican/generators.py#L548 # ARTICLE_ORDER_BY doesn't exist in 3.3, which was in Fedora 21. # (I wanted to be able to build this on F21 at the time). articles, translations = process_translations(all_articles) # , order_by=self.settings['ARTICLE_ORDER_BY']) self.articles.extend(articles) self.translations.extend(translations) # Disabled for 3.3 compatibility, great. # signals.article_generator_pretaxonomy.send(self) for article in self.articles: # only main articles are listed in categories and tags # not translations self.categories[article.category].append(article) if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) for author in getattr(article, 'authors', []): self.authors[author].append(article) # This may not technically be right, but... # Sort the articles by date too. self.articles = list(self.articles) self.dates = self.articles self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() logger.info('++++++++++++++++++++++++++++++++++++') logger.info('GOT categories %s' % str(self.categories)) logger.info('++++++++++++++++++++++++++++++++++++') self._update_context(('articles', 'dates', 'categories', 'authors'))
def add_coming_soon(article_generator): for draft in article_generator.drafts: if hasattr(draft, "visible_draft") and draft.visible_draft.strip().lower() == "true": soon_article = Article(content=None, metadata=draft.metadata) soon_article.author = draft.author soon_article.slug = draft.slug soon_article.source_path = draft.source_path article_generator.articles.append(soon_article) article_generator.articles, article_generator.translations = process_translations( article_generator.articles, order_by=article_generator.settings["ARTICLE_ORDER_BY"] )
def generate_context(self): """ Overwrites `PagesGenerator.generate_context` and builds up `HiPage`s. """ root = HiPage("", slug='../index', name="index", title='Home', parent=None, settings=self.settings) all_pages = [] hidden_pages = [] for page_path in self.settings['PAGE_PATHS']: ps, hs = self._scan_dir_r( os.path.join(self.path, page_path), exclude=self.settings['PAGE_EXCLUDES'], parent=root) all_pages.extend(ps) hidden_pages.extend(hs) self.pages, self.translations = utils.process_translations(all_pages) self.hidden_pages, self.hidden_translations = ( utils.process_translations(hidden_pages)) self.PAGES_TREE = root # Fix (hack): Go through all translations and add the missing sub-pages for page in self.translations: # find corresponding page orig_page = None for p in self.pages: if p.name == page.name: orig_page = p break page.sub_pages.update(orig_page.sub_pages) self._update_context(['pages', 'hidden_pages', 'PAGES_TREE']) # self.context['PAGES'] = self.pages # self.context['PAGES_TREE'] = root logger.info("\n" + ascii_tree(self.context['PAGES_TREE'], print_item=lambda x: x.title)) signals.page_generator_finalized.send(self)
def generate_context(self): all_pages = [] for f in self.get_files(os.sep.join((self.path, 'pages'))): content, metadatas = read_file(f) page = Page(content, metadatas, settings=self.settings, filename=f) if not is_valid_content(page, f): continue all_pages.append(page) self.pages, self.translations = process_translations(all_pages) self._update_context(('pages', )) self.context['PAGES'] = self.pages
def generate_context(self): """change the context""" # return the list of files to use files = self.get_files(self.path, exclude=['pages',]) all_articles = [] for f in files: content, metadatas = read_file(f) # if no category is set, use the name of the path as a category if 'category' not in metadatas.keys(): category = os.path.basename(os.path.dirname(f)) if category == self.path: category = self.settings['DEFAULT_CATEGORY'] if category != '': metadatas['category'] = unicode(category) if 'date' not in metadatas.keys()\ and self.settings['FALLBACK_ON_FS_DATE']: metadatas['date'] = datetime.fromtimestamp(os.stat(f).st_ctime) article = Article(content, metadatas, settings=self.settings, filename=f) if not is_valid_content(article, f): continue if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['REVERSE_ARCHIVE_ORDER']) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings.get('REVERSE_CATEGORY_ORDER')) self._update_context(('articles', 'dates', 'tags', 'categories'))
def generate_context(self): all_pages = [] for f in self.get_files(os.sep.join((self.path, 'pages'))): content, metadata = read_file(f) page = Page(content, metadata, settings=self.settings, filename=f) if not is_valid_content(page, f): continue all_pages.append(page) self.pages, self.translations = process_translations(all_pages) self._update_context(('pages', )) self.context['PAGES'] = self.pages
def test_process_translations(self): # create a bunch of articles fr_article1 = get_article(lang='fr', slug='yay', title='Un titre', content='en français') en_article1 = get_article(lang='en', slug='yay', title='A title', content='in english') articles = [fr_article1, en_article1] index, trans = utils.process_translations(articles) self.assertIn(en_article1, index) self.assertIn(fr_article1, trans) self.assertNotIn(en_article1, trans) self.assertNotIn(fr_article1, index)
def generate_context(self): all_pages = [] for f in self.get_files(os.sep.join((self.path, 'pages'))): content, metadata = read_file(f) page = Page(content, metadata, settings=self.settings, filename=f) if not is_valid_content(page, f): continue if self.settings.get('CLEAN_URLS_NO_PROXY'): # cleaning page url page.save_as = os.path.join(page.slug, 'index.html') page.url = os.path.dirname(page.save_as) + '/' all_pages.append(page) self.pages, self.translations = process_translations(all_pages) self._update_context(('pages', )) self.context['PAGES'] = self.pages
def test_process_translations(self): # create a bunch of articles # 1: no translation metadata fr_article1 = get_article(lang='fr', slug='yay', title='Un titre', content='en français') en_article1 = get_article(lang='en', slug='yay', title='A title', content='in english') # 2: reverse which one is the translation thanks to metadata fr_article2 = get_article(lang='fr', slug='yay2', title='Un titre', content='en français') en_article2 = get_article(lang='en', slug='yay2', title='A title', content='in english', extra_metadata={'translation': 'true'}) # 3: back to default language detection if all items have the # translation metadata fr_article3 = get_article(lang='fr', slug='yay3', title='Un titre', content='en français', extra_metadata={'translation': 'yep'}) en_article3 = get_article(lang='en', slug='yay3', title='A title', content='in english', extra_metadata={'translation': 'yes'}) articles = [fr_article1, en_article1, fr_article2, en_article2, fr_article3, en_article3] index, trans = utils.process_translations(articles) self.assertIn(en_article1, index) self.assertIn(fr_article1, trans) self.assertNotIn(en_article1, trans) self.assertNotIn(fr_article1, index) self.assertIn(fr_article2, index) self.assertIn(en_article2, trans) self.assertNotIn(fr_article2, trans) self.assertNotIn(en_article2, index) self.assertIn(en_article3, index) self.assertIn(fr_article3, trans) self.assertNotIn(en_article3, trans) self.assertNotIn(fr_article3, index)
def test_process_translations(self): # create a bunch of articles # 1: no translation metadata fr_article1 = get_article(lang="fr", slug="yay", title="Un titre", content="en français") en_article1 = get_article(lang="en", slug="yay", title="A title", content="in english") # 2: reverse which one is the translation thanks to metadata fr_article2 = get_article(lang="fr", slug="yay2", title="Un titre", content="en français") en_article2 = get_article( lang="en", slug="yay2", title="A title", content="in english", extra_metadata={"translation": "true"} ) # 3: back to default language detection if all items have the # translation metadata fr_article3 = get_article( lang="fr", slug="yay3", title="Un titre", content="en français", extra_metadata={"translation": "yep"} ) en_article3 = get_article( lang="en", slug="yay3", title="A title", content="in english", extra_metadata={"translation": "yes"} ) articles = [fr_article1, en_article1, fr_article2, en_article2, fr_article3, en_article3] index, trans = utils.process_translations(articles) self.assertIn(en_article1, index) self.assertIn(fr_article1, trans) self.assertNotIn(en_article1, trans) self.assertNotIn(fr_article1, index) self.assertIn(fr_article2, index) self.assertIn(en_article2, trans) self.assertNotIn(fr_article2, trans) self.assertNotIn(en_article2, index) self.assertIn(en_article3, index) self.assertIn(fr_article3, trans) self.assertNotIn(en_article3, trans) self.assertNotIn(fr_article3, index)
class ArticlesGenerator(Generator): """Generate blog articles""" def __init__(self, *args, **kwargs): """initialize properties""" self.articles = [] # only articles in default language self.translations = [] self.dates = {} self.tags = defaultdict(list) self.categories = defaultdict(list) self.authors = defaultdict(list) super(ArticlesGenerator, self).__init__(*args, **kwargs) self.drafts = [] signals.article_generator_init.send(self) def generate_feeds(self, writer): """Generate the feeds from the current context, and output files.""" if self.settings.get('FEED'): writer.write_feed(self.articles, self.context, self.settings['FEED']) if self.settings.get('FEED_RSS'): writer.write_feed(self.articles, self.context, self.settings['FEED_RSS'], feed_type='rss') for cat, arts in self.categories: arts.sort(key=attrgetter('date'), reverse=True) if self.settings.get('CATEGORY_FEED'): writer.write_feed(arts, self.context, self.settings['CATEGORY_FEED'] % cat) if self.settings.get('CATEGORY_FEED_RSS'): writer.write_feed(arts, self.context, self.settings['CATEGORY_FEED_RSS'] % cat, feed_type='rss') if self.settings.get('TAG_FEED') or self.settings.get('TAG_FEED_RSS'): for tag, arts in self.tags.items(): arts.sort(key=attrgetter('date'), reverse=True) if self.settings.get('TAG_FEED'): writer.write_feed(arts, self.context, self.settings['TAG_FEED'] % tag) if self.settings.get('TAG_FEED_RSS'): writer.write_feed(arts, self.context, self.settings['TAG_FEED_RSS'] % tag, feed_type='rss') if self.settings.get('TRANSLATION_FEED'): translations_feeds = defaultdict(list) for article in chain(self.articles, self.translations): translations_feeds[article.lang].append(article) for lang, items in translations_feeds.items(): items.sort(key=attrgetter('date'), reverse=True) writer.write_feed(items, self.context, self.settings['TRANSLATION_FEED'] % lang) def generate_articles(self, write): """Generate the articles.""" article_template = self.get_template('article') for article in chain(self.translations, self.articles): write(article.save_as, article_template, self.context, article=article, category=article.category) def generate_direct_templates(self, write): """Generate direct templates pages""" PAGINATED_TEMPLATES = self.settings.get('PAGINATED_DIRECT_TEMPLATES') for template in self.settings.get('DIRECT_TEMPLATES'): paginated = {} if template in PAGINATED_TEMPLATES: paginated = {'articles': self.articles, 'dates': self.dates} save_as = self.settings.get("%s_SAVE_AS" % template.upper(), '%s.html' % template) if not save_as: continue write(save_as, self.get_template(template), self.context, blog=True, paginated=paginated, page_name=template) def generate_tags(self, write): """Generate Tags pages.""" tag_template = self.get_template('tag') for tag, articles in self.tags.items(): articles.sort(key=attrgetter('date'), reverse=True) dates = [article for article in self.dates if article in articles] write(tag.save_as, tag_template, self.context, tag=tag, articles=articles, dates=dates, paginated={ 'articles': articles, 'dates': dates }, page_name=u'tag/%s' % tag) def generate_categories(self, write): """Generate category pages.""" category_template = self.get_template('category') for cat, articles in self.categories: dates = [article for article in self.dates if article in articles] write(cat.save_as, category_template, self.context, category=cat, articles=articles, dates=dates, paginated={ 'articles': articles, 'dates': dates }, page_name=u'category/%s' % cat) def generate_authors(self, write): """Generate Author pages.""" author_template = self.get_template('author') for aut, articles in self.authors: dates = [article for article in self.dates if article in articles] write(aut.save_as, author_template, self.context, author=aut, articles=articles, dates=dates, paginated={ 'articles': articles, 'dates': dates }, page_name=u'author/%s' % aut) def generate_drafts(self, write): """Generate drafts pages.""" article_template = self.get_template('article') for article in self.drafts: write('drafts/%s.html' % article.slug, article_template, self.context, article=article, category=article.category) def generate_pages(self, writer): """Generate the pages on the disk""" write = partial(writer.write_file, relative_urls=self.settings.get('RELATIVE_URLS')) # to minimize the number of relative path stuff modification # in writer, articles pass first self.generate_articles(write) self.generate_direct_templates(write) # and subfolders after that self.generate_tags(write) self.generate_categories(write) self.generate_authors(write) self.generate_drafts(write) def generate_context(self): """change the context""" article_path = os.path.normpath( # we have to remove trailing slashes os.path.join(self.path, self.settings['ARTICLE_DIR'])) all_articles = [] for f in self.get_files(article_path, exclude=self.settings['ARTICLE_EXCLUDES']): try: content, metadata = read_file(f, settings=self.settings) except Exception, e: logger.warning(u'Could not process %s\n%s' % (f, str(e))) continue # if no category is set, use the name of the path as a category if 'category' not in metadata: if os.path.dirname( f ) == article_path: # if the article is not in a subdirectory category = self.settings['DEFAULT_CATEGORY'] else: category = os.path.basename(os.path.dirname(f))\ .decode('utf-8') if category != '': metadata['category'] = Category(category, self.settings) if 'date' not in metadata and self.settings['FALLBACK_ON_FS_DATE']: metadata['date'] = datetime.datetime.fromtimestamp( os.stat(f).st_ctime) signals.article_generate_context.send(self, metadata=metadata) article = Article(content, metadata, settings=self.settings, filename=f) if not is_valid_content(article, f): continue if article.status == "published": if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) elif article.status == "draft": self.drafts.append(article) else: logger.warning( u"Unknown status %s for file %s, skipping it." % (repr(unicode.encode(article.status, 'utf-8')), repr(f))) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) self.authors[article.author].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['REVERSE_ARCHIVE_ORDER']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = map(itemgetter(1), tag_cloud) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ (tag, int( math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count) or 1)))) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(key=lambda item: item[0].name, reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort(key=lambda item: item[0].name) self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors'))
def generate_context(self): """Add the articles into the shared context""" all_articles = [] all_drafts = [] for f in self.get_files( self.settings['ARTICLE_PATHS'], exclude=self.settings['ARTICLE_EXCLUDES']): article_or_draft = self.get_cached_data(f, None) if article_or_draft is None: #TODO needs overhaul, maybe nomad for read_file solution, unified behaviour try: article_or_draft = self.readers.read_file( base_path=self.path, path=f, content_class=Article, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self) except Exception as e: logger.error('Could not process %s\n%s', f, e, exc_info=self.settings.get('DEBUG', False)) self._add_failed_source_path(f) continue if not is_valid_content(article_or_draft, f): self._add_failed_source_path(f) continue if article_or_draft.status.lower() == "published": all_articles.append(article_or_draft) elif article_or_draft.status.lower() == "draft": article_or_draft = self.readers.read_file( base_path=self.path, path=f, content_class=Draft, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self) self.add_source_path(article_or_draft) all_drafts.append(article_or_draft) else: logger.error("Unknown status '%s' for file %s, skipping it.", article_or_draft.status, f) self._add_failed_source_path(f) continue self.cache_data(f, article_or_draft) self.add_source_path(article_or_draft) self.articles, self.translations = process_translations(all_articles, order_by=self.settings['ARTICLE_ORDER_BY']) self.drafts, self.drafts_translations = \ process_translations(all_drafts) signals.article_generator_pretaxonomy.send(self) for article in self.articles: # only main articles are listed in categories and tags # not translations self.categories[article.category].append(article) if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) for author in getattr(article, 'authors', []): self.authors[author].append(article) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'tags', 'categories', 'authors', 'related_posts', 'drafts')) self.save_cache() self.readers.save_cache() signals.article_generator_finalized.send(self)
def generate_context(self): all_pages = [] limit = 20 urls = [] num = 0 news = [i for i in who_news.get_feeds() if i['date'] and i['title']] news.sort(key=lambda i: i['date'], reverse=True) page_num = 0 pages = math.ceil(len(news) / limit) for article in news: article['date_str'] = '' if article['date']: article['date_str'] = datetime.date.strftime( article['date'], '%d %b %Y') try: fpath = os.path.join( 'news', slugify('{}-{}-{}'.format(article['date_str'][:20], article['title'][:50], num)) + '.html') except UnicodeEncodeError: logger.info('Could not encode airline') continue num += 1 metadata = { 'title': article['title'], 'article': article, 'save_as': fpath } page = contents.Page(deepcopy(article), metadata=metadata) article['url'] = fpath all_pages.append(page) urls.append(article) if len(urls) >= limit: page_num += 1 url = 'newsroom.html' if page_num == 1 else f'newsroom-{page_num}.html' all_pages.append( contents.Page({'urls': urls}, metadata={ 'title': 'News list', 'save_as': url, 'template': 'news_list', 'urls': urls, 'pages_num': page_num, 'pages': pages })) urls = [] if urls: page_num += 1 url = 'newsroom.html' if page_num == 1 else f'newsroom-{page_num}.html' all_pages.append( contents.Page({'urls': urls}, metadata={ 'title': 'News list', 'save_as': url, 'template': 'news_list', 'urls': urls, 'pages_num': page_num, 'pages': pages })) self.pages, self.translations = process_translations(all_pages, ) signals.page_generator_finalized.send(self)
def generate_context(self): """Add the articles into the shared context""" all_articles = [] all_drafts = [] for f in self.get_files( self.settings['ARTICLE_PATHS'], exclude=self.settings['ARTICLE_EXCLUDES']): article = self.get_cached_data(f, None) if article is None: try: article = self.readers.read_file( base_path=self.path, path=f, content_class=Article, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self) except Exception as e: logger.error( 'Could not process %s\n%s', f, e, exc_info=self.settings.get('DEBUG', False)) self._add_failed_source_path(f) continue if not article.is_valid(): self._add_failed_source_path(f) continue self.cache_data(f, article) if article.status == "published": all_articles.append(article) elif article.status == "draft": all_drafts.append(article) self.add_source_path(article) self.articles, self.translations = process_translations(all_articles) self.articles = order_content( self.articles, order_by=self.settings['ARTICLE_ORDER_BY']) self.drafts, self.drafts_translations = \ process_translations(all_drafts) signals.article_generator_pretaxonomy.send(self) for article in self.articles: # only main articles are listed in categories and tags # not translations self.categories[article.category].append(article) if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) for author in getattr(article, 'authors', []): self.authors[author].append(article) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'tags', 'categories', 'authors', 'related_posts', 'drafts')) self.save_cache() self.readers.save_cache() signals.article_generator_finalized.send(self)
def generate_context(self): """Add the articles into the shared context""" all_articles = [] all_drafts = [] for f in self.get_files(self.settings["ARTICLE_PATHS"], exclude=self.settings["ARTICLE_EXCLUDES"]): article = self.get_cached_data(f, None) if article is None: try: article = self.readers.read_file( base_path=self.path, path=f, content_class=Article, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self, ) except Exception as e: logger.error("Could not process %s\n%s", f, e, exc_info=self.settings.get("DEBUG", False)) continue if not is_valid_content(article, f): continue self.cache_data(f, article) self.add_source_path(article) if article.status.lower() == "published": all_articles.append(article) elif article.status.lower() == "draft": draft = self.readers.read_file( base_path=self.path, path=f, content_class=Draft, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self, ) all_drafts.append(draft) else: logger.error("Unknown status '%s' for file %s, skipping it.", article.status, f) self.articles, self.translations = process_translations(all_articles) self.drafts, self.drafts_translations = process_translations(all_drafts) signals.article_generator_pretaxonomy.send(self) for article in self.articles: # only main articles are listed in categories and tags # not translations self.categories[article.category].append(article) if hasattr(article, "tags"): for tag in article.tags: self.tags[tag].append(article) # ignore blank authors as well as undefined for author in getattr(article, "authors", []): if author.name != "": self.authors[author].append(article) # sort the articles by date self.articles.sort(key=attrgetter("date"), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter("date"), reverse=self.context["NEWEST_FIRST_ARCHIVES"]) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, "tags", []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[: self.settings.get("TAG_CLOUD_MAX_ITEMS")] tags = list(map(itemgetter(1), tag_cloud)) if tags: max_count = max(tags) steps = self.settings.get("TAG_CLOUD_STEPS") # calculate word sizes self.tag_cloud = [ (tag, int(math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count) or 1)))) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings["REVERSE_CATEGORY_ORDER"]) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(("articles", "dates", "tags", "categories", "tag_cloud", "authors", "related_posts")) self.save_cache() self.readers.save_cache() signals.article_generator_finalized.send(self)
def generate_context(self): """Add the articles into the shared context""" all_articles = [] all_drafts = [] for f in self.get_files( self.settings['ARTICLE_PATHS'], exclude=self.settings['ARTICLE_EXCLUDES']): article = self.get_cached_data(f, None) if article is None: try: article = self.readers.read_file( base_path=self.path, path=f, content_class=Article, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self) except Exception as e: logger.error('Could not process %s\n%s', f, e, exc_info=self.settings.get('DEBUG', False)) self._add_failed_source_path(f) continue if not is_valid_content(article, f): self._add_failed_source_path(f) continue self.cache_data(f, article) self.add_source_path(article) if article.status.lower() == "published": all_articles.append(article) elif article.status.lower() == "draft": draft = self.readers.read_file( base_path=self.path, path=f, content_class=Draft, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self) self.add_source_path(draft) all_drafts.append(draft) else: logger.error("Unknown status '%s' for file %s, skipping it.", article.status, f) self.articles, self.translations = process_translations(all_articles, order_by=self.settings['ARTICLE_ORDER_BY']) self.drafts, self.drafts_translations = \ process_translations(all_drafts) signals.article_generator_pretaxonomy.send(self) for article in self.articles: # only main articles are listed in categories and tags # not translations self.categories[article.category].append(article) if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) for author in getattr(article, 'authors', []): self.authors[author].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = list(map(itemgetter(1), tag_cloud)) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ ( tag, int(math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count)or 1))) ) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors', 'related_posts')) self.save_cache() self.readers.save_cache() signals.article_generator_finalized.send(self)
def generate_context(self): """Add the articles into the shared context""" all_articles = [] for f in self.get_files(self.settings['ARTICLE_DIR'], exclude=self.settings['ARTICLE_EXCLUDES']): try: article = read_file( base_path=self.path, path=f, content_class=Article, settings=self.settings, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self) except Exception as e: logger.warning('Could not process {}\n{}'.format(f, e)) continue if not is_valid_content(article, f): continue self.add_source_path(article) if article.status == "published": all_articles.append(article) elif article.status == "draft": self.drafts.append(article) else: logger.warning("Unknown status %s for file %s, skipping it." % (repr(article.status), repr(f))) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories and tags # not translations self.categories[article.category].append(article) if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) # ignore blank authors as well as undefined if hasattr(article, 'author') and article.author.name != '': self.authors[article.author].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = list(map(itemgetter(1), tag_cloud)) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ (tag, int( math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count) or 1)))) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors', 'related_posts')) signals.article_generator_finalized.send(self)
def generate_context(self): """change the context""" # return the list of files to use files = self.get_files(self.path, exclude=['pages',]) all_articles = [] for f in files: content, metadata = read_file(f) # if no category is set, use the name of the path as a category if 'category' not in metadata.keys(): if os.path.dirname(f) == self.path: category = self.settings['DEFAULT_CATEGORY'] else: category = os.path.basename(os.path.dirname(f)) if category != '': metadata['category'] = unicode(category) if 'date' not in metadata.keys()\ and self.settings['FALLBACK_ON_FS_DATE']: metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime) article = Article(content, metadata, settings=self.settings, filename=f) if not is_valid_content(article, f): continue if article.status == "published": if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) elif article.status == "draft": self.drafts.append(article) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['REVERSE_ARCHIVE_ORDER']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key = itemgetter(1), reverse = True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = map(itemgetter(1), tag_cloud) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ ( tag, int( math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count)or 1)) ) ) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings.get('REVERSE_CATEGORY_ORDER')) self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud'))
def generate_context(self): """change the context""" # return the list of files to use files = self.get_files(self.path, exclude=[ 'pages', ]) all_articles = [] for f in files: content, metadatas = read_file(f) # if no category is set, use the name of the path as a category if 'category' not in metadatas.keys(): if os.path.dirname(f) == self.path: category = self.settings['DEFAULT_CATEGORY'] else: category = os.path.basename(os.path.dirname(f)) if category != '': metadatas['category'] = unicode(category) if 'date' not in metadatas.keys()\ and self.settings['FALLBACK_ON_FS_DATE']: metadatas['date'] = datetime.fromtimestamp(os.stat(f).st_ctime) article = Article(content, metadatas, settings=self.settings, filename=f) if not is_valid_content(article, f): continue if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['REVERSE_ARCHIVE_ORDER']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = map(itemgetter(1), tag_cloud) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ (tag, int( math.floor(steps - (steps - 1) * math.log(count) / math.log(max_count)))) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) self._update_context( ('articles', 'dates', 'tags', 'categories', 'tag_cloud'))
def generate_context(self): """Add the entities into the shared context""" all_entities = [] all_drafts = [] for f in self.get_files( self.settings['PATHS'], exclude=self.settings['EXCLUDES']): entity = self.get_cached_data(f, None) if entity is None: entity_class = EntityFactory( self.entity_type, self.settings['MANDATORY_PROPERTIES'], self.settings['DEFAULT_TEMPLATE']) try: entity = self.readers.read_file( base_path=self.path, path=f, content_class=entity_class, context=self.context, preread_signal=entity_subgenerator_preread, preread_sender=self, context_signal=entity_subgenerator_context, context_sender=self) except Exception as e: logger.error('Could not process %s\n%s', f, e, exc_info=self.settings.get('DEBUG', False)) continue if not contents.is_valid_content(entity, f): continue self.cache_data(f, entity) self.add_source_path(entity) if entity.status.lower() == "published": all_entities.append(entity) elif entity.status.lower() == "draft": all_drafts.append(entity) else: logger.warning("Unknown status '%s' for file %s, skipping it.", entity.status, f) self.entities, self.translations = process_translations(all_entities) self.drafts, self.drafts_translations = \ process_translations(all_drafts) custom_sort_attr = self.settings.get("SORT_ATTRIBUTES", []) self.sort_attrs = ["date"] if custom_sort_attr: self.sort_attrs = custom_sort_attr + self.sort_attrs self.entities.sort(key=attrgetter(*self.sort_attrs), reverse=True) entity_subgenerator_pretaxonomy.send(self) for entity in self.entities: # only main entities are listed in categories and tags # not translations if hasattr(entity, 'category'): self.categories[entity.category].append(entity) if hasattr(entity, 'tags'): for tag in entity.tags: self.tags[tag].append(entity) # ignore blank authors as well as undefined for author in getattr(entity, 'authors', []): if author.name != '': self.authors[author].append(entity) # create tag cloud tag_cloud = defaultdict(int) for entity in self.entities: for tag in getattr(entity, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = list(map(itemgetter(1), tag_cloud)) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ ( tag, int(math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count)or 1))) ) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self.save_cache() self.readers.save_cache() entity_subgenerator_finalized.send(self)
def generate_context(self): """Add the articles into the shared context""" article_path = os.path.normpath( # we have to remove trailing slashes os.path.join(self.path, self.settings["ARTICLE_DIR"]) ) all_articles = [] for f in self.get_files(article_path, exclude=self.settings["ARTICLE_EXCLUDES"]): try: signals.article_generate_preread.send(self) content, metadata = read_file(f, settings=self.settings) except Exception as e: logger.warning("Could not process %s\n%s" % (f, str(e))) continue # if no category is set, use the name of the path as a category if "category" not in metadata: if self.settings["USE_FOLDER_AS_CATEGORY"] and os.path.dirname(f) != article_path: # if the article is in a subdirectory category = os.path.basename(os.path.dirname(f)) else: # if the article is not in a subdirectory category = self.settings["DEFAULT_CATEGORY"] if category != "": metadata["category"] = Category(category, self.settings) if "date" not in metadata and self.settings.get("DEFAULT_DATE"): if self.settings["DEFAULT_DATE"] == "fs": metadata["date"] = datetime.datetime.fromtimestamp(os.stat(f).st_ctime) else: metadata["date"] = datetime.datetime(*self.settings["DEFAULT_DATE"]) signals.article_generate_context.send(self, metadata=metadata) article = Article(content, metadata, settings=self.settings, source_path=f, context=self.context) if not is_valid_content(article, f): continue self.add_source_path(article) if article.status == "published": if hasattr(article, "tags"): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) elif article.status == "draft": self.drafts.append(article) else: logger.warning("Unknown status %s for file %s, skipping it." % (repr(article.status), repr(f))) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) # ignore blank authors as well as undefined if hasattr(article, "author") and article.author.name != "": self.authors[article.author].append(article) # sort the articles by date self.articles.sort(key=attrgetter("date"), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter("date"), reverse=self.context["NEWEST_FIRST_ARCHIVES"]) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, "tags", []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[: self.settings.get("TAG_CLOUD_MAX_ITEMS")] tags = list(map(itemgetter(1), tag_cloud)) if tags: max_count = max(tags) steps = self.settings.get("TAG_CLOUD_STEPS") # calculate word sizes self.tag_cloud = [ (tag, int(math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count) or 1)))) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings["REVERSE_CATEGORY_ORDER"]) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(("articles", "dates", "tags", "categories", "tag_cloud", "authors", "related_posts")) signals.article_generator_finalized.send(self)
def generate_context(self): """Add the articles into the shared context""" article_path = os.path.normpath( # we have to remove trailing slashes os.path.join(self.path, self.settings['ARTICLE_DIR']) ) all_articles = [] for f in self.get_files( article_path, exclude=self.settings['ARTICLE_EXCLUDES']): try: signals.article_generate_preread.send(self) content, metadata = read_file(f, settings=self.settings) except Exception as e: logger.warning(u'Could not process %s\n%s' % (f, str(e))) continue # if no category is set, use the name of the path as a category if 'category' not in metadata: if (self.settings['USE_FOLDER_AS_CATEGORY'] and os.path.dirname(f) != article_path): # if the article is in a subdirectory category = os.path.basename(os.path.dirname(f))\ .decode('utf-8') else: # if the article is not in a subdirectory category = self.settings['DEFAULT_CATEGORY'] if category != '': metadata['category'] = Category(category, self.settings) if 'date' not in metadata and self.settings.get('DEFAULT_DATE'): if self.settings['DEFAULT_DATE'] == 'fs': metadata['date'] = datetime.datetime.fromtimestamp( os.stat(f).st_ctime) else: metadata['date'] = datetime.datetime( *self.settings['DEFAULT_DATE']) signals.article_generate_context.send(self, metadata=metadata) article = Article(content, metadata, settings=self.settings, filename=f, context=self.context) if not is_valid_content(article, f): continue self.add_filename(article) if article.status == "published": if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) elif article.status == "draft": self.drafts.append(article) else: logger.warning(u"Unknown status %s for file %s, skipping it." % (repr(unicode.encode(article.status, 'utf-8')), repr(f))) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) # ignore blank authors as well as undefined if hasattr(article,'author') and article.author.name != '': self.authors[article.author].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = map(itemgetter(1), tag_cloud) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ ( tag, int(math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count)or 1))) ) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( key=lambda item: item[0].name, reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort(key=lambda item: item[0].name) self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors', 'related_posts')) signals.article_generator_finalized.send(self)
def generate_context(self): # Update the context (only articles in default language) self.articles = self.context['articles'] # Complain if MBOX_PATH and MBOX_CATEGORY are not of the same length. mboxPaths = self.settings.get('MBOX_PATH') mboxCategories = self.settings.get('MBOX_CATEGORY') errMsg = 'MBOX_PATH, MBOX_CATEGORY not of equal length or non-empty.' if len(mboxPaths) != len(mboxCategories) or len(mboxPaths) <= 0: logger.error(errMsg) return all_articles = [] for i in xrange(len(mboxPaths)): mboxPath = mboxPaths[i] mboxCategory = mboxCategories[i] new_articles = self._generate_mbox_articles(mboxPath, mboxCategory) all_articles.extend(new_articles) # Log that we did stuff. print(('Read in %d messages from %s and converted to articles in ' + 'category %s.') % (len(new_articles), mboxPath, mboxCategory)) print('Read in %d messages from all mailboxes.' % (len(all_articles))) # Continue with the rest of ArticleGenerator, code adapted from: # https://github.com/getpelican/pelican/blob/master/pelican/generators.py#L548 # ARTICLE_ORDER_BY doesn't exist in 3.3, which was in Fedora 21. # (I wanted to be able to build this on F21 at the time). articles, translations = process_translations(all_articles) # , order_by=self.settings['ARTICLE_ORDER_BY']) self.articles.extend(articles) self.translations.extend(translations) # Disabled for 3.3 compatibility, great. # signals.article_generator_pretaxonomy.send(self) for article in self.articles: # only main articles are listed in categories and tags # not translations # We have to use django for this, unfortunately. if article.date.tzinfo is None: article.date = pytz.UTC.localize(article.date) self.categories[article.category].append(article) # Support for Author and Authors. if hasattr(article, 'author') and article.author.name != '': self.authors[article.author].append(article) else: for author in getattr(article, 'authors', []): self.authors[author].append(article) # This may not technically be right, but... # Sort the articles by date too. self.articles = list(self.articles) self.dates = self.articles self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'categories', 'authors'))
class ArticlesGenerator(Generator): """Generate blog articles""" def __init__(self, *args, **kwargs): """initialize properties""" self.articles = [] # only articles in default language self.translations = [] self.dates = {} self.tags = defaultdict(list) self.categories = defaultdict(list) self.authors = defaultdict(list) super(ArticlesGenerator, self).__init__(*args, **kwargs) self.drafts = [] def generate_feeds(self, writer): """Generate the feeds from the current context, and output files.""" writer.write_feed(self.articles, self.context, self.settings['FEED']) if 'FEED_RSS' in self.settings: writer.write_feed(self.articles, self.context, self.settings['FEED_RSS'], feed_type='rss') for cat, arts in self.categories: arts.sort(key=attrgetter('date'), reverse=True) writer.write_feed(arts, self.context, self.settings['CATEGORY_FEED'] % cat) if 'CATEGORY_FEED_RSS' in self.settings: writer.write_feed(arts, self.context, self.settings['CATEGORY_FEED_RSS'] % cat, feed_type='rss') if 'TAG_FEED' in self.settings: for tag, arts in self.tags.items(): arts.sort(key=attrgetter('date'), reverse=True) writer.write_feed(arts, self.context, self.settings['TAG_FEED'] % tag) if 'TAG_FEED_RSS' in self.settings: writer.write_feed(arts, self.context, self.settings['TAG_FEED_RSS'] % tag, feed_type='rss') translations_feeds = defaultdict(list) for article in chain(self.articles, self.translations): translations_feeds[article.lang].append(article) for lang, items in translations_feeds.items(): items.sort(key=attrgetter('date'), reverse=True) writer.write_feed(items, self.context, self.settings['TRANSLATION_FEED'] % lang) def generate_pages(self, writer): """Generate the pages on the disk""" write = partial(writer.write_file, relative_urls=self.settings.get('RELATIVE_URLS')) # to minimize the number of relative path stuff modification # in writer, articles pass first article_template = self.get_template('article') for article in chain(self.translations, self.articles): write(article.save_as, article_template, self.context, article=article, category=article.category) PAGINATED_TEMPLATES = self.settings.get('PAGINATED_DIRECT_TEMPLATES') for template in self.settings.get('DIRECT_TEMPLATES'): paginated = {} if template in PAGINATED_TEMPLATES: paginated = {'articles': self.articles, 'dates': self.dates} write('%s.html' % template, self.get_template(template), self.context, blog=True, paginated=paginated, page_name=template) # and subfolders after that tag_template = self.get_template('tag') for tag, articles in self.tags.items(): articles.sort(key=attrgetter('date'), reverse=True) dates = [article for article in self.dates if article in articles] write('tag/%s.html' % tag, tag_template, self.context, tag=tag, articles=articles, dates=dates, paginated={ 'articles': articles, 'dates': dates }, page_name='tag/%s' % tag) category_template = self.get_template('category') for cat, articles in self.categories: dates = [article for article in self.dates if article in articles] write('category/%s.html' % cat, category_template, self.context, category=cat, articles=articles, dates=dates, paginated={ 'articles': articles, 'dates': dates }, page_name='category/%s' % cat) author_template = self.get_template('author') for aut, articles in self.authors: dates = [article for article in self.dates if article in articles] write('author/%s.html' % aut, author_template, self.context, author=aut, articles=articles, dates=dates, paginated={ 'articles': articles, 'dates': dates }, page_name='author/%s' % aut) for article in self.drafts: write('drafts/%s.html' % article.slug, article_template, self.context, article=article, category=article.category) def generate_context(self): """change the context""" # return the list of files to use files = self.get_files(self.path, exclude=[ 'pages', ]) all_articles = [] for f in files: try: content, metadata = read_file(f, settings=self.settings) except Exception, e: warning(u'Could not process %s\n%s' % (f, str(e))) continue # if no category is set, use the name of the path as a category if 'category' not in metadata.keys(): if os.path.dirname(f) == self.path: category = self.settings['DEFAULT_CATEGORY'] else: category = os.path.basename( os.path.dirname(f)).decode('utf-8') if category != '': metadata['category'] = unicode(category) if 'date' not in metadata.keys()\ and self.settings['FALLBACK_ON_FS_DATE']: metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime) article = Article(content, metadata, settings=self.settings, filename=f) if not is_valid_content(article, f): continue add_to_url = u'' if 'ARTICLE_PERMALINK_STRUCTURE' in self.settings: article_permalink_structure = self.settings[ 'ARTICLE_PERMALINK_STRUCTURE'] article_permalink_structure = article_permalink_structure.lstrip( '/').replace('%(', "%%(") # try to substitute any python datetime directive add_to_url = article.date.strftime(article_permalink_structure) # try to substitute any article metadata in rest file add_to_url = add_to_url % article.__dict__ add_to_url = [slugify(i) for i in add_to_url.split('/')] add_to_url = os.path.join(*add_to_url) article.url = urlparse.urljoin(add_to_url, article.url) article.save_as = urlparse.urljoin(add_to_url, article.save_as) if article.status == "published": if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) elif article.status == "draft": self.drafts.append(article) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) self.authors[article.author].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['REVERSE_ARCHIVE_ORDER']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = map(itemgetter(1), tag_cloud) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ (tag, int( math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count) or 1)))) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings.get('REVERSE_CATEGORY_ORDER')) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors'))
def generate_context(self): """Add the articles into the shared context""" restrict_all_articles_to_tags = set( self.settings.get("ARTICLES_RESTRICT_TO_TAGS", [])) all_articles = [] all_drafts = [] for f in self.get_files( self.settings['ARTICLE_PATHS'], exclude=self.settings['ARTICLE_EXCLUDES']): article = self.get_cached_data(f, None) if article is None: try: article = self.readers.read_file( base_path=self.path, path=f, content_class=Article, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self) except Exception as e: logger.error( 'Could not process %s\n%s', f, e, exc_info=self.settings.get('DEBUG', False)) self._add_failed_source_path(f) continue if not article.is_valid(): self._add_failed_source_path(f) continue self.cache_data(f, article) if not restrict_all_articles_to_tags <= set( getattr(article, 'tags', [])): continue if article.status == "published": all_articles.append(article) elif article.status == "draft": all_drafts.append(article) self.add_source_path(article) self.articles, self.translations = process_translations(all_articles) self.articles = order_content( self.articles, order_by=self.settings['ARTICLE_ORDER_BY']) self.drafts, self.drafts_translations = \ process_translations(all_drafts) signals.article_generator_pretaxonomy.send(self) for article in self.articles: # only main articles are listed in categories and tags # not translations self.categories[article.category].append(article) if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) for author in getattr(article, 'authors', []): self.authors[author].append(article) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'tags', 'categories', 'authors', 'related_posts', 'drafts')) self.save_cache() self.readers.save_cache() signals.article_generator_finalized.send(self)
def generate_context(self): """Add the entities into the shared context""" all_entities = [] all_drafts = [] for f in self.get_files( self.settings['PATHS'], exclude=self.settings['EXCLUDES']): entity_or_draft = self.get_cached_data(f, None) if entity_or_draft is None: entity_class = EntityFactory( self.entity_type, self.settings['MANDATORY_PROPERTIES'], self.settings['DEFAULT_TEMPLATE']) try: entity_or_draft = self.readers.read_file( base_path=self.path, path=f, content_class=entity_class, context=self.context, preread_signal=entity_subgenerator_preread, preread_sender=self, context_signal=entity_subgenerator_context, context_sender=self) except Exception as e: logger.error('Could not process %s\n%s', f, e, exc_info=self.settings.get('DEBUG', False)) self._add_failed_source_path(f) continue if not contents.is_valid_content(entity_or_draft, f): self._add_failed_source_path(f) continue known_statuses = ("published", "draft") if entity_or_draft.status.lower() not in known_statuses: logger.warning("Unknown status '%s' for file %s, skipping it.", entity_or_draft.status, f) self._add_failed_source_path(f) continue self.cache_data(f, entity_or_draft) if entity_or_draft.status.lower() == "published": all_entities.append(entity_or_draft) else: all_drafts.append(entity_or_draft) self.add_source_path(entity_or_draft) self.entities, self.translations = process_translations( all_entities) self.drafts, self.drafts_translations = \ process_translations(all_drafts) sorter = self.settings["SORTER"] sorter(self.entities) entity_subgenerator_pretaxonomy.send(self) for entity_or_draft in self.entities: # only main entities are listed in categories and tags # not translations if hasattr(entity_or_draft, 'category'): self.categories[entity_or_draft.category].append(entity_or_draft) if hasattr(entity_or_draft, 'tags'): for tag in entity_or_draft.tags: self.tags[tag].append(entity_or_draft) for author in getattr(entity_or_draft, 'authors', []): self.authors[author].append(entity_or_draft) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self.save_cache() self.readers.save_cache() entity_subgenerator_finalized.send(self)
def test_process_translations(self): fr_articles = [] en_articles = [] # create a bunch of articles # 0: no translation metadata fr_articles.append( get_article(lang='fr', slug='yay0', title='Titre', content='en français')) en_articles.append( get_article(lang='en', slug='yay0', title='Title', content='in english')) # 1: translation metadata on default lang fr_articles.append( get_article(lang='fr', slug='yay1', title='Titre', content='en français')) en_articles.append( get_article(lang='en', slug='yay1', title='Title', content='in english', translation='true')) # 2: translation metadata not on default lang fr_articles.append( get_article(lang='fr', slug='yay2', title='Titre', content='en français', translation='true')) en_articles.append( get_article(lang='en', slug='yay2', title='Title', content='in english')) # 3: back to default language detection if all items have the # translation metadata fr_articles.append( get_article(lang='fr', slug='yay3', title='Titre', content='en français', translation='yep')) en_articles.append( get_article(lang='en', slug='yay3', title='Title', content='in english', translation='yes')) # 4-5: translation pairs with the same slug but different category fr_articles.append( get_article(lang='fr', slug='yay4', title='Titre', content='en français', category='foo')) en_articles.append( get_article(lang='en', slug='yay4', title='Title', content='in english', category='foo')) fr_articles.append( get_article(lang='fr', slug='yay4', title='Titre', content='en français', category='bar')) en_articles.append( get_article(lang='en', slug='yay4', title='Title', content='in english', category='bar')) # try adding articles in both orders for lang0_articles, lang1_articles in ((fr_articles, en_articles), (en_articles, fr_articles)): articles = lang0_articles + lang1_articles # test process_translations with falsy translation_id index, trans = utils.process_translations(articles, translation_id=None) for i in range(6): for lang_articles in [en_articles, fr_articles]: self.assertIn(lang_articles[i], index) self.assertNotIn(lang_articles[i], trans) # test process_translations with simple and complex translation_id for translation_id in ['slug', {'slug', 'category'}]: index, trans = utils.process_translations( articles, translation_id=translation_id) for a in [ en_articles[0], fr_articles[1], en_articles[2], en_articles[3], en_articles[4], en_articles[5] ]: self.assertIn(a, index) self.assertNotIn(a, trans) for a in [ fr_articles[0], en_articles[1], fr_articles[2], fr_articles[3], fr_articles[4], fr_articles[5] ]: self.assertIn(a, trans) self.assertNotIn(a, index) for i in range(6): self.assertIn(en_articles[i], fr_articles[i].translations) self.assertIn(fr_articles[i], en_articles[i].translations) for a_arts in [en_articles, fr_articles]: for b_arts in [en_articles, fr_articles]: if translation_id == 'slug': self.assertIn(a_arts[4], b_arts[5].translations) self.assertIn(a_arts[5], b_arts[4].translations) elif translation_id == {'slug', 'category'}: self.assertNotIn(a_arts[4], b_arts[5].translations) self.assertNotIn(a_arts[5], b_arts[4].translations)
def generate_context(self): """Add the entities into the shared context""" all_entities = [] all_drafts = [] for f in self.get_files( self.settings['PATHS'], exclude=self.settings['EXCLUDES']): entity_or_draft = self.get_cached_data(f, None) if entity_or_draft is None: entity_class = EntityFactory( self.entity_type, self.settings['MANDATORY_PROPERTIES'], self.settings['DEFAULT_TEMPLATE']) try: entity_or_draft = self.readers.read_file( base_path=self.path, path=f, content_class=entity_class, context=self.context, preread_signal=entity_subgenerator_preread, preread_sender=self, context_signal=entity_subgenerator_context, context_sender=self) except Exception as e: logger.error('Could not process %s\n%s', f, e, exc_info=self.settings.get('DEBUG', False)) self._add_failed_source_path(f) continue if not contents.is_valid_content(entity_or_draft, f): self._add_failed_source_path(f) continue if entity_or_draft.status.lower() == "published": all_entities.append(entity_or_draft) elif entity_or_draft.status.lower() == "draft": all_drafts.append(entity_or_draft) self.add_source_path(entity_or_draft) continue else: logger.warning("Unknown status '%s' for file %s, skipping it.", entity_or_draft.status, f) self._add_failed_source_path(f) continue self.cache_data(f, entity_or_draft) self.add_source_path(entity_or_draft) self.entities, self.translations = process_translations( all_entities) self.drafts, self.drafts_translations = \ process_translations(all_drafts) sorter = self.settings["SORTER"] sorter(self.entities) entity_subgenerator_pretaxonomy.send(self) for entity_or_draft in self.entities: # only main entities are listed in categories and tags # not translations if hasattr(entity_or_draft, 'category'): self.categories[entity_or_draft.category].append(entity_or_draft) if hasattr(entity_or_draft, 'tags'): for tag in entity_or_draft.tags: self.tags[tag].append(entity_or_draft) for author in getattr(entity_or_draft, 'authors', []): self.authors[author].append(entity_or_draft) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self.save_cache() self.readers.save_cache() entity_subgenerator_finalized.send(self)
def generate_context(self): """change the context""" # return the list of files to use files = self.get_files(self.path, exclude=[ 'pages', ]) all_articles = [] for f in files: content, metadata = read_file(f) # if no category is set, use the name of the path as a category if 'category' not in metadata.keys(): if os.path.dirname(f) == self.path: category = self.settings['DEFAULT_CATEGORY'] else: category = os.path.basename(os.path.dirname(f)) if category != '': metadata['category'] = unicode(category) if 'date' not in metadata.keys()\ and self.settings['FALLBACK_ON_FS_DATE']: metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime) article = Article(content, metadata, settings=self.settings, filename=f) if not is_valid_content(article, f): continue add_to_url = u'' if 'ARTICLE_PERMALINK_STRUCTURE' in self.settings: article_permalink_structure = self.settings[ 'ARTICLE_PERMALINK_STRUCTURE'] article_permalink_structure = article_permalink_structure.lstrip( '/') # try to substitute any python datetime directive add_to_url = article.date.strftime(article_permalink_structure) # try to substitute any article metadata in rest file add_to_url = add_to_url % article.__dict__ add_to_url = [slugify(i) for i in add_to_url.split('/')] add_to_url = os.path.join(*add_to_url) article.url = urlparse.urljoin(add_to_url, article.url) article.save_as = urlparse.urljoin(add_to_url, article.save_as) if article.status == "published": if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) elif article.status == "draft": self.drafts.append(article) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) self.authors[article.author].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['REVERSE_ARCHIVE_ORDER']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = map(itemgetter(1), tag_cloud) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ (tag, int( math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count) or 1)))) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings.get('REVERSE_CATEGORY_ORDER')) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors'))
def _process(pages): origs, translations = process_translations( pages, translation_id=self.settings['PAGE_TRANSLATION_ID']) origs = order_content(origs, self.settings['PAGE_ORDER_BY']) return origs, translations
def generate_context(self): """change the context""" # return the list of files to use files = self.get_files(self.path, exclude=['pages',]) all_articles = [] for f in files: content, metadata = read_file(f) # if no category is set, use the name of the path as a category if 'category' not in metadata.keys(): if os.path.dirname(f) == self.path: category = self.settings['DEFAULT_CATEGORY'] else: category = os.path.basename(os.path.dirname(f)) if category != '': metadata['category'] = unicode(category) if 'date' not in metadata.keys()\ and self.settings['FALLBACK_ON_FS_DATE']: metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime) article = Article(content, metadata, settings=self.settings, filename=f) if not is_valid_content(article, f): continue add_to_url = u'' if 'ARTICLE_PERMALINK_STRUCTURE' in self.settings: article_permalink_structure = self.settings['ARTICLE_PERMALINK_STRUCTURE'] article_permalink_structure = article_permalink_structure.lstrip('/') # try to substitute any python datetime directive add_to_url = article.date.strftime(article_permalink_structure) # try to substitute any article metadata in rest file add_to_url = add_to_url % article.__dict__ add_to_url = [slugify(i) for i in add_to_url.split('/')] add_to_url = os.path.join(*add_to_url) article.url = urlparse.urljoin(add_to_url, article.url) article.save_as = urlparse.urljoin(add_to_url, article.save_as) if article.status == "published": if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) all_articles.append(article) elif article.status == "draft": self.drafts.append(article) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories, not translations self.categories[article.category].append(article) self.authors[article.author].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['REVERSE_ARCHIVE_ORDER']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key = itemgetter(1), reverse = True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = map(itemgetter(1), tag_cloud) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ ( tag, int( math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count)or 1)) ) ) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings.get('REVERSE_CATEGORY_ORDER')) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors'))
def generate_context(self): # Update the context (only articles in default language) self.articles = self.context['articles'] # Complain if MBOX_PATH and MBOX_CATEGORY are not of the same length. mboxPaths = self.settings.get('MBOX_PATH') mboxCategories = self.settings.get('MBOX_CATEGORY') errMsg = 'MBOX_PATH, MBOX_CATEGORY not of equal length or non-empty.' if len(mboxPaths) != len(mboxCategories) or len(mboxPaths) <= 0: logger.error(errMsg) return all_articles = [] # To avoid pulling in a dependency, define this convenient lambda. future_range = lambda x: range( x) if not sys.version_info.major <= 2 else xrange(x) for i in future_range(len(mboxPaths)): mboxPath = mboxPaths[i] mboxCategory = mboxCategories[i] new_articles = self._generate_mbox_articles(mboxPath, mboxCategory) all_articles.extend(new_articles) # Log that we did stuff. print( ('Read in %d messages from %s and converted to articles in ' + 'category %s.') % (len(new_articles), mboxPath, mboxCategory)) print('Read in %d messages from all mailboxes.' % (len(all_articles))) # Continue with the rest of ArticleGenerator, code adapted from: # https://github.com/getpelican/pelican/blob/master/pelican/generators.py#L548 # ARTICLE_ORDER_BY doesn't exist in 3.3, which was in Fedora 21. # (I wanted to be able to build this on F21 at the time). articles, translations = process_translations(all_articles) # , order_by=self.settings['ARTICLE_ORDER_BY']) self.articles.extend(articles) self.translations.extend(translations) # Disabled for 3.3 compatibility, great. # signals.article_generator_pretaxonomy.send(self) for article in self.articles: # only main articles are listed in categories and tags # not translations # We have to use django for this, unfortunately. if article.date.tzinfo is None: article.date = pytz.UTC.localize(article.date) self.categories[article.category].append(article) # Support for Author and Authors. if hasattr(article, 'author') and article.author.name != '': self.authors[article.author].append(article) else: for author in getattr(article, 'authors', []): self.authors[author].append(article) # This may not technically be right, but... # Sort the articles by date too. self.articles = list(self.articles) self.dates = self.articles self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'categories', 'authors'))
def generate_context(self): """Add the articles into the shared context""" all_articles = [] for f in self.get_files( self.settings['ARTICLE_DIR'], exclude=self.settings['ARTICLE_EXCLUDES']): try: article = self.readers.read_file( base_path=self.path, path=f, content_class=Article, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self) except Exception as e: logger.warning('Could not process {}\n{}'.format(f, e)) continue if not is_valid_content(article, f): continue self.add_source_path(article) if article.status == "published": all_articles.append(article) elif article.status == "draft": self.drafts.append(article) else: logger.warning("Unknown status %s for file %s, skipping it." % (repr(article.status), repr(f))) self.articles, self.translations = process_translations(all_articles) for article in self.articles: # only main articles are listed in categories and tags # not translations self.categories[article.category].append(article) if hasattr(article, 'tags'): for tag in article.tags: self.tags[tag].append(article) # ignore blank authors as well as undefined if hasattr(article, 'author') and article.author.name != '': self.authors[article.author].append(article) # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) self.dates.sort(key=attrgetter('date'), reverse=self.context['NEWEST_FIRST_ARCHIVES']) # create tag cloud tag_cloud = defaultdict(int) for article in self.articles: for tag in getattr(article, 'tags', []): tag_cloud[tag] += 1 tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True) tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')] tags = list(map(itemgetter(1), tag_cloud)) if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') # calculate word sizes self.tag_cloud = [ ( tag, int(math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count)or 1))) ) for tag, count in tag_cloud ] # put words in chaos random.shuffle(self.tag_cloud) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort( reverse=self.settings['REVERSE_CATEGORY_ORDER']) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors', 'related_posts')) signals.article_generator_finalized.send(self)
def generate_context(self): """Add the articles into the shared context""" all_articles = [] all_drafts = [] for f in self.get_files(self.settings["ARTICLE_PATHS"], exclude=self.settings["ARTICLE_EXCLUDES"]): article_or_draft = self.get_cached_data(f, None) if article_or_draft is None: # TODO needs overhaul, maybe nomad for read_file # solution, unified behaviour try: article_or_draft = self.readers.read_file( base_path=self.path, path=f, content_class=Article, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self, ) except Exception as e: logger.error("Could not process %s\n%s", f, e, exc_info=self.settings.get("DEBUG", False)) self._add_failed_source_path(f) continue if not is_valid_content(article_or_draft, f): self._add_failed_source_path(f) continue if article_or_draft.status.lower() == "published": all_articles.append(article_or_draft) elif article_or_draft.status.lower() == "draft": article_or_draft = self.readers.read_file( base_path=self.path, path=f, content_class=Draft, context=self.context, preread_signal=signals.article_generator_preread, preread_sender=self, context_signal=signals.article_generator_context, context_sender=self, ) self.add_source_path(article_or_draft) all_drafts.append(article_or_draft) else: logger.error("Unknown status '%s' for file %s, skipping it.", article_or_draft.status, f) self._add_failed_source_path(f) continue self.cache_data(f, article_or_draft) self.add_source_path(article_or_draft) self.articles, self.translations = process_translations( all_articles, order_by=self.settings["ARTICLE_ORDER_BY"] ) self.drafts, self.drafts_translations = process_translations(all_drafts) signals.article_generator_pretaxonomy.send(self) for article in self.articles: # only main articles are listed in categories and tags # not translations self.categories[article.category].append(article) if hasattr(article, "tags"): for tag in article.tags: self.tags[tag].append(article) for author in getattr(article, "authors", []): self.authors[author].append(article) self.dates = list(self.articles) self.dates.sort(key=attrgetter("date"), reverse=self.context["NEWEST_FIRST_ARCHIVES"]) # and generate the output :) # order the categories per name self.categories = list(self.categories.items()) self.categories.sort(reverse=self.settings["REVERSE_CATEGORY_ORDER"]) self.authors = list(self.authors.items()) self.authors.sort() self._update_context(("articles", "dates", "tags", "categories", "authors", "related_posts", "drafts")) self.save_cache() self.readers.save_cache() signals.article_generator_finalized.send(self)
def test_process_translations(self): fr_articles = [] en_articles = [] # create a bunch of articles # 0: no translation metadata fr_articles.append(get_article(lang='fr', slug='yay0', title='Titre', content='en français')) en_articles.append(get_article(lang='en', slug='yay0', title='Title', content='in english')) # 1: translation metadata on default lang fr_articles.append(get_article(lang='fr', slug='yay1', title='Titre', content='en français')) en_articles.append(get_article(lang='en', slug='yay1', title='Title', content='in english', translation='true')) # 2: translation metadata not on default lang fr_articles.append(get_article(lang='fr', slug='yay2', title='Titre', content='en français', translation='true')) en_articles.append(get_article(lang='en', slug='yay2', title='Title', content='in english')) # 3: back to default language detection if all items have the # translation metadata fr_articles.append(get_article(lang='fr', slug='yay3', title='Titre', content='en français', translation='yep')) en_articles.append(get_article(lang='en', slug='yay3', title='Title', content='in english', translation='yes')) # 4-5: translation pairs with the same slug but different category fr_articles.append(get_article(lang='fr', slug='yay4', title='Titre', content='en français', category='foo')) en_articles.append(get_article(lang='en', slug='yay4', title='Title', content='in english', category='foo')) fr_articles.append(get_article(lang='fr', slug='yay4', title='Titre', content='en français', category='bar')) en_articles.append(get_article(lang='en', slug='yay4', title='Title', content='in english', category='bar')) # try adding articles in both orders for lang0_articles, lang1_articles in ((fr_articles, en_articles), (en_articles, fr_articles)): articles = lang0_articles + lang1_articles # test process_translations with falsy translation_id index, trans = utils.process_translations( articles, translation_id=None) for i in range(6): for lang_articles in [en_articles, fr_articles]: self.assertIn(lang_articles[i], index) self.assertNotIn(lang_articles[i], trans) # test process_translations with simple and complex translation_id for translation_id in ['slug', {'slug', 'category'}]: index, trans = utils.process_translations( articles, translation_id=translation_id) for a in [en_articles[0], fr_articles[1], en_articles[2], en_articles[3], en_articles[4], en_articles[5]]: self.assertIn(a, index) self.assertNotIn(a, trans) for a in [fr_articles[0], en_articles[1], fr_articles[2], fr_articles[3], fr_articles[4], fr_articles[5]]: self.assertIn(a, trans) self.assertNotIn(a, index) for i in range(6): self.assertIn(en_articles[i], fr_articles[i].translations) self.assertIn(fr_articles[i], en_articles[i].translations) for a_arts in [en_articles, fr_articles]: for b_arts in [en_articles, fr_articles]: if translation_id == 'slug': self.assertIn(a_arts[4], b_arts[5].translations) self.assertIn(a_arts[5], b_arts[4].translations) elif translation_id == {'slug', 'category'}: self.assertNotIn(a_arts[4], b_arts[5].translations) self.assertNotIn(a_arts[5], b_arts[4].translations)