def test_valid_save_as_passes_valid(self):
settings = get_settings()
article_kwargs = self._copy_page_kwargs()
article_kwargs['metadata']['slug'] = 'foo'
article_kwargs['settings'] = settings
article = Article(**article_kwargs)
self.assertTrue(article._has_valid_save_as())
def test_valid_save_as_detects_breakout_to_root(self):
settings = get_settings()
article_kwargs = self._copy_page_kwargs()
article_kwargs['metadata']['slug'] = '/foo'
article_kwargs['settings'] = settings
article = Article(**article_kwargs)
self.assertFalse(article._has_valid_save_as())
def generate_context(self):
"""change the context"""
# return the list of files to use
files = self.get_files(self.path, exclude=['pages',])
all_articles = []
for f in files:
try:
content, metadata = read_file(f, settings=self.settings)
except Exception, e:
warning(u'Could not process %s\n%s' % (f, str(e)))
continue
# if no category is set, use the name of the path as a category
if 'category' not in metadata.keys():
if os.path.dirname(f) == self.path:
category = self.settings['DEFAULT_CATEGORY']
else:
category = os.path.basename(os.path.dirname(f)).decode('utf-8')
if category != '':
metadata['category'] = unicode(category)
if 'date' not in metadata.keys()\
and self.settings['FALLBACK_ON_FS_DATE']:
metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime)
article = Article(content, metadata, settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
add_to_url = u''
if 'ARTICLE_PERMALINK_STRUCTURE' in self.settings:
article_permalink_structure = self.settings['ARTICLE_PERMALINK_STRUCTURE']
article_permalink_structure = article_permalink_structure.lstrip('/').replace('%(', "%%(")
# try to substitute any python datetime directive
add_to_url = article.date.strftime(article_permalink_structure)
# try to substitute any article metadata in rest file
add_to_url = add_to_url % article.__dict__
add_to_url = [slugify(i) for i in add_to_url.split('/')]
add_to_url = os.path.join(*add_to_url)
article.url = urlparse.urljoin(add_to_url, article.url)
article.save_as = urlparse.urljoin(add_to_url, article.save_as)
if article.status == "published":
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
elif article.status == "noindex":
self.noindex.append(article)
def add_coming_soon(article_generator):
for draft in article_generator.drafts:
if hasattr(draft, "visible_draft") and draft.visible_draft.strip().lower() == "true":
soon_article = Article(content=None, metadata=draft.metadata)
soon_article.author = draft.author
soon_article.slug = draft.slug
soon_article.source_path = draft.source_path
article_generator.articles.append(soon_article)
article_generator.articles, article_generator.translations = process_translations(
article_generator.articles, order_by=article_generator.settings["ARTICLE_ORDER_BY"]
)
def article2draft(article):
'''Set to draft the status of an article'''
draft = Article(article._content, article.metadata, article.settings,
article.source_path, article._context)
draft.status = 'draft'
return draft
def generate_context(self):
"""change the context"""
# return the list of files to use
files = self.get_files(self.path, exclude=['pages',])
all_articles = []
for f in files:
content, metadata = read_file(f)
# if no category is set, use the name of the path as a category
if 'category' not in metadata.keys():
if os.path.dirname(f) == self.path:
category = self.settings['DEFAULT_CATEGORY']
else:
category = os.path.basename(os.path.dirname(f))
if category != '':
metadata['category'] = unicode(category)
if 'date' not in metadata.keys()\
and self.settings['FALLBACK_ON_FS_DATE']:
metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime)
article = Article(content, metadata, settings=self.settings,
filename=f)
if not is_valid_content(article, f):
continue
add_to_url = u''
if 'ARTICLE_PERMALINK_STRUCTURE' in self.settings:
article_permalink_structure = self.settings['ARTICLE_PERMALINK_STRUCTURE']
article_permalink_structure = article_permalink_structure.lstrip('/')
# try to substitute any python datetime directive
add_to_url = article.date.strftime(article_permalink_structure)
# try to substitute any article metadata in rest file
add_to_url = add_to_url % article.__dict__
add_to_url = [slugify(i) for i in add_to_url.split('/')]
add_to_url = os.path.join(*add_to_url)
article.url = urlparse.urljoin(add_to_url, article.url)
article.save_as = urlparse.urljoin(add_to_url, article.save_as)
if article.status == "published":
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories, not translations
self.categories[article.category].append(article)
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['REVERSE_ARCHIVE_ORDER'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key = itemgetter(1), reverse = True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = map(itemgetter(1), tag_cloud)
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(
tag,
int(
math.floor(steps - (steps - 1) * math.log(count) / (math.log(max_count)or 1))
)
)
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings.get('REVERSE_CATEGORY_ORDER'))
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories', 'tag_cloud', 'authors'))
def _generate_mbox_articles(self, mboxPath, mboxCategory):
baseReader = BaseReader(self.settings)
category = baseReader.process_metadata('category', mboxCategory)
# Complain if the mbox path does not exist and is not readable.
try:
if not os.path.exists(mboxPath):
raise RuntimeError
mbox = mailbox.mbox(mboxPath)
except:
logger.error('Could not process mbox file %s', mboxPath)
return
# Retrieve some fields from the settings.
authorString = self.settings.get('MBOX_AUTHOR_STRING')
markdownify = self.settings.get('MBOX_MARKDOWNIFY')
# Loop over all messages, turn them into article objects.
all_articles = []
slugs = []
for message in mbox.itervalues():
# Get author name.
author = message['from']
if author is None:
author = 'Unknown'
else:
if '<' and '>' in author:
author = author[:author.find(' <')]
author = author.replace('"', '').replace("'", '')
# As a hack to avoid dealing with the fact that names can collide.
if authorString is not None and authorString != '':
author += ' ' + authorString
authorObject = baseReader.process_metadata('author', author)
# Get date object, using python-dateutil as an easy hack.
# If there is no date in the message, abort, we shouldn't bother.
if message['date'] is None:
continue
if parser:
date = parser.parse(message['date'])
else:
logger.error('No python-dateutil, we cannot continue as ' +
'date formats cannot be parsed. ')
continue
monthYear = date.strftime('%B-%Y').lower()
# Get title and slug; build year + month into slug.
subject = message['subject']
slugSubject = slugify(subject)
slug = os.path.join(slugify(mboxCategory), monthYear, slugSubject)
# Hack to handle multiple messages with the same subject.
if slug in slugs:
slug += "_%d"
count = 2
testSlug = slug % count
while testSlug in slugs:
count += 1
testSlug = slug % count
slug = testSlug
slugs.append(slug)
# Code adapted from Stackoverflow for parsing email messages.
# https://stackoverflow.com/questions/4824376/parse-multi-part-email-with-sub-parts-using-python
# Code is clumsy, should be refactored.
if message.is_multipart():
plaintext = None
html = None
for part in message.get_payload():
charset = message.get_content_charset()
if charset is None or charset == 'x-unknown':
charset = 'us-ascii'
payload = part.get_payload(decode=True)
if part.get_content_type() == 'text/plain':
plaintext = unicode(payload, charset, "ignore")
plaintext = plaintext.encode('ascii', 'replace')
if part.get_content_type() == 'text/html':
html = unicode(payload, charset, "ignore")
html = html.encode('ascii', 'replace')
if plaintext is None and html is None:
continue
elif plaintext is None:
content = html
else:
content = plaintext_to_html(plaintext, markdownify)
else:
charset = message.get_content_charset()
if charset is None or charset == 'x-unknown':
charset = 'us-ascii'
payload = message.get_payload(decode=True)
plaintext = unicode(payload, charset, "ignore")
plaintext = plaintext.encode('ascii', 'replace')
content = plaintext_to_html(plaintext, markdownify)
metadata = {'title': subject,
'date': date,
'category': category,
'authors': [authorObject],
'slug': slug}
article = Article(content=content,
metadata=metadata,
settings=self.settings,
source_path=mboxPath,
context=self.context)
# This seems like it cannot happen... but it does without fail.
article.author = article.authors[0]
all_articles.append(article)
return all_articles
def generate_context(self):
"""Add the articles into the shared context"""
article_path = os.path.normpath( # we have to remove trailing slashes
os.path.join(self.path, self.settings['ARTICLE_DIR']))
all_articles = []
for f in self.get_files(article_path,
exclude=self.settings['ARTICLE_EXCLUDES']):
try:
signals.article_generate_preread.send(self)
content, metadata = read_file(f, settings=self.settings)
except Exception as e:
logger.warning('Could not process %s\n%s' % (f, str(e)))
continue
# if no category is set, use the name of the path as a category
if 'category' not in metadata:
if (self.settings['USE_FOLDER_AS_CATEGORY']
and os.path.dirname(f) != article_path):
# if the article is in a subdirectory
category = os.path.basename(os.path.dirname(f))
else:
# if the article is not in a subdirectory
category = self.settings['DEFAULT_CATEGORY']
if category != '':
metadata['category'] = Category(category, self.settings)
if 'date' not in metadata and self.settings.get('DEFAULT_DATE'):
if self.settings['DEFAULT_DATE'] == 'fs':
metadata['date'] = datetime.datetime.fromtimestamp(
os.stat(f).st_ctime)
else:
metadata['date'] = datetime.datetime(
*self.settings['DEFAULT_DATE'])
signals.article_generate_context.send(self, metadata=metadata)
article = Article(content,
metadata,
settings=self.settings,
source_path=f,
context=self.context)
if not is_valid_content(article, f):
continue
self.add_source_path(article)
if article.status == "published":
all_articles.append(article)
elif article.status == "draft":
self.drafts.append(article)
else:
logger.warning("Unknown status %s for file %s, skipping it." %
(repr(article.status), repr(f)))
self.articles, self.translations = process_translations(all_articles)
for article in self.articles:
# only main articles are listed in categories and tags
# not translations
self.categories[article.category].append(article)
if hasattr(article, 'tags'):
for tag in article.tags:
self.tags[tag].append(article)
# ignore blank authors as well as undefined
if hasattr(article, 'author') and article.author.name != '':
self.authors[article.author].append(article)
# sort the articles by date
self.articles.sort(key=attrgetter('date'), reverse=True)
self.dates = list(self.articles)
self.dates.sort(key=attrgetter('date'),
reverse=self.context['NEWEST_FIRST_ARCHIVES'])
# create tag cloud
tag_cloud = defaultdict(int)
for article in self.articles:
for tag in getattr(article, 'tags', []):
tag_cloud[tag] += 1
tag_cloud = sorted(tag_cloud.items(), key=itemgetter(1), reverse=True)
tag_cloud = tag_cloud[:self.settings.get('TAG_CLOUD_MAX_ITEMS')]
tags = list(map(itemgetter(1), tag_cloud))
if tags:
max_count = max(tags)
steps = self.settings.get('TAG_CLOUD_STEPS')
# calculate word sizes
self.tag_cloud = [
(tag,
int(
math.floor(steps - (steps - 1) * math.log(count) /
(math.log(max_count) or 1))))
for tag, count in tag_cloud
]
# put words in chaos
random.shuffle(self.tag_cloud)
# and generate the output :)
# order the categories per name
self.categories = list(self.categories.items())
self.categories.sort(reverse=self.settings['REVERSE_CATEGORY_ORDER'])
self.authors = list(self.authors.items())
self.authors.sort()
self._update_context(('articles', 'dates', 'tags', 'categories',
'tag_cloud', 'authors', 'related_posts'))
signals.article_generator_finalized.send(self)
def get_article(title, slug, content, lang, extra_metadata=None):
metadata = {'slug': slug, 'title': title, 'lang': lang}
if extra_metadata is not None:
metadata.update(extra_metadata)
return Article(content, metadata=metadata)
def _handle_article_generation(self, path):
content, metadata = self.md_reader.read(path)
return Article(content=content, metadata=metadata)
def get_article(title, content, **extra_metadata):
metadata = default_metadata(settings=DEFAULT_CONFIG)
metadata['title'] = title
if extra_metadata:
metadata.update(extra_metadata)
return Article(content, metadata=metadata)