def _do_classification(self, site):
# Needed to avoid strange errors during tests
if site is not self.site:
return
# Get list of enabled taxonomy plugins and initialize data structures
taxonomies = site.taxonomy_plugins.values()
site.posts_per_classification = {}
for taxonomy in taxonomies:
site.posts_per_classification[taxonomy.classification_name] = {
lang: defaultdict(set) for lang in site.config['TRANSLATIONS'].keys()
}
# Classify posts
for post in site.timeline:
# Do classify pages, but don’t classify posts that are hidden
# (draft/private/future)
if post.is_post and not post.use_in_feeds:
continue
for taxonomy in taxonomies:
if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages:
classifications = {}
for lang in site.config['TRANSLATIONS'].keys():
# Extract classifications for this language
classifications[lang] = taxonomy.classify(post, lang)
if not taxonomy.more_than_one_classifications_per_post and len(classifications[lang]) > 1:
raise ValueError("Too many {0} classifications for post {1}".format(taxonomy.classification_name, post.source_path))
# Add post to sets
for classification in classifications[lang]:
while True:
site.posts_per_classification[taxonomy.classification_name][lang][classification].add(post)
if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy:
break
classification_path = taxonomy.extract_hierarchy(classification)
if len(classification_path) <= 1:
if len(classification_path) == 0 or not taxonomy.include_posts_into_hierarchy_root:
break
classification = taxonomy.recombine_classification_from_hierarchy(classification_path[:-1])
# Sort everything.
site.page_count_per_classification = {}
site.hierarchy_per_classification = {}
site.flat_hierarchy_per_classification = {}
site.hierarchy_lookup_per_classification = {}
for taxonomy in taxonomies:
site.page_count_per_classification[taxonomy.classification_name] = {}
# Sort post lists
for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items():
# Ensure implicit classifications are inserted
for classification in taxonomy.get_implicit_classifications(lang):
if classification not in posts_per_classification:
posts_per_classification[classification] = []
site.page_count_per_classification[taxonomy.classification_name][lang] = {}
# Convert sets to lists and sort them
for classification in list(posts_per_classification.keys()):
posts = list(posts_per_classification[classification])
posts = self.site.sort_posts_chronologically(posts, lang)
taxonomy.sort_posts(posts, classification, lang)
posts_per_classification[classification] = posts
# Create hierarchy information
if taxonomy.has_hierarchy:
site.hierarchy_per_classification[taxonomy.classification_name] = {}
site.flat_hierarchy_per_classification[taxonomy.classification_name] = {}
site.hierarchy_lookup_per_classification[taxonomy.classification_name] = {}
for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items():
# Compose hierarchy
hierarchy = {}
for classification in posts_per_classification.keys():
hier = taxonomy.extract_hierarchy(classification)
node = hierarchy
for he in hier:
if he not in node:
node[he] = {}
node = node[he]
hierarchy_lookup = {}
def create_hierarchy(hierarchy, parent=None, level=0):
"""Create hierarchy."""
result = {}
for name, children in hierarchy.items():
node = hierarchy_utils.TreeNode(name, parent)
node.children = create_hierarchy(children, node, level + 1)
node.classification_path = [pn.name for pn in node.get_path()]
node.classification_name = taxonomy.recombine_classification_from_hierarchy(node.classification_path)
hierarchy_lookup[node.classification_name] = node
result[node.name] = node
classifications = natsort.natsorted(result.keys(), alg=natsort.ns.F | natsort.ns.IC)
taxonomy.sort_classifications(classifications, lang, level=level)
return [result[classification] for classification in classifications]
root_list = create_hierarchy(hierarchy)
if '' in posts_per_classification:
node = hierarchy_utils.TreeNode('', parent=None)
node.children = root_list
node.classification_path = []
node.classification_name = ''
hierarchy_lookup[node.name] = node
root_list = [node]
flat_hierarchy = hierarchy_utils.flatten_tree_structure(root_list)
# Store result
site.hierarchy_per_classification[taxonomy.classification_name][lang] = root_list
site.flat_hierarchy_per_classification[taxonomy.classification_name][lang] = flat_hierarchy
site.hierarchy_lookup_per_classification[taxonomy.classification_name][lang] = hierarchy_lookup
taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name],
site.flat_hierarchy_per_classification[taxonomy.classification_name],
site.hierarchy_lookup_per_classification[taxonomy.classification_name])
else:
taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name])
# Check for valid paths and for collisions
taxonomy_outputs = {lang: dict() for lang in site.config['TRANSLATIONS'].keys()}
quit = False
for taxonomy in taxonomies:
# Check for collisions (per language)
for lang in site.config['TRANSLATIONS'].keys():
if not taxonomy.is_enabled(lang):
continue
for classification, posts in site.posts_per_classification[taxonomy.classification_name][lang].items():
# Do we actually generate this classification page?
filtered_posts = [x for x in posts if self.site.config["SHOW_UNTRANSLATED_POSTS"] or x.is_translation_available(lang)]
generate_list = taxonomy.should_generate_classification_page(classification, filtered_posts, lang)
if not generate_list:
continue
# Obtain path as tuple
path = site.path_handlers[taxonomy.classification_name](classification, lang)
# Check that path is OK
for path_element in path:
if len(path_element) == 0:
utils.LOGGER.error("{0} {1} yields invalid path '{2}'!".format(taxonomy.classification_name.title(), classification, '/'.join(path)))
quit = True
# Combine path
path = os.path.join(*[os.path.normpath(p) for p in path if p != '.'])
# Determine collisions
if path in taxonomy_outputs[lang]:
other_classification_name, other_classification, other_posts = taxonomy_outputs[lang][path]
if other_classification_name == taxonomy.classification_name and other_classification == classification:
taxonomy_outputs[lang][path][2].extend(filtered_posts)
else:
utils.LOGGER.error('You have classifications that are too similar: {0} "{1}" and {2} "{3}" both result in output path {4} for language {5}.'.format(
taxonomy.classification_name, classification, other_classification_name, other_classification, path, lang))
utils.LOGGER.error('{0} "{1}" is used in: {2}'.format(
taxonomy.classification_name.title(), classification, ', '.join(sorted([p.source_path for p in filtered_posts]))))
utils.LOGGER.error('{0} "{1}" is used in: {2}'.format(
other_classification_name.title(), other_classification, ', '.join(sorted([p.source_path for p in other_posts]))))
quit = True
else:
taxonomy_outputs[lang][path] = (taxonomy.classification_name, classification, list(posts))
if quit:
sys.exit(1)
blinker.signal('taxonomies_classified').send(site)
def _generate_classification_overview_kw_context(self, taxonomy, lang):
"""Create context and kw for a classification overview page."""
context, kw = taxonomy.provide_overview_context_and_uptodate(lang)
context = copy(context)
context["kind"] = "{}_index".format(taxonomy.classification_name)
sorted_links = []
sorted_links_all = []
for other_lang in sorted(self.site.config['TRANSLATIONS'].keys()):
sorted_links_all.append((other_lang, None, None))
if other_lang != lang:
sorted_links.append((other_lang, None, None))
context['has_other_languages'] = True
context['other_languages'] = sorted_links
context['all_languages'] = sorted_links_all
kw = copy(kw)
kw["messages"] = self.site.MESSAGES
kw["translations"] = self.site.config['TRANSLATIONS']
kw["filters"] = self.site.config['FILTERS']
kw["minimum_post_count"] = taxonomy.minimum_post_count_per_classification_in_overview
kw["output_folder"] = self.site.config['OUTPUT_FOLDER']
kw["pretty_urls"] = self.site.config['PRETTY_URLS']
kw["strip_indexes"] = self.site.config['STRIP_INDEXES']
kw["index_file"] = self.site.config['INDEX_FILE']
# Collect all relevant classifications
if taxonomy.has_hierarchy:
def acceptor(node):
return len(
self._filter_list(
self.site.posts_per_classification[
taxonomy.classification_name][lang][
node.classification_name],
lang)) >= kw["minimum_post_count"]
clipped_root_list = [
hierarchy_utils.clone_treenode(node,
parent=None,
acceptor=acceptor)
for node in self.site.hierarchy_per_classification[
taxonomy.classification_name][lang]
]
clipped_root_list = [node for node in clipped_root_list if node]
clipped_flat_hierarchy = hierarchy_utils.flatten_tree_structure(
clipped_root_list)
classifications = [
cat.classification_name for cat in clipped_flat_hierarchy
]
else:
classifications = natsort.natsorted([
tag for tag, posts in self.site.posts_per_classification[
taxonomy.classification_name][lang].items() if
len(self._filter_list(posts, lang)) >= kw["minimum_post_count"]
],
alg=natsort.ns.F
| natsort.ns.IC)
taxonomy.sort_classifications(classifications, lang)
# Set up classifications in context
context[taxonomy.overview_page_variable_name] = classifications
context["has_hierarchy"] = taxonomy.has_hierarchy
if taxonomy.overview_page_items_variable_name:
items = [(classification,
self.site.link(taxonomy.classification_name,
classification, lang))
for classification in classifications]
items_with_postcount = [
(classification,
self.site.link(taxonomy.classification_name, classification,
lang),
len(
self._filter_list(
self.site.posts_per_classification[
taxonomy.classification_name][lang]
[classification], lang)))
for classification in classifications
]
context[taxonomy.overview_page_items_variable_name] = items
context[taxonomy.overview_page_items_variable_name +
"_with_postcount"] = items_with_postcount
if taxonomy.has_hierarchy and taxonomy.overview_page_hierarchy_variable_name:
hier_items = [
(node.name, node.classification_name, node.classification_path,
self.site.link(taxonomy.classification_name,
node.classification_name,
lang), node.indent_levels,
node.indent_change_before, node.indent_change_after)
for node in clipped_flat_hierarchy
]
hier_items_with_postcount = [
(node.name, node.classification_name, node.classification_path,
self.site.link(taxonomy.classification_name,
node.classification_name,
lang), node.indent_levels,
node.indent_change_before, node.indent_change_after,
len(node.children),
len(
self._filter_list(
self.site.posts_per_classification[
taxonomy.classification_name][lang][
node.classification_name], lang)))
for node in clipped_flat_hierarchy
]
context[
taxonomy.overview_page_hierarchy_variable_name] = hier_items
context[taxonomy.overview_page_hierarchy_variable_name +
'_with_postcount'] = hier_items_with_postcount
return context, kw
def _do_classification(self, site):
# Needed to avoid strange errors during tests
if site is not self.site:
return
# Get list of enabled taxonomy plugins and initialize data structures
taxonomies = site.taxonomy_plugins.values()
site.posts_per_classification = {}
for taxonomy in taxonomies:
site.posts_per_classification[taxonomy.classification_name] = {
lang: defaultdict(set)
for lang in site.config['TRANSLATIONS'].keys()
}
# Classify posts
for post in site.timeline:
# Do classify pages, but don’t classify posts that are hidden
# (draft/private/future)
if post.is_post and not post.use_in_feeds:
continue
for taxonomy in taxonomies:
if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages:
classifications = {}
for lang in site.config['TRANSLATIONS'].keys():
# Extract classifications for this language
classifications[lang] = taxonomy.classify(post, lang)
if not taxonomy.more_than_one_classifications_per_post and len(
classifications[lang]) > 1:
raise ValueError(
"Too many {0} classifications for post {1}".
format(taxonomy.classification_name,
post.source_path))
# Add post to sets
for classification in classifications[lang]:
while True:
site.posts_per_classification[
taxonomy.classification_name][lang][
classification].add(post)
if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy:
break
classification_path = taxonomy.extract_hierarchy(
classification)
if len(classification_path) <= 1:
if len(
classification_path
) == 0 or not taxonomy.include_posts_into_hierarchy_root:
break
classification = taxonomy.recombine_classification_from_hierarchy(
classification_path[:-1])
# Sort everything.
site.page_count_per_classification = {}
site.hierarchy_per_classification = {}
site.flat_hierarchy_per_classification = {}
site.hierarchy_lookup_per_classification = {}
for taxonomy in taxonomies:
site.page_count_per_classification[
taxonomy.classification_name] = {}
# Sort post lists
for lang, posts_per_classification in site.posts_per_classification[
taxonomy.classification_name].items():
# Ensure implicit classifications are inserted
for classification in taxonomy.get_implicit_classifications(
lang):
if classification not in posts_per_classification:
posts_per_classification[classification] = []
site.page_count_per_classification[
taxonomy.classification_name][lang] = {}
# Convert sets to lists and sort them
for classification in list(posts_per_classification.keys()):
posts = list(posts_per_classification[classification])
posts = self.site.sort_posts_chronologically(posts, lang)
taxonomy.sort_posts(posts, classification, lang)
posts_per_classification[classification] = posts
# Create hierarchy information
if taxonomy.has_hierarchy:
site.hierarchy_per_classification[
taxonomy.classification_name] = {}
site.flat_hierarchy_per_classification[
taxonomy.classification_name] = {}
site.hierarchy_lookup_per_classification[
taxonomy.classification_name] = {}
for lang, posts_per_classification in site.posts_per_classification[
taxonomy.classification_name].items():
# Compose hierarchy
hierarchy = {}
for classification in posts_per_classification.keys():
hier = taxonomy.extract_hierarchy(classification)
node = hierarchy
for he in hier:
if he not in node:
node[he] = {}
node = node[he]
hierarchy_lookup = {}
def create_hierarchy(hierarchy, parent=None, level=0):
"""Create hierarchy."""
result = {}
for name, children in hierarchy.items():
node = hierarchy_utils.TreeNode(name, parent)
node.children = create_hierarchy(
children, node, level + 1)
node.classification_path = [
pn.name for pn in node.get_path()
]
node.classification_name = taxonomy.recombine_classification_from_hierarchy(
node.classification_path)
hierarchy_lookup[node.classification_name] = node
result[node.name] = node
classifications = natsort.natsorted(result.keys(),
alg=natsort.ns.F
| natsort.ns.IC)
taxonomy.sort_classifications(classifications,
lang,
level=level)
return [
result[classification]
for classification in classifications
]
root_list = create_hierarchy(hierarchy)
if '' in posts_per_classification:
node = hierarchy_utils.TreeNode('', parent=None)
node.children = root_list
node.classification_path = []
node.classification_name = ''
hierarchy_lookup[node.name] = node
root_list = [node]
flat_hierarchy = hierarchy_utils.flatten_tree_structure(
root_list)
# Store result
site.hierarchy_per_classification[
taxonomy.classification_name][lang] = root_list
site.flat_hierarchy_per_classification[
taxonomy.classification_name][lang] = flat_hierarchy
site.hierarchy_lookup_per_classification[
taxonomy.classification_name][lang] = hierarchy_lookup
taxonomy.postprocess_posts_per_classification(
site.posts_per_classification[
taxonomy.classification_name],
site.flat_hierarchy_per_classification[
taxonomy.classification_name],
site.hierarchy_lookup_per_classification[
taxonomy.classification_name])
else:
taxonomy.postprocess_posts_per_classification(
site.posts_per_classification[
taxonomy.classification_name])
# Check for valid paths and for collisions
taxonomy_outputs = {
lang: dict()
for lang in site.config['TRANSLATIONS'].keys()
}
quit = False
for taxonomy in taxonomies:
# Check for collisions (per language)
for lang in site.config['TRANSLATIONS'].keys():
if not taxonomy.is_enabled(lang):
continue
for classification, posts in site.posts_per_classification[
taxonomy.classification_name][lang].items():
# Do we actually generate this classification page?
filtered_posts = [
x for x in posts
if self.site.config["SHOW_UNTRANSLATED_POSTS"]
or x.is_translation_available(lang)
]
generate_list = taxonomy.should_generate_classification_page(
classification, filtered_posts, lang)
if not generate_list:
continue
# Obtain path as tuple
path = site.path_handlers[taxonomy.classification_name](
classification, lang)
# Check that path is OK
for path_element in path:
if len(path_element) == 0:
utils.LOGGER.error(
"{0} {1} yields invalid path '{2}'!".format(
taxonomy.classification_name.title(),
classification, '/'.join(path)))
quit = True
# Combine path
path = os.path.join(
*[os.path.normpath(p) for p in path if p != '.'])
# Determine collisions
if path in taxonomy_outputs[lang]:
other_classification_name, other_classification, other_posts = taxonomy_outputs[
lang][path]
if other_classification_name == taxonomy.classification_name and other_classification == classification:
taxonomy_outputs[lang][path][2].extend(
filtered_posts)
else:
utils.LOGGER.error(
'You have classifications that are too similar: {0} "{1}" and {2} "{3}" both result in output path {4} for language {5}.'
.format(taxonomy.classification_name,
classification,
other_classification_name,
other_classification, path, lang))
utils.LOGGER.error(
'{0} "{1}" is used in: {2}'.format(
taxonomy.classification_name.title(),
classification, ', '.join(
sorted([
p.source_path
for p in filtered_posts
]))))
utils.LOGGER.error(
'{0} "{1}" is used in: {2}'.format(
other_classification_name.title(),
other_classification, ', '.join(
sorted([
p.source_path for p in other_posts
]))))
quit = True
else:
taxonomy_outputs[lang][path] = (
taxonomy.classification_name, classification,
list(posts))
if quit:
sys.exit(1)
blinker.signal('taxonomies_classified').send(site)
def _generate_classification_overview_kw_context(self, taxonomy, lang):
"""Create context and kw for a classification overview page."""
context, kw = taxonomy.provide_overview_context_and_uptodate(lang)
context = copy(context)
context["kind"] = "{}_index".format(taxonomy.classification_name)
sorted_links = []
sorted_links_all = []
for other_lang in sorted(self.site.config['TRANSLATIONS'].keys()):
sorted_links_all.append((other_lang, None, None))
if other_lang != lang:
sorted_links.append((other_lang, None, None))
context['has_other_languages'] = True
context['other_languages'] = sorted_links
context['all_languages'] = sorted_links_all
kw = copy(kw)
kw["messages"] = self.site.MESSAGES
kw["translations"] = self.site.config['TRANSLATIONS']
kw["filters"] = self.site.config['FILTERS']
kw["minimum_post_count"] = taxonomy.minimum_post_count_per_classification_in_overview
kw["output_folder"] = self.site.config['OUTPUT_FOLDER']
kw["pretty_urls"] = self.site.config['PRETTY_URLS']
kw["strip_indexes"] = self.site.config['STRIP_INDEXES']
kw["index_file"] = self.site.config['INDEX_FILE']
# Collect all relevant classifications
if taxonomy.has_hierarchy:
def acceptor(node):
return len(self._filter_list(self.site.posts_per_classification[taxonomy.classification_name][lang][node.classification_name], lang)) >= kw["minimum_post_count"]
clipped_root_list = [hierarchy_utils.clone_treenode(node, parent=None, acceptor=acceptor) for node in self.site.hierarchy_per_classification[taxonomy.classification_name][lang]]
clipped_root_list = [node for node in clipped_root_list if node]
clipped_flat_hierarchy = hierarchy_utils.flatten_tree_structure(clipped_root_list)
classifications = [cat.classification_name for cat in clipped_flat_hierarchy]
else:
classifications = natsort.natsorted([tag for tag, posts in self.site.posts_per_classification[taxonomy.classification_name][lang].items()
if len(self._filter_list(posts, lang)) >= kw["minimum_post_count"]],
alg=natsort.ns.F | natsort.ns.IC)
taxonomy.sort_classifications(classifications, lang)
# Set up classifications in context
context[taxonomy.overview_page_variable_name] = classifications
context["has_hierarchy"] = taxonomy.has_hierarchy
if taxonomy.overview_page_items_variable_name:
items = [(classification,
self.site.link(taxonomy.classification_name, classification, lang))
for classification in classifications]
items_with_postcount = [
(classification,
self.site.link(taxonomy.classification_name, classification, lang),
len(self._filter_list(self.site.posts_per_classification[taxonomy.classification_name][lang][classification], lang)))
for classification in classifications
]
context[taxonomy.overview_page_items_variable_name] = items
context[taxonomy.overview_page_items_variable_name + "_with_postcount"] = items_with_postcount
if taxonomy.has_hierarchy and taxonomy.overview_page_hierarchy_variable_name:
hier_items = [
(node.name, node.classification_name, node.classification_path,
self.site.link(taxonomy.classification_name, node.classification_name, lang),
node.indent_levels, node.indent_change_before,
node.indent_change_after)
for node in clipped_flat_hierarchy
]
hier_items_with_postcount = [
(node.name, node.classification_name, node.classification_path,
self.site.link(taxonomy.classification_name, node.classification_name, lang),
node.indent_levels, node.indent_change_before,
node.indent_change_after,
len(node.children),
len(self._filter_list(self.site.posts_per_classification[taxonomy.classification_name][lang][node.classification_name], lang)))
for node in clipped_flat_hierarchy
]
context[taxonomy.overview_page_hierarchy_variable_name] = hier_items
context[taxonomy.overview_page_hierarchy_variable_name + '_with_postcount'] = hier_items_with_postcount
return context, kw