def _do_classification(self, site): # Needed to avoid strange errors during tests if site is not self.site: return # Get list of enabled taxonomy plugins and initialize data structures taxonomies = site.taxonomy_plugins.values() site.posts_per_classification = {} for taxonomy in taxonomies: site.posts_per_classification[taxonomy.classification_name] = { lang: defaultdict(set) for lang in site.config['TRANSLATIONS'].keys() } # Classify posts for post in site.timeline: # Do classify pages, but don’t classify posts that are hidden # (draft/private/future) if post.is_post and not post.use_in_feeds: continue for taxonomy in taxonomies: if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages: classifications = {} for lang in site.config['TRANSLATIONS'].keys(): # Extract classifications for this language classifications[lang] = taxonomy.classify(post, lang) if not taxonomy.more_than_one_classifications_per_post and len(classifications[lang]) > 1: raise ValueError("Too many {0} classifications for post {1}".format(taxonomy.classification_name, post.source_path)) # Add post to sets for classification in classifications[lang]: while True: site.posts_per_classification[taxonomy.classification_name][lang][classification].add(post) if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy: break classification_path = taxonomy.extract_hierarchy(classification) if len(classification_path) <= 1: if len(classification_path) == 0 or not taxonomy.include_posts_into_hierarchy_root: break classification = taxonomy.recombine_classification_from_hierarchy(classification_path[:-1]) # Sort everything. site.page_count_per_classification = {} site.hierarchy_per_classification = {} site.flat_hierarchy_per_classification = {} site.hierarchy_lookup_per_classification = {} for taxonomy in taxonomies: site.page_count_per_classification[taxonomy.classification_name] = {} # Sort post lists for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items(): # Ensure implicit classifications are inserted for classification in taxonomy.get_implicit_classifications(lang): if classification not in posts_per_classification: posts_per_classification[classification] = [] site.page_count_per_classification[taxonomy.classification_name][lang] = {} # Convert sets to lists and sort them for classification in list(posts_per_classification.keys()): posts = list(posts_per_classification[classification]) posts = self.site.sort_posts_chronologically(posts, lang) taxonomy.sort_posts(posts, classification, lang) posts_per_classification[classification] = posts # Create hierarchy information if taxonomy.has_hierarchy: site.hierarchy_per_classification[taxonomy.classification_name] = {} site.flat_hierarchy_per_classification[taxonomy.classification_name] = {} site.hierarchy_lookup_per_classification[taxonomy.classification_name] = {} for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items(): # Compose hierarchy hierarchy = {} for classification in posts_per_classification.keys(): hier = taxonomy.extract_hierarchy(classification) node = hierarchy for he in hier: if he not in node: node[he] = {} node = node[he] hierarchy_lookup = {} def create_hierarchy(hierarchy, parent=None, level=0): """Create hierarchy.""" result = {} for name, children in hierarchy.items(): node = hierarchy_utils.TreeNode(name, parent) node.children = create_hierarchy(children, node, level + 1) node.classification_path = [pn.name for pn in node.get_path()] node.classification_name = taxonomy.recombine_classification_from_hierarchy(node.classification_path) hierarchy_lookup[node.classification_name] = node result[node.name] = node classifications = natsort.natsorted(result.keys(), alg=natsort.ns.F | natsort.ns.IC) taxonomy.sort_classifications(classifications, lang, level=level) return [result[classification] for classification in classifications] root_list = create_hierarchy(hierarchy) if '' in posts_per_classification: node = hierarchy_utils.TreeNode('', parent=None) node.children = root_list node.classification_path = [] node.classification_name = '' hierarchy_lookup[node.name] = node root_list = [node] flat_hierarchy = hierarchy_utils.flatten_tree_structure(root_list) # Store result site.hierarchy_per_classification[taxonomy.classification_name][lang] = root_list site.flat_hierarchy_per_classification[taxonomy.classification_name][lang] = flat_hierarchy site.hierarchy_lookup_per_classification[taxonomy.classification_name][lang] = hierarchy_lookup taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name], site.flat_hierarchy_per_classification[taxonomy.classification_name], site.hierarchy_lookup_per_classification[taxonomy.classification_name]) else: taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name]) # Check for valid paths and for collisions taxonomy_outputs = {lang: dict() for lang in site.config['TRANSLATIONS'].keys()} quit = False for taxonomy in taxonomies: # Check for collisions (per language) for lang in site.config['TRANSLATIONS'].keys(): if not taxonomy.is_enabled(lang): continue for classification, posts in site.posts_per_classification[taxonomy.classification_name][lang].items(): # Do we actually generate this classification page? filtered_posts = [x for x in posts if self.site.config["SHOW_UNTRANSLATED_POSTS"] or x.is_translation_available(lang)] generate_list = taxonomy.should_generate_classification_page(classification, filtered_posts, lang) if not generate_list: continue # Obtain path as tuple path = site.path_handlers[taxonomy.classification_name](classification, lang) # Check that path is OK for path_element in path: if len(path_element) == 0: utils.LOGGER.error("{0} {1} yields invalid path '{2}'!".format(taxonomy.classification_name.title(), classification, '/'.join(path))) quit = True # Combine path path = os.path.join(*[os.path.normpath(p) for p in path if p != '.']) # Determine collisions if path in taxonomy_outputs[lang]: other_classification_name, other_classification, other_posts = taxonomy_outputs[lang][path] if other_classification_name == taxonomy.classification_name and other_classification == classification: taxonomy_outputs[lang][path][2].extend(filtered_posts) else: utils.LOGGER.error('You have classifications that are too similar: {0} "{1}" and {2} "{3}" both result in output path {4} for language {5}.'.format( taxonomy.classification_name, classification, other_classification_name, other_classification, path, lang)) utils.LOGGER.error('{0} "{1}" is used in: {2}'.format( taxonomy.classification_name.title(), classification, ', '.join(sorted([p.source_path for p in filtered_posts])))) utils.LOGGER.error('{0} "{1}" is used in: {2}'.format( other_classification_name.title(), other_classification, ', '.join(sorted([p.source_path for p in other_posts])))) quit = True else: taxonomy_outputs[lang][path] = (taxonomy.classification_name, classification, list(posts)) if quit: sys.exit(1) blinker.signal('taxonomies_classified').send(site)
def _generate_classification_overview_kw_context(self, taxonomy, lang): """Create context and kw for a classification overview page.""" context, kw = taxonomy.provide_overview_context_and_uptodate(lang) context = copy(context) context["kind"] = "{}_index".format(taxonomy.classification_name) sorted_links = [] sorted_links_all = [] for other_lang in sorted(self.site.config['TRANSLATIONS'].keys()): sorted_links_all.append((other_lang, None, None)) if other_lang != lang: sorted_links.append((other_lang, None, None)) context['has_other_languages'] = True context['other_languages'] = sorted_links context['all_languages'] = sorted_links_all kw = copy(kw) kw["messages"] = self.site.MESSAGES kw["translations"] = self.site.config['TRANSLATIONS'] kw["filters"] = self.site.config['FILTERS'] kw["minimum_post_count"] = taxonomy.minimum_post_count_per_classification_in_overview kw["output_folder"] = self.site.config['OUTPUT_FOLDER'] kw["pretty_urls"] = self.site.config['PRETTY_URLS'] kw["strip_indexes"] = self.site.config['STRIP_INDEXES'] kw["index_file"] = self.site.config['INDEX_FILE'] # Collect all relevant classifications if taxonomy.has_hierarchy: def acceptor(node): return len( self._filter_list( self.site.posts_per_classification[ taxonomy.classification_name][lang][ node.classification_name], lang)) >= kw["minimum_post_count"] clipped_root_list = [ hierarchy_utils.clone_treenode(node, parent=None, acceptor=acceptor) for node in self.site.hierarchy_per_classification[ taxonomy.classification_name][lang] ] clipped_root_list = [node for node in clipped_root_list if node] clipped_flat_hierarchy = hierarchy_utils.flatten_tree_structure( clipped_root_list) classifications = [ cat.classification_name for cat in clipped_flat_hierarchy ] else: classifications = natsort.natsorted([ tag for tag, posts in self.site.posts_per_classification[ taxonomy.classification_name][lang].items() if len(self._filter_list(posts, lang)) >= kw["minimum_post_count"] ], alg=natsort.ns.F | natsort.ns.IC) taxonomy.sort_classifications(classifications, lang) # Set up classifications in context context[taxonomy.overview_page_variable_name] = classifications context["has_hierarchy"] = taxonomy.has_hierarchy if taxonomy.overview_page_items_variable_name: items = [(classification, self.site.link(taxonomy.classification_name, classification, lang)) for classification in classifications] items_with_postcount = [ (classification, self.site.link(taxonomy.classification_name, classification, lang), len( self._filter_list( self.site.posts_per_classification[ taxonomy.classification_name][lang] [classification], lang))) for classification in classifications ] context[taxonomy.overview_page_items_variable_name] = items context[taxonomy.overview_page_items_variable_name + "_with_postcount"] = items_with_postcount if taxonomy.has_hierarchy and taxonomy.overview_page_hierarchy_variable_name: hier_items = [ (node.name, node.classification_name, node.classification_path, self.site.link(taxonomy.classification_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after) for node in clipped_flat_hierarchy ] hier_items_with_postcount = [ (node.name, node.classification_name, node.classification_path, self.site.link(taxonomy.classification_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after, len(node.children), len( self._filter_list( self.site.posts_per_classification[ taxonomy.classification_name][lang][ node.classification_name], lang))) for node in clipped_flat_hierarchy ] context[ taxonomy.overview_page_hierarchy_variable_name] = hier_items context[taxonomy.overview_page_hierarchy_variable_name + '_with_postcount'] = hier_items_with_postcount return context, kw
def _do_classification(self, site): # Needed to avoid strange errors during tests if site is not self.site: return # Get list of enabled taxonomy plugins and initialize data structures taxonomies = site.taxonomy_plugins.values() site.posts_per_classification = {} for taxonomy in taxonomies: site.posts_per_classification[taxonomy.classification_name] = { lang: defaultdict(set) for lang in site.config['TRANSLATIONS'].keys() } # Classify posts for post in site.timeline: # Do classify pages, but don’t classify posts that are hidden # (draft/private/future) if post.is_post and not post.use_in_feeds: continue for taxonomy in taxonomies: if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages: classifications = {} for lang in site.config['TRANSLATIONS'].keys(): # Extract classifications for this language classifications[lang] = taxonomy.classify(post, lang) if not taxonomy.more_than_one_classifications_per_post and len( classifications[lang]) > 1: raise ValueError( "Too many {0} classifications for post {1}". format(taxonomy.classification_name, post.source_path)) # Add post to sets for classification in classifications[lang]: while True: site.posts_per_classification[ taxonomy.classification_name][lang][ classification].add(post) if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy: break classification_path = taxonomy.extract_hierarchy( classification) if len(classification_path) <= 1: if len( classification_path ) == 0 or not taxonomy.include_posts_into_hierarchy_root: break classification = taxonomy.recombine_classification_from_hierarchy( classification_path[:-1]) # Sort everything. site.page_count_per_classification = {} site.hierarchy_per_classification = {} site.flat_hierarchy_per_classification = {} site.hierarchy_lookup_per_classification = {} for taxonomy in taxonomies: site.page_count_per_classification[ taxonomy.classification_name] = {} # Sort post lists for lang, posts_per_classification in site.posts_per_classification[ taxonomy.classification_name].items(): # Ensure implicit classifications are inserted for classification in taxonomy.get_implicit_classifications( lang): if classification not in posts_per_classification: posts_per_classification[classification] = [] site.page_count_per_classification[ taxonomy.classification_name][lang] = {} # Convert sets to lists and sort them for classification in list(posts_per_classification.keys()): posts = list(posts_per_classification[classification]) posts = self.site.sort_posts_chronologically(posts, lang) taxonomy.sort_posts(posts, classification, lang) posts_per_classification[classification] = posts # Create hierarchy information if taxonomy.has_hierarchy: site.hierarchy_per_classification[ taxonomy.classification_name] = {} site.flat_hierarchy_per_classification[ taxonomy.classification_name] = {} site.hierarchy_lookup_per_classification[ taxonomy.classification_name] = {} for lang, posts_per_classification in site.posts_per_classification[ taxonomy.classification_name].items(): # Compose hierarchy hierarchy = {} for classification in posts_per_classification.keys(): hier = taxonomy.extract_hierarchy(classification) node = hierarchy for he in hier: if he not in node: node[he] = {} node = node[he] hierarchy_lookup = {} def create_hierarchy(hierarchy, parent=None, level=0): """Create hierarchy.""" result = {} for name, children in hierarchy.items(): node = hierarchy_utils.TreeNode(name, parent) node.children = create_hierarchy( children, node, level + 1) node.classification_path = [ pn.name for pn in node.get_path() ] node.classification_name = taxonomy.recombine_classification_from_hierarchy( node.classification_path) hierarchy_lookup[node.classification_name] = node result[node.name] = node classifications = natsort.natsorted(result.keys(), alg=natsort.ns.F | natsort.ns.IC) taxonomy.sort_classifications(classifications, lang, level=level) return [ result[classification] for classification in classifications ] root_list = create_hierarchy(hierarchy) if '' in posts_per_classification: node = hierarchy_utils.TreeNode('', parent=None) node.children = root_list node.classification_path = [] node.classification_name = '' hierarchy_lookup[node.name] = node root_list = [node] flat_hierarchy = hierarchy_utils.flatten_tree_structure( root_list) # Store result site.hierarchy_per_classification[ taxonomy.classification_name][lang] = root_list site.flat_hierarchy_per_classification[ taxonomy.classification_name][lang] = flat_hierarchy site.hierarchy_lookup_per_classification[ taxonomy.classification_name][lang] = hierarchy_lookup taxonomy.postprocess_posts_per_classification( site.posts_per_classification[ taxonomy.classification_name], site.flat_hierarchy_per_classification[ taxonomy.classification_name], site.hierarchy_lookup_per_classification[ taxonomy.classification_name]) else: taxonomy.postprocess_posts_per_classification( site.posts_per_classification[ taxonomy.classification_name]) # Check for valid paths and for collisions taxonomy_outputs = { lang: dict() for lang in site.config['TRANSLATIONS'].keys() } quit = False for taxonomy in taxonomies: # Check for collisions (per language) for lang in site.config['TRANSLATIONS'].keys(): if not taxonomy.is_enabled(lang): continue for classification, posts in site.posts_per_classification[ taxonomy.classification_name][lang].items(): # Do we actually generate this classification page? filtered_posts = [ x for x in posts if self.site.config["SHOW_UNTRANSLATED_POSTS"] or x.is_translation_available(lang) ] generate_list = taxonomy.should_generate_classification_page( classification, filtered_posts, lang) if not generate_list: continue # Obtain path as tuple path = site.path_handlers[taxonomy.classification_name]( classification, lang) # Check that path is OK for path_element in path: if len(path_element) == 0: utils.LOGGER.error( "{0} {1} yields invalid path '{2}'!".format( taxonomy.classification_name.title(), classification, '/'.join(path))) quit = True # Combine path path = os.path.join( *[os.path.normpath(p) for p in path if p != '.']) # Determine collisions if path in taxonomy_outputs[lang]: other_classification_name, other_classification, other_posts = taxonomy_outputs[ lang][path] if other_classification_name == taxonomy.classification_name and other_classification == classification: taxonomy_outputs[lang][path][2].extend( filtered_posts) else: utils.LOGGER.error( 'You have classifications that are too similar: {0} "{1}" and {2} "{3}" both result in output path {4} for language {5}.' .format(taxonomy.classification_name, classification, other_classification_name, other_classification, path, lang)) utils.LOGGER.error( '{0} "{1}" is used in: {2}'.format( taxonomy.classification_name.title(), classification, ', '.join( sorted([ p.source_path for p in filtered_posts ])))) utils.LOGGER.error( '{0} "{1}" is used in: {2}'.format( other_classification_name.title(), other_classification, ', '.join( sorted([ p.source_path for p in other_posts ])))) quit = True else: taxonomy_outputs[lang][path] = ( taxonomy.classification_name, classification, list(posts)) if quit: sys.exit(1) blinker.signal('taxonomies_classified').send(site)
def _generate_classification_overview_kw_context(self, taxonomy, lang): """Create context and kw for a classification overview page.""" context, kw = taxonomy.provide_overview_context_and_uptodate(lang) context = copy(context) context["kind"] = "{}_index".format(taxonomy.classification_name) sorted_links = [] sorted_links_all = [] for other_lang in sorted(self.site.config['TRANSLATIONS'].keys()): sorted_links_all.append((other_lang, None, None)) if other_lang != lang: sorted_links.append((other_lang, None, None)) context['has_other_languages'] = True context['other_languages'] = sorted_links context['all_languages'] = sorted_links_all kw = copy(kw) kw["messages"] = self.site.MESSAGES kw["translations"] = self.site.config['TRANSLATIONS'] kw["filters"] = self.site.config['FILTERS'] kw["minimum_post_count"] = taxonomy.minimum_post_count_per_classification_in_overview kw["output_folder"] = self.site.config['OUTPUT_FOLDER'] kw["pretty_urls"] = self.site.config['PRETTY_URLS'] kw["strip_indexes"] = self.site.config['STRIP_INDEXES'] kw["index_file"] = self.site.config['INDEX_FILE'] # Collect all relevant classifications if taxonomy.has_hierarchy: def acceptor(node): return len(self._filter_list(self.site.posts_per_classification[taxonomy.classification_name][lang][node.classification_name], lang)) >= kw["minimum_post_count"] clipped_root_list = [hierarchy_utils.clone_treenode(node, parent=None, acceptor=acceptor) for node in self.site.hierarchy_per_classification[taxonomy.classification_name][lang]] clipped_root_list = [node for node in clipped_root_list if node] clipped_flat_hierarchy = hierarchy_utils.flatten_tree_structure(clipped_root_list) classifications = [cat.classification_name for cat in clipped_flat_hierarchy] else: classifications = natsort.natsorted([tag for tag, posts in self.site.posts_per_classification[taxonomy.classification_name][lang].items() if len(self._filter_list(posts, lang)) >= kw["minimum_post_count"]], alg=natsort.ns.F | natsort.ns.IC) taxonomy.sort_classifications(classifications, lang) # Set up classifications in context context[taxonomy.overview_page_variable_name] = classifications context["has_hierarchy"] = taxonomy.has_hierarchy if taxonomy.overview_page_items_variable_name: items = [(classification, self.site.link(taxonomy.classification_name, classification, lang)) for classification in classifications] items_with_postcount = [ (classification, self.site.link(taxonomy.classification_name, classification, lang), len(self._filter_list(self.site.posts_per_classification[taxonomy.classification_name][lang][classification], lang))) for classification in classifications ] context[taxonomy.overview_page_items_variable_name] = items context[taxonomy.overview_page_items_variable_name + "_with_postcount"] = items_with_postcount if taxonomy.has_hierarchy and taxonomy.overview_page_hierarchy_variable_name: hier_items = [ (node.name, node.classification_name, node.classification_path, self.site.link(taxonomy.classification_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after) for node in clipped_flat_hierarchy ] hier_items_with_postcount = [ (node.name, node.classification_name, node.classification_path, self.site.link(taxonomy.classification_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after, len(node.children), len(self._filter_list(self.site.posts_per_classification[taxonomy.classification_name][lang][node.classification_name], lang))) for node in clipped_flat_hierarchy ] context[taxonomy.overview_page_hierarchy_variable_name] = hier_items context[taxonomy.overview_page_hierarchy_variable_name + '_with_postcount'] = hier_items_with_postcount return context, kw