def _process_comments(self, comments): """Given a list of comments, rearranges them according to hierarchy and returns ordered list with indentation information.""" # First, build tree structure out of TreeNode with comments attached root_list = [] comment_nodes = dict() for comment in comments: node = utils.TreeNode(comment.id) node.comment = comment comment_nodes[comment.id] = node for comment in comments: node = comment_nodes[comment.id] parent_node = comment_nodes.get(node.comment.parent_id) if parent_node is not None: parent_node.children.append(node) else: root_list.append(node) # Then flatten structure and add indent information comment_nodes = utils.flatten_tree_structure(root_list) for node in comment_nodes: comment = node.comment comment.indent_levels = node.indent_levels comment.indent_change_before = node.indent_change_before comment.indent_change_after = node.indent_change_after return [node.comment for node in comment_nodes]
def _do_classification(self, site): # Needed to avoid strange errors during tests if site is not self.site: return # Get list of enabled taxonomy plugins and initialize data structures taxonomies = site.taxonomy_plugins.values() site.posts_per_classification = {} for taxonomy in taxonomies: site.posts_per_classification[taxonomy.classification_name] = { lang: defaultdict(set) for lang in site.config['TRANSLATIONS'].keys() } # Classify posts for post in site.timeline: if not post.use_in_feeds: continue for taxonomy in taxonomies: if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages: classifications = {} for lang in site.config['TRANSLATIONS'].keys(): # Extract classifications for this language classifications[lang] = taxonomy.classify(post, lang) if not taxonomy.more_than_one_classifications_per_post and len(classifications[lang]) > 1: raise ValueError("Too many {0} classifications for post {1}".format(taxonomy.classification_name, post.source_path)) # Add post to sets for classification in classifications[lang]: while True: site.posts_per_classification[taxonomy.classification_name][lang][classification].add(post) if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy: break classification_path = taxonomy.extract_hierarchy(classification) if len(classification_path) <= 1: if len(classification_path) == 0 or not taxonomy.include_posts_into_hierarchy_root: break classification = taxonomy.recombine_classification_from_hierarchy(classification_path[:-1]) # Sort everything. site.page_count_per_classification = {} site.hierarchy_per_classification = {} site.flat_hierarchy_per_classification = {} site.hierarchy_lookup_per_classification = {} for taxonomy in taxonomies: site.page_count_per_classification[taxonomy.classification_name] = {} # Sort post lists for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items(): # Ensure implicit classifications are inserted for classification in taxonomy.get_implicit_classifications(lang): if classification not in posts_per_classification: posts_per_classification[classification] = [] site.page_count_per_classification[taxonomy.classification_name][lang] = {} # Convert sets to lists and sort them for classification in list(posts_per_classification.keys()): posts = list(posts_per_classification[classification]) posts.sort(key=lambda p: (int(p.meta('priority')) if p.meta('priority') else 0, p.date, p.source_path)) posts.reverse() taxonomy.sort_posts(posts, classification, lang) posts_per_classification[classification] = posts # Create hierarchy information if taxonomy.has_hierarchy: site.hierarchy_per_classification[taxonomy.classification_name] = {} site.flat_hierarchy_per_classification[taxonomy.classification_name] = {} site.hierarchy_lookup_per_classification[taxonomy.classification_name] = {} for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items(): # Compose hierarchy hierarchy = {} for classification in posts_per_classification.keys(): hier = taxonomy.extract_hierarchy(classification) node = hierarchy for he in hier: if he not in node: node[he] = {} node = node[he] hierarchy_lookup = {} def create_hierarchy(hierarchy, parent=None, level=0): """Create hierarchy.""" result = {} for name, children in hierarchy.items(): node = utils.TreeNode(name, parent) node.children = create_hierarchy(children, node, level + 1) node.classification_path = [pn.name for pn in node.get_path()] node.classification_name = taxonomy.recombine_classification_from_hierarchy(node.classification_path) hierarchy_lookup[node.classification_name] = node result[node.name] = node classifications = natsort.natsorted(result.keys(), alg=natsort.ns.F | natsort.ns.IC) taxonomy.sort_classifications(classifications, lang, level=level) return [result[classification] for classification in classifications] root_list = create_hierarchy(hierarchy) if '' in posts_per_classification: node = utils.TreeNode('', parent=None) node.children = root_list node.classification_path = [] node.classification_name = '' hierarchy_lookup[node.name] = node root_list = [node] flat_hierarchy = utils.flatten_tree_structure(root_list) # Store result site.hierarchy_per_classification[taxonomy.classification_name][lang] = root_list site.flat_hierarchy_per_classification[taxonomy.classification_name][lang] = flat_hierarchy site.hierarchy_lookup_per_classification[taxonomy.classification_name][lang] = hierarchy_lookup taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name], site.flat_hierarchy_per_classification[taxonomy.classification_name], site.hierarchy_lookup_per_classification[taxonomy.classification_name]) else: taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name]) # Check for valid paths and for collisions taxonomy_outputs = {lang: dict() for lang in site.config['TRANSLATIONS'].keys()} quit = False for taxonomy in taxonomies: # Check for collisions (per language) for lang in site.config['TRANSLATIONS'].keys(): if not taxonomy.is_enabled(lang): continue for tlang in site.config['TRANSLATIONS'].keys(): if lang != tlang and not taxonomy.also_create_classifications_from_other_languages: continue for classification, posts in site.posts_per_classification[taxonomy.classification_name][tlang].items(): # Obtain path as tuple path = site.path_handlers[taxonomy.classification_name](classification, lang) # Check that path is OK for path_element in path: if len(path_element) == 0: utils.LOGGER.error("{0} {1} yields invalid path '{2}'!".format(taxonomy.classification_name.title(), classification, '/'.join(path))) quit = True # Combine path path = os.path.join(*[os.path.normpath(p) for p in path if p != '.']) # Determine collisions if path in taxonomy_outputs[lang]: other_classification_name, other_classification, other_posts = taxonomy_outputs[lang][path] if other_classification_name == taxonomy.classification_name and other_classification == classification: taxonomy_outputs[lang][path][2].extend(posts) else: utils.LOGGER.error('You have classifications that are too similar: {0} "{1}" and {2} "{3}" both result in output path {4} for language {5}.'.format( taxonomy.classification_name, classification, other_classification_name, other_classification, path, lang)) utils.LOGGER.error('{0} {1} is used in: {2}'.format( taxonomy.classification_name.title(), classification, ', '.join(sorted([p.source_path for p in posts])))) utils.LOGGER.error('{0} {1} is used in: {2}'.format( other_classification_name.title(), other_classification, ', '.join(sorted([p.source_path for p in other_posts])))) quit = True else: taxonomy_outputs[lang][path] = (taxonomy.classification_name, classification, list(posts)) if quit: sys.exit(1)
def _do_classification(self, site): # Needed to avoid strange errors during tests if site is not self.site: return # Get list of enabled taxonomy plugins and initialize data structures taxonomies = site.taxonomy_plugins.values() site.posts_per_classification = {} for taxonomy in taxonomies: site.posts_per_classification[taxonomy.classification_name] = { lang: defaultdict(set) for lang in site.config['TRANSLATIONS'].keys() } # Classify posts for post in site.timeline: if not post.use_in_feeds: continue for taxonomy in taxonomies: if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages: classifications = {} for lang in site.config['TRANSLATIONS'].keys(): # Extract classifications for this language classifications[lang] = taxonomy.classify(post, lang) if not taxonomy.more_than_one_classifications_per_post and len( classifications[lang]) > 1: raise ValueError( "Too many {0} classifications for post {1}". format(taxonomy.classification_name, post.source_path)) # Add post to sets for classification in classifications[lang]: while True: site.posts_per_classification[ taxonomy.classification_name][lang][ classification].add(post) if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy: break classification_path = taxonomy.extract_hierarchy( classification) if len(classification_path) <= 1: if len( classification_path ) == 0 or not taxonomy.include_posts_into_hierarchy_root: break classification = taxonomy.recombine_classification_from_hierarchy( classification_path[:-1]) # Sort everything. site.page_count_per_classification = {} site.hierarchy_per_classification = {} site.flat_hierarchy_per_classification = {} site.hierarchy_lookup_per_classification = {} for taxonomy in taxonomies: site.page_count_per_classification[ taxonomy.classification_name] = {} # Sort post lists for lang, posts_per_classification in site.posts_per_classification[ taxonomy.classification_name].items(): # Ensure implicit classifications are inserted for classification in taxonomy.get_implicit_classifications( lang): if classification not in posts_per_classification: posts_per_classification[classification] = [] site.page_count_per_classification[ taxonomy.classification_name][lang] = {} # Convert sets to lists and sort them for classification in list(posts_per_classification.keys()): posts = list(posts_per_classification[classification]) posts.sort( key=lambda p: (int(p.meta('priority')) if p.meta( 'priority') else 0, p.date, p.source_path)) posts.reverse() taxonomy.sort_posts(posts, classification, lang) posts_per_classification[classification] = posts # Create hierarchy information if taxonomy.has_hierarchy: site.hierarchy_per_classification[ taxonomy.classification_name] = {} site.flat_hierarchy_per_classification[ taxonomy.classification_name] = {} site.hierarchy_lookup_per_classification[ taxonomy.classification_name] = {} for lang, posts_per_classification in site.posts_per_classification[ taxonomy.classification_name].items(): # Compose hierarchy hierarchy = {} for classification in posts_per_classification.keys(): hier = taxonomy.extract_hierarchy(classification) node = hierarchy for he in hier: if he not in node: node[he] = {} node = node[he] hierarchy_lookup = {} def create_hierarchy(hierarchy, parent=None, level=0): """Create hierarchy.""" result = {} for name, children in hierarchy.items(): node = utils.TreeNode(name, parent) node.children = create_hierarchy( children, node, level + 1) node.classification_path = [ pn.name for pn in node.get_path() ] node.classification_name = taxonomy.recombine_classification_from_hierarchy( node.classification_path) hierarchy_lookup[node.classification_name] = node result[node.name] = node classifications = natsort.natsorted(result.keys(), alg=natsort.ns.F | natsort.ns.IC) taxonomy.sort_classifications(classifications, lang, level=level) return [ result[classification] for classification in classifications ] root_list = create_hierarchy(hierarchy) if '' in posts_per_classification: node = utils.TreeNode('', parent=None) node.children = root_list node.classification_path = [] node.classification_name = '' hierarchy_lookup[node.name] = node root_list = [node] flat_hierarchy = utils.flatten_tree_structure(root_list) # Store result site.hierarchy_per_classification[ taxonomy.classification_name][lang] = root_list site.flat_hierarchy_per_classification[ taxonomy.classification_name][lang] = flat_hierarchy site.hierarchy_lookup_per_classification[ taxonomy.classification_name][lang] = hierarchy_lookup taxonomy.postprocess_posts_per_classification( site.posts_per_classification[ taxonomy.classification_name], site.flat_hierarchy_per_classification[ taxonomy.classification_name], site.hierarchy_lookup_per_classification[ taxonomy.classification_name]) else: taxonomy.postprocess_posts_per_classification( site.posts_per_classification[ taxonomy.classification_name]) # Check for valid paths and for collisions taxonomy_outputs = { lang: dict() for lang in site.config['TRANSLATIONS'].keys() } quit = False for taxonomy in taxonomies: # Check for collisions (per language) for lang in site.config['TRANSLATIONS'].keys(): if not taxonomy.is_enabled(lang): continue for tlang in site.config['TRANSLATIONS'].keys(): if lang != tlang and not taxonomy.also_create_classifications_from_other_languages: continue for classification, posts in site.posts_per_classification[ taxonomy.classification_name][tlang].items(): # Obtain path as tuple path = site.path_handlers[ taxonomy.classification_name](classification, lang) # Check that path is OK for path_element in path: if len(path_element) == 0: utils.LOGGER.error( "{0} {1} yields invalid path '{2}'!". format( taxonomy.classification_name.title(), classification, '/'.join(path))) quit = True # Combine path path = os.path.join( *[os.path.normpath(p) for p in path if p != '.']) # Determine collisions if path in taxonomy_outputs[lang]: other_classification_name, other_classification, other_posts = taxonomy_outputs[ lang][path] if other_classification_name == taxonomy.classification_name and other_classification == classification: taxonomy_outputs[lang][path][2].extend(posts) else: utils.LOGGER.error( 'You have classifications that are too similar: {0} "{1}" and {2} "{3}" both result in output path {4} for language {5}.' .format(taxonomy.classification_name, classification, other_classification_name, other_classification, path, lang)) utils.LOGGER.error( '{0} {1} is used in: {2}'.format( taxonomy.classification_name.title(), classification, ', '.join( sorted([ p.source_path for p in posts ])))) utils.LOGGER.error( '{0} {1} is used in: {2}'.format( other_classification_name.title(), other_classification, ', '.join( sorted([ p.source_path for p in other_posts ])))) quit = True else: taxonomy_outputs[lang][path] = ( taxonomy.classification_name, classification, list(posts)) if quit: sys.exit(1) blinker.signal('taxonomies_classified').send(site)
def _generate_classification_overview_kw_context(self, taxonomy, lang): """Create context and kw for a classification overview page.""" context, kw = taxonomy.provide_overview_context_and_uptodate(lang) context = copy(context) kw = copy(kw) kw["messages"] = self.site.MESSAGES kw["translations"] = self.site.config['TRANSLATIONS'] kw["filters"] = self.site.config['FILTERS'] kw["minimum_post_count"] = taxonomy.minimum_post_count_per_classification_in_overview kw["output_folder"] = self.site.config['OUTPUT_FOLDER'] kw["pretty_urls"] = self.site.config['PRETTY_URLS'] kw["strip_indexes"] = self.site.config['STRIP_INDEXES'] kw["index_file"] = self.site.config['INDEX_FILE'] # Collect all relevant classifications if taxonomy.has_hierarchy: def acceptor(node): return len(self._filter_list(self.site.posts_per_classification[taxonomy.classification_name][lang][node.classification_name], lang)) >= kw["minimum_post_count"] clipped_root_list = [utils.clone_treenode(node, parent=None, acceptor=acceptor) for node in self.site.hierarchy_per_classification[taxonomy.classification_name][lang]] clipped_root_list = [node for node in clipped_root_list if node] clipped_flat_hierarchy = utils.flatten_tree_structure(clipped_root_list) classifications = [cat.classification_name for cat in clipped_flat_hierarchy] else: classifications = natsort.natsorted([tag for tag, posts in self.site.posts_per_classification[taxonomy.classification_name][lang].items() if len(self._filter_list(posts, lang)) >= kw["minimum_post_count"]], alg=natsort.ns.F | natsort.ns.IC) taxonomy.sort_classifications(classifications, lang) # Set up classifications in context context[taxonomy.overview_page_variable_name] = classifications context["has_hierarchy"] = taxonomy.has_hierarchy if taxonomy.overview_page_items_variable_name: items = [(classification, self.site.link(taxonomy.classification_name, classification, lang)) for classification in classifications] items_with_postcount = [ (classification, self.site.link(taxonomy.classification_name, classification, lang), len(self._filter_list(self.site.posts_per_classification[taxonomy.classification_name][lang][classification], lang))) for classification in classifications ] context[taxonomy.overview_page_items_variable_name] = items context[taxonomy.overview_page_items_variable_name + "_with_postcount"] = items_with_postcount if taxonomy.has_hierarchy and taxonomy.overview_page_hierarchy_variable_name: hier_items = [ (node.name, node.classification_name, node.classification_path, self.site.link(taxonomy.classification_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after) for node in clipped_flat_hierarchy ] hier_items_with_postcount = [ (node.name, node.classification_name, node.classification_path, self.site.link(taxonomy.classification_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after, len(node.children), len(self._filter_list(self.site.posts_per_classification[taxonomy.classification_name][lang][node.classification_name], lang))) for node in clipped_flat_hierarchy ] context[taxonomy.overview_page_hierarchy_variable_name] = hier_items context[taxonomy.overview_page_hierarchy_variable_name + '_with_postcount'] = hier_items_with_postcount return context, kw
def _build_taxonomy_list_and_hierarchy(self, taxonomy_name, lang): """Build taxonomy list and hierarchy for the given taxnonmy name and language.""" if taxonomy_name not in self.site.posts_per_classification or taxonomy_name not in self.site.taxonomy_plugins: return None, None posts_per_tag = self.site.posts_per_classification[taxonomy_name][lang] taxonomy = self.site.taxonomy_plugins[taxonomy_name] def acceptor(post): return True if self.site.config[ 'SHOW_UNTRANSLATED_POSTS'] else post.is_translation_available( lang) # Build classification list classifications = [(taxonomy.get_classification_friendly_name( tag, lang, only_last_component=False), tag) for tag in posts_per_tag.keys()] if classifications: # Sort classifications classifications = natsort.humansorted(classifications) # Build items list result = list() for classification_name, classification in classifications: count = len([ post for post in posts_per_tag[classification] if acceptor(post) ]) result.append((classification_name, count, self.site.link(taxonomy_name, classification, lang))) # Build hierarchy if taxonomy.has_hierarchy: # Special post-processing for archives: get rid of root and cut off tree at month level if taxonomy_name == 'archive': root_list = self.site.hierarchy_per_classification[ taxonomy_name][lang] root_list = utils.clone_treenode(root_list[0]).children def cut_depth(node, cutoff): if cutoff <= 1: node.children = [] else: for node in node.children: cut_depth(node, cutoff - 1) def invert_order(node): node.children.reverse() for node in node.children: invert_order(node) # Make sure that days don't creep in for node in root_list: cut_depth(node, 2) invert_order(node) root_list.reverse() flat_hierarchy = utils.flatten_tree_structure(root_list) else: flat_hierarchy = self.site.flat_hierarchy_per_classification[ taxonomy_name][lang] else: root_list = [] for classification_name, classification in classifications: node = utils.TreeNode(classification_name) node.classification_name = classification node.classification_path = taxonomy.extract_hierarchy( classification) root_list.append(node) flat_hierarchy = utils.flatten_tree_structure(root_list) # Build flattened hierarchy list hierarchy = [ (taxonomy.get_classification_friendly_name( node.classification_name, lang, only_last_component=False), node.classification_name, node.classification_path, self.site.link(taxonomy_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after, len(node.children), len([ post for post in posts_per_tag[node.classification_name] if acceptor(post) ])) for node in flat_hierarchy ] return result, hierarchy else: return None, None
def _generate_classification_overview_kw_context(self, taxonomy, lang): """Create context and kw for a classification overview page.""" context, kw = taxonomy.provide_overview_context_and_uptodate(lang) context = copy(context) kw = copy(kw) kw["messages"] = self.site.MESSAGES kw["translations"] = self.site.config['TRANSLATIONS'] kw["filters"] = self.site.config['FILTERS'] kw["minimum_post_count"] = taxonomy.minimum_post_count_per_classification_in_overview kw["output_folder"] = self.site.config['OUTPUT_FOLDER'] kw["pretty_urls"] = self.site.config['PRETTY_URLS'] kw["strip_indexes"] = self.site.config['STRIP_INDEXES'] kw["index_file"] = self.site.config['INDEX_FILE'] # Collect all relevant classifications if taxonomy.has_hierarchy: def acceptor(node): return len( self._filter_list( self.site.posts_per_classification[ taxonomy.classification_name][lang][ node.classification_name], lang)) >= kw["minimum_post_count"] clipped_root_list = [ utils.clone_treenode(node, parent=None, acceptor=acceptor) for node in self.site.hierarchy_per_classification[ taxonomy.classification_name][lang] ] clipped_root_list = [node for node in clipped_root_list if node] clipped_flat_hierarchy = utils.flatten_tree_structure( clipped_root_list) classifications = [ cat.classification_name for cat in clipped_flat_hierarchy ] else: classifications = natsort.natsorted([ tag for tag, posts in self.site.posts_per_classification[ taxonomy.classification_name][lang].items() if len(self._filter_list(posts, lang)) >= kw["minimum_post_count"] ], alg=natsort.ns.F | natsort.ns.IC) taxonomy.sort_classifications(classifications, lang) # Set up classifications in context context[taxonomy.overview_page_variable_name] = classifications context["has_hierarchy"] = taxonomy.has_hierarchy if taxonomy.overview_page_items_variable_name: items = [(classification, self.site.link(taxonomy.classification_name, classification, lang)) for classification in classifications] items_with_postcount = [ (classification, self.site.link(taxonomy.classification_name, classification, lang), len( self._filter_list( self.site.posts_per_classification[ taxonomy.classification_name][lang] [classification], lang))) for classification in classifications ] context[taxonomy.overview_page_items_variable_name] = items context[taxonomy.overview_page_items_variable_name + "_with_postcount"] = items_with_postcount if taxonomy.has_hierarchy and taxonomy.overview_page_hierarchy_variable_name: hier_items = [ (node.name, node.classification_name, node.classification_path, self.site.link(taxonomy.classification_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after) for node in clipped_flat_hierarchy ] hier_items_with_postcount = [ (node.name, node.classification_name, node.classification_path, self.site.link(taxonomy.classification_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after, len(node.children), len( self._filter_list( self.site.posts_per_classification[ taxonomy.classification_name][lang][ node.classification_name], lang))) for node in clipped_flat_hierarchy ] context[ taxonomy.overview_page_hierarchy_variable_name] = hier_items context[taxonomy.overview_page_hierarchy_variable_name + '_with_postcount'] = hier_items_with_postcount return context, kw
def _build_taxonomy_list_and_hierarchy(self, taxonomy_name, lang): """Build taxonomy list and hierarchy for the given taxnonmy name and language.""" if taxonomy_name not in self.site.posts_per_classification or taxonomy_name not in self.site.taxonomy_plugins: return None, None posts_per_tag = self.site.posts_per_classification[taxonomy_name][lang] taxonomy = self.site.taxonomy_plugins[taxonomy_name] def acceptor(post): return True if self.site.config['SHOW_UNTRANSLATED_POSTS'] else post.is_translation_available(lang) # Build classification list classifications = [(taxonomy.get_classification_friendly_name(tag, lang, only_last_component=False), tag) for tag in posts_per_tag.keys()] if classifications: # Sort classifications classifications = natsort.humansorted(classifications) # Build items list result = list() for classification_name, classification in classifications: count = len([post for post in posts_per_tag[classification] if acceptor(post)]) result.append((classification_name, count, self.site.link(taxonomy_name, classification, lang))) # Build hierarchy if taxonomy.has_hierarchy: # Special post-processing for archives: get rid of root and cut off tree at month level if taxonomy_name == 'archive': root_list = self.site.hierarchy_per_classification[taxonomy_name][lang] root_list = utils.clone_treenode(root_list[0]).children def cut_depth(node, cutoff): if cutoff <= 1: node.children = [] else: for node in node.children: cut_depth(node, cutoff - 1) def invert_order(node): node.children.reverse() for node in node.children: invert_order(node) # Make sure that days don't creep in for node in root_list: cut_depth(node, 2) invert_order(node) root_list.reverse() flat_hierarchy = utils.flatten_tree_structure(root_list) else: flat_hierarchy = self.site.flat_hierarchy_per_classification[taxonomy_name][lang] else: root_list = [] for classification_name, classification in classifications: node = utils.TreeNode(classification_name) node.classification_name = classification node.classification_path = taxonomy.extract_hierarchy(classification) root_list.append(node) flat_hierarchy = utils.flatten_tree_structure(root_list) # Build flattened hierarchy list hierarchy = [(taxonomy.get_classification_friendly_name(node.classification_name, lang, only_last_component=False), node.classification_name, node.classification_path, self.site.link(taxonomy_name, node.classification_name, lang), node.indent_levels, node.indent_change_before, node.indent_change_after, len(node.children), len([post for post in posts_per_tag[node.classification_name] if acceptor(post)])) for node in flat_hierarchy] return result, hierarchy else: return None, None