def get_items(self): timer = g.stats.get_timer("CommentBuilder.get_items") timer.start() r = link_comments_and_sort(self.link, self.sort.col) cids, cid_tree, depth, parents, sorter = r timer.intermediate("load_storage") if self.comment and not self.comment._id in depth: g.log.error( "Hack - self.comment (%d) not in depth. Defocusing..." % self.comment._id) self.comment = None more_recursions = {} dont_collapse = [] candidates = [] offset_depth = 0 if self.children: # requested specific child comments children = [cid for cid in self.children if cid in cids] self.update_candidates(candidates, sorter, children) dont_collapse.extend(comment for sort_val, comment in candidates) elif self.comment: # requested the tree from a specific comment # construct path back to top level from this comment, a maximum of # `context` levels comment = self.comment._id path = [] while comment and len(path) <= self.context: path.append(comment) comment = parents[comment] dont_collapse.extend(path) # rewrite cid_tree so the parents lead only to the requested comment for comment in path: parent = parents[comment] cid_tree[parent] = [comment] # start building comment tree from earliest comment self.update_candidates(candidates, sorter, path[-1]) # set offset_depth because we may not be at the top level and can # show deeper levels offset_depth = depth.get(path[-1], 0) else: # full tree requested, start with the top level comments top_level_comments = cid_tree.get(None, ()) self.update_candidates(candidates, sorter, top_level_comments) timer.intermediate("pick_candidates") if not candidates: timer.stop() return [] # choose which comments to show items = [] while (self.num is None or len(items) < self.num) and candidates: sort_val, comment_id = heapq.heappop(candidates) if comment_id not in cids: continue comment_depth = depth[comment_id] - offset_depth if comment_depth < self.max_depth: items.append(comment_id) # add children if comment_id in cid_tree: children = cid_tree[comment_id] self.update_candidates(candidates, sorter, children) elif (self.continue_this_thread and parents.get(comment_id) is not None): # the comment is too deep to add, so add a MoreRecursion for # its parent parent_id = parents[comment_id] if parent_id not in more_recursions: w = Wrapped( MoreRecursion(self.link, depth=0, parent_id=parent_id)) else: w = more_recursions[parent_id] w.children.append(comment_id) more_recursions[parent_id] = w timer.intermediate("pick_comments") # retrieve num_children for the visible comments top_level_candidates = [ comment for sort_val, comment in candidates if depth.get(comment, 0) == 0 ] needs_num_children = items + top_level_candidates num_children = get_num_children(needs_num_children, cid_tree) timer.intermediate("calc_num_children") comments = Comment._byID(items, data=True, return_dict=False, stale=self.stale) timer.intermediate("lookup_comments") wrapped = self.wrap_items(comments) timer.intermediate("wrap_comments") wrapped_by_id = {comment._id: comment for comment in wrapped} final = [] # We have some special collapsing rules for the Q&A sort type. # However, we want to show everything when we're building a specific # set of children (like from "load more" links) or when viewing a # comment permalink. qa_sort_hiding = ((self.sort.col == '_qa') and not self.children and self.comment is None) if qa_sort_hiding: special_responder_ids = self.link.responder_ids else: special_responder_ids = () max_relation_walks = g.max_comment_parent_walk for comment in wrapped: # skip deleted comments with no children if (comment.deleted and not cid_tree.has_key(comment._id) and not self.show_deleted): comment.hidden_completely = True continue comment.num_children = num_children[comment._id] comment.edits_visible = self.edits_visible # In the Q&A sort type, we want to collapse all comments other than # those that are: # # 1. Top-level comments, # 2. Responses from the OP(s), # 3. Responded to by the OP(s) (dealt with below), or # 4. Otherwise normally prevented from collapse (eg distinguished # comments). if (qa_sort_hiding and depth[comment._id] != 0 and # (1) comment.author_id not in special_responder_ids and # (2) not comment.prevent_collapse): # (4) comment.hidden = True if comment.collapsed and comment._id in dont_collapse: comment.collapsed = False comment.hidden = False parent = wrapped_by_id.get(comment.parent_id) if parent: if (qa_sort_hiding and comment.author_id in special_responder_ids): # Un-collapse parents as necessary. It's a lot easier to # do this here, upwards, than to check through all the # children when we were iterating at the parent. ancestor = parent counter = 0 while (ancestor and not getattr(ancestor, 'walked', False) and counter < max_relation_walks): ancestor.hidden = False # In case we haven't processed this comment yet. ancestor.prevent_collapse = True # This allows us to short-circuit when the rest of the # tree has already been uncollapsed. ancestor.walked = True ancestor = wrapped_by_id.get(ancestor.parent_id) counter += 1 # One more time through to actually add things to the final list. We # couldn't do that the first time because in the Q&A sort we don't know # if a comment should be visible until after we've processed all its # children. for comment in wrapped: if getattr(comment, 'hidden_completely', False): # Don't add it to the tree, don't put it in "load more", don't # acknowledge its existence at all. continue if getattr(comment, 'hidden', False): # Remove it from the list of visible comments so it'll # automatically be a candidate for the "load more" links. del wrapped_by_id[comment._id] # And don't add it to the tree. continue # add the comment as a child of its parent or to the top level of # the tree if it has no parent parent = wrapped_by_id.get(comment.parent_id) if parent: if not hasattr(parent, 'child'): add_child_listing(parent, comment) else: parent.child.things.append(comment) else: final.append(comment) for parent_id, more_recursion in more_recursions.iteritems(): if parent_id not in wrapped_by_id: continue parent = wrapped_by_id[parent_id] add_child_listing(parent, more_recursion) timer.intermediate("build_comments") if not self.load_more: timer.stop() return final # build MoreChildren for visible comments visible_comments = wrapped_by_id.keys() for visible_id in visible_comments: if visible_id in more_recursions: # don't add a MoreChildren if we already have a MoreRecursion continue children = cid_tree.get(visible_id, ()) missing_children = [ child for child in children if child not in visible_comments ] if missing_children: visible_children = (child for child in children if child in visible_comments) visible_count = sum(1 + num_children[child] for child in visible_children) missing_count = num_children[visible_id] - visible_count missing_depth = depth.get(visible_id, 0) + 1 - offset_depth if missing_depth < self.max_depth: mc = MoreChildren(self.link, self.sort, depth=missing_depth, parent_id=visible_id) mc.children.extend(missing_children) w = Wrapped(mc) w.count = missing_count else: mr = MoreRecursion(self.link, depth=missing_depth, parent_id=visible_id) w = Wrapped(mr) # attach the MoreChildren parent = wrapped_by_id[visible_id] if hasattr(parent, 'child'): parent.child.things.append(w) else: add_child_listing(parent, w) # build MoreChildren for missing root level comments if top_level_candidates: mc = MoreChildren(self.link, self.sort, depth=0, parent_id=None) mc.children.extend(top_level_candidates) w = Wrapped(mc) w.count = sum(1 + num_children[comment] for comment in top_level_candidates) final.append(w) if isinstance(self.sort, operators.shuffled): shuffle(final) timer.intermediate("build_morechildren") timer.stop() return final
def _make_wrapped_tree(self): timer = self.timer comments = self.comments cid_tree = self.cid_tree top_level_candidates = self.top_level_candidates depth = self.depth more_recursions = self.more_recursions offset_depth = self.offset_depth dont_collapse = self.dont_collapse timer.intermediate("waiting") if not comments and not top_level_candidates: timer.stop() return [] # retrieve num_children for the visible comments needs_num_children = [c._id for c in comments] + top_level_candidates num_children = get_num_children(needs_num_children, cid_tree) timer.intermediate("calc_num_children") wrapped = self.wrap_items(comments) timer.intermediate("wrap_comments") wrapped_by_id = {comment._id: comment for comment in wrapped} final = [] # We have some special collapsing rules for the Q&A sort type. # However, we want to show everything when we're building a specific # set of children (like from "load more" links) or when viewing a # comment permalink. qa_sort_hiding = ((self.sort.col == '_qa') and not self.children and self.comment is None) if qa_sort_hiding: special_responder_ids = self.link.responder_ids else: special_responder_ids = () max_relation_walks = g.max_comment_parent_walk for comment in wrapped: # skip deleted comments with no children if (comment.deleted and not cid_tree.has_key(comment._id) and not self.show_deleted): comment.hidden_completely = True continue comment.num_children = num_children[comment._id] comment.edits_visible = self.edits_visible # In the Q&A sort type, we want to collapse all comments other than # those that are: # # 1. Top-level comments, # 2. Responses from the OP(s), # 3. Responded to by the OP(s) (dealt with below), or # 4. Otherwise normally prevented from collapse (eg distinguished # comments). if (qa_sort_hiding and depth[comment._id] != 0 and # (1) comment.author_id not in special_responder_ids and # (2) not comment.prevent_collapse): # (4) comment.hidden = True if comment.collapsed and comment._id in dont_collapse: comment.collapsed = False comment.hidden = False parent = wrapped_by_id.get(comment.parent_id) if parent: if (qa_sort_hiding and comment.author_id in special_responder_ids): # Un-collapse parents as necessary. It's a lot easier to # do this here, upwards, than to check through all the # children when we were iterating at the parent. ancestor = parent counter = 0 while (ancestor and not getattr(ancestor, 'walked', False) and counter < max_relation_walks): ancestor.hidden = False # In case we haven't processed this comment yet. ancestor.prevent_collapse = True # This allows us to short-circuit when the rest of the # tree has already been uncollapsed. ancestor.walked = True ancestor = wrapped_by_id.get(ancestor.parent_id) counter += 1 # One more time through to actually add things to the final list. We # couldn't do that the first time because in the Q&A sort we don't know # if a comment should be visible until after we've processed all its # children. for comment in wrapped: if getattr(comment, 'hidden_completely', False): # Don't add it to the tree, don't put it in "load more", don't # acknowledge its existence at all. continue if getattr(comment, 'hidden', False): # Remove it from the list of visible comments so it'll # automatically be a candidate for the "load more" links. del wrapped_by_id[comment._id] # And don't add it to the tree. continue # add the comment as a child of its parent or to the top level of # the tree if it has no parent parent = wrapped_by_id.get(comment.parent_id) if parent: if not hasattr(parent, 'child'): add_child_listing(parent, comment) else: parent.child.things.append(comment) else: final.append(comment) for parent_id, more_recursion in more_recursions.iteritems(): if parent_id not in wrapped_by_id: continue parent = wrapped_by_id[parent_id] add_child_listing(parent, more_recursion) timer.intermediate("build_comments") if not self.load_more: timer.stop() return final # build MoreChildren for visible comments visible_comments = wrapped_by_id.keys() for visible_id in visible_comments: if visible_id in more_recursions: # don't add a MoreChildren if we already have a MoreRecursion continue children = cid_tree.get(visible_id, ()) missing_children = [child for child in children if child not in visible_comments] if missing_children: visible_children = (child for child in children if child in visible_comments) visible_count = sum(1 + num_children[child] for child in visible_children) missing_count = num_children[visible_id] - visible_count missing_depth = depth.get(visible_id, 0) + 1 - offset_depth if missing_depth < self.max_depth: mc = MoreChildren(self.link, self.sort, depth=missing_depth, parent_id=visible_id) mc.children.extend(missing_children) w = Wrapped(mc) w.count = missing_count else: mr = MoreRecursion(self.link, depth=missing_depth, parent_id=visible_id) w = Wrapped(mr) # attach the MoreChildren parent = wrapped_by_id[visible_id] if hasattr(parent, 'child'): parent.child.things.append(w) else: add_child_listing(parent, w) # build MoreChildren for missing root level comments if top_level_candidates: mc = MoreChildren(self.link, self.sort, depth=0, parent_id=None) mc.children.extend(top_level_candidates) w = Wrapped(mc) w.count = sum(1 + num_children[comment] for comment in top_level_candidates) final.append(w) if isinstance(self.sort, operators.shuffled): shuffle(final) timer.intermediate("build_morechildren") timer.stop() return final
def get_items(self): timer = g.stats.get_timer("CommentBuilder.get_items") timer.start() r = link_comments_and_sort(self.link, self.sort.col) cids, cid_tree, depth, parents, sorter = r timer.intermediate("load_storage") if self.comment and not self.comment._id in depth: g.log.error("Hack - self.comment (%d) not in depth. Defocusing..." % self.comment._id) self.comment = None more_recursions = {} dont_collapse = [] candidates = [] offset_depth = 0 if self.children: # requested specific child comments children = [child._id for child in self.children if child._id in cids] self.update_candidates(candidates, sorter, children) dont_collapse.extend(comment for sort_val, comment in candidates) elif self.comment: # requested the tree from a specific comment # construct path back to top level from this comment, a maximum of # `context` levels comment = self.comment._id path = [] while comment and len(path) <= self.context: path.append(comment) comment = parents[comment] dont_collapse.extend(path) # rewrite cid_tree so the parents lead only to the requested comment for comment in path: parent = parents[comment] cid_tree[parent] = [comment] # start building comment tree from earliest comment self.update_candidates(candidates, sorter, path[-1]) # set offset_depth because we may not be at the top level and can # show deeper levels offset_depth = depth.get(path[-1], 0) else: # full tree requested, start with the top level comments top_level_comments = cid_tree.get(None, ()) self.update_candidates(candidates, sorter, top_level_comments) timer.intermediate("pick_candidates") if not candidates: timer.stop() return [] # choose which comments to show items = [] while (self.num is None or len(items) < self.num) and candidates: sort_val, comment_id = heapq.heappop(candidates) if comment_id not in cids: continue comment_depth = depth[comment_id] - offset_depth if comment_depth < self.max_depth: items.append(comment_id) # add children if comment_id in cid_tree: children = cid_tree[comment_id] self.update_candidates(candidates, sorter, children) elif (self.continue_this_thread and parents.get(comment_id) is not None): # the comment is too deep to add, so add a MoreRecursion for # its parent parent_id = parents[comment_id] if parent_id not in more_recursions: w = Wrapped(MoreRecursion(self.link, depth=0, parent_id=parent_id)) else: w = more_recursions[parent_id] w.children.append(comment_id) more_recursions[parent_id] = w timer.intermediate("pick_comments") # retrieve num_children for the visible comments top_level_candidates = [comment for sort_val, comment in candidates if depth.get(comment, 0) == 0] needs_num_children = items + top_level_candidates num_children = get_num_children(needs_num_children, cid_tree) timer.intermediate("calc_num_children") comments = Comment._byID(items, data=True, return_dict=False, stale=self.stale) timer.intermediate("lookup_comments") wrapped = self.wrap_items(comments) timer.intermediate("wrap_comments") wrapped_by_id = {comment._id: comment for comment in wrapped} final = [] for comment in wrapped: # skip deleted comments with no children if (comment.deleted and not cid_tree.has_key(comment._id) and not c.user_is_admin): continue comment.num_children = num_children[comment._id] if comment.collapsed and comment._id in dont_collapse: comment.collapsed = False # add the comment as a child of its parent or to the top level of # the tree if it has no parent parent = wrapped_by_id.get(comment.parent_id) if parent: if not hasattr(parent, 'child'): parent.child = empty_listing() if not parent.deleted: parent.child.parent_name = parent._fullname parent.child.things.append(comment) else: final.append(comment) for parent_id, more_recursion in more_recursions.iteritems(): if parent_id not in wrapped_by_id: continue parent = wrapped_by_id[parent_id] parent.child = empty_listing(more_recursion) if not parent.deleted: parent.child.parent_name = parent._fullname timer.intermediate("build_comments") if not self.load_more: timer.stop() return final # build MoreChildren for visible comments visible_comments = wrapped_by_id.keys() for visible_id in visible_comments: if visible_id in more_recursions: # don't add a MoreChildren if we already have a MoreRecursion continue children = cid_tree.get(visible_id, ()) missing_children = [child for child in children if child not in visible_comments] if missing_children: visible_children = (child for child in children if child in visible_comments) visible_count = sum(1 + num_children[child] for child in visible_children) missing_count = num_children[visible_id] - visible_count missing_depth = depth.get(visible_id, 0) + 1 - offset_depth mc = MoreChildren(self.link, depth=missing_depth, parent_id=visible_id) mc.children.extend(missing_children) w = Wrapped(mc) w.count = missing_count # attach the MoreChildren parent = wrapped_by_id[visible_id] if hasattr(parent, 'child'): parent.child.things.append(w) else: parent.child = empty_listing(w) if not parent.deleted: parent.child.parent_name = parent._fullname # build MoreChildren for missing root level comments if top_level_candidates: mc = MoreChildren(self.link, depth=0, parent_id=None) mc.children.extend(top_level_candidates) w = Wrapped(mc) w.count = sum(1 + num_children[comment] for comment in top_level_candidates) final.append(w) if isinstance(self.sort, operators.shuffled): shuffle(final) timer.intermediate("build_morechildren") timer.stop() return final
def _get_comments(self): timer = g.stats.get_timer("CommentBuilder.get_items") timer.start() r = link_comments_and_sort(self.link, self.sort.col) cids, cid_tree, depth, parents, sorter = r timer.intermediate("load_storage") if self.comment and not self.comment._id in depth: g.log.error("Hack - self.comment (%d) not in depth. Defocusing..." % self.comment._id) self.comment = None more_recursions = {} dont_collapse = [] candidates = [] offset_depth = 0 if self.children: # requested specific child comments children = [cid for cid in self.children if cid in cids] self.update_candidates(candidates, sorter, children) dont_collapse.extend(comment for sort_val, comment in candidates) elif self.comment: # requested the tree from a specific comment # construct path back to top level from this comment, a maximum of # `context` levels comment = self.comment._id path = [] while comment and len(path) <= self.context: path.append(comment) comment = parents[comment] dont_collapse.extend(path) # rewrite cid_tree so the parents lead only to the requested comment for comment in path: parent = parents[comment] cid_tree[parent] = [comment] # start building comment tree from earliest comment self.update_candidates(candidates, sorter, path[-1]) # set offset_depth because we may not be at the top level and can # show deeper levels offset_depth = depth.get(path[-1], 0) else: # full tree requested, start with the top level comments top_level_comments = cid_tree.get(None, ()) self.update_candidates(candidates, sorter, top_level_comments) timer.intermediate("pick_candidates") # choose which comments to show items = [] while (self.num is None or len(items) < self.num) and candidates: sort_val, comment_id = heapq.heappop(candidates) if comment_id not in cids: continue comment_depth = depth[comment_id] - offset_depth if comment_depth < self.max_depth: items.append(comment_id) # add children if comment_id in cid_tree: children = cid_tree[comment_id] self.update_candidates(candidates, sorter, children) elif (self.continue_this_thread and parents.get(comment_id) is not None): # the comment is too deep to add, so add a MoreRecursion for # its parent parent_id = parents[comment_id] if parent_id not in more_recursions: w = Wrapped(MoreRecursion(self.link, depth=0, parent_id=parent_id)) else: w = more_recursions[parent_id] w.children.append(comment_id) more_recursions[parent_id] = w timer.intermediate("pick_comments") self.top_level_candidates = [comment for sort_val, comment in candidates if depth.get(comment, 0) == 0] self.comments = Comment._byID( items, data=True, return_dict=False, stale=self.stale) timer.intermediate("lookup_comments") self.timer = timer self.cid_tree = cid_tree self.depth = depth self.more_recursions = more_recursions self.offset_depth = offset_depth self.dont_collapse = dont_collapse