def get_items(self):
timer = g.stats.get_timer("CommentBuilder.get_items")
timer.start()
r = link_comments_and_sort(self.link, self.sort.col)
cids, cid_tree, depth, parents, sorter = r
timer.intermediate("load_storage")
if self.comment and not self.comment._id in depth:
g.log.error("Hack - self.comment (%d) not in depth. Defocusing..."
% self.comment._id)
self.comment = None
more_recursions = {}
dont_collapse = []
candidates = []
offset_depth = 0
if self.children:
# requested specific child comments
children = [child._id for child in self.children
if child._id in cids]
self.update_candidates(candidates, sorter, children)
dont_collapse.extend(comment for sort_val, comment in candidates)
elif self.comment:
# requested the tree from a specific comment
# construct path back to top level from this comment, a maximum of
# `context` levels
comment = self.comment._id
path = []
while comment and len(path) <= self.context:
path.append(comment)
comment = parents[comment]
dont_collapse.extend(path)
# rewrite cid_tree so the parents lead only to the requested comment
for comment in path:
parent = parents[comment]
cid_tree[parent] = [comment]
# start building comment tree from earliest comment
self.update_candidates(candidates, sorter, path[-1])
# set offset_depth because we may not be at the top level and can
# show deeper levels
offset_depth = depth.get(path[-1], 0)
else:
# full tree requested, start with the top level comments
top_level_comments = cid_tree.get(None, ())
self.update_candidates(candidates, sorter, top_level_comments)
timer.intermediate("pick_candidates")
if not candidates:
timer.stop()
return []
# choose which comments to show
items = []
while (self.num is None or len(items) < self.num) and candidates:
sort_val, comment_id = heapq.heappop(candidates)
if comment_id not in cids:
continue
comment_depth = depth[comment_id] - offset_depth
if comment_depth < self.max_depth:
items.append(comment_id)
# add children
if comment_id in cid_tree:
children = cid_tree[comment_id]
self.update_candidates(candidates, sorter, children)
elif (self.continue_this_thread and
parents.get(comment_id) is not None):
# the comment is too deep to add, so add a MoreRecursion for
# its parent
parent_id = parents[comment_id]
if parent_id not in more_recursions:
w = Wrapped(MoreRecursion(self.link, depth=0,
parent_id=parent_id))
else:
w = more_recursions[parent_id]
w.children.append(comment_id)
more_recursions[parent_id] = w
timer.intermediate("pick_comments")
# retrieve num_children for the visible comments
top_level_candidates = [comment for sort_val, comment in candidates
if depth.get(comment, 0) == 0]
needs_num_children = items + top_level_candidates
num_children = get_num_children(needs_num_children, cid_tree)
timer.intermediate("calc_num_children")
comments = Comment._byID(items, data=True, return_dict=False,
stale=self.stale)
timer.intermediate("lookup_comments")
wrapped = self.wrap_items(comments)
timer.intermediate("wrap_comments")
wrapped_by_id = {comment._id: comment for comment in wrapped}
final = []
for comment in wrapped:
# skip deleted comments with no children
if (comment.deleted and not cid_tree.has_key(comment._id)
and not c.user_is_admin):
continue
comment.num_children = num_children[comment._id]
if comment.collapsed and comment._id in dont_collapse:
comment.collapsed = False
# add the comment as a child of its parent or to the top level of
# the tree if it has no parent
parent = wrapped_by_id.get(comment.parent_id)
if parent:
if not hasattr(parent, 'child'):
parent.child = empty_listing()
if not parent.deleted:
parent.child.parent_name = parent._fullname
parent.child.things.append(comment)
else:
final.append(comment)
for parent_id, more_recursion in more_recursions.iteritems():
if parent_id not in wrapped_by_id:
continue
parent = wrapped_by_id[parent_id]
parent.child = empty_listing(more_recursion)
if not parent.deleted:
parent.child.parent_name = parent._fullname
timer.intermediate("build_comments")
if not self.load_more:
timer.stop()
return final
# build MoreChildren for visible comments
visible_comments = wrapped_by_id.keys()
for visible_id in visible_comments:
if visible_id in more_recursions:
# don't add a MoreChildren if we already have a MoreRecursion
continue
children = cid_tree.get(visible_id, ())
missing_children = [child for child in children
if child not in visible_comments]
if missing_children:
visible_children = (child for child in children
if child in visible_comments)
visible_count = sum(1 + num_children[child]
for child in visible_children)
missing_count = num_children[visible_id] - visible_count
missing_depth = depth.get(visible_id, 0) + 1 - offset_depth
mc = MoreChildren(self.link, depth=missing_depth,
parent_id=visible_id)
mc.children.extend(missing_children)
w = Wrapped(mc)
w.count = missing_count
# attach the MoreChildren
parent = wrapped_by_id[visible_id]
if hasattr(parent, 'child'):
parent.child.things.append(w)
else:
parent.child = empty_listing(w)
if not parent.deleted:
parent.child.parent_name = parent._fullname
# build MoreChildren for missing root level comments
if top_level_candidates:
mc = MoreChildren(self.link, depth=0, parent_id=None)
mc.children.extend(top_level_candidates)
w = Wrapped(mc)
w.count = sum(1 + num_children[comment]
for comment in top_level_candidates)
final.append(w)
if isinstance(self.sort, operators.shuffled):
shuffle(final)
timer.intermediate("build_morechildren")
timer.stop()
return final
def get_items(self):
timer = g.stats.get_timer("CommentBuilder.get_items")
timer.start()
r = link_comments_and_sort(self.link, self.sort.col)
cids, cid_tree, depth, parents, sorter = r
timer.intermediate("load_storage")
if self.comment and not self.comment._id in depth:
g.log.error(
"Hack - self.comment (%d) not in depth. Defocusing..." %
self.comment._id)
self.comment = None
more_recursions = {}
dont_collapse = []
candidates = []
offset_depth = 0
if self.children:
# requested specific child comments
children = [cid for cid in self.children if cid in cids]
self.update_candidates(candidates, sorter, children)
dont_collapse.extend(comment for sort_val, comment in candidates)
elif self.comment:
# requested the tree from a specific comment
# construct path back to top level from this comment, a maximum of
# `context` levels
comment = self.comment._id
path = []
while comment and len(path) <= self.context:
path.append(comment)
comment = parents[comment]
dont_collapse.extend(path)
# rewrite cid_tree so the parents lead only to the requested comment
for comment in path:
parent = parents[comment]
cid_tree[parent] = [comment]
# start building comment tree from earliest comment
self.update_candidates(candidates, sorter, path[-1])
# set offset_depth because we may not be at the top level and can
# show deeper levels
offset_depth = depth.get(path[-1], 0)
else:
# full tree requested, start with the top level comments
top_level_comments = cid_tree.get(None, ())
self.update_candidates(candidates, sorter, top_level_comments)
timer.intermediate("pick_candidates")
if not candidates:
timer.stop()
return []
# choose which comments to show
items = []
while (self.num is None or len(items) < self.num) and candidates:
sort_val, comment_id = heapq.heappop(candidates)
if comment_id not in cids:
continue
comment_depth = depth[comment_id] - offset_depth
if comment_depth < self.max_depth:
items.append(comment_id)
# add children
if comment_id in cid_tree:
children = cid_tree[comment_id]
self.update_candidates(candidates, sorter, children)
elif (self.continue_this_thread
and parents.get(comment_id) is not None):
# the comment is too deep to add, so add a MoreRecursion for
# its parent
parent_id = parents[comment_id]
if parent_id not in more_recursions:
w = Wrapped(
MoreRecursion(self.link, depth=0, parent_id=parent_id))
else:
w = more_recursions[parent_id]
w.children.append(comment_id)
more_recursions[parent_id] = w
timer.intermediate("pick_comments")
# retrieve num_children for the visible comments
top_level_candidates = [
comment for sort_val, comment in candidates
if depth.get(comment, 0) == 0
]
needs_num_children = items + top_level_candidates
num_children = get_num_children(needs_num_children, cid_tree)
timer.intermediate("calc_num_children")
comments = Comment._byID(items,
data=True,
return_dict=False,
stale=self.stale)
timer.intermediate("lookup_comments")
wrapped = self.wrap_items(comments)
timer.intermediate("wrap_comments")
wrapped_by_id = {comment._id: comment for comment in wrapped}
final = []
# We have some special collapsing rules for the Q&A sort type.
# However, we want to show everything when we're building a specific
# set of children (like from "load more" links) or when viewing a
# comment permalink.
qa_sort_hiding = ((self.sort.col == '_qa') and not self.children
and self.comment is None)
if qa_sort_hiding:
special_responder_ids = self.link.responder_ids
else:
special_responder_ids = ()
max_relation_walks = g.max_comment_parent_walk
for comment in wrapped:
# skip deleted comments with no children
if (comment.deleted and not cid_tree.has_key(comment._id)
and not self.show_deleted):
comment.hidden_completely = True
continue
comment.num_children = num_children[comment._id]
comment.edits_visible = self.edits_visible
# In the Q&A sort type, we want to collapse all comments other than
# those that are:
#
# 1. Top-level comments,
# 2. Responses from the OP(s),
# 3. Responded to by the OP(s) (dealt with below), or
# 4. Otherwise normally prevented from collapse (eg distinguished
# comments).
if (qa_sort_hiding and depth[comment._id] != 0 and # (1)
comment.author_id not in special_responder_ids and # (2)
not comment.prevent_collapse): # (4)
comment.hidden = True
if comment.collapsed and comment._id in dont_collapse:
comment.collapsed = False
comment.hidden = False
parent = wrapped_by_id.get(comment.parent_id)
if parent:
if (qa_sort_hiding
and comment.author_id in special_responder_ids):
# Un-collapse parents as necessary. It's a lot easier to
# do this here, upwards, than to check through all the
# children when we were iterating at the parent.
ancestor = parent
counter = 0
while (ancestor and not getattr(ancestor, 'walked', False)
and counter < max_relation_walks):
ancestor.hidden = False
# In case we haven't processed this comment yet.
ancestor.prevent_collapse = True
# This allows us to short-circuit when the rest of the
# tree has already been uncollapsed.
ancestor.walked = True
ancestor = wrapped_by_id.get(ancestor.parent_id)
counter += 1
# One more time through to actually add things to the final list. We
# couldn't do that the first time because in the Q&A sort we don't know
# if a comment should be visible until after we've processed all its
# children.
for comment in wrapped:
if getattr(comment, 'hidden_completely', False):
# Don't add it to the tree, don't put it in "load more", don't
# acknowledge its existence at all.
continue
if getattr(comment, 'hidden', False):
# Remove it from the list of visible comments so it'll
# automatically be a candidate for the "load more" links.
del wrapped_by_id[comment._id]
# And don't add it to the tree.
continue
# add the comment as a child of its parent or to the top level of
# the tree if it has no parent
parent = wrapped_by_id.get(comment.parent_id)
if parent:
if not hasattr(parent, 'child'):
add_child_listing(parent, comment)
else:
parent.child.things.append(comment)
else:
final.append(comment)
for parent_id, more_recursion in more_recursions.iteritems():
if parent_id not in wrapped_by_id:
continue
parent = wrapped_by_id[parent_id]
add_child_listing(parent, more_recursion)
timer.intermediate("build_comments")
if not self.load_more:
timer.stop()
return final
# build MoreChildren for visible comments
visible_comments = wrapped_by_id.keys()
for visible_id in visible_comments:
if visible_id in more_recursions:
# don't add a MoreChildren if we already have a MoreRecursion
continue
children = cid_tree.get(visible_id, ())
missing_children = [
child for child in children if child not in visible_comments
]
if missing_children:
visible_children = (child for child in children
if child in visible_comments)
visible_count = sum(1 + num_children[child]
for child in visible_children)
missing_count = num_children[visible_id] - visible_count
missing_depth = depth.get(visible_id, 0) + 1 - offset_depth
if missing_depth < self.max_depth:
mc = MoreChildren(self.link,
self.sort,
depth=missing_depth,
parent_id=visible_id)
mc.children.extend(missing_children)
w = Wrapped(mc)
w.count = missing_count
else:
mr = MoreRecursion(self.link,
depth=missing_depth,
parent_id=visible_id)
w = Wrapped(mr)
# attach the MoreChildren
parent = wrapped_by_id[visible_id]
if hasattr(parent, 'child'):
parent.child.things.append(w)
else:
add_child_listing(parent, w)
# build MoreChildren for missing root level comments
if top_level_candidates:
mc = MoreChildren(self.link, self.sort, depth=0, parent_id=None)
mc.children.extend(top_level_candidates)
w = Wrapped(mc)
w.count = sum(1 + num_children[comment]
for comment in top_level_candidates)
final.append(w)
if isinstance(self.sort, operators.shuffled):
shuffle(final)
timer.intermediate("build_morechildren")
timer.stop()
return final
def _make_wrapped_tree(self):
timer = self.timer
comments = self.comments
cid_tree = self.cid_tree
top_level_candidates = self.top_level_candidates
depth = self.depth
more_recursions = self.more_recursions
offset_depth = self.offset_depth
dont_collapse = self.dont_collapse
timer.intermediate("waiting")
if not comments and not top_level_candidates:
timer.stop()
return []
# retrieve num_children for the visible comments
needs_num_children = [c._id for c in comments] + top_level_candidates
num_children = get_num_children(needs_num_children, cid_tree)
timer.intermediate("calc_num_children")
wrapped = self.wrap_items(comments)
timer.intermediate("wrap_comments")
wrapped_by_id = {comment._id: comment for comment in wrapped}
final = []
# We have some special collapsing rules for the Q&A sort type.
# However, we want to show everything when we're building a specific
# set of children (like from "load more" links) or when viewing a
# comment permalink.
qa_sort_hiding = ((self.sort.col == '_qa') and not self.children and
self.comment is None)
if qa_sort_hiding:
special_responder_ids = self.link.responder_ids
else:
special_responder_ids = ()
max_relation_walks = g.max_comment_parent_walk
for comment in wrapped:
# skip deleted comments with no children
if (comment.deleted and not cid_tree.has_key(comment._id)
and not self.show_deleted):
comment.hidden_completely = True
continue
comment.num_children = num_children[comment._id]
comment.edits_visible = self.edits_visible
# In the Q&A sort type, we want to collapse all comments other than
# those that are:
#
# 1. Top-level comments,
# 2. Responses from the OP(s),
# 3. Responded to by the OP(s) (dealt with below), or
# 4. Otherwise normally prevented from collapse (eg distinguished
# comments).
if (qa_sort_hiding and
depth[comment._id] != 0 and # (1)
comment.author_id not in special_responder_ids and # (2)
not comment.prevent_collapse): # (4)
comment.hidden = True
if comment.collapsed and comment._id in dont_collapse:
comment.collapsed = False
comment.hidden = False
parent = wrapped_by_id.get(comment.parent_id)
if parent:
if (qa_sort_hiding and
comment.author_id in special_responder_ids):
# Un-collapse parents as necessary. It's a lot easier to
# do this here, upwards, than to check through all the
# children when we were iterating at the parent.
ancestor = parent
counter = 0
while (ancestor and
not getattr(ancestor, 'walked', False) and
counter < max_relation_walks):
ancestor.hidden = False
# In case we haven't processed this comment yet.
ancestor.prevent_collapse = True
# This allows us to short-circuit when the rest of the
# tree has already been uncollapsed.
ancestor.walked = True
ancestor = wrapped_by_id.get(ancestor.parent_id)
counter += 1
# One more time through to actually add things to the final list. We
# couldn't do that the first time because in the Q&A sort we don't know
# if a comment should be visible until after we've processed all its
# children.
for comment in wrapped:
if getattr(comment, 'hidden_completely', False):
# Don't add it to the tree, don't put it in "load more", don't
# acknowledge its existence at all.
continue
if getattr(comment, 'hidden', False):
# Remove it from the list of visible comments so it'll
# automatically be a candidate for the "load more" links.
del wrapped_by_id[comment._id]
# And don't add it to the tree.
continue
# add the comment as a child of its parent or to the top level of
# the tree if it has no parent
parent = wrapped_by_id.get(comment.parent_id)
if parent:
if not hasattr(parent, 'child'):
add_child_listing(parent, comment)
else:
parent.child.things.append(comment)
else:
final.append(comment)
for parent_id, more_recursion in more_recursions.iteritems():
if parent_id not in wrapped_by_id:
continue
parent = wrapped_by_id[parent_id]
add_child_listing(parent, more_recursion)
timer.intermediate("build_comments")
if not self.load_more:
timer.stop()
return final
# build MoreChildren for visible comments
visible_comments = wrapped_by_id.keys()
for visible_id in visible_comments:
if visible_id in more_recursions:
# don't add a MoreChildren if we already have a MoreRecursion
continue
children = cid_tree.get(visible_id, ())
missing_children = [child for child in children
if child not in visible_comments]
if missing_children:
visible_children = (child for child in children
if child in visible_comments)
visible_count = sum(1 + num_children[child]
for child in visible_children)
missing_count = num_children[visible_id] - visible_count
missing_depth = depth.get(visible_id, 0) + 1 - offset_depth
if missing_depth < self.max_depth:
mc = MoreChildren(self.link, self.sort, depth=missing_depth,
parent_id=visible_id)
mc.children.extend(missing_children)
w = Wrapped(mc)
w.count = missing_count
else:
mr = MoreRecursion(self.link, depth=missing_depth,
parent_id=visible_id)
w = Wrapped(mr)
# attach the MoreChildren
parent = wrapped_by_id[visible_id]
if hasattr(parent, 'child'):
parent.child.things.append(w)
else:
add_child_listing(parent, w)
# build MoreChildren for missing root level comments
if top_level_candidates:
mc = MoreChildren(self.link, self.sort, depth=0, parent_id=None)
mc.children.extend(top_level_candidates)
w = Wrapped(mc)
w.count = sum(1 + num_children[comment]
for comment in top_level_candidates)
final.append(w)
if isinstance(self.sort, operators.shuffled):
shuffle(final)
timer.intermediate("build_morechildren")
timer.stop()
return final