def rebuild(cls, link):
# fetch all comments and sort by parent_id, so parents are added to the
# tree before their children
q = Comment._query(Comment.c.link_id == link._id,
Comment.c._deleted == (True, False),
Comment.c._spam == (True, False),
optimize_rules=True,
data=True)
comments = sorted(q, key=lambda c: c.parent_id)
# remove any comments with missing parents
comment_ids = {comment._id for comment in comments}
comments = [
comment for comment in comments
if not comment.parent_id or comment.parent_id in comment_ids
]
# build tree from scratch (for V2 results in double-counting in cass)
tree = cls(link, cids=[], tree={}, depth={}, parents={})
impl = cls.IMPLEMENTATIONS[link.comment_tree_version]
impl.rebuild(tree, comments)
link.num_comments = sum(1 for c in comments if not c._deleted)
link._commit()
return tree
def run():
STEP = 100
thing = Link
max_id = max_thing_id(thing)
id_start = 0
for id_low in xrange(id_start, max_id + 1, STEP):
print "Add desc karma for links %s to %s" % (id_low, id_low + STEP)
links = list(query_thing_id_range(thing, id_low, id_low + STEP))
for link in links:
if not link._loaded:
link._load()
comments = list(Comment._query(Comment.c.link_id == link._id, eager_load = True))
link_descendant_karma = 0
for comment in comments:
if not comment._loaded:
comment._load()
if hasattr(comment, 'parent_id') and comment.parent_id:
Comment._byID(comment.parent_id).incr_descendant_karma([], comment._ups - comment._downs)
link_descendant_karma += (comment._ups - comment._downs)
link._incr('_descendant_karma', link_descendant_karma)
def run():
STEP = 100
thing = Link
max_id = max_thing_id(thing)
id_start = 0
for id_low in xrange(id_start, max_id + 1, STEP):
print "Add desc karma for links %s to %s" % (id_low, id_low + STEP)
links = list(query_thing_id_range(thing, id_low, id_low + STEP))
for link in links:
if not link._loaded:
link._load()
comments = list(
Comment._query(Comment.c.link_id == link._id, eager_load=True))
link_descendant_karma = 0
for comment in comments:
if not comment._loaded:
comment._load()
if hasattr(comment, 'parent_id') and comment.parent_id:
Comment._byID(comment.parent_id).incr_descendant_karma(
[], comment._ups - comment._downs)
link_descendant_karma += (comment._ups - comment._downs)
link._incr('_descendant_karma', link_descendant_karma)
def _populate(after_id=None, estimate=54301242):
from r2.models import desc
from r2.lib.db import tdb_cassandra
from r2.lib import utils
# larger has a chance to decrease the number of Cassandra writes,
# but the probability is low
chunk_size = 5000
q = Comment._query(Comment.c._spam == (True, False), Comment.c._deleted == (True, False), sort=desc("_date"))
if after_id is not None:
q._after(Comment._byID(after_id))
q = utils.fetch_things2(q, chunk_size=chunk_size)
q = utils.progress(q, verbosity=chunk_size, estimate=estimate)
for chunk in utils.in_chunks(q, chunk_size):
chunk = filter(lambda x: hasattr(x, "link_id"), chunk)
update_comment_votes(chunk)
def rebuild(cls, link):
# retrieve all the comments for the link
q = Comment._query(
Comment.c.link_id == link._id,
Comment.c._deleted == (True, False),
Comment.c._spam == (True, False),
optimize_rules=True,
)
comments = list(q)
# remove any comments with missing parents
comment_ids = {comment._id for comment in comments}
comments = [
comment for comment in comments
if not comment.parent_id or comment.parent_id in comment_ids
]
CommentTreePermacache.rebuild(link, comments)
link.num_comments = sum(1 for c in comments if not c._deleted)
link._commit()
def rebuild(cls, link):
# retrieve all the comments for the link
q = Comment._query(
Comment.c.link_id == link._id,
Comment.c._deleted == (True, False),
Comment.c._spam == (True, False),
optimize_rules=True,
)
comments = list(q)
# remove any comments with missing parents
comment_ids = {comment._id for comment in comments}
comments = [
comment for comment in comments
if not comment.parent_id or comment.parent_id in comment_ids
]
CommentTreePermacache.rebuild(link, comments)
link.num_comments = sum(1 for c in comments if not c._deleted)
link._commit()
def _populate(after_id=None, estimate=54301242):
from r2.models import desc
from r2.lib.db import tdb_cassandra
from r2.lib import utils
# larger has a chance to decrease the number of Cassandra writes,
# but the probability is low
chunk_size = 5000
q = Comment._query(Comment.c._spam == (True, False),
Comment.c._deleted == (True, False),
sort=desc('_date'))
if after_id is not None:
q._after(Comment._byID(after_id))
q = utils.fetch_things2(q, chunk_size=chunk_size)
q = utils.progress(q, verbosity=chunk_size, estimate=estimate)
for chunk in utils.in_chunks(q, chunk_size):
chunk = filter(lambda x: hasattr(x, 'link_id'), chunk)
update_comment_votes(chunk)
