def get_all_comments(self): from r2.lib.db import queries from r2.models import Comment from r2.controllers.errors import UserRequiredException if not c.user_is_loggedin: raise UserRequiredException friends = self.get_important_friends(c.user._id) if not friends: return [] if g.use_query_cache: # with the precomputer enabled, this Subreddit only supports # being sorted by 'new'. it would be nice to have a # cleaner UI than just blatantly ignoring their sort, # though sort = 'new' time = 'all' friends = Account._byID(friends, return_dict=False) crs = [ queries.get_comments(friend, sort, time) for friend in friends ] return queries.MergedCachedResults(crs) else: q = Comment._query(Comment.c.author_id == friends, sort=desc('_date'), data=True) return q
def get_deleted_comments(user_id): return Comment._query( Comment.c.author_id == user_id, Comment.c._deleted == True, Comment.c._spam == (True, False), sort=db_sort("new"), )
def get_all_comments(self): from r2.lib.db import queries from r2.models import Comment from r2.controllers.errors import UserRequiredException if not c.user_is_loggedin: raise UserRequiredException friends = self.get_important_friends(c.user._id) if not friends: return [] if g.use_query_cache: # with the precomputer enabled, this Subreddit only supports # being sorted by 'new'. it would be nice to have a # cleaner UI than just blatantly ignoring their sort, # though sort = "new" time = "all" friends = Account._byID(friends, return_dict=False) crs = [queries.get_comments(friend, sort, time) for friend in friends] return queries.MergedCachedResults(crs) else: q = Comment._query(Comment.c.author_id == friends, sort=desc("_date"), data=True) return q
def post_process_post(self, post): """Perform post processsing to rewrite URLs and generate mapping between old and new permalinks""" post.article = self.rewrite_ob_urls(post.article) post._commit() comments = Comment._query(Comment.c.link_id == post._id, data=True) for comment in comments: comment.body = self.rewrite_ob_urls(comment.body) comment._commit()
def post_process_post(self, post): """Perform post processsing to rewrite URLs and generate mapping between old and new permalinks""" post.article = self.rewrite_ob_urls(post.article) post._commit() comments = Comment._query(Comment.c.link_id == post._id, data = True) for comment in comments: comment.body = self.rewrite_ob_urls(comment.body) comment._commit()
def comment_exists(post, comment): # Check if this comment already exists using brutal compare on content # BeautifulSoup is used to parse as HTML in order to remove markup content = ''.join(BeautifulSoup(comment['body']).findAll(text=True)) key = re_non_alphanum.sub('', content) existing_comments = Comment._query(Comment.c.link_id == post._id, Comment.c.ob_imported == True, data=True) for existing_comment in existing_comments: author = Account._byID(existing_comment.author_id, data=True) content = ''.join(BeautifulSoup(existing_comment.body).findAll(text=True)) existing_key = re_non_alphanum.sub('', content) if key == existing_key: print " Skipping existing %s" % comment_excerpt(comment) return True # else: # print "%s *|NOT|* %s" % (key, existing_key) return False
def _load_link_comments(link_id): from r2.models import Comment q = Comment._query(Comment.c.link_id == link_id, Comment.c._deleted == (True, False), Comment.c._spam == (True, False), optimize_rules=True, data = True) comments = list(q) cids = [c._id for c in comments] #make a tree comment_tree = {} for cm in comments: p_id = cm.parent_id comment_tree.setdefault(p_id, []).append(cm._id) #calculate the depths depth = {} level = 0 cur_level = comment_tree.get(None, ()) while cur_level: next_level = [] for cm_id in cur_level: depth[cm_id] = level next_level.extend(comment_tree.get(cm_id, ())) cur_level = next_level level += 1 #calc the number of children num_children = {} for cm_id in cids: num = 0 todo = [cm_id] iteration_count = 0 while todo: if iteration_count > MAX_ITERATIONS: raise Exception("bad comment tree for link %s" % link_id) more = comment_tree.get(todo.pop(0), ()) num += len(more) todo.extend(more) iteration_count += 1 num_children[cm_id] = num num_comments = sum(1 for c in comments if not c._deleted) return cids, comment_tree, depth, num_children, num_comments
def _load_link_comments(link_id): from r2.models import Comment q = Comment._query(Comment.c.link_id == link_id, Comment.c._deleted == (True, False), Comment.c._spam == (True, False), optimize_rules=True, data=True) comments = list(q) cids = [c._id for c in comments] #make a tree comment_tree = {} for cm in comments: p_id = cm.parent_id comment_tree.setdefault(p_id, []).append(cm._id) #calculate the depths depth = {} level = 0 cur_level = comment_tree.get(None, ()) while cur_level: next_level = [] for cm_id in cur_level: depth[cm_id] = level next_level.extend(comment_tree.get(cm_id, ())) cur_level = next_level level += 1 #calc the number of children num_children = {} for cm_id in cids: num = 0 todo = [cm_id] iteration_count = 0 while todo: if iteration_count > MAX_ITERATIONS: raise Exception("bad comment tree for link %s" % link_id) more = comment_tree.get(todo.pop(0), ()) num += len(more) todo.extend(more) iteration_count += 1 num_children[cm_id] = num num_comments = sum(1 for c in comments if not c._deleted) return cids, comment_tree, depth, num_children, num_comments
def fix_images(dryrun=True): from r2.models import Link, Comment links = Link._query(Link.c.ob_permalink != None, data=True) for link in links: ob_url = link.ob_permalink.strip() print "Processing %s" % ob_url new_content = process_content(link.article) if not dryrun: link.article = new_content link._commit() comments = Comment._query(Comment.c.link_id == link._id, data=True) for comment in comments: new_content = process_content(comment.body) if not dryrun: comment.body = new_content comment._commit()
def fix_images(dryrun=True): from r2.models import Link, Comment links = Link._query(Link.c.ob_permalink != None, data = True) for link in links: ob_url = link.ob_permalink.strip() print "Processing %s" % ob_url new_content = process_content(link.article) if not dryrun: link.article = new_content link._commit() comments = Comment._query(Comment.c.link_id == link._id, data = True) for comment in comments: new_content = process_content(comment.body) if not dryrun: comment.body = new_content comment._commit()
def _load_br_criticisms(link_id): from r2.models import Comment q = Comment._query(Comment.c.link_id == link_id, Comment.c._deleted == (True, False), Comment.c._spam == (True, False), optimize_rules=True, data = True) comments = list(q) brs = [c for c in comments if c.bestresponse==True] #print "SQUIRRREL!" #print comments #print link_id outs = [c._id for c in brs] ups = [c.parent_id for c in brs if c.parent_id] while ups: brs = [c for c in comments if c._id in ups] outs.extend([c._id for c in brs]) ups = [c.parent_id for c in brs if c.parent_id] return outs
def _populate(after_id=None, estimate=54301242): from r2.models import Comment, CommentSortsCache, desc from r2.lib.db import tdb_cassandra from r2.lib import utils # larger has a chance to decrease the number of Cassandra writes, # but the probability is low chunk_size = 5000 q = Comment._query(Comment.c._spam == (True, False), Comment.c._deleted == (True, False), sort=desc("_date")) if after_id is not None: q._after(Comment._byID(after_id)) q = utils.fetch_things2(q, chunk_size=chunk_size) q = utils.progress(q, verbosity=chunk_size, estimate=estimate) for chunk in utils.in_chunks(q, chunk_size): chunk = filter(lambda x: hasattr(x, "link_id"), chunk) update_comment_votes(chunk, write_consistency_level=tdb_cassandra.CL.ONE)
def spam_account_comments(self, account, query_limit=10000, spam_limit=500): from r2.lib.db.operators import asc, desc, timeago q = Comment._query(Comment.c.author_id == account._id, Link.c._spam == False, sort=desc('_date'), data=False) q._limit = query_limit things = list(q) processed = 0 for item in things: if processed < spam_limit: verdict = getattr(item, "verdict", None) if not verdict or not verdict.endswith("-approved"): processed += 1 admintools.spam(item, auto=False, moderator_banned=False, banner=None, train_spam=True)
def _load_link_comments(link_id): from r2.models import Comment q = Comment._query( Comment.c.link_id == link_id, Comment.c._deleted == (True, False), Comment.c._spam == (True, False), data=True) comments = list(q) cids = [c._id for c in comments] #make a tree comment_tree = {} for cm in comments: p_id = cm.parent_id comment_tree.setdefault(p_id, []).append(cm._id) #calculate the depths depth = {} level = 0 cur_level = comment_tree.get(None, ()) while cur_level: next_level = [] for cm_id in cur_level: depth[cm_id] = level next_level.extend(comment_tree.get(cm_id, ())) cur_level = next_level level += 1 #calc the number of children num_children = {} for cm_id in cids: num = 0 todo = [cm_id] while todo: more = comment_tree.get(todo.pop(0), ()) num += len(more) todo.extend(more) num_children[cm_id] = num return cids, comment_tree, depth, num_children
def _load_link_comments(link_id): from r2.models import Comment q = Comment._query(Comment.c.link_id == link_id, Comment.c._deleted == (True, False), Comment.c._spam == (True, False), data=True) comments = list(q) cids = [c._id for c in comments] #make a tree comment_tree = {} for cm in comments: p_id = cm.parent_id comment_tree.setdefault(p_id, []).append(cm._id) #calculate the depths depth = {} level = 0 cur_level = comment_tree.get(None, ()) while cur_level: next_level = [] for cm_id in cur_level: depth[cm_id] = level next_level.extend(comment_tree.get(cm_id, ())) cur_level = next_level level += 1 #calc the number of children num_children = {} for cm_id in cids: num = 0 todo = [cm_id] while todo: more = comment_tree.get(todo.pop(0), ()) num += len(more) todo.extend(more) num_children[cm_id] = num return cids, comment_tree, depth, num_children
def _populate(after_id = None, estimate=54301242): from r2.models import Comment, CommentSortsCache, desc from r2.lib.db import tdb_cassandra from r2.lib import utils # larger has a chance to decrease the number of Cassandra writes, # but the probability is low chunk_size = 5000 q = Comment._query(Comment.c._spam==(True,False), Comment.c._deleted==(True,False), sort=desc('_date')) if after_id is not None: q._after(Comment._byID(after_id)) q = utils.fetch_things2(q, chunk_size=chunk_size) q = utils.progress(q, verbosity=chunk_size, estimate = estimate) for chunk in utils.in_chunks(q, chunk_size): chunk = filter(lambda x: hasattr(x, 'link_id'), chunk) update_comment_votes(chunk, write_consistency_level = tdb_cassandra.CL.ONE)
def get_spam_comments(sr_id): return Comment._query(Comment.c.sr_id == sr_id, Comment.c._spam == True, sort = db_sort('new'))
def _query_comments(self, *args): kwargs = {'data': True} q = Comment._query(*args, **kwargs) comments = list(q) return comments
Subreddit, ) LINK_GILDING_START = datetime(2014, 2, 1, 0, 0, tzinfo=g.tz) COMMENT_GILDING_START = datetime(2012, 10, 1, 0, 0, tzinfo=g.tz) queries = [ Link._query( Link.c.gildings != 0, Link.c._date > LINK_GILDING_START, data=True, sort=desc('_date'), ), Comment._query( Comment.c.gildings != 0, Comment.c._date > COMMENT_GILDING_START, data=True, sort=desc('_date'), ), ] seconds_by_srid = defaultdict(int) gilding_price = g.gold_month_price.pennies for q in queries: for things in fetch_things2(q, chunks=True, chunk_size=100): print things[0]._fullname for thing in things: seconds_per_gilding = calculate_server_seconds( gilding_price, thing._date) seconds_by_srid[thing.sr_id] += int(thing.gildings *
def get_all_comments(): """the master /comments page""" return Comment._query(sort=desc('_date'))
def get_all_comments(): """the master /comments page""" q = Comment._query(sort=desc('_date')) return make_results(q)
def get_all_comments(): """the master /comments page""" q = Comment._query(sort = desc('_date')) return make_results(q)
def get_all_comments(): """the master /comments page""" return Comment._query(sort=desc('_date'))
def get_spam_comments(sr): q_c = Comment._query(Comment.c.sr_id == sr._id, Comment.c._spam == True, sort=db_sort("new")) return make_results(q_c)
def get_reported_comments(sr): q_c = Comment._query( Comment.c.reported != 0, Comment.c.sr_id == sr._id, Comment.c._spam == False, sort=db_sort("new") ) return make_results(q_c)
def _query_comments(self, *args): kwargs = {'data': True} q = Comment._query(*args, **kwargs) comments = list(q) return comments
def get_spam_comments(sr_id): return Comment._query(Comment.c.sr_id == sr_id, Comment.c._spam == True, sort = db_sort('new'))
def get_spam_filtered_comments(sr_id): return Comment._query(Comment.c.sr_id == sr_id, Comment.c._spam == True, Comment.c.verdict != 'mod-removed', sort = db_sort('new'))
def get_reported_comments(sr_id): return Comment._query(Comment.c.reported != 0, Comment.c.sr_id == sr_id, Comment.c._spam == False, sort = db_sort('new'))
def _get_criticisms(user_id, sort, time): q = Comment._query(Comment.c.author_id == user_id, Comment.c.criticism==True, sort = db_sort(sort),data=True) return make_results(q)
def get_spam_filtered_comments(sr_id): return Comment._query(Comment.c.sr_id == sr_id, Comment.c._spam == True, Comment.c.verdict != 'mod-removed', sort = db_sort('new'))
def get_deleted_comments(user_id): return Comment._query(Comment.c.author_id == user_id, Comment.c._deleted == True, Comment.c._spam == (True, False), sort=db_sort('new'))
def get_reported_comments(sr_id): return Comment._query(Comment.c.reported != 0, Comment.c.sr_id == sr_id, Comment.c._spam == False, sort = db_sort('new'))
def get_spam_comments(sr): q_c = Comment._query(Comment.c.sr_id == sr._id, Comment.c._spam == True, sort=db_sort('new')) return make_results(q_c)
def _get_sr_comments(sr_id): """the subreddit /r/foo/comments page""" q = Comment._query(Comment.c.sr_id == sr_id, sort = desc('_date')) return make_results(q)
def get_reported_comments(sr): q_c = Comment._query(Comment.c.reported != 0, Comment.c.sr_id == sr._id, Comment.c._spam == False, sort=db_sort('new')) return make_results(q_c)
from collections import defaultdict from datetime import datetime from pylons import g from r2.lib.db.operators import desc from r2.lib.utils import fetch_things2 from r2.models import calculate_server_seconds, Comment, Link, Subreddit LINK_GILDING_START = datetime(2014, 2, 1, 0, 0, tzinfo=g.tz) COMMENT_GILDING_START = datetime(2012, 10, 1, 0, 0, tzinfo=g.tz) queries = [ Link._query(Link.c.gildings != 0, Link.c._date > LINK_GILDING_START, data=True, sort=desc("_date")), Comment._query(Comment.c.gildings != 0, Comment.c._date > COMMENT_GILDING_START, data=True, sort=desc("_date")), ] seconds_by_srid = defaultdict(int) gilding_price = g.gold_month_price.pennies for q in queries: for things in fetch_things2(q, chunks=True, chunk_size=100): print things[0]._fullname for thing in things: seconds_per_gilding = calculate_server_seconds(gilding_price, thing._date) seconds_by_srid[thing.sr_id] += int(thing.gildings * seconds_per_gilding) for sr_id, seconds in seconds_by_srid: sr = Subreddit._byID(sr_id, data=True) print "%s: %s seconds" % (sr.name, seconds)
def _get_sr_comments(sr_id): """the subreddit /r/foo/comments page""" q = Comment._query(Comment.c.sr_id == sr_id, sort = desc('_date')) return make_results(q)