def populate_spam_filtered(): from r2.lib.db.queries import get_spam_links, get_spam_comments from r2.lib.db.queries import get_spam_filtered_links, get_spam_filtered_comments from r2.models.query_cache import CachedQueryMutator def was_filtered(thing): if thing._spam and not thing._deleted and \ getattr(thing, 'verdict', None) != 'mod-removed': return True else: return False q = Subreddit._query(sort=asc('_date')) for sr in fetch_things2(q): print 'Processing %s' % sr.name links = Thing._by_fullname(get_spam_links(sr), data=True, return_dict=False) comments = Thing._by_fullname(get_spam_comments(sr), data=True, return_dict=False) insert_links = [l for l in links if was_filtered(l)] insert_comments = [c for c in comments if was_filtered(c)] with CachedQueryMutator() as m: m.insert(get_spam_filtered_links(sr), insert_links) m.insert(get_spam_filtered_comments(sr), insert_comments)
def gen_keys(): yield promoted_memo_key # just let this one do its own writing load_all_reddits() yield queries.get_all_comments().iden l_q = Link._query(Link.c._spam == (True, False), Link.c._deleted == (True, False), sort=desc('_date'), data=True, ) for link in fetch_things2(l_q, verbosity): yield comments_key(link._id) yield last_modified_key(link, 'comments') a_q = Account._query(Account.c._spam == (True, False), sort=desc('_date'), ) for account in fetch_things2(a_q, verbosity): yield messages_key(account._id) yield last_modified_key(account, 'overview') yield last_modified_key(account, 'commented') yield last_modified_key(account, 'submitted') yield last_modified_key(account, 'liked') yield last_modified_key(account, 'disliked') yield queries.get_comments(account, 'new', 'all').iden yield queries.get_submitted(account, 'new', 'all').iden yield queries.get_liked(account).iden yield queries.get_disliked(account).iden yield queries.get_hidden(account).iden yield queries.get_saved(account).iden yield queries.get_inbox_messages(account).iden yield queries.get_unread_messages(account).iden yield queries.get_inbox_comments(account).iden yield queries.get_unread_comments(account).iden yield queries.get_inbox_selfreply(account).iden yield queries.get_unread_selfreply(account).iden yield queries.get_sent(account).iden sr_q = Subreddit._query(Subreddit.c._spam == (True, False), sort=desc('_date'), ) for sr in fetch_things2(sr_q, verbosity): yield last_modified_key(sr, 'stylesheet_contents') yield queries.get_links(sr, 'hot', 'all').iden yield queries.get_links(sr, 'new', 'all').iden for sort in 'top', 'controversial': for time in 'hour', 'day', 'week', 'month', 'year', 'all': yield queries.get_links(sr, sort, time, merge_batched=False).iden yield queries.get_spam_links(sr).iden yield queries.get_spam_comments(sr).iden yield queries.get_reported_links(sr).iden yield queries.get_reported_comments(sr).iden yield queries.get_subreddit_messages(sr).iden yield queries.get_unread_subreddit_messages(sr).iden
def gen_keys(): yield promoted_memo_key # just let this one do its own writing load_all_reddits() yield queries.get_all_comments().iden l_q = Link._query( Link.c._spam == (True, False), Link.c._deleted == (True, False), sort=desc("_date"), data=True ) for link in fetch_things2(l_q, verbosity): yield comments_key(link._id) yield last_modified_key(link, "comments") a_q = Account._query(Account.c._spam == (True, False), sort=desc("_date")) for account in fetch_things2(a_q, verbosity): yield messages_key(account._id) yield last_modified_key(account, "overview") yield last_modified_key(account, "commented") yield last_modified_key(account, "submitted") yield last_modified_key(account, "liked") yield last_modified_key(account, "disliked") yield queries.get_comments(account, "new", "all").iden yield queries.get_submitted(account, "new", "all").iden yield queries.get_liked(account).iden yield queries.get_disliked(account).iden yield queries.get_hidden(account).iden yield queries.get_saved(account).iden yield queries.get_inbox_messages(account).iden yield queries.get_unread_messages(account).iden yield queries.get_inbox_comments(account).iden yield queries.get_unread_comments(account).iden yield queries.get_inbox_selfreply(account).iden yield queries.get_unread_selfreply(account).iden yield queries.get_sent(account).iden sr_q = Subreddit._query(Subreddit.c._spam == (True, False), sort=desc("_date")) for sr in fetch_things2(sr_q, verbosity): yield last_modified_key(sr, "stylesheet_contents") yield queries.get_links(sr, "hot", "all").iden yield queries.get_links(sr, "new", "all").iden for sort in "top", "controversial": for time in "hour", "day", "week", "month", "year", "all": yield queries.get_links(sr, sort, time, merge_batched=False).iden yield queries.get_spam_links(sr).iden yield queries.get_spam_comments(sr).iden yield queries.get_reported_links(sr).iden yield queries.get_reported_comments(sr).iden yield queries.get_subreddit_messages(sr).iden yield queries.get_unread_subreddit_messages(sr).iden
def populate_spam_filtered(): from r2.lib.db.queries import get_spam_links, get_spam_comments from r2.lib.db.queries import get_spam_filtered_links, get_spam_filtered_comments from r2.models.query_cache import CachedQueryMutator def was_filtered(thing): if thing._spam and not thing._deleted and getattr(thing, "verdict", None) != "mod-removed": return True else: return False q = Subreddit._query(sort=asc("_date")) for sr in fetch_things2(q): print "Processing %s" % sr.name links = Thing._by_fullname(get_spam_links(sr), data=True, return_dict=False) comments = Thing._by_fullname(get_spam_comments(sr), data=True, return_dict=False) insert_links = [l for l in links if was_filtered(l)] insert_comments = [c for c in comments if was_filtered(c)] with CachedQueryMutator() as m: m.insert(get_spam_filtered_links(sr), insert_links) m.insert(get_spam_filtered_comments(sr), insert_comments)