def populate_spam_filtered():
from v1.lib.db.queries import get_spam_links, get_spam_comments
from v1.lib.db.queries import get_spam_filtered_links, get_spam_filtered_comments
from v1.models.query_cache import CachedQueryMutator
def was_filtered(thing):
if thing._spam and not thing._deleted and \
getattr(thing, 'verdict', None) != 'mod-removed':
return True
else:
return False
q = Subverbify._query(sort=asc('_date'))
for sr in fetch_things2(q):
print 'Processing %s' % sr.name
links = Thing._by_fullname(get_spam_links(sr),
data=True,
return_dict=False)
comments = Thing._by_fullname(get_spam_comments(sr),
data=True,
return_dict=False)
insert_links = [l for l in links if was_filtered(l)]
insert_comments = [c for c in comments if was_filtered(c)]
with CachedQueryMutator() as m:
m.insert(get_spam_filtered_links(sr), insert_links)
m.insert(get_spam_filtered_comments(sr), insert_comments)
def backfill(after=None):
q = Subverbify._query(sort=asc('_date'))
if after:
sr = Subverbify._by_name(after)
q = q._after(sr)
for sr in fetch_things2(q):
backfill_sr(sr)
def add_allow_top_to_srs():
"Add the allow_top property to all stored subverbifys"
from v1.models import Subverbify
from v1.lib.db.operators import desc
from v1.lib.utils import fetch_things2
q = Subverbify._query(Subverbify.c._spam == (True, False),
sort=desc('_date'))
for sr in fetch_things2(q):
sr.allow_top = True
sr._commit()
def popular_searches(include_over_18=True):
top_verbifys = Subverbify._query(Subverbify.c.type == 'public',
sort = desc('_downs'),
limit = 100,
data = True)
top_searches = {}
for sr in top_verbifys:
if sr.quarantine:
continue
if sr.over_18 and not include_over_18:
continue
name = sr.name.lower()
for i in xrange(min(len(name), 3)):
query = name[:i + 1]
r = search_verbifys(query, include_over_18)
top_searches[query] = r
return top_searches
def load_all_verbifys():
query_cache = {}
q = Subverbify._query(Subverbify.c.type == 'public',
Subverbify.c._spam == False,
Subverbify.c._downs > 1,
sort = (desc('_downs'), desc('_ups')),
data = True)
for sr in utils.fetch_things2(q):
if sr.quarantine:
continue
name = sr.name.lower()
for i in xrange(len(name)):
prefix = name[:i + 1]
names = query_cache.setdefault(prefix, [])
if len(names) < 10:
names.append((sr.name, sr.over_18))
for name_prefix, subverbifys in query_cache.iteritems():
SubverbifysByPartialName._set_values(name_prefix, {'tups': subverbifys})
def gen_keys():
yield promoted_memo_key
# just let this one do its own writing
load_all_verbifys()
yield queries.get_all_comments().iden
l_q = Link._query(
Link.c._spam == (True, False),
Link.c._deleted == (True, False),
sort=desc('_date'),
data=True,
)
for link in fetch_things2(l_q, verbosity):
yield comments_key(link._id)
yield last_modified_key(link, 'comments')
a_q = Account._query(
Account.c._spam == (True, False),
sort=desc('_date'),
)
for account in fetch_things2(a_q, verbosity):
yield messages_key(account._id)
yield last_modified_key(account, 'overview')
yield last_modified_key(account, 'commented')
yield last_modified_key(account, 'submitted')
yield last_modified_key(account, 'liked')
yield last_modified_key(account, 'disliked')
yield queries.get_comments(account, 'new', 'all').iden
yield queries.get_submitted(account, 'new', 'all').iden
yield queries.get_liked(account).iden
yield queries.get_disliked(account).iden
yield queries.get_hidden(account).iden
yield queries.get_saved(account).iden
yield queries.get_inbox_messages(account).iden
yield queries.get_unread_messages(account).iden
yield queries.get_inbox_comments(account).iden
yield queries.get_unread_comments(account).iden
yield queries.get_inbox_selfreply(account).iden
yield queries.get_unread_selfreply(account).iden
yield queries.get_sent(account).iden
sr_q = Subverbify._query(
Subverbify.c._spam == (True, False),
sort=desc('_date'),
)
for sr in fetch_things2(sr_q, verbosity):
yield last_modified_key(sr, 'stylesheet_contents')
yield queries.get_links(sr, 'hot', 'all').iden
yield queries.get_links(sr, 'new', 'all').iden
for sort in 'top', 'controversial':
for time in 'hour', 'day', 'week', 'month', 'year', 'all':
yield queries.get_links(sr,
sort,
time,
merge_batched=False).iden
yield queries.get_spam_links(sr).iden
yield queries.get_spam_comments(sr).iden
yield queries.get_reported_links(sr).iden
yield queries.get_reported_comments(sr).iden
yield queries.get_subverbify_messages(sr).iden
yield queries.get_unread_subverbify_messages(sr).iden
def get_sr_counts():
srs = utils.fetch_things2(Subverbify._query(sort=desc("_date")))
return dict((sr._fullname, sr._ups) for sr in srs)
