def new_report(thing):
if isinstance(thing, Link):
sr = Subsciteit._byID(thing.sr_id)
add_queries([get_reported_links(sr)], insert_items = thing)
elif isinstance(thing, Comment):
sr = Subsciteit._byID(thing.sr_id)
add_queries([get_reported_comments(sr)], insert_items = thing)
def get_subsciteit(self):
"""checks if the current url refers to a subsciteit and returns
that subsciteit object. The cases here are:
* the hostname is unset or is g.domain, in which case it
looks for /r/XXXX or /sciteits. The default in this case
is Default.
* the hostname is a cname to a known subsciteit.
On failure to find a subsciteit, returns None.
"""
from pylons import g
from r2.models import Subsciteit, Sub, NotFound, DefaultSR
try:
if not self.hostname or self.hostname.startswith(g.domain):
if self.path.startswith('/r/'):
return Subsciteit._by_name(self.path.split('/')[2])
elif self.path.startswith('/sciteits/'):
return Sub
else:
return DefaultSR()
elif self.hostname:
return Subsciteit._by_domain(self.hostname)
except NotFound:
pass
return None
def build_sr_tree(root_id):
"""Builds the tree below this point."""
from r2.models import Subsciteit
tree=[root_id]
undone=Subsciteit._byID(root_id).children
while undone:
tree.extend(undone)
tmp=Subsciteit._byID(undone)
undone=[]
for k in tmp:
if tmp[k].children:
undone.extend(tmp[k].children)
return tree
def run():
#rss = get_sr_rss()
#names = rss.keys()
#Build tree order, this will be from root to leaves.
order=build_sr_tree(Subsciteit._by_name(g.default_sr)._id)
#Populate RSS in the other order...
order.reverse()
for sr_id in order:
sr = Subsciteit._byID(sr_id)
if sr.rss_source:
#ac=Account._byID(srob.moderators[0])
ac=Account._by_name(g.system_user)
print "Populating %s as %s using feed |%s|" % (sr.name,ac.name,sr.rss_source)
submit_rss_links(sr.name,sr.rss_source,user=ac._id)
def submit_rss_links(srname,rss,user,titlefield='title',linkfield='link'):
#F**k the API, let's just do it the way we would if we were really doing it. This avoids screwing around with cookies and so forth...
feed=fetch_feed(rss)
if feed is None:
return
ac=Account._byID(user)
sr=Subsciteit._by_name(srname)
ip='0.0.0.0'
niceify=False
if domain(rss)=="arxiv.org":
niceify=dict(find="\(arXiv:.*?\)",replace="")
#Let's randomize why not...
random.shuffle(feed.entries)
for article in feed.entries:
#This can take all night if it has to, we don't want to hammer the server into oblivios
sleep(1)
kw = fetch_article(article,titlefield=titlefield,linkfield=linkfield,niceify=niceify)
if kw is None:
continue
l = Link._submit(kw['title'],kw['link'],ac,sr,ip,spam=False)
l._commit()
l.set_url_cache()
#We don't really need auto-submitted links to be vote on...
queries.queue_vote(ac,l,True,ip,cheater=False)
queries.new_link(l)
changed(l)
print "Submitted %s" % article[titlefield]
sleep(.1)
return
def update_karmas():
for pair in to_update():
user = Account._byID(pair[0], True)
sr = Subsciteit._byID(pair[1], True)
print user.name, sr.name
user.incr_karma('comment', sr, 20)
def filter_links(links, filter_spam = False, multiple = True):
# run the list through a builder to remove any that the user
# isn't allowed to see
from pylons import c
from r2.models import IDBuilder, Link, Subsciteit, NotFound
links = IDBuilder([link._fullname for link in links],
skip = False).get_items()[0]
if not links:
return
if filter_spam:
# first, try to remove any spam
links_nonspam = [ link for link in links
if not link._spam ]
if links_nonspam:
links = links_nonspam
# if it occurs in one or more of their subscriptions, show them
# that one first
subs = set(Subsciteit.user_subsciteits(c.user, limit = None))
def cmp_links(a, b):
if a.sr_id in subs and b.sr_id not in subs:
return -1
elif a.sr_id not in subs and b.sr_id in subs:
return 1
else:
return cmp(b._hot, a._hot)
links = sorted(links, cmp = cmp_links)
# among those, show them the hottest one
return links if multiple else links[0]
def gen_keys():
yield promoted_memo_key
# just let this one do its own writing
load_all_sciteits()
yield queries.get_all_comments().iden
l_q = Link._query(Link.c._spam == (True, False),
Link.c._deleted == (True, False),
sort=desc('_date'),
data=True,
)
for link in fetch_things2(l_q, verbosity):
yield comments_key(link._id)
yield last_modified_key(link, 'comments')
a_q = Account._query(Account.c._spam == (True, False),
sort=desc('_date'),
)
for account in fetch_things2(a_q, verbosity):
yield messages_key(account._id)
yield last_modified_key(account, 'overview')
yield last_modified_key(account, 'commented')
yield last_modified_key(account, 'submitted')
yield last_modified_key(account, 'liked')
yield last_modified_key(account, 'disliked')
yield queries.get_comments(account, 'new', 'all').iden
yield queries.get_submitted(account, 'new', 'all').iden
yield queries.get_liked(account).iden
yield queries.get_disliked(account).iden
yield queries.get_hidden(account).iden
yield queries.get_saved(account).iden
yield queries.get_inbox_messages(account).iden
yield queries.get_unread_messages(account).iden
yield queries.get_inbox_comments(account).iden
yield queries.get_unread_comments(account).iden
yield queries.get_inbox_selfreply(account).iden
yield queries.get_unread_selfreply(account).iden
yield queries.get_sent(account).iden
sr_q = Subsciteit._query(Subsciteit.c._spam == (True, False),
sort=desc('_date'),
)
for sr in fetch_things2(sr_q, verbosity):
yield last_modified_key(sr, 'stylesheet_contents')
yield queries.get_links(sr, 'hot', 'all').iden
yield queries.get_links(sr, 'new', 'all').iden
for sort in 'top', 'controversial':
for time in 'hour', 'day', 'week', 'month', 'year', 'all':
yield queries.get_links(sr, sort, time,
merge_batched=False).iden
yield queries.get_spam_links(sr).iden
yield queries.get_spam_comments(sr).iden
yield queries.get_reported_links(sr).iden
yield queries.get_reported_comments(sr).iden
yield queries.get_subsciteit_messages(sr).iden
yield queries.get_unread_subsciteit_messages(sr).iden
def multi_load_tree(sr_ids):
srs = Subsciteit._byID(sr_ids, return_dict = False)
res = {}
for sr in srs:
trees = subsciteit_messages_nocache(sr)
if trees:
res[sr._id] = trees
return res
def get_spam(sr):
if isinstance(sr, ModContribSR):
srs = Subsciteit._byID(sr.sr_ids(), return_dict=False)
results = [ get_spam_links(sr) for sr in srs ]
return merge_results(*results)
else:
return merge_results(get_spam_links(sr),
get_spam_comments(sr))
def add_all_ban_report_srs():
"""Adds the initial spam/reported pages to the report queue"""
q = Subsciteit._query(sort = asc('_date'))
for sr in fetch_things2(q):
add_queries([get_spam_links(sr),
get_spam_comments(sr),
get_reported_links(sr),
get_reported_comments(sr),
])
def set_downs():
sr_counts = count.get_sr_counts()
names = [k for k, v in sr_counts.iteritems() if v != 0]
srs = Subsciteit._by_fullname(names)
for name in names:
sr,c = srs[name], sr_counts[name]
if c != sr._downs and c > 0:
sr._downs = max(c, 0)
sr._commit()
def assign_trial(account, juries_already_on, ip, slash16):
from r2.models import Jury, Subsciteit, Trial
from r2.lib.db import queries
defendants_assigned_to = []
for jury in juries_already_on:
defendants_assigned_to.append(jury._thing2_id)
subscribed_sr_ids = Subsciteit.user_subsciteits(account, ids=True, limit=None)
# Pull defendants, except ones which already have lots of juryvotes
defs = Trial.all_defendants(quench=True)
# Filter out defendants outside this user's subscribed SRs
defs = filter (lambda d: d.sr_id in subscribed_sr_ids, defs)
# Dictionary of sr_id => SR for all defendants' SRs
srs = Subsciteit._byID(set([ d.sr_id for d in defs ]))
# Dictionary of sr_id => eligibility bool
submit_srs = {}
for sr_id, sr in srs.iteritems():
submit_srs[sr_id] = sr.can_submit(account) and not sr._spam
# Filter out defendants with ineligible SRs
defs = filter (lambda d: submit_srs.get(d.sr_id), defs)
likes = queries.get_likes(account, defs)
if not g.debug:
# Filter out things that the user has upvoted or downvoted
defs = filter (lambda d: likes.get((account, d)) is None, defs)
# Prefer oldest trials
defs.sort(key=lambda x: x._date)
for defendant in defs:
sr = srs[defendant.sr_id]
if voir_dire(account, ip, slash16, defendants_assigned_to, defendant, sr):
j = Jury._new(account, defendant)
return defendant
return None
def add_allow_top_to_srs():
"Add the allow_top property to all stored subsciteits"
from r2.models import Subsciteit
from r2.lib.db.operators import desc
from r2.lib.utils import fetch_things2
q = Subsciteit._query(Subsciteit.c._spam == (True,False),
sort = desc('_date'))
for sr in fetch_things2(q):
sr.allow_top = True; sr._commit()
def get_reported(sr):
if isinstance(sr, ModContribSR):
srs = Subsciteit._byID(sr.sr_ids(), return_dict=False)
results = []
results.extend(get_reported_links(sr) for sr in srs)
results.extend(get_reported_comments(sr) for sr in srs)
return merge_results(*results)
else:
return merge_results(get_reported_links(sr),
get_reported_comments(sr))
def degolden(self, account, severe=False):
if severe:
account.gold_charter = False
Award.take_away("charter_subscriber", account)
Award.take_away("sciteit_gold", account)
account.gold = False
account._commit()
if g.lounge_sciteit and not getattr(account, "gold_charter", False):
sr = Subsciteit._by_name(g.lounge_sciteit)
sr.remove_contributor(account)
def popular_searches():
top_sciteits = Subsciteit._query(Subsciteit.c.type == 'public',
sort = desc('_downs'),
limit = 100,
data = True)
top_searches = {}
for sr in top_sciteits:
name = sr.name.lower()
for i in xrange(min(len(name), 3)):
query = name[:i + 1]
r = search_sciteits(query)
top_searches[query] = r
return top_searches
def thing_attr(self, thing, attr):
from r2.models import Subsciteit
if attr == "has_mail":
if c.user_is_loggedin and thing._id == c.user._id:
return bool(c.have_messages)
return None
if attr == "has_mod_mail":
if c.user_is_loggedin and thing._id == c.user._id:
return bool(c.have_mod_messages)
return None
if attr == "is_mod":
return bool(Subsciteit.reverse_moderator_ids(thing))
return ThingJsonTemplate.thing_attr(self, thing, attr)
def add_all_srs():
"""Recalculates every listing query for every subsciteit. Very,
very slow."""
q = Subsciteit._query(sort = asc('_date'))
for sr in fetch_things2(q):
for q in all_queries(get_links, sr, ('hot', 'new'), ['all'],no_children=True):
q.update()
for q in all_queries(get_links, sr, time_filtered_sorts, db_times.keys(),no_children=True):
q.update()
get_spam_links(sr).update()
get_spam_comments(sr).update()
get_reported_links(sr).update()
get_reported_comments(sr).update()
def subscribe_to_blog_and_annoucements(filename):
import re
from time import sleep
from r2.models import Account, Subsciteit
r_blog = Subsciteit._by_name("blog")
r_announcements = Subsciteit._by_name("announcements")
contents = file(filename).read()
numbers = [ int(s) for s in re.findall("\d+", contents) ]
# d = Account._byID(numbers, data=True)
# for i, account in enumerate(d.values()):
for i, account_id in enumerate(numbers):
account = Account._byID(account_id, data=True)
for sr in r_blog, r_announcements:
if sr.add_subscriber(account):
sr._incr("_ups", 1)
print ("%d: subscribed %s to %s" % (i, account.name, sr.name))
else:
print ("%d: didn't subscribe %s to %s" % (i, account.name, sr.name))
def set_last_sr_ban(self, things):
by_srid = {}
for thing in things:
if getattr(thing, 'sr_id', None) is not None:
by_srid.setdefault(thing.sr_id, []).append(thing)
if by_srid:
srs = Subsciteit._byID(by_srid.keys(), data=True, return_dict=True)
for sr_id, sr_things in by_srid.iteritems():
sr = srs[sr_id]
sr.last_mod_action = datetime.now(g.tz)
sr._commit()
sr._incr('mod_actions', len(sr_things))
def _by_srid(things,srs=True):
"""Takes a list of things and returns them in a dict separated by
sr_id, in addition to the looked-up subsciteits"""
ret = {}
for thing in tup(things):
if getattr(thing, 'sr_id', None) is not None:
ret.setdefault(thing.sr_id, []).append(thing)
if srs:
_srs = Subsciteit._byID(ret.keys(), return_dict=True) if ret else {}
return ret, _srs
else:
return ret
def store_keys(key, maxes):
# we're building queries using queries.py, but we could make the
# queries ourselves if we wanted to avoid the individual lookups
# for accounts and subsciteits.
# Note that we're only generating the 'sr-' type queries here, but
# we're also able to process the other listings generated by the
# old migrate.mr_permacache for convenience
userrel_fns = dict(liked = queries.get_liked,
disliked = queries.get_disliked,
saved = queries.get_saved,
hidden = queries.get_hidden)
if key.startswith('user-'):
acc_str, keytype, account_id = key.split('-')
account_id = int(account_id)
fn = queries.get_submitted if keytype == 'submitted' else queries.get_comments
q = fn(Account._byID(account_id), 'new', 'all')
q._insert_tuples([(fname, float(timestamp))
for (timestamp, fname)
in maxes])
elif key.startswith('sr-'):
sr_str, sort, time, sr_id = key.split('-')
sr_id = int(sr_id)
if sort == 'controversy':
# I screwed this up in the mapper and it's too late to fix
# it
sort = 'controversial'
q = queries.get_links(Subsciteit._byID(sr_id), sort, time)
q._insert_tuples([tuple([item[-1]] + map(float, item[:-1]))
for item in maxes])
elif key.startswith('domain/'):
d_str, sort, time, domain = key.split('/')
q = queries.get_domain_links(domain, sort, time)
q._insert_tuples([tuple([item[-1]] + map(float, item[:-1]))
for item in maxes])
elif key.split('-')[0] in userrel_fns:
key_type, account_id = key.split('-')
account_id = int(account_id)
fn = userrel_fns[key_type]
q = fn(Account._byID(account_id))
q._insert_tuples([tuple([item[-1]] + map(float, item[:-1]))
for item in maxes])
def by_sr_merged(cls, sr, _update=False):
if sr.name == g.default_sr:
return cls.by_sr(sr)
my_adsrs = cls.by_sr(sr)
global_adsrs = cls.by_sr(Subsciteit._by_name(g.default_sr, stale=True))
seen = {}
for adsr in my_adsrs:
seen[adsr._thing1.codename] = True
for adsr in global_adsrs:
if adsr._thing1.codename not in seen:
my_adsrs.append(adsr)
return my_adsrs
def default_queries():
from r2.models import Link, Subsciteit
from r2.lib.db.operators import desc
from copy import deepcopy
queries = []
q = Link._query(Link.c.sr_id == Subsciteit.user_subsciteits(None),
sort = desc('_hot'),
limit = 37)
queries.append(q)
#add a higher limit one too
q = deepcopy(q)
q._limit = 75
queries.append(q)
return queries
def load_all_sciteits():
query_cache = {}
q = Subsciteit._query(Subsciteit.c.type == 'public',
Subsciteit.c._downs > 1,
sort = (desc('_downs'), desc('_ups')),
data = True)
for sr in utils.fetch_things2(q):
name = sr.name.lower()
for i in xrange(len(name)):
prefix = name[:i + 1]
names = query_cache.setdefault(prefix, [])
if len(names) < 10:
names.append(sr.name)
for name_prefix, subsciteits in query_cache.iteritems():
SubsciteitsByPartialName._set_values(name_prefix, {'srs': subsciteits})
def moderator_messages(user):
from r2.models import Subsciteit
sr_ids = Subsciteit.reverse_moderator_ids(user)
def multi_load_tree(sr_ids):
srs = Subsciteit._byID(sr_ids, return_dict = False)
res = {}
for sr in srs:
trees = subsciteit_messages_nocache(sr)
if trees:
res[sr._id] = trees
return res
res = sgm(g.permacache, sr_ids, miss_fn = multi_load_tree,
prefix = sr_messages_key(""))
return sorted(chain(*res.values()), key = tree_sort_fn, reverse = True)
def cache_lists():
def _chop(srs):
srs.sort(key=lambda s: s._downs, reverse=True)
return srs[:limit]
# bylang =:= dict((lang, over18_state) -> [Subsciteit])
# lang =:= all | lang()
# nsfwstate =:= no_over18 | allow_over18 | only_over18
bylang = {}
for sr in fetch_things2(Subsciteit._query(sort=desc('_date'),
data=True)):
aid = getattr(sr, 'author_id', None)
if aid is not None and aid < 0:
# skip special system sciteits like promos
continue
type = getattr(sr, 'type', 'private')
if type not in ('public', 'restricted'):
# skips sciteits that can't appear in the default list
# because of permissions
continue
for lang in 'all', sr.lang:
over18s = ['allow_over18']
if sr.over_18:
over18s.append('only_over18')
else:
over18s.append('no_over18')
for over18 in over18s:
k = (lang, over18)
bylang.setdefault(k, []).append(sr)
# keep the lists small while we work
if len(bylang[k]) > limit*2:
bylang[k] = _chop(bylang[k])
for (lang, over18), srs in bylang.iteritems():
srs = _chop(srs)
sr_tuples = map(lambda sr: (sr._downs, sr.allow_top, sr._id), srs)
print "For %s/%s setting %s" % (lang, over18,
map(lambda sr: sr.name, srs[:50]))
SubsciteitPopularityByLanguage._set_values(lang, {over18: sr_tuples})
def new_comment(comment, inbox_rels):
author = Account._byID(comment.author_id)
job = [get_comments(author, 'new', 'all'),
get_comments(author, 'top', 'all'),
get_comments(author, 'controversial', 'all')]
sr = Subsciteit._byID(comment.sr_id)
if comment._deleted:
job_key = "delete_items"
job.append(get_sr_comments(sr))
job.append(get_all_comments())
else:
job_key = "insert_items"
if comment._spam:
job.append(get_spam_comments(sr))
amqp.add_item('new_comment', comment._fullname)
if not g.amqp_host:
add_comment_tree([comment])
job_dict = { job_key: comment }
add_queries(job, **job_dict)
# note that get_all_comments() is updated by the amqp process
# r2.lib.db.queries.run_new_comments (to minimise lock contention)
if inbox_rels:
for inbox_rel in tup(inbox_rels):
inbox_owner = inbox_rel._thing1
job_dict = { job_key: inbox_rel }
if inbox_rel._name == "inbox":
inbox_func = get_inbox_comments
unread_func = get_unread_comments
elif inbox_rel._name == "selfreply":
inbox_func = get_inbox_selfreply
unread_func = get_unread_selfreply
else:
raise ValueError("wtf is " + inbox_rel._name)
add_queries([inbox_func(inbox_owner)], **job_dict)
if comment._deleted:
add_queries([unread_func(inbox_owner)], **job_dict)
else:
set_unread(comment, inbox_owner, True)
def new_link(link):
"Called on the submission and deletion of links"
sr = Subsciteit._byID(link.sr_id)
author = Account._byID(link.author_id)
results = [get_links(sr, 'new', 'all',no_children=True)]
# we don't have to do hot/top/controversy because new_vote will do
# that
results.append(get_submitted(author, 'new', 'all'))
for domain in utils.UrlParser(link.url).domain_permutations():
results.append(get_domain_links(domain, 'new', "all"))
if link._spam:
results.append(get_spam_links(sr))
add_queries(results, insert_items = link)
amqp.add_item('new_link', link._fullname)
