def _run_changed(msgs, chan):
'''Consume the cloudsearch_changes queue, and print reporting information
on how long it took and how many remain
'''
start = datetime.now(g.tz)
changed = [pickle.loads(msg.body) for msg in msgs]
fullnames = set()
fullnames.update(LinkUploader.desired_fullnames(changed))
fullnames.update(SubredditUploader.desired_fullnames(changed))
things = Thing._by_fullname(fullnames, data=True, return_dict=False)
link_uploader = LinkUploader(g.CLOUDSEARCH_DOC_API, things=things)
subreddit_uploader = SubredditUploader(g.CLOUDSEARCH_SUBREDDIT_DOC_API,
things=things)
link_time = link_uploader.inject()
subreddit_time = subreddit_uploader.inject()
cloudsearch_time = link_time + subreddit_time
totaltime = (datetime.now(g.tz) - start).total_seconds()
print(
"%s: %d messages in %.2fs seconds (%.2fs secs waiting on "
"cloudsearch); %d duplicates, %s remaining)" %
(start, len(changed), totaltime, cloudsearch_time, len(changed) -
len(things), msgs[-1].delivery_info.get('message_count', 'unknown')))
def _run_changed(msgs, chan):
'''Consume the cloudsearch_changes queue, and print reporting information
on how long it took and how many remain
'''
start = datetime.now(g.tz)
changed = [pickle.loads(msg.body) for msg in msgs]
fullnames = set()
fullnames.update(LinkUploader.desired_fullnames(changed))
fullnames.update(SubredditUploader.desired_fullnames(changed))
things = Thing._by_fullname(fullnames, data=True, return_dict=False)
link_uploader = LinkUploader(g.CLOUDSEARCH_DOC_API, things=things)
subreddit_uploader = SubredditUploader(g.CLOUDSEARCH_SUBREDDIT_DOC_API,
things=things)
link_time = link_uploader.inject()
subreddit_time = subreddit_uploader.inject()
cloudsearch_time = link_time + subreddit_time
totaltime = (datetime.now(g.tz) - start).total_seconds()
print ("%s: %d messages in %.2fs seconds (%.2fs secs waiting on "
"cloudsearch); %d duplicates, %s remaining)" %
(start, len(changed), totaltime, cloudsearch_time,
len(changed) - len(things),
msgs[-1].delivery_info.get('message_count', 'unknown')))
def get_things(codes):
"""Fetch relevant things for a list of ad codenames in batch."""
fullnames = [
AdvertTrafficSummary.split_codename(code)[0] for code in codes
if code.startswith(Thing._type_prefix)
]
return Thing._by_fullname(fullnames, data=True, return_dict=True)
def process_message(msg):
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
vote_data = json.loads(msg.body)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.error(e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
def _get_newest_link(sr):
for fullname in sr.get_links('new', 'all'):
link = Thing._by_fullname(fullname, data=True)
if not link._spam and not link._deleted:
return link
return None
def process_message(msg):
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
vote_data = json.loads(msg.body)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
def _get_newest_link(sr):
for fullname in sr.get_links('new', 'all'):
link = Thing._by_fullname(fullname, data=True)
if not link._spam and not link._deleted:
return link
return None
def batch_lookups(self):
try:
self.things = Thing._by_fullname(self.fullnames, data=True, return_dict=False)
except NotFound:
if self.use_safe_get:
self.things = safe_get(Thing._by_fullname, self.fullnames, data=True, return_dict=False)
else:
raise
def send_gift(buyer, recipient, months, days, signed, giftmessage,
thing_fullname):
admintools.engolden(recipient, days)
if thing_fullname:
thing = Thing._by_fullname(thing_fullname, data=True)
thing._gild(buyer)
else:
thing = None
if signed:
sender = buyer.name
md_sender = "[%s](/user/%s)" % (sender, sender)
else:
sender = _("An anonymous redditor")
md_sender = _("An anonymous redditor")
create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed)
if months == 1:
amount = "a month"
else:
amount = "%d months" % months
if not thing:
subject = _(
'Let there be gold! %s just sent you reddit gold!') % sender
message = strings.youve_got_gold % dict(sender=md_sender,
amount=amount)
if giftmessage and giftmessage.strip():
message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----'
else:
url = thing.make_permalink_slow()
if isinstance(thing, Comment):
subject = _('Your comment has been gilded!')
message = strings.youve_been_gilded_comment % {'url': url}
else:
subject = _('Your submission has been gilded!')
message = strings.youve_been_gilded_link % {'url': url}
message += '\n\n' + strings.gold_benefits_msg
if g.lounge_reddit:
message += '\n* ' + strings.lounge_msg
message = append_random_bottlecap_phrase(message)
try:
send_system_message(recipient,
subject,
message,
distinguished='gold-auto')
except MessageError:
g.log.error('send_gift: could not send system message')
g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
return thing
def from_queue(self, max_date, batch_limit=50, kind=None):
from r2.models import Account, Thing
keep_trying = True
min_id = None
s = self.queue_table
while keep_trying:
where = [s.c.date < max_date]
if min_id:
where.append(s.c.uid > min_id)
if kind:
where.append(s.c.kind == kind)
res = sa.select([
s.c.to_addr,
s.c.account_id,
s.c.from_name,
s.c.fullname,
s.c.body,
s.c.kind,
s.c.ip,
s.c.date,
s.c.uid,
s.c.msg_hash,
s.c.fr_addr,
s.c.reply_to,
s.c.html_body,
s.c.list_unsubscribe_header,
],
sa.and_(*where),
order_by=s.c.uid,
limit=batch_limit).execute()
res = res.fetchall()
if not res: break
# batch load user accounts
aids = [x[1] for x in res if x[1] > 0]
accts = Account._byID(aids, data=True,
return_dict=True) if aids else {}
# batch load things
tids = [x[3] for x in res if x[3]]
things = Thing._by_fullname(tids, data=True,
return_dict=True) if tids else {}
# get the lower bound date for next iteration
min_id = max(x[8] for x in res)
# did we not fetch them all?
keep_trying = (len(res) == batch_limit)
for (addr, acct, fname, fulln, body, kind, ip, date, uid, msg_hash,
fr_addr, reply_to, html_body, list_unsubscribe_header) in res:
yield (accts.get(acct), things.get(fulln), addr, fname, date,
ip, kind, msg_hash, body, fr_addr, reply_to, html_body,
list_unsubscribe_header, uid)
def batch_lookups(self):
try:
self.things = Thing._by_fullname(self.fullnames, data=True,
return_dict=False)
except NotFound:
if self.use_safe_get:
self.things = safe_get(Thing._by_fullname, self.fullnames,
data=True, return_dict=False)
else:
raise
def send_gift(buyer, recipient, months, days, signed, giftmessage,
thing_fullname):
admintools.engolden(recipient, days)
if thing_fullname:
thing = Thing._by_fullname(thing_fullname, data=True)
thing._gild(buyer)
else:
thing = None
if signed:
sender = buyer.name
md_sender = "[%s](/user/%s)" % (sender, sender)
else:
sender = _("An anonymous redditor")
md_sender = _("An anonymous redditor")
create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed)
if months == 1:
amount = "a month"
else:
amount = "%d months" % months
if not thing:
subject = _(
'Let there be gold! %s just sent you reddit gold!') % sender
message = strings.youve_got_gold % dict(
sender=md_sender, amount=amount)
if giftmessage and giftmessage.strip():
message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----'
else:
url = thing.make_permalink_slow()
if isinstance(thing, Comment):
subject = _('Your comment has been gilded!')
message = strings.youve_been_gilded_comment % {'url': url}
else:
subject = _('Your submission has been gilded!')
message = strings.youve_been_gilded_link % {'url': url}
message += '\n\n' + strings.gold_benefits_msg
if g.lounge_reddit:
message += '\n* ' + strings.lounge_msg
message = append_random_bottlecap_phrase(message)
try:
send_system_message(
recipient, subject, message, distinguished='gold-auto')
except MessageError:
g.log.error('send_gift: could not send system message')
g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
return thing
def add_props(cls, user, wrapped):
user_fullnames = {w.user_fullname for w in wrapped}
target_fullnames = {w.target_fullname for w in wrapped}
users = Account._by_fullname(user_fullnames,
data=True,
return_dict=True)
targets = Thing._by_fullname(target_fullnames,
data=True,
return_dict=True)
author_ids = {
t.author_id
for t in targets.itervalues() if hasattr(t, 'author_id')
}
link_ids = {
t.link_id
for t in targets.itervalues() if hasattr(t, 'link_id')
}
sr_ids = {t.sr_id for t in targets.itervalues() if hasattr(t, 'sr_id')}
authors = Account._byID(author_ids, data=True, return_dict=True)
links = Link._byID(link_ids, data=True, return_dict=True)
subreddits = Subreddit._byID(sr_ids, data=True, return_dict=True)
target_things = {}
for fullname, target in targets.iteritems():
if isinstance(target, (Comment, Link)):
author = authors[target.author_id]
if isinstance(target, Link):
subreddit = subreddits[target.sr_id]
path = target.make_permalink(subreddit)
else:
link = links[target.link_id]
subreddit = subreddits[link.sr_id]
path = target.make_permalink(link, subreddit)
target_things[fullname] = GameLogTarget(
target, path, author, subreddit)
elif isinstance(target, Account):
target_things[fullname] = WrappedUser(target)
for w in wrapped:
w.is_self = (c.user_is_loggedin
and w.user_fullname == c.user._fullname)
w.user = WrappedUser(users[w.user_fullname])
w.target = target_things[w.target_fullname]
w.item = g.f2pitems[w.item]
w.user_team = scores.get_user_team(users[w.user_fullname])
if isinstance(w.target, WrappedUser):
target_user = targets[w.target.fullname]
else:
target_user = authors[targets[w.target_fullname].author_id]
w.target_team = scores.get_user_team(target_user)
def process_message(msg):
# msg is *PROBABLY* json
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
# json being loaded into a python object
# it has the fields "user_id", "thing_fullname"
# a thing is a database object
# it's a link, comment, post, whatever, everything can be upvoted/downvoted
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
# this gets the user from database/cache (either memcached or postgres, whatever)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
# this gets a servers-wide lock
# I mean, a bunch of consumers might be consuming items that use the same "thing" (same database object)
# so, you want a global lock to avoid them from f*****g eachother up
# memcachd stores the lock, atomically
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
def validate_blob(custom):
"""Validate payment_blob and return a dict with everything looked up."""
ret = {}
if not custom:
raise GoldException('no custom')
payment_blob = g.hardcache.get('payment_blob-%s' % str(custom))
if not payment_blob:
raise GoldException('no payment_blob')
if 'account_id' in payment_blob and 'account_name' in payment_blob:
try:
buyer = Account._byID(payment_blob['account_id'], data=True)
ret['buyer'] = buyer
except NotFound:
raise GoldException('bad account_id')
if not buyer.name.lower() == payment_blob['account_name'].lower():
raise GoldException('buyer mismatch')
elif 'email' in payment_blob:
ret['email'] = payment_blob['email']
else:
raise GoldException('no account_id or email')
goldtype = payment_blob['goldtype']
ret['goldtype'] = goldtype
if goldtype == 'gift':
recipient_name = payment_blob.get('recipient', None)
if not recipient_name:
raise GoldException('gift missing recpient')
try:
recipient = Account._by_name(recipient_name)
ret['recipient'] = recipient
except NotFound:
raise GoldException('bad recipient')
thing_fullname = payment_blob.get('thing', None)
if thing_fullname:
try:
ret['thing'] = Thing._by_fullname(thing_fullname)
except NotFound:
raise GoldException('bad thing')
ret['signed'] = payment_blob.get('signed', False)
giftmessage = payment_blob.get('giftmessage')
giftmessage = _force_unicode(giftmessage) if giftmessage else None
ret['giftmessage'] = giftmessage
elif goldtype not in ('onetime', 'autorenew', 'creddits', 'code'):
raise GoldException('bad goldtype')
return ret
def validate_blob(custom):
"""Validate payment_blob and return a dict with everything looked up."""
ret = {}
if not custom:
raise GoldException('no custom')
payment_blob = g.hardcache.get('payment_blob-%s' % str(custom))
if not payment_blob:
raise GoldException('no payment_blob')
if 'account_id' in payment_blob and 'account_name' in payment_blob:
try:
buyer = Account._byID(payment_blob['account_id'], data=True)
ret['buyer'] = buyer
except NotFound:
raise GoldException('bad account_id')
if not buyer.name.lower() == payment_blob['account_name'].lower():
raise GoldException('buyer mismatch')
elif 'email' in payment_blob:
ret['email'] = payment_blob['email']
else:
raise GoldException('no account_id or email')
goldtype = payment_blob['goldtype']
ret['goldtype'] = goldtype
if goldtype == 'gift':
recipient_name = payment_blob.get('recipient', None)
if not recipient_name:
raise GoldException('gift missing recpient')
try:
recipient = Account._by_name(recipient_name)
ret['recipient'] = recipient
except NotFound:
raise GoldException('bad recipient')
thing_fullname = payment_blob.get('thing', None)
if thing_fullname:
try:
ret['thing'] = Thing._by_fullname(thing_fullname)
except NotFound:
raise GoldException('bad thing')
ret['signed'] = payment_blob.get('signed', False)
giftmessage = payment_blob.get('giftmessage')
giftmessage = _force_unicode(giftmessage) if giftmessage else None
ret['giftmessage'] = giftmessage
elif goldtype not in ('onetime', 'autorenew', 'creddits', 'code'):
raise GoldException('bad goldtype')
return ret
def test_get_links(self):
from r2.lib.db import queries
from r2.models import Subreddit, Account, Link, Thing
account = Account._byID(1, data=True)
sr = Subreddit._by_name("reddit_test0")
link_url = self.make_unique_url()
new_link = Link._submit("test_get_links", link_url, account, sr, "127.0.0.1", kind="link")
queries.new_link(new_link, foreground=True)
res = Thing._by_fullname(queries.get_links(sr, "new", "all"), return_dict=False)
self.assert_true(len(res) > 0, "no links returned")
self.assert_equal(new_link._id, res[0]._id)
def validate_blob(custom):
"""Validate payment_blob and return a dict with everything looked up."""
ret = {}
if not custom:
raise GoldException("no custom")
payment_blob = g.hardcache.get("payment_blob-%s" % str(custom))
if not payment_blob:
raise GoldException("no payment_blob")
if not ("account_id" in payment_blob and "account_name" in payment_blob):
raise GoldException("no account_id")
try:
buyer = Account._byID(payment_blob["account_id"], data=True)
ret["buyer"] = buyer
except NotFound:
raise GoldException("bad account_id")
if not buyer.name.lower() == payment_blob["account_name"].lower():
raise GoldException("buyer mismatch")
goldtype = payment_blob["goldtype"]
ret["goldtype"] = goldtype
if goldtype == "gift":
recipient_name = payment_blob.get("recipient", None)
if not recipient_name:
raise GoldException("gift missing recpient")
try:
recipient = Account._by_name(recipient_name)
ret["recipient"] = recipient
except NotFound:
raise GoldException("bad recipient")
thing_fullname = payment_blob.get("thing", None)
if thing_fullname:
try:
ret["thing"] = Thing._by_fullname(thing_fullname)
except NotFound:
raise GoldException("bad thing")
ret["signed"] = payment_blob.get("signed", False)
giftmessage = payment_blob.get("giftmessage")
giftmessage = _force_unicode(giftmessage) if giftmessage else None
ret["giftmessage"] = giftmessage
elif goldtype not in ("onetime", "autorenew", "creddits", "code"):
raise GoldException("bad goldtype")
return ret
def all_defendants(cls, quench=False, _update=False):
all = cls.all_defendants_cache(_update=_update)
defs = Thing._by_fullname(all, data=True).values()
if quench:
# Used for the spotlight, to filter out trials with over 20 votes;
# otherwise, hung juries would hog the spotlight for an hour as
# their vote counts continued to skyrocket
return filter(
lambda d: not g.cache.get("quench_jurors-" + d._fullname),
defs)
else:
return defs
def all_defendants(cls, quench=False, _update=False):
all = cls.all_defendants_cache(_update=_update)
defs = Thing._by_fullname(all, data=True).values()
if quench:
# Used for the spotlight, to filter out trials with over 20 votes;
# otherwise, hung juries would hog the spotlight for an hour as
# their vote counts continued to skyrocket
return filter (lambda d:
not g.cache.get("quench_jurors-" + d._fullname),
defs)
else:
return defs
def get_recent_name_submissions():
link_fullnames = list(queries.get_links(SERVERNAME_SR, "new", "all"))
links = chain.from_iterable(Thing._by_fullname(chunk, return_dict=False)
for chunk in in_chunks(link_fullnames))
for link in links:
if link._deleted or link._spam:
continue
# OH GOD WHAT HAVE YOU POSTED IN MY LOVELY AUTOMATED SUBREDDIT!?
if (not hasattr(link, "revenue_date") or
not hasattr(link, "revenue_bucket") or
not hasattr(link, "server_names")):
continue
yield link
def from_queue(self, max_date, batch_limit = 50, kind = None):
from r2.models import is_banned_IP, Account, Thing
keep_trying = True
min_id = None
s = self.queue_table
while keep_trying:
where = [s.c.date < max_date]
if min_id:
where.append(s.c.uid > min_id)
if kind:
where.append(s.c.kind == kind)
res = sa.select([s.c.to_addr, s.c.account_id,
s.c.from_name, s.c.fullname, s.c.body,
s.c.kind, s.c.ip, s.c.date, s.c.uid,
s.c.msg_hash, s.c.fr_addr, s.c.reply_to],
sa.and_(*where),
order_by = s.c.uid, limit = batch_limit).execute()
res = res.fetchall()
if not res: break
# batch load user accounts
aids = [x[1] for x in res if x[1] > 0]
accts = Account._byID(aids, data = True,
return_dict = True) if aids else {}
# batch load things
tids = [x[3] for x in res if x[3]]
things = Thing._by_fullname(tids, data = True,
return_dict = True) if tids else {}
# make sure no IPs have been banned in the mean time
ips = set(x[6] for x in res)
ips = dict((ip, is_banned_IP(ip)) for ip in ips)
# get the lower bound date for next iteration
min_id = max(x[8] for x in res)
# did we not fetch them all?
keep_trying = (len(res) == batch_limit)
for (addr, acct, fname, fulln, body, kind, ip, date, uid,
msg_hash, fr_addr, reply_to) in res:
yield (accts.get(acct), things.get(fulln), addr,
fname, date, ip, ips[ip], kind, msg_hash, body,
fr_addr, reply_to)
def add_props(cls, user, wrapped):
user_fullnames = {w.user_fullname for w in wrapped}
target_fullnames = {w.target_fullname for w in wrapped}
users = Account._by_fullname(user_fullnames, data=True,
return_dict=True)
targets = Thing._by_fullname(target_fullnames, data=True,
return_dict=True)
author_ids = {t.author_id for t in targets.itervalues()
if hasattr(t, 'author_id')}
link_ids = {t.link_id for t in targets.itervalues()
if hasattr(t, 'link_id')}
sr_ids = {t.sr_id for t in targets.itervalues() if hasattr(t, 'sr_id')}
authors = Account._byID(author_ids, data=True, return_dict=True)
links = Link._byID(link_ids, data=True, return_dict=True)
subreddits = Subreddit._byID(sr_ids, data=True, return_dict=True)
target_things = {}
for fullname, target in targets.iteritems():
if isinstance(target, (Comment, Link)):
author = authors[target.author_id]
if isinstance(target, Link):
subreddit = subreddits[target.sr_id]
path = target.make_permalink(subreddit)
else:
link = links[target.link_id]
subreddit = subreddits[link.sr_id]
path = target.make_permalink(link, subreddit)
target_things[fullname] = GameLogTarget(target, path, author,
subreddit)
elif isinstance(target, Account):
target_things[fullname] = WrappedUser(target)
for w in wrapped:
w.is_self = (c.user_is_loggedin and
w.user_fullname == c.user._fullname)
w.user = WrappedUser(users[w.user_fullname])
w.target = target_things[w.target_fullname]
w.item = g.f2pitems[w.item]
w.user_team = scores.get_user_team(users[w.user_fullname])
if isinstance(w.target, WrappedUser):
target_user = targets[w.target.fullname]
else:
target_user = authors[targets[w.target_fullname].author_id]
w.target_team = scores.get_user_team(target_user)
def process_message(msg):
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
# if it's an old-style vote, convert to the new format
if "uid" in vote_data:
vote_data = convert_old_vote_data(vote_data, msg.timestamp)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id):
admintools.engolden(recipient, days)
if comment_id:
comment = Thing._by_fullname(comment_id, data=True)
comment._gild(buyer)
else:
comment = None
if signed:
sender = buyer.name
md_sender = "[%s](/user/%s)" % (sender, sender)
else:
sender = _("An anonymous redditor")
md_sender = _("An anonymous redditor")
create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed)
if months == 1:
amount = "a month"
else:
amount = "%d months" % months
if not comment:
subject = _(
'Let there be gold! %s just sent you reddit gold!') % sender
message = strings.youve_got_gold % dict(sender=md_sender,
amount=amount)
if giftmessage and giftmessage.strip():
message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----'
else:
subject = _('Your comment has been gilded.')
message = strings.youve_got_comment_gold % dict(
url=comment.make_permalink_slow(), )
message += '\n\n' + strings.gold_benefits_msg
message += '\n\n' + strings.lounge_msg
try:
send_system_message(recipient, subject, message)
except MessageError:
g.log.error('send_gift: could not send system message')
g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
return comment
def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id):
admintools.engolden(recipient, days)
if comment_id:
comment = Thing._by_fullname(comment_id, data=True)
comment._gild(buyer)
else:
comment = None
if signed:
sender = buyer.name
md_sender = "[%s](/user/%s)" % (sender, sender)
else:
sender = _("An anonymous redditor")
md_sender = _("An anonymous redditor")
create_gift_gold (buyer._id, recipient._id, days, c.start_time, signed)
if months == 1:
amount = "a month"
else:
amount = "%d months" % months
if not comment:
subject = _('Let there be gold! %s just sent you reddit gold!') % sender
message = strings.youve_got_gold % dict(sender=md_sender, amount=amount)
if giftmessage and giftmessage.strip():
message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----'
else:
subject = _('Your comment has been gilded.')
message = strings.youve_got_comment_gold % dict(
url=comment.make_permalink_slow(),
)
message += '\n\n' + strings.gold_benefits_msg
message += '\n\n' + strings.lounge_msg
try:
send_system_message(recipient, subject, message)
except MessageError:
g.log.error('send_gift: could not send system message')
g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
return comment
def process_message(msg):
if not ACCOUNT:
return
fullname = msg.body
item = Thing._by_fullname(fullname, data=True)
if not isinstance(item, (Link, Comment)):
return
subreddit = item.subreddit_slow
wiki_page_id = wiki_id(subreddit._id36, "config/automoderator")
wiki_page_fullname = "WikiPage_%s" % wiki_page_id
last_edited = LastModified.get(wiki_page_fullname, "Edit")
if not last_edited:
return
# initialize rules for the subreddit if we haven't already
# or if the page has been edited since we last initialized
need_to_init = False
if subreddit._id not in rules_by_subreddit:
need_to_init = True
else:
rules = rules_by_subreddit[subreddit._id]
if last_edited > rules.init_time:
need_to_init = True
if need_to_init:
wp = WikiPage.get(subreddit, "config/automoderator")
rules = Ruleset(wp.content)
rules_by_subreddit[subreddit._id] = rules
if not rules:
return
try:
TimeoutFunction(rules.apply_to_item, 2)(item)
print "Checked %s from /r/%s" % (item, subreddit.name)
except TimeoutFunctionException:
print "Timed out on %s from /r/%s" % (item, subreddit.name)
except KeyboardInterrupt:
raise
except:
print "Error on %s from /r/%s" % (item, subreddit.name)
print traceback.format_exc()
def process_message(msg):
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'), vote_data)
return
# if it's an old-style vote, convert to the new format
if "uid" in vote_data:
vote_data = convert_old_vote_data(vote_data, msg.timestamp)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
def update_trending_subreddits():
try:
trending_sr = Subreddit._by_name(g.config['trending_sr'])
except NotFound:
g.log.info("Unknown trending subreddit %r or trending_sr config "
"not set. Not updating.", g.config['trending_sr'])
return
iq = iter(trending_sr.get_links('new', 'all'))
link = Thing._by_fullname(next(iq), data=True)
subreddit_names = _SUBREDDIT_RE.findall(link.title)
trending_data = {
'subreddit_names': subreddit_names,
'permalink': link.make_permalink(trending_sr),
'link_id': link._id,
}
NamedGlobals.set(TRENDING_SUBREDDITS_KEY, trending_data)
g.log.debug("Trending subreddit data set to %r", trending_data)
def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id):
admintools.engolden(recipient, days)
if comment_id:
comment = Thing._by_fullname(comment_id, data=True)
comment._gild(buyer)
else:
comment = None
if signed:
sender = buyer.name
md_sender = "[%s](/user/%s)" % (sender, sender)
else:
sender = "someone"
md_sender = "An anonymous lightnet user"
create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed)
if months == 1:
amount = "a month"
else:
amount = "%d months" % months
if not comment:
message = strings.youve_got_gold % dict(sender=md_sender, amount=amount)
if giftmessage and giftmessage.strip():
message += "\n\n" + strings.giftgold_note + giftmessage + "\n\n----"
else:
message = strings.youve_got_comment_gold % dict(url=comment.make_permalink_slow())
message += "\n\n" + strings.gold_benefits_msg
message += "\n\n" + strings.lounge_msg % {"link": "/space/" + g.lounge_reddit}
subject = sender + " just sent you reddit gold!"
try:
send_system_message(recipient, subject, message)
except MessageError:
g.log.error("send_gift: could not send system message")
g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
return comment
def test_get_files(self):
from r2.lib.db import queries
from r2.models import Subreddit, Account, Link, Thing
account = Account._byID(1, data=True)
sr = Subreddit._by_name("reddit_test0")
link_url = self.make_unique_url()
new_link = Link._submit("test_get_files", link_url, account, sr, "127.0.0.1", kind="file")
queries.new_link(new_link, foreground=True)
# make sure it returns like a normal link
res = Thing._by_fullname(queries.get_links(sr, "new", "all"), return_dict=False)
self.assert_true(len(res) > 0, "no links returned")
self.assert_equal(new_link._id, res[0]._id)
# should return with a kind = 'file' filter
res = list(queries.get_files(sr))
self.assert_true(len(res) > 0, "no links returned")
self.assert_equal(new_link._id, res[0]._id)
def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id):
admintools.engolden(recipient, days)
if comment_id:
comment = Thing._by_fullname(comment_id, data=True)
comment._gild(buyer)
else:
comment = None
if signed:
sender = buyer.name
md_sender = "[%s](/user/%s)" % (sender, sender)
else:
sender = "someone"
md_sender = "An anonymous redditor"
create_gift_gold (buyer._id, recipient._id, days, c.start_time, signed)
if months == 1:
amount = "a month"
else:
amount = "%d months" % months
if not comment:
message = strings.youve_got_gold % dict(sender=md_sender, amount=amount)
if giftmessage and giftmessage.strip():
message += "\n\n" + strings.giftgold_note + giftmessage
else:
message = strings.youve_got_comment_gold % dict(
url=comment.make_permalink_slow(),
)
subject = sender + " just sent you reddit gold!"
send_system_message(recipient, subject, message)
g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name))
return comment
def _run_changed(msgs, chan):
'''Consume the cloudsearch_changes queue, and print reporting information
on how long it took and how many remain
'''
start = datetime.now(g.tz)
changed = [pickle.loads(msg.body) for msg in msgs]
# Only handle links to start with
fullnames = _desired_things(changed, (Link, ))
things = Thing._by_fullname(fullnames, data=True, return_dict=False)
cloudsearch_time = inject(things)
totaltime = (datetime.now(g.tz) - start).total_seconds()
print(
"%s: %d messages in %.2fs seconds (%.2fs secs waiting on "
"cloudsearch); %d duplicates, %s remaining)" %
(start, len(changed), totaltime, cloudsearch_time, len(changed) -
len(things), msgs[-1].delivery_info.get('message_count', 'unknown')))
def _run_changed(msgs, chan):
'''Consume the cloudsearch_changes queue, and print reporting information
on how long it took and how many remain
'''
start = datetime.now(g.tz)
changed = [pickle.loads(msg.body) for msg in msgs]
# Only handle links to start with
fullnames = _desired_things(changed, (Link,))
things = Thing._by_fullname(fullnames, data=True, return_dict=False)
cloudsearch_time = inject(things)
totaltime = (datetime.now(g.tz) - start).total_seconds()
print ("%s: %d messages in %.2fs seconds (%.2fs secs waiting on "
"cloudsearch); %d duplicates, %s remaining)" %
(start, len(changed), totaltime, cloudsearch_time,
len(changed) - len(things),
msgs[-1].delivery_info.get('message_count', 'unknown')))
import datetime
from pylons import g
from r2.lib.db.queries import get_gilded_user_comments
from r2.lib.utils import Storage
from r2.models import GildingsByDay, Thing, Comment
from r2.models.query_cache import CachedQueryMutator
date = datetime.datetime.now(g.tz)
earliest_date = datetime.datetime(2012, 10, 01, tzinfo=g.tz)
already_seen = set()
with CachedQueryMutator() as m:
while date > earliest_date:
gildings = GildingsByDay.get_gildings(date)
fullnames = [x["thing"] for x in gildings]
things = Thing._by_fullname(fullnames, data=True, return_dict=False)
comments = {t._fullname: t for t in things if isinstance(t, Comment)}
for gilding in gildings:
fullname = gilding["thing"]
if fullname in comments and fullname not in already_seen:
thing = gilding["thing"] = comments[fullname]
gilding_object = Storage(gilding)
m.insert(get_gilded_user_comments(thing.author_id),
[gilding_object])
already_seen.add(fullname)
date -= datetime.timedelta(days=1)
def get_things(codes):
"""Fetch relevant things for a list of ad codenames in batch."""
fullnames = [AdvertTrafficSummary.split_codename(code)[0]
for code in codes
if code.startswith(Thing._type_prefix)]
return Thing._by_fullname(fullnames, data=True, return_dict=True)
def _get_modmail_userinfo(self, conversation, sr=None):
if conversation.is_internal:
raise ValueError('Cannot get userinfo for internal conversations')
if not sr:
sr = Subreddit._by_fullname(conversation.owner_fullname)
# Retrieve the participant associated with the conversation
try:
account = conversation.get_participant_account()
if not account:
raise ValueError('No account associated with convo')
permatimeout = (account.in_timeout
and account.days_remaining_in_timeout == 0)
if account._deleted or permatimeout:
raise ValueError('User info is inaccessible')
except NotFound:
raise NotFound('Unable to retrieve conversation participant')
# Fetch the mute and ban status of the participant as it relates
# to the subreddit associated with the conversation.
mute_status = sr.is_muted(account)
ban_status = sr.is_banned(account)
# Parse the ban status and retrieve the length of the ban,
# then output the data into a serialiazable dict
ban_result = {
'isBanned': bool(ban_status),
'reason': '',
'endDate': None,
'isPermanent': False
}
if ban_status:
ban_result['reason'] = getattr(ban_status, 'note', '')
ban_duration = sr.get_tempbans('banned', account.name)
ban_duration = ban_duration.get(account.name)
if ban_duration:
ban_result['endDate'] = ban_duration.isoformat()
else:
ban_result['isPermanent'] = True
ban_result['endDate'] = None
# Parse the mute status and retrieve the length of the ban,
# then output the data into the serialiazable dict
mute_result = {
'isMuted': bool(mute_status),
'endDate': None,
'reason': ''
}
if mute_status:
mute_result['reason'] = getattr(mute_status, 'note', '')
muted_items = sr.get_muted_items(account.name)
mute_duration = muted_items.get(account.name)
if mute_duration:
mute_result['endDate'] = mute_duration.isoformat()
# Retrieve the participants post and comment fullnames from cache
post_fullnames = []
comment_fullnames = []
if not account._spam:
post_fullnames = list(queries.get_submitted(account, 'new',
'all'))[:100]
comment_fullnames = list(
queries.get_comments(account, 'new', 'all'))[:100]
# Retrieve the associated link objects for posts and comments
# using the retrieve fullnames, afer the link objects are retrieved
# create a serializable dict with the the necessary information from
# the endpoint.
lookup_fullnames = list(set(post_fullnames) | set(comment_fullnames))
posts = Thing._by_fullname(lookup_fullnames)
serializable_posts = {}
for fullname in post_fullnames:
if len(serializable_posts) == 3:
break
post = posts[fullname]
if post.sr_id == sr._id and not post._deleted:
serializable_posts[fullname] = {
'title': post.title,
'permalink': post.make_permalink(sr, force_domain=True),
'date': post._date.isoformat(),
}
# Extract the users most recent comments associated with the
# subreddit
sr_comments = []
for fullname in comment_fullnames:
if len(sr_comments) == 3:
break
comment = posts[fullname]
if comment.sr_id == sr._id and not comment._deleted:
sr_comments.append(comment)
# Retrieve all associated link objects (combines lookup)
comment_links = Link._byID(
[sr_comment.link_id for sr_comment in sr_comments])
# Serialize all of the user's sr comments
serializable_comments = {}
for sr_comment in sr_comments:
comment_link = comment_links[sr_comment.link_id]
comment_body = sr_comment.body
if len(comment_body) > 140:
comment_body = '{:.140}...'.format(comment_body)
serializable_comments[sr_comment._fullname] = {
'title':
comment_link.title,
'comment':
comment_body,
'permalink':
sr_comment.make_permalink(comment_link, sr, force_domain=True),
'date':
sr_comment._date.isoformat(),
}
return {
'id': account._fullname,
'name': account.name,
'created': account._date.isoformat(),
'banStatus': ban_result,
'isShadowBanned': account._spam,
'muteStatus': mute_result,
'recentComments': serializable_comments,
'recentPosts': serializable_posts,
}
def get_things(codes):
fullnames = [
code for code in codes if code.startswith(Thing._type_prefix)
]
return Thing._by_fullname(fullnames, data=True, return_dict=True)
def get_things(codes):
fullnames = [code for code in codes
if code.startswith(Thing._type_prefix)]
return Thing._by_fullname(fullnames, data=True, return_dict=True)
def thing_lookup(self, names):
return Thing._by_fullname(names, data=True, return_dict=False,
stale=self.stale)
from pylons import app_globals as g
from r2.lib.db.queries import get_gilded_user_comments
from r2.lib.utils import Storage
from r2.models import GildingsByDay, Thing, Comment
from r2.models.query_cache import CachedQueryMutator
date = datetime.datetime.now(g.tz)
earliest_date = datetime.datetime(2012, 10, 01, tzinfo=g.tz)
already_seen = set()
with CachedQueryMutator() as m:
while date > earliest_date:
gildings = GildingsByDay.get_gildings(date)
fullnames = [x["thing"] for x in gildings]
things = Thing._by_fullname(fullnames, data=True, return_dict=False)
comments = {t._fullname: t for t in things if isinstance(t, Comment)}
for gilding in gildings:
fullname = gilding["thing"]
if fullname in comments and fullname not in already_seen:
thing = gilding["thing"] = comments[fullname]
gilding_object = Storage(gilding)
m.insert(get_gilded_user_comments(thing.author_id),
[gilding_object])
already_seen.add(fullname)
date -= datetime.timedelta(days=1)
def process_message(msg):
fname = msg.body
item = Thing._by_fullname(fname, data=True)
monitor_mentions(item)
def timeout_forbidden_event(self, action_name, details_text,
target=None, target_fullname=None, subreddit=None,
request=None, context=None):
"""Create a timeout-related 'forbidden_actions' for event-collector.
action_name: the action taken by a user in timeout
details_text: this provides more details about the action
target: The intended item the action was to be taken on
target_fullname: The fullname used to convert to a target
subreddit: The Subreddit the action was taken in. If target is of the
type Subreddit, then this won't be passed in
request, context: Should be pylons.request & pylons.c respectively;
"""
if not action_name:
request_vars = request.environ["pylons.routes_dict"]
action_name = request_vars.get('action_name')
# type of vote
if action_name == "vote":
direction = int(request.POST.get("dir", 0))
if direction == 1:
action_name = "upvote"
elif direction == -1:
action_name = "downvote"
else:
action_name = "clearvote"
# set or unset for contest mode and subreddit sticky
elif action_name in ("set_contest_mode", "set_subreddit_sticky"):
action_name = action_name.replace("_", "")
if request.POST.get('state') == "False":
action_name = "un" + action_name
# set or unset for suggested sort
elif action_name == "set_suggested_sort":
action_name = action_name.replace("_", "")
if request.POST.get("sort") in ("", "clear"):
action_name = "un" + action_name
# action for viewing /about/reports, /about/spam, /about/modqueue
elif action_name == "spamlisting":
action_name = "pageview"
details_text = request_vars.get("location")
elif action_name == "clearflairtemplates":
action_name = "editflair"
details_text = "flair_clear_template"
elif action_name in ("flairconfig", "flaircsv", "flairlisting"):
details_text = action_name.replace("flair", "flair_")
action_name = "editflair"
if not target:
if not target_fullname:
if action_name in ("wiki_settings", "wiki_edit"):
target = context.site
elif action_name in ("wiki_allow_editor"):
target = Account._by_name(request.POST.get("username"))
elif action_name in ("delete_sr_header", "delete_sr_icon",
"delete_sr_banner"):
details_text = "%s" % action_name.replace("ete_sr", "")
action_name = "editsettings"
target = context.site
elif action_name in ("bannedlisting", "mutedlisting",
"wikibannedlisting", "wikicontributorslisting"):
target = context.site
if target_fullname:
from r2.models import Thing
target = Thing._by_fullname(
target_fullname,
return_dict=False,
data=True,
)
event = Event(
topic="forbidden_actions",
event_type="ss.forbidden_%s" % action_name,
request=request,
context=context,
)
event.add("details_text", details_text)
event.add("process_notes", "IN_TIMEOUT")
from r2.models import Comment, Link, Subreddit
if not subreddit:
if isinstance(context.site, Subreddit):
subreddit = context.site
elif isinstance(target, (Comment, Link)):
subreddit = target.subreddit_slow
elif isinstance(target, Subreddit):
subreddit = target
event.add_subreddit_fields(subreddit)
event.add_target_fields(target)
self.save_event(event)
def thing_lookup(self, names):
return Thing._by_fullname(names,
data=True,
return_dict=False,
stale=self.stale)
def timeout_forbidden_event(self, action_name, details_text,
target=None, target_fullname=None, subreddit=None,
request=None, context=None):
"""Create a timeout-related 'forbidden_actions' for event-collector.
action_name: the action taken by a user in timeout
details_text: this provides more details about the action
target: The intended item the action was to be taken on
target_fullname: The fullname used to convert to a target
subreddit: The Subreddit the action was taken in. If target is of the
type Subreddit, then this won't be passed in
request, context: Should be pylons.request & pylons.c respectively;
"""
if not action_name:
request_vars = request.environ["pylons.routes_dict"]
action_name = request_vars.get('action_name')
# type of vote
if action_name == "vote":
direction = int(request.POST.get("dir", 0))
if direction == 1:
action_name = "upvote"
elif direction == -1:
action_name = "downvote"
else:
action_name = "clearvote"
# set or unset for contest mode and subreddit sticky
elif action_name in ("set_contest_mode", "set_subreddit_sticky"):
action_name = action_name.replace("_", "")
if request.POST.get('state') == "False":
action_name = "un" + action_name
# set or unset for suggested sort
elif action_name == "set_suggested_sort":
action_name = action_name.replace("_", "")
if request.POST.get("sort") in ("", "clear"):
action_name = "un" + action_name
# action for viewing /about/reports, /about/spam, /about/modqueue
elif action_name == "spamlisting":
action_name = "pageview"
details_text = request_vars.get("location")
elif action_name == "clearflairtemplates":
action_name = "editflair"
details_text = "flair_clear_template"
elif action_name in ("flairconfig", "flaircsv", "flairlisting"):
details_text = action_name.replace("flair", "flair_")
action_name = "editflair"
if not target:
if not target_fullname:
if action_name in ("wiki_settings", "wiki_edit"):
target = context.site
elif action_name in ("wiki_allow_editor"):
target = Account._by_name(request.POST.get("username"))
elif action_name in ("delete_sr_header", "delete_sr_icon",
"delete_sr_banner"):
details_text = "%s" % action_name.replace("ete_sr", "")
action_name = "editsettings"
target = context.site
elif action_name in ("bannedlisting", "mutedlisting",
"wikibannedlisting", "wikicontributorslisting"):
target = context.site
if target_fullname:
from r2.models import Thing
target = Thing._by_fullname(
target_fullname,
return_dict=False,
data=True,
)
event = Event(
topic="forbidden_actions",
event_type="ss.forbidden_%s" % action_name,
request=request,
context=context,
)
event.add("details_text", details_text)
event.add("process_notes", "IN_TIMEOUT")
from r2.models import Comment, Link, Subreddit
if not subreddit:
if isinstance(context.site, Subreddit):
subreddit = context.site
elif isinstance(target, (Comment, Link)):
subreddit = target.subreddit_slow
elif isinstance(target, Subreddit):
subreddit = target
event.add_subreddit_fields(subreddit)
event.add_target_fields(target)
self.save_event(event)
def process_message(msg):
fname = msg.body
item = Thing._by_fullname(fname, data=True)
monitor_mentions(item)
def get_things(codes):
fullnames = [
AdvertTrafficSummary.split_codename(code)[0] for code in codes
if code.startswith(Thing._type_prefix)
]
return Thing._by_fullname(fullnames, data=True, return_dict=True)
