def process_message(msg):
from r2.lib.db.queries import (
add_queries,
add_to_commentstree_q,
get_comments,
)
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'), vote_data)
return
timer = g.stats.get_timer("comment_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
comment = Comment._by_fullname(vote_data.pop("thing_fullname"))
print "Processing vote by %s on %s %s" % (user, comment, vote_data)
try:
vote = Vote(
user,
comment,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
comment_valid = not (comment._spam or comment._deleted)
if vote_valid and comment_valid:
author = Account._byID(comment.author_id)
add_queries(
queries=[get_comments(author, sort, 'all') for sort in SORTS],
insert_items=comment,
)
timer.intermediate("author_queries")
# update the score periodically when a comment has many votes
update_threshold = g.live_config['comment_vote_update_threshold']
update_period = g.live_config['comment_vote_update_period']
num_votes = comment.num_votes
if num_votes <= update_threshold or num_votes % update_period == 0:
add_to_commentstree_q(comment)
timer.stop()
timer.flush()
def _run_new_comment(msg):
fname = msg.body
comment = Comment._by_fullname(fname, data=True)
sr = Subreddit._byID(comment.sr_id)
add_queries(
[get_all_comments(), get_sr_comments(sr)], insert_items=[comment])
def _run_commentstree(msgs, chan):
comments = Comment._by_fullname([msg.body for msg in msgs],
data=True,
return_dict=False)
print 'Processing %r' % (comments, )
add_comment_tree(comments)
def _run_new_comment(msg):
fname = msg.body
comment = Comment._by_fullname(fname,data=True)
sr = Subreddit._byID(comment.sr_id)
add_queries([get_all_comments(),
get_sr_comments(sr)],
insert_items = [comment])
def _run_new_comments(msgs, chan):
fnames = [msg.body for msg in msgs]
comments = Comment._by_fullname(fnames, data=True, return_dict=False)
add_queries([get_all_comments()], insert_items=comments)
bysrid = _by_srid(comments, False)
for srid, sr_comments in bysrid.iteritems():
add_queries([_get_sr_comments(srid)], insert_items=sr_comments)
def _run_new_comments(msgs, chan):
fnames = [msg.body for msg in msgs]
comments = Comment._by_fullname(fnames, data=True, return_dict=False)
add_queries([get_all_comments()], insert_items=comments)
bysrid = _by_srid(comments, False)
for srid, sr_comments in bysrid.iteritems():
add_queries([_get_sr_comments(srid)], insert_items=sr_comments)
def validate_blob(custom):
"""Validate payment_blob and return a dict with everything looked up."""
ret = {}
if not custom:
raise GoldException('no custom')
payment_blob = g.hardcache.get('payment_blob-%s' % str(custom))
if not payment_blob:
raise GoldException('no payment_blob')
if not ('account_id' in payment_blob and
'account_name' in payment_blob):
raise GoldException('no account_id')
try:
buyer = Account._byID(payment_blob['account_id'], data=True)
ret['buyer'] = buyer
except NotFound:
raise GoldException('bad account_id')
if not buyer.name.lower() == payment_blob['account_name'].lower():
raise GoldException('buyer mismatch')
goldtype = payment_blob['goldtype']
ret['goldtype'] = goldtype
if goldtype == 'gift':
recipient_name = payment_blob.get('recipient', None)
if not recipient_name:
raise GoldException('gift missing recpient')
try:
recipient = Account._by_name(recipient_name)
ret['recipient'] = recipient
except NotFound:
raise GoldException('bad recipient')
comment_fullname = payment_blob.get('comment', None)
if comment_fullname:
try:
ret['comment'] = Comment._by_fullname(comment_fullname)
except NotFound:
raise GoldException('bad comment')
ret['signed'] = payment_blob.get('signed', False)
giftmessage = payment_blob.get('giftmessage')
giftmessage = _force_unicode(giftmessage) if giftmessage else None
ret['giftmessage'] = giftmessage
elif goldtype not in ('onetime', 'autorenew', 'creddits'):
raise GoldException('bad goldtype')
return ret
def validate_blob(custom):
"""Validate payment_blob and return a dict with everything looked up."""
ret = {}
if not custom:
raise GoldException('no custom')
payment_blob = g.hardcache.get('payment_blob-%s' % str(custom))
if not payment_blob:
raise GoldException('no payment_blob')
if not ('account_id' in payment_blob and
'account_name' in payment_blob):
raise GoldException('no account_id')
try:
buyer = Account._byID(payment_blob['account_id'], data=True)
ret['buyer'] = buyer
except NotFound:
raise GoldException('bad account_id')
if not buyer.name.lower() == payment_blob['account_name'].lower():
raise GoldException('buyer mismatch')
goldtype = payment_blob['goldtype']
ret['goldtype'] = goldtype
if goldtype == 'gift':
recipient_name = payment_blob.get('recipient', None)
if not recipient_name:
raise GoldException('gift missing recpient')
try:
recipient = Account._by_name(recipient_name)
ret['recipient'] = recipient
except NotFound:
raise GoldException('bad recipient')
comment_fullname = payment_blob.get('comment', None)
if comment_fullname:
try:
ret['comment'] = Comment._by_fullname(comment_fullname)
except NotFound:
raise GoldException('bad comment')
ret['signed'] = payment_blob.get('signed', False)
giftmessage = payment_blob.get('giftmessage')
giftmessage = _force_unicode(giftmessage) if giftmessage else None
ret['giftmessage'] = giftmessage
elif goldtype not in ('onetime', 'autorenew', 'creddits'):
raise GoldException('bad goldtype')
return ret
def _run_commentstree(msg):
fname = msg.body
comment = Comment._by_fullname(fname, data=True)
link = Link._byID(comment.link_id, data=True)
try:
add_comment_tree(comment, link)
except KeyError:
# Hackity hack. Try to recover from a corrupted comment
# tree
print "Trying to fix broken comments-tree."
link_comments(link._id, _update=True)
add_comment_tree(comment, link)
def validate_blob(custom):
"""Validate payment_blob and return a dict with everything looked up."""
ret = {}
if not custom:
raise GoldException("no custom")
payment_blob = g.hardcache.get("payment_blob-%s" % str(custom))
if not payment_blob:
raise GoldException("no payment_blob")
if not ("account_id" in payment_blob and "account_name" in payment_blob):
raise GoldException("no account_id")
try:
buyer = Account._byID(payment_blob["account_id"], data=True)
ret["buyer"] = buyer
except NotFound:
raise GoldException("bad account_id")
if not buyer.name.lower() == payment_blob["account_name"].lower():
raise GoldException("buyer mismatch")
goldtype = payment_blob["goldtype"]
ret["goldtype"] = goldtype
if goldtype == "gift":
recipient_name = payment_blob.get("recipient", None)
if not recipient_name:
raise GoldException("gift missing recpient")
try:
recipient = Account._by_name(recipient_name)
ret["recipient"] = recipient
except NotFound:
raise GoldException("bad recipient")
comment_fullname = payment_blob.get("comment", None)
if comment_fullname:
try:
ret["comment"] = Comment._by_fullname(comment_fullname)
except NotFound:
raise GoldException("bad comment")
ret["signed"] = payment_blob.get("signed", False)
giftmessage = payment_blob.get("giftmessage")
giftmessage = _force_unicode(giftmessage) if giftmessage else None
ret["giftmessage"] = giftmessage
elif goldtype not in ("onetime", "autorenew", "creddits"):
raise GoldException("bad goldtype")
return ret
def _run_commentstree(msg):
fname = msg.body
comment = Comment._by_fullname(fname, data=True)
link = Link._byID(comment.link_id,
data=True)
try:
add_comment_tree(comment, link)
except KeyError:
# Hackity hack. Try to recover from a corrupted comment
# tree
print "Trying to fix broken comments-tree."
link_comments(link._id, _update=True)
add_comment_tree(comment, link)
def message_notification_email(data):
"""Queues a system email for a new message notification."""
from r2.lib.pages import MessageNotificationEmail
MAX_EMAILS_PER_DAY = 1000
MESSAGE_THROTTLE_KEY = 'message_notification_emails'
# If our counter's expired, initialize it again.
g.cache.add(MESSAGE_THROTTLE_KEY, 0, time=24 * 60 * 60)
for datum in data.itervalues():
datum = json.loads(datum)
user = Account._byID36(datum['to'], data=True)
comment = Comment._by_fullname(datum['comment'], data=True)
# In case a user has enabled the preference while it was enabled for
# them, but we've since turned it off. We need to explicitly state the
# user because we're not in the context of an HTTP request from them.
if not feature.is_enabled('orangereds_as_emails', user=user):
continue
if g.cache.get(MESSAGE_THROTTLE_KEY) > MAX_EMAILS_PER_DAY:
raise Exception(
'Message notification emails: safety limit exceeded!')
mac = generate_notification_email_unsubscribe_token(
datum['to'],
user_email=user.email,
user_password_hash=user.password)
base = g.https_endpoint or g.origin
unsubscribe_link = base + '/mail/unsubscribe/%s/%s' % (datum['to'],
mac)
templateData = {
'sender_username': datum.get('from', ''),
'comment': comment,
'permalink': datum['permalink'],
'unsubscribe_link': unsubscribe_link,
}
_system_email(
user.email,
MessageNotificationEmail(**templateData).render(style='email'),
Email.Kind.MESSAGE_NOTIFICATION,
from_address=g.notification_email)
g.stats.simple_event('email.message_notification.queued')
g.cache.incr(MESSAGE_THROTTLE_KEY)
def _run_commentstree(msgs, chan):
fnames = [msg.body for msg in msgs]
comments = Comment._by_fullname(fnames, data=True, return_dict=False)
links = Link._byID(set(cm.link_id for cm in comments), data=True, return_dict=True)
# add the comment to the comments-tree
for comment in comments:
l = links[comment.link_id]
try:
add_comment_tree(comment, l)
except KeyError:
# Hackity hack. Try to recover from a corrupted
# comment tree
print "Trying to fix broken comments-tree."
link_comments(l._id, _update=True)
add_comment_tree(comment, l)
def message_notification_email(data):
"""Queues a system email for a new message notification."""
from r2.lib.pages import MessageNotificationEmail
MAX_EMAILS_PER_DAY = 1000
MESSAGE_THROTTLE_KEY = 'message_notification_emails'
# If our counter's expired, initialize it again.
g.cache.add(MESSAGE_THROTTLE_KEY, 0, time=24*60*60)
for datum in data.itervalues():
datum = json.loads(datum)
user = Account._byID36(datum['to'], data=True)
comment = Comment._by_fullname(datum['comment'], data=True)
# In case a user has enabled the preference while it was enabled for
# them, but we've since turned it off. We need to explicitly state the
# user because we're not in the context of an HTTP request from them.
if not feature.is_enabled('orangereds_as_emails', user=user):
continue
if g.cache.get(MESSAGE_THROTTLE_KEY) > MAX_EMAILS_PER_DAY:
raise Exception(
'Message notification emails: safety limit exceeded!')
mac = generate_notification_email_unsubscribe_token(
datum['to'], user_email=user.email,
user_password_hash=user.password)
base = g.https_endpoint or g.origin
unsubscribe_link = base + '/mail/unsubscribe/%s/%s' % (datum['to'], mac)
templateData = {
'sender_username': datum.get('from', ''),
'comment': comment,
'permalink': datum['permalink'],
'unsubscribe_link': unsubscribe_link,
}
_system_email(user.email,
MessageNotificationEmail(**templateData).render(style='email'),
Email.Kind.MESSAGE_NOTIFICATION,
from_address=g.notification_email)
g.stats.simple_event('email.message_notification.queued')
g.cache.incr(MESSAGE_THROTTLE_KEY)
def _run_commentstree(msgs, chan):
fnames = [msg.body for msg in msgs]
comments = Comment._by_fullname(fnames, data=True, return_dict=False)
links = Link._byID(set(cm.link_id for cm in comments),
data=True,
return_dict=True)
# add the comment to the comments-tree
for comment in comments:
l = links[comment.link_id]
try:
add_comment_tree(comment, l)
except KeyError:
# Hackity hack. Try to recover from a corrupted
# comment tree
print "Trying to fix broken comments-tree."
link_comments(l._id, _update=True)
add_comment_tree(comment, l)
def _run_new_comments(msgs, chan):
fnames = [msg.body for msg in msgs]
comments = Comment._by_fullname(fnames, data=True, return_dict=False)
add_queries([get_all_comments()], insert_items=comments)
def process_message(msg):
from r2.lib.comment_tree import write_comment_scores
from r2.lib.db.queries import (
add_queries,
add_to_commentstree_q,
get_comments,
)
from r2.models.builder import get_active_sort_orders_for_link
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'), vote_data)
return
timer = g.stats.get_timer("comment_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
comment = Comment._by_fullname(vote_data.pop("thing_fullname"))
print "Processing vote by %s on %s %s" % (user, comment, vote_data)
try:
vote = Vote(
user,
comment,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
vote_invalid = (not vote.effects.affects_score
and not vote.is_automatic_initial_vote)
comment_invalid = comment._spam or comment._deleted
if vote_invalid or comment_invalid:
timer.stop()
timer.flush()
return
author = Account._byID(comment.author_id)
add_queries(
queries=[get_comments(author, sort, 'all') for sort in SORTS],
insert_items=comment,
)
timer.intermediate("author_queries")
update_threshold = g.live_config['comment_vote_update_threshold']
update_period = g.live_config['comment_vote_update_period']
skip_score_update = (comment.num_votes > update_threshold
and comment.num_votes % update_period != 0)
# skip updating scores if this was the automatic initial vote. those
# updates will be handled by new_comment. Also only update scores
# periodically once a comment has many votes.
if not vote.is_automatic_initial_vote and not skip_score_update:
# check whether this link is using precomputed sorts, if it is
# we'll need to push an update to commentstree_q
link = Link._byID(comment.link_id)
if get_active_sort_orders_for_link(link):
# send this comment to commentstree_q where we will update
# CommentScoresByLink, CommentTree (noop), and CommentOrderer
add_to_commentstree_q(comment)
else:
# the link isn't using precomputed sorts, so just update the
# scores
write_comment_scores(link, [comment])
timer.intermediate("update_scores")
timer.stop()
timer.flush()
def _run_new_comments(msgs, chan):
fnames = [msg.body for msg in msgs]
comments = Comment._by_fullname(fnames, data=True, return_dict=False)
add_queries([get_all_comments()], insert_items=comments)
def process_message(msg):
from r2.lib.db.queries import (
add_queries,
add_to_commentstree_q,
get_comments,
)
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("comment_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
comment = Comment._by_fullname(vote_data.pop("thing_fullname"))
print "Processing vote by %s on %s %s" % (user, comment, vote_data)
try:
vote = Vote(
user,
comment,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
comment_valid = not (comment._spam or comment._deleted)
if vote_valid and comment_valid:
author = Account._byID(comment.author_id)
add_queries(
queries=[get_comments(author, sort, 'all') for sort in SORTS],
insert_items=comment,
)
timer.intermediate("author_queries")
# update the score periodically when a comment has many votes
update_threshold = g.live_config['comment_vote_update_threshold']
update_period = g.live_config['comment_vote_update_period']
num_votes = comment.num_votes
if num_votes <= update_threshold or num_votes % update_period == 0:
add_to_commentstree_q(comment)
timer.stop()
timer.flush()
def _run_new_comment(msg):
fname = msg.body
comment = Comment._by_fullname(fname)
add_queries([get_all_comments()], insert_items=[comment])
def _run_new_comment(msg):
fname = msg.body
comment = Comment._by_fullname(fname)
add_queries([get_all_comments()], insert_items=[comment])
def _run_commentstree(msgs, chan):
comments = Comment._by_fullname([msg.body for msg in msgs],
data = True, return_dict = False)
print 'Processing %r' % (comments,)
add_comment_tree(comments)
def process_message(msg):
from r2.lib.comment_tree import write_comment_scores
from r2.lib.db.queries import (
add_queries,
add_to_commentstree_q,
get_comments,
)
from r2.models.builder import get_active_sort_orders_for_link
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("comment_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
comment = Comment._by_fullname(vote_data.pop("thing_fullname"))
print "Processing vote by %s on %s %s" % (user, comment, vote_data)
try:
vote = Vote(
user,
comment,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
vote_invalid = (not vote.effects.affects_score and
not vote.is_automatic_initial_vote)
comment_invalid = comment._spam or comment._deleted
if vote_invalid or comment_invalid:
timer.stop()
timer.flush()
return
author = Account._byID(comment.author_id)
add_queries(
queries=[get_comments(author, sort, 'all') for sort in SORTS],
insert_items=comment,
)
timer.intermediate("author_queries")
update_threshold = g.live_config['comment_vote_update_threshold']
update_period = g.live_config['comment_vote_update_period']
skip_score_update = (comment.num_votes > update_threshold and
comment.num_votes % update_period != 0)
# skip updating scores if this was the automatic initial vote. those
# updates will be handled by new_comment. Also only update scores
# periodically once a comment has many votes.
if not vote.is_automatic_initial_vote and not skip_score_update:
# check whether this link is using precomputed sorts, if it is
# we'll need to push an update to commentstree_q
link = Link._byID(comment.link_id)
if get_active_sort_orders_for_link(link):
# send this comment to commentstree_q where we will update
# CommentScoresByLink, CommentTree (noop), and CommentOrderer
add_to_commentstree_q(comment)
else:
# the link isn't using precomputed sorts, so just update the
# scores
write_comment_scores(link, [comment])
timer.intermediate("update_scores")
timer.stop()
timer.flush()
