def process_votes_single(qname, limit=0):
# limit is taken but ignored for backwards compatibility
@g.stats.amqp_processor(qname)
def _handle_vote(msg):
#assert(len(msgs) == 1)
r = pickle.loads(msg.body)
uid, tid, dir, ip, organic, cheater = r
voter = Account._byID(uid, data=True)
votee = Thing._by_fullname(tid, data=True)
if isinstance(votee, Comment):
update_comment_votes([votee])
# I don't know how, but somebody is sneaking in votes
# for subreddits
if isinstance(votee, (Link, Comment)):
print(voter, votee, dir, ip, organic, cheater)
handle_vote(voter,
votee,
dir,
ip,
organic,
cheater=cheater,
foreground=True)
amqp.consume_items(qname, _handle_vote, verbose=False)
def process_liveupdate_scraper_q():
@g.stats.amqp_processor('liveupdate_scraper_q')
def _handle_q(msg):
d = json.loads(msg.body)
event_id = d.get(
"event_id") or d["event_fullname"][len("LiveUpdateEvent_"):]
liveupdate_id = d["liveupdate_id"]
try:
fn = TimeoutFunction(parse_embeds, 10)
liveupdate = fn(event_id, liveupdate_id)
except TimeoutFunctionException:
g.log.warning("Timed out on %s::%s", event_id, liveupdate_id)
return
except Exception as e:
g.log.warning("Failed to scrape %s::%s: %r", event_id,
liveupdate_id, e)
return
if not liveupdate:
return
if not liveupdate.embeds and not liveupdate.mobile_embeds:
return
payload = {
"liveupdate_id": "LiveUpdate_" + liveupdate_id,
"media_embeds": liveupdate.embeds,
"mobile_embeds": liveupdate.mobile_embeds,
}
send_event_broadcast(event_id, type="embeds_ready", payload=payload)
amqp.consume_items('liveupdate_scraper_q', _handle_q, verbose=False)
def process_adzerk():
@g.stats.amqp_processor('adzerk_q')
def _handle_adzerk(msg):
data = json.loads(msg.body)
g.log.debug('data: %s' % data)
action = data.get('action')
if action == 'deactivate_orphaned_flight':
_deactivate_orphaned_flight(data['flight'])
return
link = Link._by_fullname(data['link'], data=True)
if data['campaign']:
campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
else:
campaign = None
if action == 'update_adzerk':
if 'triggered_by' in data and data['triggered_by'] is not None:
triggered_by = Account._by_fullname(data['triggered_by'], data=True)
else:
triggered_by = None
_update_adzerk(link, campaign, triggered_by)
elif action == 'deactivate_overdelivered':
_deactivate_overdelivered(link, campaign)
amqp.consume_items('adzerk_q', _handle_adzerk, verbose=False)
def process_liveupdate_scraper_q():
@g.stats.amqp_processor('liveupdate_scraper_q')
def _handle_q(msg):
d = json.loads(msg.body)
try:
fn = TimeoutFunction(parse_embeds, 10)
liveupdate = fn(d['event_id'], d['liveupdate_id'])
except TimeoutFunctionException:
g.log.warning("Timed out on %s::%s", d["event_id"],
d["liveupdate_id"])
return
except Exception as e:
g.log.warning("Failed to scrape %s::%s: %r", d["event_id"],
d["liveupdate_id"], e)
return
payload = {
"liveupdate_id": "LiveUpdate_" + d['liveupdate_id'],
"media_embeds": liveupdate.embeds,
"mobile_embeds": liveupdate.mobile_embeds,
}
send_event_broadcast(d['event_id'],
type="embeds_ready",
payload=payload)
amqp.consume_items('liveupdate_scraper_q', _handle_q, verbose=False)
def process_votes(qname, limit=0):
# limit is taken but ignored for backwards compatibility
@g.stats.amqp_processor(qname)
def _handle_vote(msg):
timer = stats.get_timer("service_time." + qname)
timer.start()
#assert(len(msgs) == 1)
r = pickle.loads(msg.body)
uid, tid, dir, ip, organic, cheater = r
voter = Account._byID(uid, data=True)
votee = Thing._by_fullname(tid, data = True)
timer.intermediate("preamble")
if isinstance(votee, Comment):
update_comment_votes([votee])
timer.intermediate("update_comment_votes")
# I don't know how, but somebody is sneaking in votes
# for subreddits
if isinstance(votee, (Link, Comment)):
print (voter, votee, dir, ip, organic, cheater)
handle_vote(voter, votee, dir, ip, organic,
cheater = cheater, foreground=True, timer=timer)
amqp.consume_items(qname, _handle_vote, verbose = False)
def process_liveupdate_scraper_q():
@g.stats.amqp_processor('liveupdate_scraper_q')
def _handle_q(msg):
d = json.loads(msg.body)
try:
fn = TimeoutFunction(parse_embeds, 10)
liveupdate = fn(d['event_id'], d['liveupdate_id'])
except TimeoutFunctionException:
g.log.warning(
"Timed out on %s::%s", d["event_id"], d["liveupdate_id"])
return
except Exception as e:
g.log.warning("Failed to scrape %s::%s: %r",
d["event_id"], d["liveupdate_id"], e)
return
payload = {
"liveupdate_id": "LiveUpdate_" + d['liveupdate_id'],
"media_embeds": liveupdate.embeds,
"mobile_embeds": liveupdate.mobile_embeds,
}
send_event_broadcast(d['event_id'],
type="embeds_ready",
payload=payload)
amqp.consume_items('liveupdate_scraper_q', _handle_q, verbose=False)
def run():
@g.stats.amqp_processor("butler_q")
def process_message(msg):
fname = msg.body
item = Thing._by_fullname(fname, data=True)
monitor_mentions(item)
amqp.consume_items("butler_q", process_message, verbose=True)
def run():
@g.stats.amqp_processor("butler_q")
def process_message(msg):
fname = msg.body
item = Thing._by_fullname(fname, data=True)
monitor_mentions(item)
amqp.consume_items("butler_q", process_message, verbose=True)
def consume_link_vote_queue(qname="vote_link_q"):
@g.stats.amqp_processor(qname)
def process_message(msg):
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("link_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
link = Link._by_fullname(vote_data.pop("thing_fullname"))
# create the vote and update the voter's liked/disliked under lock so
# that the vote state and cached query are consistent
lock_key = "vote-%s-%s" % (user._id36, link._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, link, vote_data)
try:
vote = Vote(
user,
link,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
# CUSTOM: voting model
vote_direction=vote_data["vote_direction"],
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
update_user_liked(vote)
timer.intermediate("voter_likes")
vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
link_valid = not (link._spam or link._deleted)
if vote_valid and link_valid:
add_to_author_query_q(link)
add_to_subreddit_query_q(link)
add_to_domain_query_q(link)
timer.stop()
timer.flush()
amqp.consume_items(qname, process_message, verbose=False)
def run_summary_email_q(verbose=False):
queue_name = 'summary_email_q'
@g.stats.amqp_processor(queue_name)
def _run_summary_email(msg):
account_thing_id = int(msg.body)
send_account_summary_email(account_thing_id)
amqp.consume_items(queue_name, _run_summary_email, verbose=verbose)
def consume_vote_queue(queue):
@g.stats.amqp_processor(queue)
def process_message(msg):
# msg is *PROBABLY* json
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
# json being loaded into a python object
# it has the fields "user_id", "thing_fullname"
# a thing is a database object
# it's a link, comment, post, whatever, everything can be upvoted/downvoted
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
# this gets the user from database/cache (either memcached or postgres, whatever)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
# this gets a servers-wide lock
# I mean, a bunch of consumers might be consuming items that use the same "thing" (same database object)
# so, you want a global lock to avoid them from f*****g eachother up
# memcachd stores the lock, atomically
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
amqp.consume_items(queue, process_message, verbose=False)
def consume_link_vote_queue(qname="vote_link_q"):
@g.stats.amqp_processor(qname)
def process_message(msg):
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("link_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
link = Link._by_fullname(vote_data.pop("thing_fullname"))
# create the vote and update the voter's liked/disliked under lock so
# that the vote state and cached query are consistent
lock_key = "vote-%s-%s" % (user._id36, link._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, link, vote_data)
try:
vote = Vote(
user,
link,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
update_user_liked(vote)
timer.intermediate("voter_likes")
vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
link_valid = not (link._spam or link._deleted)
if vote_valid and link_valid:
add_to_author_query_q(link)
add_to_subreddit_query_q(link)
add_to_domain_query_q(link)
timer.stop()
timer.flush()
amqp.consume_items(qname, process_message, verbose=False)
def run_subreddit_maker():
@g.stats.amqp_processor('robin_subreddit_maker_q')
def process_subreddit_maker(msg):
room_id = msg.body
try:
room = RobinRoom._byID(room_id)
except tdb_cassandra.NotFound:
try:
room = RobinRoomDead._byID(room_id)
except tdb_cassandra.NotFound:
print "can't find room %s, giving up" % room_id
print 'creating sr for room %s' % room
subreddit = room.create_sr()
print 'got %s from room.create_sr()' % subreddit
if subreddit:
g.stats.simple_event("robin.subreddit.created")
participant_ids = room.get_all_participants()
participants = [
Account._byID(participant_id)
for participant_id in participant_ids
]
moderators = participants[:5]
print 'adding moderators to %s' % subreddit
for moderator in moderators:
subreddit.add_moderator(moderator)
print 'adding contributors to %s' % subreddit
g.stats.simple_event(
"robin.subreddit.contributors_added",
delta=len(participants),
)
for participant in participants:
# To be replaced with UserRel hacking?
subreddit.add_contributor(participant)
send_sr_message(subreddit, participant)
payload = {
"body": subreddit.name,
}
websockets.send_broadcast(
namespace="/robin/" + room.id,
type="continue",
payload=payload,
)
else:
g.stats.simple_event("robin.subreddit.creation_failed")
print 'subreddit creation failed for room %s' % room.id
amqp.consume_items('robin_subreddit_maker_q', process_subreddit_maker)
def run_new_comments():
"""Add new incoming comments to the /comments page"""
# this is done as a queue because otherwise the contention for the
# lock on the query would be very high
def _run_new_comment(msg):
fname = msg.body
comment = Comment._by_fullname(fname)
add_queries([get_all_comments()], insert_items=[comment])
amqp.consume_items("newcomments_q", _run_new_comment)
def run_new_comments():
"""Add new incoming comments to the /comments page"""
# this is done as a queue because otherwise the contention for the
# lock on the query would be very high
def _run_new_comment(msg):
fname = msg.body
comment = Comment._by_fullname(fname)
add_queries([get_all_comments()], insert_items=[comment])
amqp.consume_items('newcomments_q', _run_new_comment)
def run():
# Add watch to only update kwl when the keyword list changes
@g.zookeeper.DataWatch("/keyword-targets")
def watch_keywords(data, stats):
global kwl
kwl = json.loads(data)
@g.stats.amqp_processor("keyword_target_q")
def process_message(msg):
fname = msg.body
link = Link._by_fullname(fname, data=True)
extract_keywords(link)
amqp.consume_items("keyword_target_q", process_message, verbose=True)
def consume_vote_queue(queue):
@g.stats.amqp_processor(queue)
def process_message(msg):
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
# if it's an old-style vote, convert to the new format
if "uid" in vote_data:
vote_data = convert_old_vote_data(vote_data, msg.timestamp)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
amqp.consume_items(queue, process_message, verbose=False)
def process_votes(limit=None):
# limit is taken but ignored for backwards compatibility
def _handle_vote(msg):
r = pickle.loads(msg.body)
uid, tid, dir, ip, organic, cheater = r
voter = Account._byID(uid, data=True)
votee = Thing._by_fullname(tid, data=True)
print(voter, votee, dir, ip, organic, cheater)
handle_vote(voter, votee, dir, ip, organic, cheater=cheater)
amqp.consume_items('register_vote_q', _handle_vote)
def run():
@g.stats.amqp_processor("automoderator_q")
def process_message(msg):
if not ACCOUNT:
return
fullname = msg.body
item = Thing._by_fullname(fullname, data=True)
if not isinstance(item, (Link, Comment)):
return
subreddit = item.subreddit_slow
wiki_page_id = wiki_id(subreddit._id36, "config/automoderator")
wiki_page_fullname = "WikiPage_%s" % wiki_page_id
last_edited = LastModified.get(wiki_page_fullname, "Edit")
if not last_edited:
return
# initialize rules for the subreddit if we haven't already
# or if the page has been edited since we last initialized
need_to_init = False
if subreddit._id not in rules_by_subreddit:
need_to_init = True
else:
rules = rules_by_subreddit[subreddit._id]
if last_edited > rules.init_time:
need_to_init = True
if need_to_init:
wp = WikiPage.get(subreddit, "config/automoderator")
rules = Ruleset(wp.content)
rules_by_subreddit[subreddit._id] = rules
if not rules:
return
try:
TimeoutFunction(rules.apply_to_item, 2)(item)
print "Checked %s from /r/%s" % (item, subreddit.name)
except TimeoutFunctionException:
print "Timed out on %s from /r/%s" % (item, subreddit.name)
except KeyboardInterrupt:
raise
except:
print "Error on %s from /r/%s" % (item, subreddit.name)
print traceback.format_exc()
amqp.consume_items("automoderator_q", process_message, verbose=False)
def consume_vote_queue(queue):
@g.stats.amqp_processor(queue)
def process_message(msg):
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'), vote_data)
return
# if it's an old-style vote, convert to the new format
if "uid" in vote_data:
vote_data = convert_old_vote_data(vote_data, msg.timestamp)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
amqp.consume_items(queue, process_message, verbose=False)
def run_new_comments():
"""Add new incoming comments to the /comments page"""
# this is done as a queue because otherwise the contention for the
# lock on the query would be very high
def _run_new_comment(msg):
fname = msg.body
comment = Comment._by_fullname(fname,data=True)
sr = Subreddit._byID(comment.sr_id)
add_queries([get_all_comments(),
get_sr_comments(sr)],
insert_items = [comment])
amqp.consume_items('newcomments_q', _run_new_comment)
def process_votes(limit=None):
# limit is taken but ignored for backwards compatibility
def _handle_vote(msg):
r = pickle.loads(msg.body)
uid, tid, dir, ip, organic, cheater = r
voter = Account._byID(uid, data=True)
votee = Thing._by_fullname(tid, data = True)
print (voter, votee, dir, ip, organic, cheater)
handle_vote(voter, votee, dir, ip, organic,
cheater = cheater)
amqp.consume_items('register_vote_q', _handle_vote)
def run():
# Add watch to only update kwl when the keyword list changes
@g.zookeeper.DataWatch("/keyword-targets")
def watch_keywords(data, stats):
global kwl
kwl = json.loads(data)
@g.stats.amqp_processor("keyword_target_q")
def process_message(msg):
fname = msg.body
link = Link._by_fullname(fname, data=True)
extract_keywords(link)
amqp.consume_items("keyword_target_q",
process_message,
verbose=True)
def process_report_q():
@g.stats.amqp_processor('adzerk_reporting_q')
def _processor(message):
data = json.loads(message.body)
action = data.get("action")
if action == "generate_daily_link_report":
_handle_generate_daily_link_report(link_id=data.get("link_id"), )
elif action == "generate_lifetime_campaign_report":
_handle_generate_lifetime_campaign_report(
campaign_id=data.get("campaign_id"), )
else:
g.log.warning("adzerk_reporting_q: unknown action - \"%s\"" %
action)
amqp.consume_items("adzerk_reporting_q", _processor, verbose=False)
def run():
@g.stats.amqp_processor('scraper_q')
def process_link(msg):
fname = msg.body
link = Link._by_fullname(msg.body, data=True)
try:
TimeoutFunction(_set_media, 30)(link)
except TimeoutFunctionException:
print "Timed out on %s" % fname
except KeyboardInterrupt:
raise
except:
print "Error fetching %s" % fname
print traceback.format_exc()
amqp.consume_items('scraper_q', process_link)
def process():
processor = Processor()
processor.register("upsert_promotion", _handle_upsert_promotion)
processor.register("upsert_campaign", _handle_upsert_campaign)
processor.register("deactivate_campaign", _handle_deactivate_campaign)
@g.stats.amqp_processor(DFP_QUEUE)
def _handler(message):
data = json.loads(message.body)
g.log.debug("processing action: %s" % data)
action = data.get("action")
payload = data.get("payload")
processor.call(action, payload)
amqp.consume_items(DFP_QUEUE, _handler, verbose=False)
def run():
@g.stats.amqp_processor('scraper_q')
def process_link(msg):
fname = msg.body
link = Link._by_fullname(fname, data=True)
try:
TimeoutFunction(_set_media, 30)(link, use_cache=True)
except TimeoutFunctionException:
print "Timed out on %s" % fname
except KeyboardInterrupt:
raise
except:
print "Error fetching %s" % fname
print traceback.format_exc()
amqp.consume_items('scraper_q', process_link)
def run_steam_q():
import requests
session = requests.Session()
session.headers.update({
"User-Agent": g.useragent,
})
@g.stats.amqp_processor(QNAME)
def _claim_hat(msg):
data = json.loads(msg.body)
account = Account._byID(int(data["user-id"]), data=True)
if account.f2p != "claiming":
g.log.warning("%r attempted to claim twice!", account)
return
user_team = scores.get_user_team(account)
promo_id = g.steam_promo_items[user_team]
g.stats.event_count("f2p.claim_hat", "item_%s" % promo_id)
response = session.post(GRANT_URL,
data={
"SteamID": data["steam-id"],
"PromoID": promo_id,
"key": g.steam_api_key,
"format": "json",
})
# band-aid for requests API change in v1.0.0
if callable(response.json):
response_data = response.json()
else:
response_data = response.json
if response_data["result"]["status"] != 1:
g.log.warning("Steam Promo for %r -> %r failed: %s", account,
data["steam-id"], response_data["statusDetail"])
raise Exception
account.f2p = "claimed"
account._commit()
amqp.consume_items(QNAME, _claim_hat, verbose=True)
def run():
def process_link(msg):
def _process_link(fname):
link = Link._by_fullname(fname, data=True)
set_media(link)
fname = msg.body
try:
TimeoutFunction(_process_link, 30)(fname)
except TimeoutFunctionException:
print "Timed out on %s" % fname
except KeyboardInterrupt:
raise
except:
print "Error fetching %s" % fname
print traceback.format_exc()
amqp.consume_items('scraper_q', process_link)
def run():
def process_link(msg):
def _process_link(fname):
link = Link._by_fullname(fname, data=True)
set_media(link)
fname = msg.body
try:
TimeoutFunction(_process_link, 30)(fname)
except TimeoutFunctionException:
print "Timed out on %s" % fname
except KeyboardInterrupt:
raise
except:
print "Error fetching %s" % fname
print traceback.format_exc()
amqp.consume_items('scraper_q', process_link)
def process_modmail_email():
@g.stats.amqp_processor("modmail_email_q")
def process_message(msg):
msg_dict = json.loads(msg.body)
if msg_dict["event"] == "new_message":
message_id36 = msg_dict["message_id36"]
message = Message._byID36(message_id36, data=True)
send_modmail_email(message)
elif msg_dict["event"] == "blocked_muted":
subreddit_id36 = msg_dict["subreddit_id36"]
sr = Subreddit._byID36(subreddit_id36, data=True)
parent_id36 = msg_dict["parent_id36"]
parent = Message._byID36(parent_id36, data=True)
sender_email = msg_dict["sender_email"]
incoming_email_id = msg_dict["incoming_email_id"]
send_blocked_muted_email(sr, parent, sender_email, incoming_email_id)
amqp.consume_items("modmail_email_q", process_message)
def process_modmail_email():
@g.stats.amqp_processor("modmail_email_q")
def process_message(msg):
msg_dict = json.loads(msg.body)
if msg_dict["event"] == "new_message":
message_id36 = msg_dict["message_id36"]
message = Message._byID36(message_id36, data=True)
send_modmail_email(message)
elif msg_dict["event"] == "blocked_muted":
subreddit_id36 = msg_dict["subreddit_id36"]
sr = Subreddit._byID36(subreddit_id36, data=True)
parent_id36 = msg_dict["parent_id36"]
parent = Message._byID36(parent_id36, data=True)
sender_email = msg_dict["sender_email"]
incoming_email_id = msg_dict["incoming_email_id"]
send_blocked_muted_email(sr, parent, sender_email, incoming_email_id)
amqp.consume_items("modmail_email_q", process_message)
def process_report_q():
@g.stats.amqp_processor('adzerk_reporting_q')
def _processor(message):
data = json.loads(message.body)
action = data.get("action")
if action == "generate_daily_link_report":
_handle_generate_daily_link_report(
link_id=data.get("link_id"),
)
elif action == "generate_lifetime_campaign_report":
_handle_generate_lifetime_campaign_report(
campaign_id=data.get("campaign_id"),
)
else:
g.log.warning("adzerk_reporting_q: unknown action - \"%s\"" % action)
amqp.consume_items("adzerk_reporting_q", _processor, verbose=False)
def process_adzerk():
@g.stats.amqp_processor('adzerk_q')
def _handle_adzerk(msg):
data = json.loads(msg.body)
g.log.debug('data: %s' % data)
action = data.get('action')
if action == 'deactivate_link':
link = Link._by_fullname(data['link'], data=True)
_deactivate_link(link)
elif action == 'deactivate_campaign':
link = Link._by_fullname(data['link'], data=True)
campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
_deactivate_campaign(link, campaign)
elif action == 'update_adzerk':
link = Link._by_fullname(data['link'], data=True)
campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
_update_adzerk(link, campaign)
amqp.consume_items('adzerk_q', _handle_adzerk, verbose=False)
def run_commentstree():
"""Add new incoming comments to their respective comments trees"""
def _run_commentstree(msg):
fname = msg.body
comment = Comment._by_fullname(fname, data=True)
link = Link._byID(comment.link_id, data=True)
try:
add_comment_tree(comment, link)
except KeyError:
# Hackity hack. Try to recover from a corrupted comment
# tree
print "Trying to fix broken comments-tree."
link_comments(link._id, _update=True)
add_comment_tree(comment, link)
amqp.consume_items('commentstree_q', _run_commentstree)
def run_steam_q():
import requests
session = requests.Session()
session.headers.update({
"User-Agent": g.useragent,
})
@g.stats.amqp_processor(QNAME)
def _claim_hat(msg):
data = json.loads(msg.body)
account = Account._byID(int(data["user-id"]), data=True)
if account.f2p != "claiming":
g.log.warning("%r attempted to claim twice!", account)
return
user_team = scores.get_user_team(account)
promo_id = g.steam_promo_items[user_team]
g.stats.event_count("f2p.claim_hat", "item_%s" % promo_id)
response = session.post(GRANT_URL, data={
"SteamID": data["steam-id"],
"PromoID": promo_id,
"key": g.steam_api_key,
"format": "json",
})
# band-aid for requests API change in v1.0.0
if callable(response.json):
response_data = response.json()
else:
response_data = response.json
if response_data["result"]["status"] != 1:
g.log.warning("Steam Promo for %r -> %r failed: %r",
account, data["steam-id"],
response_data)
raise Exception
account.f2p = "claimed"
account._commit()
amqp.consume_items(QNAME, _claim_hat, verbose=True)
def run():
@g.stats.amqp_processor("robin_presence_q")
def process_presence_update(msg):
message_type = msg.delivery_info["routing_key"]
payload = json.loads(msg.body)
namespace = payload["namespace"]
if not namespace.startswith("/robin/"):
return
user_id36 = posixpath.basename(namespace)
room_namespace = posixpath.dirname(namespace)
room_id = posixpath.basename(room_namespace)
account = Account._byID36(user_id36, data=True, stale=True)
try:
room = RobinRoom._byID(room_id)
except tdb_cassandra.NotFoundException:
return
if not room.is_participant(account):
return
presence_type = "join" if message_type == "websocket.connect" else "part"
websockets.send_broadcast(
namespace=room_namespace,
type=presence_type,
payload={
"user": account.name,
},
)
if presence_type == "join":
ParticipantPresenceByRoom.mark_joined(room, account)
else:
ParticipantPresenceByRoom.mark_exited(room, account)
amqp.consume_items(
"robin_presence_q",
process_presence_update,
verbose=True,
)
def run():
def callback(msg):
# cr is a r2.lib.db.queries.CachedResults
cr = pickle.loads(msg.body)
iden = cr.query._iden()
working_key = working_prefix + iden
key = prefix + iden
last_time = g.memcache.get(key)
# check to see if we've computed this job since it was
# added to the queue
if last_time and last_time > msg.timestamp:
print 'skipping, already computed ', key
return
if not cr.preflight_check():
print 'skipping, preflight check failed', key
return
# check if someone else is working on this
elif not g.memcache.add(working_key, 1, TIMEOUT):
print 'skipping, someone else is working', working_key
return
print 'working: ', iden, cr.query._rules, cr.query._sort
start = datetime.now()
try:
cr.update()
g.memcache.set(key, datetime.now())
cr.postflight()
finally:
g.memcache.delete(working_key)
done = datetime.now()
q_time_s = (done - msg.timestamp).seconds
proc_time_s = (done - start).seconds + (
(done - start).microseconds / 1000000.0)
print('processed %s in %.6f seconds after %d seconds in queue' %
(iden, proc_time_s, q_time_s))
amqp.consume_items('prec_links', callback)
def run():
@g.stats.amqp_processor("robin_presence_q")
def process_presence_update(msg):
message_type = msg.delivery_info["routing_key"]
payload = json.loads(msg.body)
namespace = payload["namespace"]
if not namespace.startswith("/robin/"):
return
user_id36 = posixpath.basename(namespace)
room_namespace = posixpath.dirname(namespace)
room_id = posixpath.basename(room_namespace)
account = Account._byID36(user_id36, data=True, stale=True)
try:
room = RobinRoom._byID(room_id)
except tdb_cassandra.NotFoundException:
return
if not room.is_participant(account):
return
presence_type = "join" if message_type == "websocket.connect" else "part"
websockets.send_broadcast(
namespace=room_namespace,
type=presence_type,
payload={
"user": account.name,
},
)
if presence_type == "join":
ParticipantPresenceByRoom.mark_joined(room, account)
else:
ParticipantPresenceByRoom.mark_exited(room, account)
amqp.consume_items(
"robin_presence_q",
process_presence_update,
verbose=True,
)
def run():
def callback(msg):
# cr is a r2.lib.db.queries.CachedResults
cr = pickle.loads(msg.body)
iden = cr.query._iden()
working_key = working_prefix + iden
key = prefix + iden
last_time = g.memcache.get(key)
# check to see if we've computed this job since it was
# added to the queue
if last_time and last_time > msg.timestamp:
print 'skipping, already computed ', key
return
if not cr.preflight_check():
print 'skipping, preflight check failed', key
return
# check if someone else is working on this
elif not g.memcache.add(working_key, 1, TIMEOUT):
print 'skipping, someone else is working', working_key
return
print 'working: ', iden, cr.query._rules, cr.query._sort
start = datetime.now()
try:
cr.update()
g.memcache.set(key, datetime.now())
cr.postflight()
finally:
g.memcache.delete(working_key)
done = datetime.now()
q_time_s = (done - msg.timestamp).seconds
proc_time_s = (done - start).seconds + ((done - start).microseconds/1000000.0)
print ('processed %s in %.6f seconds after %d seconds in queue'
% (iden, proc_time_s, q_time_s))
amqp.consume_items('prec_links', callback)
def run_commentstree():
"""Add new incoming comments to their respective comments trees"""
def _run_commentstree(msg):
fname = msg.body
comment = Comment._by_fullname(fname, data=True)
link = Link._byID(comment.link_id,
data=True)
try:
add_comment_tree(comment, link)
except KeyError:
# Hackity hack. Try to recover from a corrupted comment
# tree
print "Trying to fix broken comments-tree."
link_comments(link._id, _update=True)
add_comment_tree(comment, link)
amqp.consume_items('commentstree_q', _run_commentstree)
def run_waitinglist():
@g.stats.amqp_processor("robin_waitinglist_q")
def process_waitinglist(msg):
user_id36 = msg.body
user = Account._byID36(user_id36, data=True, stale=True)
if RobinRoom.get_room_for_user(user):
print "%s already in room" % user.name
return
with g.make_lock("robin_room", "global"):
current_room_id = g.gencache.get("current_robin_room")
if not current_room_id:
current_room = make_new_room()
else:
try:
current_room = RobinRoom._byID(current_room_id)
except tdb_cassandra.NotFoundException:
current_room_id = None
current_room = make_new_room()
if not current_room.is_alive or current_room.is_continued:
current_room_id = None
current_room = make_new_room()
current_room.add_participants([user])
print "added %s to %s" % (user.name, current_room.id)
if current_room_id:
g.gencache.delete("current_robin_room")
current_room.persist_computed_name()
websockets.send_broadcast(
namespace="/robin/" + current_room.id,
type="updated_name",
payload={
"room_name": current_room.name,
},
)
else:
g.gencache.set("current_robin_room", current_room.id)
amqp.consume_items("robin_waitinglist_q", process_waitinglist)
def run_waitinglist():
@g.stats.amqp_processor("robin_waitinglist_q")
def process_waitinglist(msg):
user_id36 = msg.body
user = Account._byID36(user_id36, data=True, stale=True)
if RobinRoom.get_room_for_user(user):
print "%s already in room" % user.name
return
with g.make_lock("robin_room", "global"):
current_room_id = g.cache.get("current_robin_room")
if not current_room_id:
current_room = make_new_room()
else:
try:
current_room = RobinRoom._byID(current_room_id)
except tdb_cassandra.NotFoundException:
current_room_id = None
current_room = make_new_room()
if not current_room.is_alive or current_room.is_continued:
current_room_id = None
current_room = make_new_room()
current_room.add_participants([user])
print "added %s to %s" % (user.name, current_room.id)
if current_room_id:
g.cache.delete("current_robin_room")
current_room.persist_computed_name()
websockets.send_broadcast(
namespace="/robin/" + current_room.id,
type="updated_name",
payload={
"room_name": current_room.name,
},
)
else:
g.cache.set("current_robin_room", current_room.id)
amqp.consume_items("robin_waitinglist_q", process_waitinglist)
def process_votes_single(qname, limit=0):
# limit is taken but ignored for backwards compatibility
def _handle_vote(msg):
#assert(len(msgs) == 1)
r = pickle.loads(msg.body)
uid, tid, dir, ip, organic, cheater = r
voter = Account._byID(uid, data=True)
votee = Thing._by_fullname(tid, data = True)
if isinstance(votee, Comment):
update_comment_votes([votee])
# I don't know how, but somebody is sneaking in votes
# for subreddits
if isinstance(votee, (Link, Comment)):
print (voter, votee, dir, ip, organic, cheater)
handle_vote(voter, votee, dir, ip, organic,
cheater = cheater, foreground=True)
amqp.consume_items(qname, _handle_vote, verbose = False)
def consume_vote_queue(queue):
@g.stats.amqp_processor(queue)
def process_message(msg):
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
vote_data = json.loads(msg.body)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
amqp.consume_items(queue, process_message, verbose=False)
def consume_vote_queue(queue):
@g.stats.amqp_processor(queue)
def process_message(msg):
timer = g.stats.get_timer("new_voting.%s" % queue)
timer.start()
vote_data = json.loads(msg.body)
user = Account._byID(vote_data.pop("user_id"), data=True)
thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True)
timer.intermediate("preamble")
lock_key = "vote-%s-%s" % (user._id36, thing._fullname)
with g.make_lock("voting", lock_key, timeout=5):
print "Processing vote by %s on %s %s" % (user, thing, vote_data)
try:
vote = Vote(
user,
thing,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.error(e.message)
return
timer.intermediate("create_vote_obj")
vote.commit()
timer.flush()
amqp.consume_items(queue, process_message, verbose=False)
def process_liveupdate_scraper_q():
@g.stats.amqp_processor('liveupdate_scraper_q')
def _handle_q(msg):
d = json.loads(msg.body)
try:
liveupdate = parse_embeds(d['event_id'], d['liveupdate_id'])
except Exception as e:
g.log.warning("Failed to scrape %s::%s: %r",
d["event_id"], d["liveupdate_id"], e)
return
if not liveupdate.media_objects:
return
payload = {
"liveupdate_id": "LiveUpdate_" + d['liveupdate_id'],
"media_embeds": liveupdate.embeds
}
send_event_broadcast(d['event_id'],
type="embeds_ready",
payload=payload)
amqp.consume_items('liveupdate_scraper_q', _handle_q, verbose=False)
def process_liveupdate_scraper_q():
@g.stats.amqp_processor('liveupdate_scraper_q')
def _handle_q(msg):
d = json.loads(msg.body)
try:
liveupdate = parse_embeds(d['event_id'], d['liveupdate_id'])
except Exception as e:
g.log.warning("Failed to scrape %s::%s: %r", d["event_id"],
d["liveupdate_id"], e)
return
if not liveupdate.media_objects:
return
payload = {
"liveupdate_id": "LiveUpdate_" + d['liveupdate_id'],
"media_embeds": liveupdate.embeds
}
send_event_broadcast(d['event_id'],
type="embeds_ready",
payload=payload)
amqp.consume_items('liveupdate_scraper_q', _handle_q, verbose=False)
def process_liveupdate_scraper_q():
@g.stats.amqp_processor('liveupdate_scraper_q')
def _handle_q(msg):
d = json.loads(msg.body)
event_id = d.get("event_id") or d["event_fullname"][len("LiveUpdateEvent_"):]
liveupdate_id = d["liveupdate_id"]
try:
fn = TimeoutFunction(parse_embeds, 10)
liveupdate = fn(event_id, liveupdate_id)
except TimeoutFunctionException:
g.log.warning(
"Timed out on %s::%s", event_id, liveupdate_id)
return
except Exception as e:
g.log.warning("Failed to scrape %s::%s: %r",
event_id, liveupdate_id, e)
return
if not liveupdate:
return
if not liveupdate.embeds and not liveupdate.mobile_embeds:
return
payload = {
"liveupdate_id": "LiveUpdate_" + liveupdate_id,
"media_embeds": liveupdate.embeds,
"mobile_embeds": liveupdate.mobile_embeds,
}
send_event_broadcast(event_id,
type="embeds_ready",
payload=payload)
amqp.consume_items('liveupdate_scraper_q', _handle_q, verbose=False)
def process_adzerk():
@g.stats.amqp_processor('adzerk_q')
def _handle_adzerk(msg):
data = json.loads(msg.body)
g.log.debug('data: %s' % data)
action = data.get('action')
if action == 'deactivate_orphaned_flight':
_deactivate_orphaned_flight(data['flight'])
return
link = Link._by_fullname(data['link'], data=True)
if data['campaign']:
campaign = PromoCampaign._by_fullname(data['campaign'], data=True)
else:
campaign = None
if action == 'update_adzerk':
_update_adzerk(link, campaign)
elif action == 'deactivate_overdelivered':
_deactivate_overdelivered(link, campaign)
amqp.consume_items('adzerk_q', _handle_adzerk, verbose=False)
def process_flair():
amqp.consume_items('buttonflair_q', update_flairs)
def consume_comment_vote_queue(qname="vote_comment_q"):
@g.stats.amqp_processor(qname)
def process_message(msg):
from r2.lib.db.queries import (
add_queries,
add_to_commentstree_q,
get_comments,
)
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'),
vote_data)
return
timer = g.stats.get_timer("comment_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
comment = Comment._by_fullname(vote_data.pop("thing_fullname"))
print "Processing vote by %s on %s %s" % (user, comment, vote_data)
try:
vote = Vote(
user,
comment,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
vote_valid = vote.is_automatic_initial_vote or vote.effects.affects_score
comment_valid = not (comment._spam or comment._deleted)
if vote_valid and comment_valid:
author = Account._byID(comment.author_id)
add_queries(
queries=[get_comments(author, sort, 'all') for sort in SORTS],
insert_items=comment,
)
timer.intermediate("author_queries")
# update the score periodically when a comment has many votes
update_threshold = g.live_config['comment_vote_update_threshold']
update_period = g.live_config['comment_vote_update_period']
num_votes = comment.num_votes
if num_votes <= update_threshold or num_votes % update_period == 0:
add_to_commentstree_q(comment)
timer.stop()
timer.flush()
amqp.consume_items(qname, process_message, verbose=False)
def run(streamfile=None, verbose=False):
if streamfile:
stream_fp = open(streamfile, "a")
else:
stream_fp = None
def streamlog(msg, important=False):
if stream_fp:
stream_fp.write(msg + "\n")
stream_fp.flush()
if important:
print msg
def add_timestamps(d):
d['hms'] = d['time'].strftime("%H:%M:%S")
d['occ'] = "<%s:%s, pid=%-5s, %s>" % (
d['host'], d['port'], d['pid'],
d['time'].strftime("%Y-%m-%d %H:%M:%S"))
def limited_append(l, item):
if len(l) >= 25:
l.pop(12)
l.append(item)
def log_exception(d, daystring):
exc_desc = d['exception_desc']
exc_type = d['exception_type']
exc_str = "%s: %s" % (exc_type, exc_desc)
add_timestamps(d)
tb = []
key_material = exc_type
pretty_lines = []
make_lock_seen = False
flaky_db_seen = False
cassandra_seen = False
for tpl in d['traceback']:
tb.append(tpl)
filename, lineno, funcname, text = tpl
if text is None:
pass
elif (text.startswith("with g.make_lock(")
or text.startswith("with make_lock(")):
make_lock_seen = True
elif (text.startswith(
"(ProgrammingError) server closed the connection")):
flaky_db_seen = True
if '/cassandra/' in filename:
cassandra_seen = True
key_material += "%s %s " % (filename, funcname)
pretty_lines.append("%s:%s: %s()" % (filename, lineno, funcname))
pretty_lines.append(" %s" % text)
if exc_desc.startswith("QueuePool limit of size"):
fingerprint = "QueuePool_overflow"
elif exc_desc.startswith("error 2 from memcached_get: HOSTNAME "):
fingerprint = "memcache_suckitude"
elif exc_type == "TimeoutExpired" and make_lock_seen:
fingerprint = "make_lock_timeout"
elif exc_desc.startswith("(OperationalError) FATAL: the database " +
"system is in recovery mode"):
fingerprint = "recovering_db"
elif exc_desc.startswith("(OperationalError) could not connect " +
"to server"):
fingerprint = "unconnectable_db"
elif exc_desc.startswith("(OperationalError) server closed the " +
"connection unexpectedly"):
fingerprint = "flaky_db_op"
elif exc_type == "ProgrammingError" and flaky_db_seen:
fingerprint = "flaky_db_prog"
# SQLAlchemy includes the entire query in the exception
# description which can sometimes be gigantic, in the case of
# SELECTs. Get rid of it.
select_pos = exc_str.find("SELECT")
if select_pos > 0:
exc_str = exc_str[pos]
elif exc_type == "NoServerAvailable":
fingerprint = "cassandra_suckitude"
elif exc_type == "TimedOutException" and cassandra_seen:
fingerprint = "cassandra_suckitude #2"
else:
fingerprint = md5(key_material).hexdigest()
nickname_key = "error_nickname-" + fingerprint
status_key = "error_status-" + fingerprint
nickname = g.hardcache.get(nickname_key)
if nickname is None:
nickname = '"%s" Exception' % randword().capitalize()
news = ("A new kind of thing just happened! " +
"I'm going to call it a %s\n\n" % nickname)
news += "Where and when: %s\n\n" % d['occ']
news += "Traceback:\n"
news += "\n".join(pretty_lines)
news += exc_str
news += "\n"
emailer.nerds_email(news, "Exception Watcher")
g.hardcache.set(nickname_key, nickname, 86400 * 365)
g.hardcache.set(status_key, "new", 86400)
if g.hardcache.get(status_key) == "fixed":
g.hardcache.set(status_key, "new", 86400)
news = "This was marked as fixed: %s\n" % nickname
news += "But it just occurred, so I'm marking it new again."
emailer.nerds_email(news, "Exception Watcher")
err_key = "-".join(["error", daystring, fingerprint])
existing = g.hardcache.get(err_key)
if not existing:
existing = dict(exception=exc_str, traceback=tb, occurrences=[])
limited_append(existing['occurrences'], d['occ'])
g.hardcache.set(err_key, existing, 7 * 86400)
streamlog("%s [X] %-70s" % (d['hms'], nickname), verbose)
def log_text(d, daystring):
add_timestamps(d)
char = d['level'][0].upper()
streamlog("%s [%s] %r" % (d['hms'], char, d['text']), verbose)
logclass_key = "logclass-" + d['classification']
if not g.hardcache.get(logclass_key):
g.hardcache.set(logclass_key, True, 86400 * 90)
if d['level'] != 'debug':
news = "The code just generated a [%s] message.\n" % \
d['classification']
news += "I don't remember ever seeing one of those before.\n"
news += "\n"
news += "It happened on: %s\n" % d['occ']
news += "The log level was: %s\n" % d['level']
news += "The complete text was:\n"
news += repr(d['text'])
emailer.nerds_email(news, "reddit secretary")
occ_key = "-".join(
["logtext", daystring, d['level'], d['classification']])
occurrences = g.hardcache.get(occ_key)
if occurrences is None:
occurrences = []
d2 = {}
d2['occ'] = d['occ']
d2['text'] = repr(d['text'])
limited_append(occurrences, d2)
g.hardcache.set(occ_key, occurrences, 86400 * 7)
def myfunc(msg):
daystring = datetime.now(g.display_tz).strftime("%Y/%m/%d")
try:
d = pickle.loads(msg.body)
except TypeError:
streamlog("wtf is %r" % msg.body, True)
return
if not 'type' in d:
streamlog("wtf is %r" % d, True)
elif d['type'] == 'exception':
try:
log_exception(d, daystring)
except Exception as e:
print "Error in log_exception(): %r" % e
elif d['type'] == 'text':
try:
log_text(d, daystring)
except Exception as e:
print "Error in log_text(): %r" % e
else:
streamlog("wtf is %r" % d['type'], True)
amqp.consume_items(q, myfunc, verbose=verbose)
def run(streamfile=None, verbose=False):
if streamfile:
stream_fp = open(streamfile, "a")
else:
stream_fp = None
def streamlog(msg, important=False):
if stream_fp:
stream_fp.write(msg + "\n")
stream_fp.flush()
if important:
print msg
def add_timestamps (d):
d['hms'] = d['time'].strftime("%H:%M:%S")
d['occ'] = "<%s:%s, pid=%-5s, %s>" % (d['host'], d['port'], d['pid'],
d['time'].strftime("%Y-%m-%d %H:%M:%S"))
def limited_append(l, item):
if len(l) >= 25:
l.pop(12)
l.append(item)
def log_exception(d, daystring):
exc_desc = d['exception_desc']
exc_type = d['exception_type']
exc_str = "%s: %s" % (exc_type, exc_desc)
add_timestamps(d)
tb = []
key_material = exc_type
pretty_lines = []
make_lock_seen = False
flaky_db_seen = False
cassandra_seen = False
for tpl in d['traceback']:
tb.append(tpl)
filename, lineno, funcname, text = tpl
if text is None:
pass
elif (text.startswith("with g.make_lock(") or
text.startswith("with make_lock(")):
make_lock_seen = True
elif (text.startswith("(ProgrammingError) server closed the connection")):
flaky_db_seen = True
if '/cassandra/' in filename.lower():
cassandra_seen = True
if '/pycassa/' in filename.lower():
cassandra_seen = True
key_material += "%s %s " % (filename, funcname)
pretty_lines.append ("%s:%s: %s()" % (filename, lineno, funcname))
pretty_lines.append (" %s" % text)
if exc_desc.startswith("QueuePool limit of size"):
fingerprint = "QueuePool_overflow"
elif exc_desc.startswith("error 2 from memcached_get: HOSTNAME "):
fingerprint = "memcache_suckitude"
elif exc_type == "TimeoutExpired" and make_lock_seen:
fingerprint = "make_lock_timeout"
elif exc_desc.startswith("(OperationalError) FATAL: the database " +
"system is in recovery mode"):
fingerprint = "recovering_db"
elif exc_desc.startswith("(OperationalError) could not connect " +
"to server"):
fingerprint = "unconnectable_db"
elif exc_desc.startswith("(OperationalError) server closed the " +
"connection unexpectedly"):
fingerprint = "flaky_db_op"
elif cassandra_seen:
fingerprint = "something's wrong with cassandra"
else:
fingerprint = md5(key_material).hexdigest()
nickname_key = "error_nickname-" + fingerprint
status_key = "error_status-" + fingerprint
nickname = g.hardcache.get(nickname_key)
if nickname is None:
nickname = '"%s" Exception' % randword().capitalize()
news = ("A new kind of thing just happened! " +
"I'm going to call it a %s\n\n" % nickname)
news += "Where and when: %s\n\n" % d['occ']
news += "Traceback:\n"
news += "\n".join(pretty_lines)
news += exc_str
news += "\n"
emailer.nerds_email(news, "Exception Watcher")
g.hardcache.set(nickname_key, nickname, 86400 * 365)
g.hardcache.set(status_key, "new", 86400)
if g.hardcache.get(status_key) == "fixed":
g.hardcache.set(status_key, "new", 86400)
news = "This was marked as fixed: %s\n" % nickname
news += "But it just occurred, so I'm marking it new again."
emailer.nerds_email(news, "Exception Watcher")
err_key = "-".join(["error", daystring, fingerprint])
existing = g.hardcache.get(err_key)
if not existing:
existing = dict(exception=exc_str, traceback=tb, occurrences=[])
existing.setdefault('times_seen', 0)
existing['times_seen'] += 1
limited_append(existing['occurrences'], d['occ'])
g.hardcache.set(err_key, existing, 7 * 86400)
streamlog ("%s [X] %-70s" % (d['hms'], nickname), verbose)
def log_text(d, daystring):
add_timestamps(d)
char = d['level'][0].upper()
streamlog ("%s [%s] %r" % (d['hms'], char, d['text']), verbose)
logclass_key = "logclass-" + d['classification']
if not g.hardcache.get(logclass_key):
g.hardcache.set(logclass_key, True, 86400 * 90)
if d['level'] != 'debug':
news = "The code just generated a [%s] message.\n" % \
d['classification']
news += "I don't remember ever seeing one of those before.\n"
news += "\n"
news += "It happened on: %s\n" % d['occ']
news += "The log level was: %s\n" % d['level']
news += "The complete text was:\n"
news += repr(d['text'])
emailer.nerds_email (news, "reddit secretary")
occ_key = "-".join(["logtext", daystring,
d['level'], d['classification']])
occurrences = g.hardcache.get(occ_key)
if occurrences is None:
occurrences = []
d2 = {}
d2['occ'] = d['occ']
d2['text'] = repr(d['text'])
limited_append(occurrences, d2)
g.hardcache.set(occ_key, occurrences, 86400 * 7)
def myfunc(msg):
daystring = datetime.now(g.display_tz).strftime("%Y/%m/%d")
try:
d = pickle.loads(msg.body)
except TypeError:
streamlog ("wtf is %r" % msg.body, True)
return
if not 'type' in d:
streamlog ("wtf is %r" % d, True)
elif d['type'] == 'exception':
try:
log_exception(d, daystring)
except Exception as e:
print "Error in log_exception(): %r" % e
elif d['type'] == 'text':
try:
log_text(d, daystring)
except Exception as e:
print "Error in log_text(): %r" % e
else:
streamlog ("wtf is %r" % d['type'], True)
amqp.consume_items(q, myfunc, verbose=verbose)
def consume_comment_vote_queue(qname="vote_comment_q"):
@g.stats.amqp_processor(qname)
def process_message(msg):
from r2.lib.comment_tree import write_comment_scores
from r2.lib.db.queries import (
add_queries,
add_to_commentstree_q,
get_comments,
)
from r2.models.builder import get_active_sort_orders_for_link
vote_data = json.loads(msg.body)
hook = hooks.get_hook('vote.validate_vote_data')
if hook.call_until_return(msg=msg, vote_data=vote_data) is False:
# Corrupt records in the queue. Ignore them.
print "Ignoring invalid vote by %s on %s %s" % (
vote_data.get('user_id', '<unknown>'),
vote_data.get('thing_fullname', '<unknown>'), vote_data)
return
timer = g.stats.get_timer("comment_vote_processor")
timer.start()
user = Account._byID(vote_data.pop("user_id"))
comment = Comment._by_fullname(vote_data.pop("thing_fullname"))
print "Processing vote by %s on %s %s" % (user, comment, vote_data)
try:
vote = Vote(
user,
comment,
direction=vote_data["direction"],
date=datetime.utcfromtimestamp(vote_data["date"]),
data=vote_data["data"],
event_data=vote_data.get("event_data"),
)
except TypeError as e:
# a vote on an invalid type got in the queue, just skip it
g.log.exception("Invalid type: %r", e.message)
return
vote.commit()
timer.intermediate("create_vote_object")
vote_invalid = (not vote.effects.affects_score
and not vote.is_automatic_initial_vote)
comment_invalid = comment._spam or comment._deleted
if vote_invalid or comment_invalid:
timer.stop()
timer.flush()
return
author = Account._byID(comment.author_id)
add_queries(
queries=[get_comments(author, sort, 'all') for sort in SORTS],
insert_items=comment,
)
timer.intermediate("author_queries")
update_threshold = g.live_config['comment_vote_update_threshold']
update_period = g.live_config['comment_vote_update_period']
skip_score_update = (comment.num_votes > update_threshold
and comment.num_votes % update_period != 0)
# skip updating scores if this was the automatic initial vote. those
# updates will be handled by new_comment. Also only update scores
# periodically once a comment has many votes.
if not vote.is_automatic_initial_vote and not skip_score_update:
# check whether this link is using precomputed sorts, if it is
# we'll need to push an update to commentstree_q
link = Link._byID(comment.link_id)
if get_active_sort_orders_for_link(link):
# send this comment to commentstree_q where we will update
# CommentScoresByLink, CommentTree (noop), and CommentOrderer
add_to_commentstree_q(comment)
else:
# the link isn't using precomputed sorts, so just update the
# scores
write_comment_scores(link, [comment])
timer.intermediate("update_scores")
timer.stop()
timer.flush()
amqp.consume_items(qname, process_message, verbose=False)
def process():
from r2.models import (
Account,
Link,
NotFound,
PromoCampaign,
)
from reddit_dfp.services import (
creatives_service,
lineitems_service,
)
def _handle_upsert_promotion(payload):
link = Link._by_fullname(payload["link"], data=True)
author = Account._byID(link.author_id)
creatives_service.upsert_creative(author, link)
def _handle_upsert_campaign(payload):
link = Link._by_fullname(payload["link"], data=True)
campaign = PromoCampaign._by_fullname(payload["campaign"], data=True)
owner = Account._byID(campaign.owner_id)
author = Account._byID(link.author_id)
try:
lineitem = lineitems_service.upsert_lineitem(owner, campaign)
except ValueError as e:
g.log.error("unable to upsert lineitem: %s" % e)
return
creative = creatives_service.upsert_creative(author, link)
lineitems_service.associate_with_creative(
lineitem=lineitem, creative=creative)
def _handle_deactivate(payload):
campaign_ids = payload["campaigns"] and payload["campaigns"].split(",")
if not campaign_ids:
return
campaigns = PromoCampaign._by_fullname(campaign_ids, data=True)
lineitems_service.deactivate(campaign_ids)
def _handle_activate(payload):
campaign_ids = payload["campaigns"] and payload["campaigns"].split(",")
if not campaign_ids:
return
campaigns = PromoCampaign._by_fullname(campaign_ids, data=True)
lineitems_service.activate(campaign_ids)
def _handle_check_edits(payload):
existing = Link._by_fullname(payload["link"], data=True)
creative = creatives_service.get_creative(existing)
link = utils.dfp_creative_to_link(
creative, link=Link._by_fullname(payload["link"], data=True))
link.dfp_checking_edits = False
link._commit()
processor = Processor()
if feature.is_enabled("dfp_selfserve"):
g.log.debug("dfp enabled, registering %s processors", DFP_QUEUE)
processor.register("upsert_promotion", _handle_upsert_promotion)
processor.register("upsert_campaign", _handle_upsert_campaign)
processor.register("activate", _handle_activate)
processor.register("deactivate", _handle_deactivate)
processor.register("check_edits", _handle_check_edits)
@g.stats.amqp_processor(DFP_QUEUE)
def _handler(message):
rate_limit_ends_at = g.cache.get(RATE_LIMIT_ENDS_AT)
now_utc = datetime.utcnow()
if rate_limit_ends_at:
if now_utc > rate_limit_ends_at:
g.cache.delete(RATE_LIMIT_ENDS_AT)
else:
raise RateLimitException("waiting until %s" % rate_limit_ends_at)
data = json.loads(message.body)
g.log.debug("processing action: %s" % data)
action = data.get("action")
payload = data.get("payload")
try:
processor.call(action, payload)
except RateLimitException as e:
g.cache.set(RATE_LIMIT_ENDS_AT, datetime.utcnow() + timedelta(minutes=1))
raise e
amqp.consume_items(DFP_QUEUE, _handler, verbose=False)
