def _run_changed(msgs, chan): '''Consume the cloudsearch_changes queue, and print reporting information on how long it took and how many remain ''' start = datetime.now(g.tz) changed = [pickle.loads(msg.body) for msg in msgs] fullnames = set() fullnames.update(LinkUploader.desired_fullnames(changed)) fullnames.update(SubredditUploader.desired_fullnames(changed)) things = Thing._by_fullname(fullnames, data=True, return_dict=False) link_uploader = LinkUploader(g.CLOUDSEARCH_DOC_API, things=things) subreddit_uploader = SubredditUploader(g.CLOUDSEARCH_SUBREDDIT_DOC_API, things=things) link_time = link_uploader.inject() subreddit_time = subreddit_uploader.inject() cloudsearch_time = link_time + subreddit_time totaltime = (datetime.now(g.tz) - start).total_seconds() print( "%s: %d messages in %.2fs seconds (%.2fs secs waiting on " "cloudsearch); %d duplicates, %s remaining)" % (start, len(changed), totaltime, cloudsearch_time, len(changed) - len(things), msgs[-1].delivery_info.get('message_count', 'unknown')))
def _run_changed(msgs, chan): '''Consume the cloudsearch_changes queue, and print reporting information on how long it took and how many remain ''' start = datetime.now(g.tz) changed = [pickle.loads(msg.body) for msg in msgs] fullnames = set() fullnames.update(LinkUploader.desired_fullnames(changed)) fullnames.update(SubredditUploader.desired_fullnames(changed)) things = Thing._by_fullname(fullnames, data=True, return_dict=False) link_uploader = LinkUploader(g.CLOUDSEARCH_DOC_API, things=things) subreddit_uploader = SubredditUploader(g.CLOUDSEARCH_SUBREDDIT_DOC_API, things=things) link_time = link_uploader.inject() subreddit_time = subreddit_uploader.inject() cloudsearch_time = link_time + subreddit_time totaltime = (datetime.now(g.tz) - start).total_seconds() print ("%s: %d messages in %.2fs seconds (%.2fs secs waiting on " "cloudsearch); %d duplicates, %s remaining)" % (start, len(changed), totaltime, cloudsearch_time, len(changed) - len(things), msgs[-1].delivery_info.get('message_count', 'unknown')))
def get_things(codes): """Fetch relevant things for a list of ad codenames in batch.""" fullnames = [ AdvertTrafficSummary.split_codename(code)[0] for code in codes if code.startswith(Thing._type_prefix) ] return Thing._by_fullname(fullnames, data=True, return_dict=True)
def process_message(msg): timer = g.stats.get_timer("new_voting.%s" % queue) timer.start() vote_data = json.loads(msg.body) user = Account._byID(vote_data.pop("user_id"), data=True) thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True) timer.intermediate("preamble") lock_key = "vote-%s-%s" % (user._id36, thing._fullname) with g.make_lock("voting", lock_key, timeout=5): print "Processing vote by %s on %s %s" % (user, thing, vote_data) try: vote = Vote( user, thing, direction=vote_data["direction"], date=datetime.utcfromtimestamp(vote_data["date"]), data=vote_data["data"], event_data=vote_data.get("event_data"), ) except TypeError as e: # a vote on an invalid type got in the queue, just skip it g.log.error(e.message) return timer.intermediate("create_vote_obj") vote.commit() timer.flush()
def _get_newest_link(sr): for fullname in sr.get_links('new', 'all'): link = Thing._by_fullname(fullname, data=True) if not link._spam and not link._deleted: return link return None
def process_message(msg): timer = g.stats.get_timer("new_voting.%s" % queue) timer.start() vote_data = json.loads(msg.body) user = Account._byID(vote_data.pop("user_id"), data=True) thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True) timer.intermediate("preamble") lock_key = "vote-%s-%s" % (user._id36, thing._fullname) with g.make_lock("voting", lock_key, timeout=5): print "Processing vote by %s on %s %s" % (user, thing, vote_data) try: vote = Vote( user, thing, direction=vote_data["direction"], date=datetime.utcfromtimestamp(vote_data["date"]), data=vote_data["data"], event_data=vote_data.get("event_data"), ) except TypeError as e: # a vote on an invalid type got in the queue, just skip it g.log.exception("Invalid type: %r", e.message) return timer.intermediate("create_vote_obj") vote.commit() timer.flush()
def batch_lookups(self): try: self.things = Thing._by_fullname(self.fullnames, data=True, return_dict=False) except NotFound: if self.use_safe_get: self.things = safe_get(Thing._by_fullname, self.fullnames, data=True, return_dict=False) else: raise
def send_gift(buyer, recipient, months, days, signed, giftmessage, thing_fullname): admintools.engolden(recipient, days) if thing_fullname: thing = Thing._by_fullname(thing_fullname, data=True) thing._gild(buyer) else: thing = None if signed: sender = buyer.name md_sender = "[%s](/user/%s)" % (sender, sender) else: sender = _("An anonymous redditor") md_sender = _("An anonymous redditor") create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed) if months == 1: amount = "a month" else: amount = "%d months" % months if not thing: subject = _( 'Let there be gold! %s just sent you reddit gold!') % sender message = strings.youve_got_gold % dict(sender=md_sender, amount=amount) if giftmessage and giftmessage.strip(): message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----' else: url = thing.make_permalink_slow() if isinstance(thing, Comment): subject = _('Your comment has been gilded!') message = strings.youve_been_gilded_comment % {'url': url} else: subject = _('Your submission has been gilded!') message = strings.youve_been_gilded_link % {'url': url} message += '\n\n' + strings.gold_benefits_msg if g.lounge_reddit: message += '\n* ' + strings.lounge_msg message = append_random_bottlecap_phrase(message) try: send_system_message(recipient, subject, message, distinguished='gold-auto') except MessageError: g.log.error('send_gift: could not send system message') g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name)) return thing
def from_queue(self, max_date, batch_limit=50, kind=None): from r2.models import Account, Thing keep_trying = True min_id = None s = self.queue_table while keep_trying: where = [s.c.date < max_date] if min_id: where.append(s.c.uid > min_id) if kind: where.append(s.c.kind == kind) res = sa.select([ s.c.to_addr, s.c.account_id, s.c.from_name, s.c.fullname, s.c.body, s.c.kind, s.c.ip, s.c.date, s.c.uid, s.c.msg_hash, s.c.fr_addr, s.c.reply_to, s.c.html_body, s.c.list_unsubscribe_header, ], sa.and_(*where), order_by=s.c.uid, limit=batch_limit).execute() res = res.fetchall() if not res: break # batch load user accounts aids = [x[1] for x in res if x[1] > 0] accts = Account._byID(aids, data=True, return_dict=True) if aids else {} # batch load things tids = [x[3] for x in res if x[3]] things = Thing._by_fullname(tids, data=True, return_dict=True) if tids else {} # get the lower bound date for next iteration min_id = max(x[8] for x in res) # did we not fetch them all? keep_trying = (len(res) == batch_limit) for (addr, acct, fname, fulln, body, kind, ip, date, uid, msg_hash, fr_addr, reply_to, html_body, list_unsubscribe_header) in res: yield (accts.get(acct), things.get(fulln), addr, fname, date, ip, kind, msg_hash, body, fr_addr, reply_to, html_body, list_unsubscribe_header, uid)
def send_gift(buyer, recipient, months, days, signed, giftmessage, thing_fullname): admintools.engolden(recipient, days) if thing_fullname: thing = Thing._by_fullname(thing_fullname, data=True) thing._gild(buyer) else: thing = None if signed: sender = buyer.name md_sender = "[%s](/user/%s)" % (sender, sender) else: sender = _("An anonymous redditor") md_sender = _("An anonymous redditor") create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed) if months == 1: amount = "a month" else: amount = "%d months" % months if not thing: subject = _( 'Let there be gold! %s just sent you reddit gold!') % sender message = strings.youve_got_gold % dict( sender=md_sender, amount=amount) if giftmessage and giftmessage.strip(): message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----' else: url = thing.make_permalink_slow() if isinstance(thing, Comment): subject = _('Your comment has been gilded!') message = strings.youve_been_gilded_comment % {'url': url} else: subject = _('Your submission has been gilded!') message = strings.youve_been_gilded_link % {'url': url} message += '\n\n' + strings.gold_benefits_msg if g.lounge_reddit: message += '\n* ' + strings.lounge_msg message = append_random_bottlecap_phrase(message) try: send_system_message( recipient, subject, message, distinguished='gold-auto') except MessageError: g.log.error('send_gift: could not send system message') g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name)) return thing
def add_props(cls, user, wrapped): user_fullnames = {w.user_fullname for w in wrapped} target_fullnames = {w.target_fullname for w in wrapped} users = Account._by_fullname(user_fullnames, data=True, return_dict=True) targets = Thing._by_fullname(target_fullnames, data=True, return_dict=True) author_ids = { t.author_id for t in targets.itervalues() if hasattr(t, 'author_id') } link_ids = { t.link_id for t in targets.itervalues() if hasattr(t, 'link_id') } sr_ids = {t.sr_id for t in targets.itervalues() if hasattr(t, 'sr_id')} authors = Account._byID(author_ids, data=True, return_dict=True) links = Link._byID(link_ids, data=True, return_dict=True) subreddits = Subreddit._byID(sr_ids, data=True, return_dict=True) target_things = {} for fullname, target in targets.iteritems(): if isinstance(target, (Comment, Link)): author = authors[target.author_id] if isinstance(target, Link): subreddit = subreddits[target.sr_id] path = target.make_permalink(subreddit) else: link = links[target.link_id] subreddit = subreddits[link.sr_id] path = target.make_permalink(link, subreddit) target_things[fullname] = GameLogTarget( target, path, author, subreddit) elif isinstance(target, Account): target_things[fullname] = WrappedUser(target) for w in wrapped: w.is_self = (c.user_is_loggedin and w.user_fullname == c.user._fullname) w.user = WrappedUser(users[w.user_fullname]) w.target = target_things[w.target_fullname] w.item = g.f2pitems[w.item] w.user_team = scores.get_user_team(users[w.user_fullname]) if isinstance(w.target, WrappedUser): target_user = targets[w.target.fullname] else: target_user = authors[targets[w.target_fullname].author_id] w.target_team = scores.get_user_team(target_user)
def process_message(msg): # msg is *PROBABLY* json timer = g.stats.get_timer("new_voting.%s" % queue) timer.start() # json being loaded into a python object # it has the fields "user_id", "thing_fullname" # a thing is a database object # it's a link, comment, post, whatever, everything can be upvoted/downvoted vote_data = json.loads(msg.body) hook = hooks.get_hook('vote.validate_vote_data') if hook.call_until_return(msg=msg, vote_data=vote_data) is False: # Corrupt records in the queue. Ignore them. print "Ignoring invalid vote by %s on %s %s" % ( vote_data.get('user_id', '<unknown>'), vote_data.get('thing_fullname', '<unknown>'), vote_data) return # this gets the user from database/cache (either memcached or postgres, whatever) user = Account._byID(vote_data.pop("user_id"), data=True) thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True) timer.intermediate("preamble") # this gets a servers-wide lock # I mean, a bunch of consumers might be consuming items that use the same "thing" (same database object) # so, you want a global lock to avoid them from f*****g eachother up # memcachd stores the lock, atomically lock_key = "vote-%s-%s" % (user._id36, thing._fullname) with g.make_lock("voting", lock_key, timeout=5): print "Processing vote by %s on %s %s" % (user, thing, vote_data) try: vote = Vote( user, thing, direction=vote_data["direction"], date=datetime.utcfromtimestamp(vote_data["date"]), data=vote_data["data"], event_data=vote_data.get("event_data"), ) except TypeError as e: # a vote on an invalid type got in the queue, just skip it g.log.exception("Invalid type: %r", e.message) return timer.intermediate("create_vote_obj") vote.commit() timer.flush()
def validate_blob(custom): """Validate payment_blob and return a dict with everything looked up.""" ret = {} if not custom: raise GoldException('no custom') payment_blob = g.hardcache.get('payment_blob-%s' % str(custom)) if not payment_blob: raise GoldException('no payment_blob') if 'account_id' in payment_blob and 'account_name' in payment_blob: try: buyer = Account._byID(payment_blob['account_id'], data=True) ret['buyer'] = buyer except NotFound: raise GoldException('bad account_id') if not buyer.name.lower() == payment_blob['account_name'].lower(): raise GoldException('buyer mismatch') elif 'email' in payment_blob: ret['email'] = payment_blob['email'] else: raise GoldException('no account_id or email') goldtype = payment_blob['goldtype'] ret['goldtype'] = goldtype if goldtype == 'gift': recipient_name = payment_blob.get('recipient', None) if not recipient_name: raise GoldException('gift missing recpient') try: recipient = Account._by_name(recipient_name) ret['recipient'] = recipient except NotFound: raise GoldException('bad recipient') thing_fullname = payment_blob.get('thing', None) if thing_fullname: try: ret['thing'] = Thing._by_fullname(thing_fullname) except NotFound: raise GoldException('bad thing') ret['signed'] = payment_blob.get('signed', False) giftmessage = payment_blob.get('giftmessage') giftmessage = _force_unicode(giftmessage) if giftmessage else None ret['giftmessage'] = giftmessage elif goldtype not in ('onetime', 'autorenew', 'creddits', 'code'): raise GoldException('bad goldtype') return ret
def test_get_links(self): from r2.lib.db import queries from r2.models import Subreddit, Account, Link, Thing account = Account._byID(1, data=True) sr = Subreddit._by_name("reddit_test0") link_url = self.make_unique_url() new_link = Link._submit("test_get_links", link_url, account, sr, "127.0.0.1", kind="link") queries.new_link(new_link, foreground=True) res = Thing._by_fullname(queries.get_links(sr, "new", "all"), return_dict=False) self.assert_true(len(res) > 0, "no links returned") self.assert_equal(new_link._id, res[0]._id)
def validate_blob(custom): """Validate payment_blob and return a dict with everything looked up.""" ret = {} if not custom: raise GoldException("no custom") payment_blob = g.hardcache.get("payment_blob-%s" % str(custom)) if not payment_blob: raise GoldException("no payment_blob") if not ("account_id" in payment_blob and "account_name" in payment_blob): raise GoldException("no account_id") try: buyer = Account._byID(payment_blob["account_id"], data=True) ret["buyer"] = buyer except NotFound: raise GoldException("bad account_id") if not buyer.name.lower() == payment_blob["account_name"].lower(): raise GoldException("buyer mismatch") goldtype = payment_blob["goldtype"] ret["goldtype"] = goldtype if goldtype == "gift": recipient_name = payment_blob.get("recipient", None) if not recipient_name: raise GoldException("gift missing recpient") try: recipient = Account._by_name(recipient_name) ret["recipient"] = recipient except NotFound: raise GoldException("bad recipient") thing_fullname = payment_blob.get("thing", None) if thing_fullname: try: ret["thing"] = Thing._by_fullname(thing_fullname) except NotFound: raise GoldException("bad thing") ret["signed"] = payment_blob.get("signed", False) giftmessage = payment_blob.get("giftmessage") giftmessage = _force_unicode(giftmessage) if giftmessage else None ret["giftmessage"] = giftmessage elif goldtype not in ("onetime", "autorenew", "creddits", "code"): raise GoldException("bad goldtype") return ret
def all_defendants(cls, quench=False, _update=False): all = cls.all_defendants_cache(_update=_update) defs = Thing._by_fullname(all, data=True).values() if quench: # Used for the spotlight, to filter out trials with over 20 votes; # otherwise, hung juries would hog the spotlight for an hour as # their vote counts continued to skyrocket return filter( lambda d: not g.cache.get("quench_jurors-" + d._fullname), defs) else: return defs
def all_defendants(cls, quench=False, _update=False): all = cls.all_defendants_cache(_update=_update) defs = Thing._by_fullname(all, data=True).values() if quench: # Used for the spotlight, to filter out trials with over 20 votes; # otherwise, hung juries would hog the spotlight for an hour as # their vote counts continued to skyrocket return filter (lambda d: not g.cache.get("quench_jurors-" + d._fullname), defs) else: return defs
def get_recent_name_submissions(): link_fullnames = list(queries.get_links(SERVERNAME_SR, "new", "all")) links = chain.from_iterable(Thing._by_fullname(chunk, return_dict=False) for chunk in in_chunks(link_fullnames)) for link in links: if link._deleted or link._spam: continue # OH GOD WHAT HAVE YOU POSTED IN MY LOVELY AUTOMATED SUBREDDIT!? if (not hasattr(link, "revenue_date") or not hasattr(link, "revenue_bucket") or not hasattr(link, "server_names")): continue yield link
def from_queue(self, max_date, batch_limit = 50, kind = None): from r2.models import is_banned_IP, Account, Thing keep_trying = True min_id = None s = self.queue_table while keep_trying: where = [s.c.date < max_date] if min_id: where.append(s.c.uid > min_id) if kind: where.append(s.c.kind == kind) res = sa.select([s.c.to_addr, s.c.account_id, s.c.from_name, s.c.fullname, s.c.body, s.c.kind, s.c.ip, s.c.date, s.c.uid, s.c.msg_hash, s.c.fr_addr, s.c.reply_to], sa.and_(*where), order_by = s.c.uid, limit = batch_limit).execute() res = res.fetchall() if not res: break # batch load user accounts aids = [x[1] for x in res if x[1] > 0] accts = Account._byID(aids, data = True, return_dict = True) if aids else {} # batch load things tids = [x[3] for x in res if x[3]] things = Thing._by_fullname(tids, data = True, return_dict = True) if tids else {} # make sure no IPs have been banned in the mean time ips = set(x[6] for x in res) ips = dict((ip, is_banned_IP(ip)) for ip in ips) # get the lower bound date for next iteration min_id = max(x[8] for x in res) # did we not fetch them all? keep_trying = (len(res) == batch_limit) for (addr, acct, fname, fulln, body, kind, ip, date, uid, msg_hash, fr_addr, reply_to) in res: yield (accts.get(acct), things.get(fulln), addr, fname, date, ip, ips[ip], kind, msg_hash, body, fr_addr, reply_to)
def add_props(cls, user, wrapped): user_fullnames = {w.user_fullname for w in wrapped} target_fullnames = {w.target_fullname for w in wrapped} users = Account._by_fullname(user_fullnames, data=True, return_dict=True) targets = Thing._by_fullname(target_fullnames, data=True, return_dict=True) author_ids = {t.author_id for t in targets.itervalues() if hasattr(t, 'author_id')} link_ids = {t.link_id for t in targets.itervalues() if hasattr(t, 'link_id')} sr_ids = {t.sr_id for t in targets.itervalues() if hasattr(t, 'sr_id')} authors = Account._byID(author_ids, data=True, return_dict=True) links = Link._byID(link_ids, data=True, return_dict=True) subreddits = Subreddit._byID(sr_ids, data=True, return_dict=True) target_things = {} for fullname, target in targets.iteritems(): if isinstance(target, (Comment, Link)): author = authors[target.author_id] if isinstance(target, Link): subreddit = subreddits[target.sr_id] path = target.make_permalink(subreddit) else: link = links[target.link_id] subreddit = subreddits[link.sr_id] path = target.make_permalink(link, subreddit) target_things[fullname] = GameLogTarget(target, path, author, subreddit) elif isinstance(target, Account): target_things[fullname] = WrappedUser(target) for w in wrapped: w.is_self = (c.user_is_loggedin and w.user_fullname == c.user._fullname) w.user = WrappedUser(users[w.user_fullname]) w.target = target_things[w.target_fullname] w.item = g.f2pitems[w.item] w.user_team = scores.get_user_team(users[w.user_fullname]) if isinstance(w.target, WrappedUser): target_user = targets[w.target.fullname] else: target_user = authors[targets[w.target_fullname].author_id] w.target_team = scores.get_user_team(target_user)
def process_message(msg): timer = g.stats.get_timer("new_voting.%s" % queue) timer.start() vote_data = json.loads(msg.body) hook = hooks.get_hook('vote.validate_vote_data') if hook.call_until_return(msg=msg, vote_data=vote_data) is False: # Corrupt records in the queue. Ignore them. print "Ignoring invalid vote by %s on %s %s" % ( vote_data.get('user_id', '<unknown>'), vote_data.get('thing_fullname', '<unknown>'), vote_data) return # if it's an old-style vote, convert to the new format if "uid" in vote_data: vote_data = convert_old_vote_data(vote_data, msg.timestamp) user = Account._byID(vote_data.pop("user_id"), data=True) thing = Thing._by_fullname(vote_data.pop("thing_fullname"), data=True) timer.intermediate("preamble") lock_key = "vote-%s-%s" % (user._id36, thing._fullname) with g.make_lock("voting", lock_key, timeout=5): print "Processing vote by %s on %s %s" % (user, thing, vote_data) try: vote = Vote( user, thing, direction=vote_data["direction"], date=datetime.utcfromtimestamp(vote_data["date"]), data=vote_data["data"], event_data=vote_data.get("event_data"), ) except TypeError as e: # a vote on an invalid type got in the queue, just skip it g.log.exception("Invalid type: %r", e.message) return timer.intermediate("create_vote_obj") vote.commit() timer.flush()
def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id): admintools.engolden(recipient, days) if comment_id: comment = Thing._by_fullname(comment_id, data=True) comment._gild(buyer) else: comment = None if signed: sender = buyer.name md_sender = "[%s](/user/%s)" % (sender, sender) else: sender = _("An anonymous redditor") md_sender = _("An anonymous redditor") create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed) if months == 1: amount = "a month" else: amount = "%d months" % months if not comment: subject = _( 'Let there be gold! %s just sent you reddit gold!') % sender message = strings.youve_got_gold % dict(sender=md_sender, amount=amount) if giftmessage and giftmessage.strip(): message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----' else: subject = _('Your comment has been gilded.') message = strings.youve_got_comment_gold % dict( url=comment.make_permalink_slow(), ) message += '\n\n' + strings.gold_benefits_msg message += '\n\n' + strings.lounge_msg try: send_system_message(recipient, subject, message) except MessageError: g.log.error('send_gift: could not send system message') g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name)) return comment
def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id): admintools.engolden(recipient, days) if comment_id: comment = Thing._by_fullname(comment_id, data=True) comment._gild(buyer) else: comment = None if signed: sender = buyer.name md_sender = "[%s](/user/%s)" % (sender, sender) else: sender = _("An anonymous redditor") md_sender = _("An anonymous redditor") create_gift_gold (buyer._id, recipient._id, days, c.start_time, signed) if months == 1: amount = "a month" else: amount = "%d months" % months if not comment: subject = _('Let there be gold! %s just sent you reddit gold!') % sender message = strings.youve_got_gold % dict(sender=md_sender, amount=amount) if giftmessage and giftmessage.strip(): message += "\n\n" + strings.giftgold_note + giftmessage + '\n\n----' else: subject = _('Your comment has been gilded.') message = strings.youve_got_comment_gold % dict( url=comment.make_permalink_slow(), ) message += '\n\n' + strings.gold_benefits_msg message += '\n\n' + strings.lounge_msg try: send_system_message(recipient, subject, message) except MessageError: g.log.error('send_gift: could not send system message') g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name)) return comment
def process_message(msg): if not ACCOUNT: return fullname = msg.body item = Thing._by_fullname(fullname, data=True) if not isinstance(item, (Link, Comment)): return subreddit = item.subreddit_slow wiki_page_id = wiki_id(subreddit._id36, "config/automoderator") wiki_page_fullname = "WikiPage_%s" % wiki_page_id last_edited = LastModified.get(wiki_page_fullname, "Edit") if not last_edited: return # initialize rules for the subreddit if we haven't already # or if the page has been edited since we last initialized need_to_init = False if subreddit._id not in rules_by_subreddit: need_to_init = True else: rules = rules_by_subreddit[subreddit._id] if last_edited > rules.init_time: need_to_init = True if need_to_init: wp = WikiPage.get(subreddit, "config/automoderator") rules = Ruleset(wp.content) rules_by_subreddit[subreddit._id] = rules if not rules: return try: TimeoutFunction(rules.apply_to_item, 2)(item) print "Checked %s from /r/%s" % (item, subreddit.name) except TimeoutFunctionException: print "Timed out on %s from /r/%s" % (item, subreddit.name) except KeyboardInterrupt: raise except: print "Error on %s from /r/%s" % (item, subreddit.name) print traceback.format_exc()
def update_trending_subreddits(): try: trending_sr = Subreddit._by_name(g.config['trending_sr']) except NotFound: g.log.info("Unknown trending subreddit %r or trending_sr config " "not set. Not updating.", g.config['trending_sr']) return iq = iter(trending_sr.get_links('new', 'all')) link = Thing._by_fullname(next(iq), data=True) subreddit_names = _SUBREDDIT_RE.findall(link.title) trending_data = { 'subreddit_names': subreddit_names, 'permalink': link.make_permalink(trending_sr), 'link_id': link._id, } NamedGlobals.set(TRENDING_SUBREDDITS_KEY, trending_data) g.log.debug("Trending subreddit data set to %r", trending_data)
def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id): admintools.engolden(recipient, days) if comment_id: comment = Thing._by_fullname(comment_id, data=True) comment._gild(buyer) else: comment = None if signed: sender = buyer.name md_sender = "[%s](/user/%s)" % (sender, sender) else: sender = "someone" md_sender = "An anonymous lightnet user" create_gift_gold(buyer._id, recipient._id, days, c.start_time, signed) if months == 1: amount = "a month" else: amount = "%d months" % months if not comment: message = strings.youve_got_gold % dict(sender=md_sender, amount=amount) if giftmessage and giftmessage.strip(): message += "\n\n" + strings.giftgold_note + giftmessage + "\n\n----" else: message = strings.youve_got_comment_gold % dict(url=comment.make_permalink_slow()) message += "\n\n" + strings.gold_benefits_msg message += "\n\n" + strings.lounge_msg % {"link": "/space/" + g.lounge_reddit} subject = sender + " just sent you reddit gold!" try: send_system_message(recipient, subject, message) except MessageError: g.log.error("send_gift: could not send system message") g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name)) return comment
def test_get_files(self): from r2.lib.db import queries from r2.models import Subreddit, Account, Link, Thing account = Account._byID(1, data=True) sr = Subreddit._by_name("reddit_test0") link_url = self.make_unique_url() new_link = Link._submit("test_get_files", link_url, account, sr, "127.0.0.1", kind="file") queries.new_link(new_link, foreground=True) # make sure it returns like a normal link res = Thing._by_fullname(queries.get_links(sr, "new", "all"), return_dict=False) self.assert_true(len(res) > 0, "no links returned") self.assert_equal(new_link._id, res[0]._id) # should return with a kind = 'file' filter res = list(queries.get_files(sr)) self.assert_true(len(res) > 0, "no links returned") self.assert_equal(new_link._id, res[0]._id)
def send_gift(buyer, recipient, months, days, signed, giftmessage, comment_id): admintools.engolden(recipient, days) if comment_id: comment = Thing._by_fullname(comment_id, data=True) comment._gild(buyer) else: comment = None if signed: sender = buyer.name md_sender = "[%s](/user/%s)" % (sender, sender) else: sender = "someone" md_sender = "An anonymous redditor" create_gift_gold (buyer._id, recipient._id, days, c.start_time, signed) if months == 1: amount = "a month" else: amount = "%d months" % months if not comment: message = strings.youve_got_gold % dict(sender=md_sender, amount=amount) if giftmessage and giftmessage.strip(): message += "\n\n" + strings.giftgold_note + giftmessage else: message = strings.youve_got_comment_gold % dict( url=comment.make_permalink_slow(), ) subject = sender + " just sent you reddit gold!" send_system_message(recipient, subject, message) g.log.info("%s gifted %s to %s" % (buyer.name, amount, recipient.name)) return comment
def _run_changed(msgs, chan): '''Consume the cloudsearch_changes queue, and print reporting information on how long it took and how many remain ''' start = datetime.now(g.tz) changed = [pickle.loads(msg.body) for msg in msgs] # Only handle links to start with fullnames = _desired_things(changed, (Link, )) things = Thing._by_fullname(fullnames, data=True, return_dict=False) cloudsearch_time = inject(things) totaltime = (datetime.now(g.tz) - start).total_seconds() print( "%s: %d messages in %.2fs seconds (%.2fs secs waiting on " "cloudsearch); %d duplicates, %s remaining)" % (start, len(changed), totaltime, cloudsearch_time, len(changed) - len(things), msgs[-1].delivery_info.get('message_count', 'unknown')))
def _run_changed(msgs, chan): '''Consume the cloudsearch_changes queue, and print reporting information on how long it took and how many remain ''' start = datetime.now(g.tz) changed = [pickle.loads(msg.body) for msg in msgs] # Only handle links to start with fullnames = _desired_things(changed, (Link,)) things = Thing._by_fullname(fullnames, data=True, return_dict=False) cloudsearch_time = inject(things) totaltime = (datetime.now(g.tz) - start).total_seconds() print ("%s: %d messages in %.2fs seconds (%.2fs secs waiting on " "cloudsearch); %d duplicates, %s remaining)" % (start, len(changed), totaltime, cloudsearch_time, len(changed) - len(things), msgs[-1].delivery_info.get('message_count', 'unknown')))
import datetime from pylons import g from r2.lib.db.queries import get_gilded_user_comments from r2.lib.utils import Storage from r2.models import GildingsByDay, Thing, Comment from r2.models.query_cache import CachedQueryMutator date = datetime.datetime.now(g.tz) earliest_date = datetime.datetime(2012, 10, 01, tzinfo=g.tz) already_seen = set() with CachedQueryMutator() as m: while date > earliest_date: gildings = GildingsByDay.get_gildings(date) fullnames = [x["thing"] for x in gildings] things = Thing._by_fullname(fullnames, data=True, return_dict=False) comments = {t._fullname: t for t in things if isinstance(t, Comment)} for gilding in gildings: fullname = gilding["thing"] if fullname in comments and fullname not in already_seen: thing = gilding["thing"] = comments[fullname] gilding_object = Storage(gilding) m.insert(get_gilded_user_comments(thing.author_id), [gilding_object]) already_seen.add(fullname) date -= datetime.timedelta(days=1)
def get_things(codes): """Fetch relevant things for a list of ad codenames in batch.""" fullnames = [AdvertTrafficSummary.split_codename(code)[0] for code in codes if code.startswith(Thing._type_prefix)] return Thing._by_fullname(fullnames, data=True, return_dict=True)
def _get_modmail_userinfo(self, conversation, sr=None): if conversation.is_internal: raise ValueError('Cannot get userinfo for internal conversations') if not sr: sr = Subreddit._by_fullname(conversation.owner_fullname) # Retrieve the participant associated with the conversation try: account = conversation.get_participant_account() if not account: raise ValueError('No account associated with convo') permatimeout = (account.in_timeout and account.days_remaining_in_timeout == 0) if account._deleted or permatimeout: raise ValueError('User info is inaccessible') except NotFound: raise NotFound('Unable to retrieve conversation participant') # Fetch the mute and ban status of the participant as it relates # to the subreddit associated with the conversation. mute_status = sr.is_muted(account) ban_status = sr.is_banned(account) # Parse the ban status and retrieve the length of the ban, # then output the data into a serialiazable dict ban_result = { 'isBanned': bool(ban_status), 'reason': '', 'endDate': None, 'isPermanent': False } if ban_status: ban_result['reason'] = getattr(ban_status, 'note', '') ban_duration = sr.get_tempbans('banned', account.name) ban_duration = ban_duration.get(account.name) if ban_duration: ban_result['endDate'] = ban_duration.isoformat() else: ban_result['isPermanent'] = True ban_result['endDate'] = None # Parse the mute status and retrieve the length of the ban, # then output the data into the serialiazable dict mute_result = { 'isMuted': bool(mute_status), 'endDate': None, 'reason': '' } if mute_status: mute_result['reason'] = getattr(mute_status, 'note', '') muted_items = sr.get_muted_items(account.name) mute_duration = muted_items.get(account.name) if mute_duration: mute_result['endDate'] = mute_duration.isoformat() # Retrieve the participants post and comment fullnames from cache post_fullnames = [] comment_fullnames = [] if not account._spam: post_fullnames = list(queries.get_submitted(account, 'new', 'all'))[:100] comment_fullnames = list( queries.get_comments(account, 'new', 'all'))[:100] # Retrieve the associated link objects for posts and comments # using the retrieve fullnames, afer the link objects are retrieved # create a serializable dict with the the necessary information from # the endpoint. lookup_fullnames = list(set(post_fullnames) | set(comment_fullnames)) posts = Thing._by_fullname(lookup_fullnames) serializable_posts = {} for fullname in post_fullnames: if len(serializable_posts) == 3: break post = posts[fullname] if post.sr_id == sr._id and not post._deleted: serializable_posts[fullname] = { 'title': post.title, 'permalink': post.make_permalink(sr, force_domain=True), 'date': post._date.isoformat(), } # Extract the users most recent comments associated with the # subreddit sr_comments = [] for fullname in comment_fullnames: if len(sr_comments) == 3: break comment = posts[fullname] if comment.sr_id == sr._id and not comment._deleted: sr_comments.append(comment) # Retrieve all associated link objects (combines lookup) comment_links = Link._byID( [sr_comment.link_id for sr_comment in sr_comments]) # Serialize all of the user's sr comments serializable_comments = {} for sr_comment in sr_comments: comment_link = comment_links[sr_comment.link_id] comment_body = sr_comment.body if len(comment_body) > 140: comment_body = '{:.140}...'.format(comment_body) serializable_comments[sr_comment._fullname] = { 'title': comment_link.title, 'comment': comment_body, 'permalink': sr_comment.make_permalink(comment_link, sr, force_domain=True), 'date': sr_comment._date.isoformat(), } return { 'id': account._fullname, 'name': account.name, 'created': account._date.isoformat(), 'banStatus': ban_result, 'isShadowBanned': account._spam, 'muteStatus': mute_result, 'recentComments': serializable_comments, 'recentPosts': serializable_posts, }
def get_things(codes): fullnames = [ code for code in codes if code.startswith(Thing._type_prefix) ] return Thing._by_fullname(fullnames, data=True, return_dict=True)
def get_things(codes): fullnames = [code for code in codes if code.startswith(Thing._type_prefix)] return Thing._by_fullname(fullnames, data=True, return_dict=True)
def thing_lookup(self, names): return Thing._by_fullname(names, data=True, return_dict=False, stale=self.stale)
from pylons import app_globals as g from r2.lib.db.queries import get_gilded_user_comments from r2.lib.utils import Storage from r2.models import GildingsByDay, Thing, Comment from r2.models.query_cache import CachedQueryMutator date = datetime.datetime.now(g.tz) earliest_date = datetime.datetime(2012, 10, 01, tzinfo=g.tz) already_seen = set() with CachedQueryMutator() as m: while date > earliest_date: gildings = GildingsByDay.get_gildings(date) fullnames = [x["thing"] for x in gildings] things = Thing._by_fullname(fullnames, data=True, return_dict=False) comments = {t._fullname: t for t in things if isinstance(t, Comment)} for gilding in gildings: fullname = gilding["thing"] if fullname in comments and fullname not in already_seen: thing = gilding["thing"] = comments[fullname] gilding_object = Storage(gilding) m.insert(get_gilded_user_comments(thing.author_id), [gilding_object]) already_seen.add(fullname) date -= datetime.timedelta(days=1)
def process_message(msg): fname = msg.body item = Thing._by_fullname(fname, data=True) monitor_mentions(item)
def timeout_forbidden_event(self, action_name, details_text, target=None, target_fullname=None, subreddit=None, request=None, context=None): """Create a timeout-related 'forbidden_actions' for event-collector. action_name: the action taken by a user in timeout details_text: this provides more details about the action target: The intended item the action was to be taken on target_fullname: The fullname used to convert to a target subreddit: The Subreddit the action was taken in. If target is of the type Subreddit, then this won't be passed in request, context: Should be pylons.request & pylons.c respectively; """ if not action_name: request_vars = request.environ["pylons.routes_dict"] action_name = request_vars.get('action_name') # type of vote if action_name == "vote": direction = int(request.POST.get("dir", 0)) if direction == 1: action_name = "upvote" elif direction == -1: action_name = "downvote" else: action_name = "clearvote" # set or unset for contest mode and subreddit sticky elif action_name in ("set_contest_mode", "set_subreddit_sticky"): action_name = action_name.replace("_", "") if request.POST.get('state') == "False": action_name = "un" + action_name # set or unset for suggested sort elif action_name == "set_suggested_sort": action_name = action_name.replace("_", "") if request.POST.get("sort") in ("", "clear"): action_name = "un" + action_name # action for viewing /about/reports, /about/spam, /about/modqueue elif action_name == "spamlisting": action_name = "pageview" details_text = request_vars.get("location") elif action_name == "clearflairtemplates": action_name = "editflair" details_text = "flair_clear_template" elif action_name in ("flairconfig", "flaircsv", "flairlisting"): details_text = action_name.replace("flair", "flair_") action_name = "editflair" if not target: if not target_fullname: if action_name in ("wiki_settings", "wiki_edit"): target = context.site elif action_name in ("wiki_allow_editor"): target = Account._by_name(request.POST.get("username")) elif action_name in ("delete_sr_header", "delete_sr_icon", "delete_sr_banner"): details_text = "%s" % action_name.replace("ete_sr", "") action_name = "editsettings" target = context.site elif action_name in ("bannedlisting", "mutedlisting", "wikibannedlisting", "wikicontributorslisting"): target = context.site if target_fullname: from r2.models import Thing target = Thing._by_fullname( target_fullname, return_dict=False, data=True, ) event = Event( topic="forbidden_actions", event_type="ss.forbidden_%s" % action_name, request=request, context=context, ) event.add("details_text", details_text) event.add("process_notes", "IN_TIMEOUT") from r2.models import Comment, Link, Subreddit if not subreddit: if isinstance(context.site, Subreddit): subreddit = context.site elif isinstance(target, (Comment, Link)): subreddit = target.subreddit_slow elif isinstance(target, Subreddit): subreddit = target event.add_subreddit_fields(subreddit) event.add_target_fields(target) self.save_event(event)
def get_things(codes): fullnames = [ AdvertTrafficSummary.split_codename(code)[0] for code in codes if code.startswith(Thing._type_prefix) ] return Thing._by_fullname(fullnames, data=True, return_dict=True)