def batch_lookups(self): super(LinkUploader, self).batch_lookups() author_ids = [ thing.author_id for thing in self.things if hasattr(thing, 'author_id') ] try: self.accounts = Account._byID(author_ids, data=True, return_dict=True) except NotFound: if self.use_safe_get: self.accounts = safe_get(Account._byID, author_ids, data=True, return_dict=True) else: raise sr_ids = [ thing.sr_id for thing in self.things if hasattr(thing, 'sr_id') ] try: self.srs = Subverbify._byID(sr_ids, data=True, return_dict=True) except NotFound: if self.use_safe_get: self.srs = safe_get(Subverbify._byID, sr_ids, data=True, return_dict=True) else: raise
def new_promotion(is_self, title, content, author, ip): """ Creates a new promotion with the provided title, etc, and sets it status to be 'unpaid'. """ sr = Subverbify._byID(Subverbify.get_promote_srid()) l = Link._submit( is_self=is_self, title=title, content=content, author=author, sr=sr, ip=ip, ) l.promoted = True l.disable_comments = False l.sendreplies = True PromotionLog.add(l, 'promotion created') update_promote_status(l, PROMOTE_STATUS.unpaid) # the user has posted a promotion, so enable the promote menu unless # they have already opted out if author.pref_show_promote is not False: author.pref_show_promote = True author._commit() # notify of new promo emailer.new_promo(l) return l
def process_message(msgs, chan): """Update get_links(), the Links by Subverbify precomputed query. get_links() is a CachedResult which is stored in permacache. To update these objects we need to do a read-modify-write which requires obtaining a lock. Sharding these updates by subverbify allows us to run multiple consumers (but ideally just one per shard) to avoid lock contention. """ from v1.lib.db.queries import add_queries, get_links link_names = {msg.body for msg in msgs} links = Link._by_fullname(link_names, return_dict=False) print 'Processing %r' % (links,) links_by_sr_id = defaultdict(list) for link in links: links_by_sr_id[link.sr_id].append(link) srs_by_id = Subverbify._byID(links_by_sr_id.keys(), stale=True) for sr_id, links in links_by_sr_id.iteritems(): with g.stats.get_timer("link_vote_processor.subverbify_queries"): sr = srs_by_id[sr_id] add_queries( queries=[get_links(sr, sort, "all") for sort in SORTS], insert_items=links, )
def get_reply_to_address(message): """Construct a reply-to address that encodes the message id. The address is of the form: zendeskreply+{message_id36}-{email_mac} where the mac is generated from {message_id36} using the `modmail_email_secret` The reply address should be configured with the inbound email service so that replies to our messages are routed back to the app somehow. For mailgun this involves adding a Routes filter for messages sent to "zendeskreply\+*@". to be forwarded to POST /api/zendeskreply. """ # all email replies are treated as replies to the first message in the # conversation. this is to get around some peculiarities of zendesk if message.first_message: first_message = Message._byID(message.first_message, data=True) else: first_message = message email_id = first_message._id36 email_mac = hmac.new(g.secrets['modmail_email_secret'], email_id, hashlib.sha256).hexdigest() reply_id = "zendeskreply+{email_id}-{email_mac}".format( email_id=email_id, email_mac=email_mac) sr = Subverbify._byID(message.sr_id, data=True) return "r/{subverbify} mail <{reply_id}@{domain}>".format( subverbify=sr.name, reply_id=reply_id, domain=g.modmail_email_domain)
def get_message_subject(message): sr = Subverbify._byID(message.sr_id, data=True) if message.first_message: first_message = Message._byID(message.first_message, data=True) conversation_subject = first_message.subject else: conversation_subject = message.subject return u"[r/{subverbify} mail]: {subject}".format( subverbify=sr.name, subject=_force_unicode(conversation_subject))
def get_rising_items(omit_sr_ids, count=4): """Get links that are rising right now.""" all_rising = rising.get_all_rising() candidate_sr_ids = {sr_id for link, score, sr_id in all_rising}.difference(omit_sr_ids) link_fullnames = [link for link, score, sr_id in all_rising if sr_id in candidate_sr_ids] link_fullnames_to_show = random_sample(link_fullnames, count) rising_links = Link._by_fullname(link_fullnames_to_show, return_dict=False, data=True) rising_items = [ExploreItem(TYPE_RISING, 'ris', Subverbify._byID(l.sr_id), l) for l in rising_links] return rising_items
def set_last_sr_ban(self, things): by_srid = {} for thing in things: if getattr(thing, 'sr_id', None) is not None: by_srid.setdefault(thing.sr_id, []).append(thing) if by_srid: srs = Subverbify._byID(by_srid.keys(), data=True, return_dict=True) for sr_id, sr_things in by_srid.iteritems(): sr = srs[sr_id] sr.last_mod_action = datetime.now(g.tz) sr._commit() sr._incr('mod_actions', len(sr_things))
def store_keys(key, maxes): # we're building queries using queries.py, but we could make the # queries ourselves if we wanted to avoid the individual lookups # for accounts and subverbifys. # Note that we're only generating the 'sr-' type queries here, but # we're also able to process the other listings generated by the # old migrate.mr_permacache for convenience userrel_fns = dict(liked=queries.get_liked, disliked=queries.get_disliked, saved=queries.get_saved, hidden=queries.get_hidden) if key.startswith('user-'): acc_str, keytype, account_id = key.split('-') account_id = int(account_id) fn = queries.get_submitted if keytype == 'submitted' else queries.get_comments q = fn(Account._byID(account_id), 'new', 'all') q._insert_tuples([(fname, float(timestamp)) for (timestamp, fname) in maxes]) elif key.startswith('sr-'): sr_str, sort, time, sr_id = key.split('-') sr_id = int(sr_id) if sort == 'controversy': # I screwed this up in the mapper and it's too late to fix # it sort = 'controversial' q = queries.get_links(Subverbify._byID(sr_id), sort, time) q._insert_tuples( [tuple([item[-1]] + map(float, item[:-1])) for item in maxes]) elif key.startswith('domain/'): d_str, sort, time, domain = key.split('/') q = queries.get_domain_links(domain, sort, time) q._insert_tuples( [tuple([item[-1]] + map(float, item[:-1])) for item in maxes]) elif key.split('-')[0] in userrel_fns: key_type, account_id = key.split('-') account_id = int(account_id) fn = userrel_fns[key_type] q = fn(Account._byID(account_id)) q._insert_tuples( [tuple([item[-1]] + map(float, item[:-1])) for item in maxes])
def get_comment_items(srs, src, count=4): """Get hot links from srs, plus top comment from each link.""" link_fullnames = normalized_hot([sr._id for sr in srs]) hot_links = Link._by_fullname(link_fullnames[:count], return_dict=False) top_comments = [] for link in hot_links: builder = CommentBuilder(link, operators.desc('_confidence'), comment=None, context=None, num=1, load_more=False) listing = NestedListing(builder, parent_name=link._fullname).listing() top_comments.extend(listing.things) srs = Subverbify._byID([com.sr_id for com in top_comments]) links = Link._byID([com.link_id for com in top_comments]) comment_items = [ExploreItem(TYPE_COMMENT, src, srs[com.sr_id], links[com.link_id], com) for com in top_comments] return comment_items
def moderator_messages(sr_ids): from v1.models import Subverbify srs = Subverbify._byID(sr_ids) sr_ids = [ sr_id for sr_id, sr in srs.iteritems() if sr.is_moderator_with_perms(c.user, 'mail') ] def multi_load_tree(sr_ids): res = {} for sr_id in sr_ids: trees = subverbify_messages_nocache(srs[sr_id]) if trees: res[sr_id] = trees return res res = sgm(g.permacache, sr_ids, miss_fn=multi_load_tree, prefix=sr_messages_key("")) return sorted(chain(*res.values()), key=tree_sort_fn, reverse=True)
def POST_zendeskreply(self): request_body = request.POST recipient = request_body["recipient"] sender_email = request_body["sender"] from_ = request_body["from"] subject = request_body["subject"] body_plain = request_body["body-plain"] stripped_text = request_body["stripped-text"] timestamp = request_body["timestamp"] token = request_body["token"] signature = request_body["signature"] email_id = request_body["Message-Id"] if not validate_mailgun_webhook(timestamp, token, signature): # per Mailgun docs send a 406 so the message won't be retried abort(406, "invalid signature") message_id36 = parse_and_validate_reply_to_address(recipient) if not message_id36: # per Mailgun docs send a 406 so the message won't be retried abort(406, "invalid message") parent = Message._byID36(message_id36, data=True) to = Account._byID(parent.author_id, data=True) sr = Subverbify._byID(parent.sr_id, data=True) if stripped_text.startswith(ZENDESK_PREFIX): stripped_text = stripped_text[len(ZENDESK_PREFIX):].lstrip() if len(stripped_text) > 10000: body = stripped_text[:10000] + "\n\n--snipped--" else: body = stripped_text try: markdown_souptest(body) except SoupError: g.log.warning("bad markdown in modmail email: %s", body) abort(406, "invalid body") if parent.get_muted_user_in_conversation(): queue_blocked_muted_email(sr, parent, sender_email, email_id) return # keep the subject consistent message_subject = parent.subject if not message_subject.startswith("re: "): message_subject = "re: " + message_subject # from_ is like '"NAME (GROUP)" <*****@*****.**>' match = re.search("\"(?P<name>\w+) [\w ()]*\"", from_) from_sr = True author = Account.system_user() if match and match.group( "name") in g.live_config['modmail_account_map']: zendesk_name = match.group("name") moderator_name = g.live_config['modmail_account_map'][zendesk_name] moderator = Account._by_name(moderator_name) if sr.is_moderator_with_perms(moderator, "mail"): author = moderator from_sr = False message, inbox_rel = Message._new( author=author, to=to, subject=message_subject, body=body, ip='0.0.0.0', parent=parent, sr=sr, from_sr=from_sr, can_send_email=False, sent_via_email=True, email_id=email_id, ) message._commit() queries.new_message(message, inbox_rel) g.stats.simple_event("mailgun.incoming.success") g.stats.simple_event("modmail_email.incoming_email")
Link.c.sildings != 0, Link.c._date > LINK_SILDING_START, data=True, sort=desc('_date'), ), Comment._query( Comment.c.sildings != 0, Comment.c._date > COMMENT_SILDING_START, data=True, sort=desc('_date'), ), ] seconds_by_srid = defaultdict(int) silding_price = g.sodium_month_price.pennies for q in queries: for things in fetch_things2(q, chunks=True, chunk_size=100): print things[0]._fullname for thing in things: seconds_per_silding = calculate_server_seconds( silding_price, thing._date) seconds_by_srid[thing.sr_id] += int(thing.sildings * seconds_per_silding) for sr_id, seconds in seconds_by_srid: sr = Subverbify._byID(sr_id, data=True) print "%s: %s seconds" % (sr.name, seconds) sr._incr("silding_server_seconds", seconds)
def send_modmail_email(message): if not message.sr_id: return sr = Subverbify._byID(message.sr_id, data=True) forwarding_email = g.live_config['modmail_forwarding_email'].get(sr.name) if not forwarding_email: return sender = Account._byID(message.author_id, data=True) if sender.name in g.admins: distinguish = "[A]" elif sr.is_moderator(sender): distinguish = "[M]" else: distinguish = None if distinguish: from_address = "u/{username} {distinguish} <{sender_email}>".format( username=sender.name, distinguish=distinguish, sender_email=g.modmail_sender_email) else: from_address = "u/{username} <{sender_email}>".format( username=sender.name, sender_email=g.modmail_sender_email) reply_to = get_reply_to_address(message) parent_email_id, other_email_ids = get_email_ids(message) subject = get_message_subject(message) if message.from_sr and not message.first_message: # this is a message from the subverbify to a user. add some text that # shows the recipient recipient = Account._byID(message.to_id, data=True) sender_text = ("This message was sent from r/{subverbify} to " "u/{user}").format(subverbify=sr.name, user=recipient.name) else: userlink = add_sr("/u/{name}".format(name=sender.name), sr_path=False) sender_text = "This message was sent by {userlink}".format( userlink=userlink, ) reply_footer = ( "\n\n-\n{sender_text}\n\n" "Reply to this email directly or view it on verbify: {link}") reply_footer = reply_footer.format( sender_text=sender_text, link=message.make_permalink(force_domain=True), ) message_text = message.body + reply_footer email_id = g.email_provider.send_email( to_address=forwarding_email, from_address=from_address, subject=subject, text=message_text, reply_to=reply_to, parent_email_id=parent_email_id, other_email_ids=other_email_ids, ) if email_id: g.log.info("sent %s as %s", message._id36, email_id) message.email_id = email_id message._commit() g.stats.simple_event("modmail_email.outgoing_email")