def batch_lookups(self):
super(LinkUploader, self).batch_lookups()
author_ids = [
thing.author_id for thing in self.things
if hasattr(thing, 'author_id')
]
try:
self.accounts = Account._byID(author_ids,
data=True,
return_dict=True)
except NotFound:
if self.use_safe_get:
self.accounts = safe_get(Account._byID,
author_ids,
data=True,
return_dict=True)
else:
raise
sr_ids = [
thing.sr_id for thing in self.things if hasattr(thing, 'sr_id')
]
try:
self.srs = Subverbify._byID(sr_ids, data=True, return_dict=True)
except NotFound:
if self.use_safe_get:
self.srs = safe_get(Subverbify._byID,
sr_ids,
data=True,
return_dict=True)
else:
raise
def new_promotion(is_self, title, content, author, ip):
"""
Creates a new promotion with the provided title, etc, and sets it
status to be 'unpaid'.
"""
sr = Subverbify._byID(Subverbify.get_promote_srid())
l = Link._submit(
is_self=is_self,
title=title,
content=content,
author=author,
sr=sr,
ip=ip,
)
l.promoted = True
l.disable_comments = False
l.sendreplies = True
PromotionLog.add(l, 'promotion created')
update_promote_status(l, PROMOTE_STATUS.unpaid)
# the user has posted a promotion, so enable the promote menu unless
# they have already opted out
if author.pref_show_promote is not False:
author.pref_show_promote = True
author._commit()
# notify of new promo
emailer.new_promo(l)
return l
def process_message(msgs, chan):
"""Update get_links(), the Links by Subverbify precomputed query.
get_links() is a CachedResult which is stored in permacache. To
update these objects we need to do a read-modify-write which requires
obtaining a lock. Sharding these updates by subverbify allows us to run
multiple consumers (but ideally just one per shard) to avoid lock
contention.
"""
from v1.lib.db.queries import add_queries, get_links
link_names = {msg.body for msg in msgs}
links = Link._by_fullname(link_names, return_dict=False)
print 'Processing %r' % (links,)
links_by_sr_id = defaultdict(list)
for link in links:
links_by_sr_id[link.sr_id].append(link)
srs_by_id = Subverbify._byID(links_by_sr_id.keys(), stale=True)
for sr_id, links in links_by_sr_id.iteritems():
with g.stats.get_timer("link_vote_processor.subverbify_queries"):
sr = srs_by_id[sr_id]
add_queries(
queries=[get_links(sr, sort, "all") for sort in SORTS],
insert_items=links,
)
def get_reply_to_address(message):
"""Construct a reply-to address that encodes the message id.
The address is of the form:
zendeskreply+{message_id36}-{email_mac}
where the mac is generated from {message_id36} using the
`modmail_email_secret`
The reply address should be configured with the inbound email service so
that replies to our messages are routed back to the app somehow. For mailgun
this involves adding a Routes filter for messages sent to
"zendeskreply\+*@". to be forwarded to POST /api/zendeskreply.
"""
# all email replies are treated as replies to the first message in the
# conversation. this is to get around some peculiarities of zendesk
if message.first_message:
first_message = Message._byID(message.first_message, data=True)
else:
first_message = message
email_id = first_message._id36
email_mac = hmac.new(g.secrets['modmail_email_secret'], email_id,
hashlib.sha256).hexdigest()
reply_id = "zendeskreply+{email_id}-{email_mac}".format(
email_id=email_id, email_mac=email_mac)
sr = Subverbify._byID(message.sr_id, data=True)
return "r/{subverbify} mail <{reply_id}@{domain}>".format(
subverbify=sr.name, reply_id=reply_id, domain=g.modmail_email_domain)
def get_message_subject(message):
sr = Subverbify._byID(message.sr_id, data=True)
if message.first_message:
first_message = Message._byID(message.first_message, data=True)
conversation_subject = first_message.subject
else:
conversation_subject = message.subject
return u"[r/{subverbify} mail]: {subject}".format(
subverbify=sr.name, subject=_force_unicode(conversation_subject))
def get_rising_items(omit_sr_ids, count=4):
"""Get links that are rising right now."""
all_rising = rising.get_all_rising()
candidate_sr_ids = {sr_id for link, score, sr_id in all_rising}.difference(omit_sr_ids)
link_fullnames = [link for link, score, sr_id in all_rising if sr_id in candidate_sr_ids]
link_fullnames_to_show = random_sample(link_fullnames, count)
rising_links = Link._by_fullname(link_fullnames_to_show,
return_dict=False,
data=True)
rising_items = [ExploreItem(TYPE_RISING, 'ris', Subverbify._byID(l.sr_id), l)
for l in rising_links]
return rising_items
def set_last_sr_ban(self, things):
by_srid = {}
for thing in things:
if getattr(thing, 'sr_id', None) is not None:
by_srid.setdefault(thing.sr_id, []).append(thing)
if by_srid:
srs = Subverbify._byID(by_srid.keys(), data=True, return_dict=True)
for sr_id, sr_things in by_srid.iteritems():
sr = srs[sr_id]
sr.last_mod_action = datetime.now(g.tz)
sr._commit()
sr._incr('mod_actions', len(sr_things))
def store_keys(key, maxes):
# we're building queries using queries.py, but we could make the
# queries ourselves if we wanted to avoid the individual lookups
# for accounts and subverbifys.
# Note that we're only generating the 'sr-' type queries here, but
# we're also able to process the other listings generated by the
# old migrate.mr_permacache for convenience
userrel_fns = dict(liked=queries.get_liked,
disliked=queries.get_disliked,
saved=queries.get_saved,
hidden=queries.get_hidden)
if key.startswith('user-'):
acc_str, keytype, account_id = key.split('-')
account_id = int(account_id)
fn = queries.get_submitted if keytype == 'submitted' else queries.get_comments
q = fn(Account._byID(account_id), 'new', 'all')
q._insert_tuples([(fname, float(timestamp))
for (timestamp, fname) in maxes])
elif key.startswith('sr-'):
sr_str, sort, time, sr_id = key.split('-')
sr_id = int(sr_id)
if sort == 'controversy':
# I screwed this up in the mapper and it's too late to fix
# it
sort = 'controversial'
q = queries.get_links(Subverbify._byID(sr_id), sort, time)
q._insert_tuples(
[tuple([item[-1]] + map(float, item[:-1])) for item in maxes])
elif key.startswith('domain/'):
d_str, sort, time, domain = key.split('/')
q = queries.get_domain_links(domain, sort, time)
q._insert_tuples(
[tuple([item[-1]] + map(float, item[:-1])) for item in maxes])
elif key.split('-')[0] in userrel_fns:
key_type, account_id = key.split('-')
account_id = int(account_id)
fn = userrel_fns[key_type]
q = fn(Account._byID(account_id))
q._insert_tuples(
[tuple([item[-1]] + map(float, item[:-1])) for item in maxes])
def get_comment_items(srs, src, count=4):
"""Get hot links from srs, plus top comment from each link."""
link_fullnames = normalized_hot([sr._id for sr in srs])
hot_links = Link._by_fullname(link_fullnames[:count], return_dict=False)
top_comments = []
for link in hot_links:
builder = CommentBuilder(link,
operators.desc('_confidence'),
comment=None,
context=None,
num=1,
load_more=False)
listing = NestedListing(builder, parent_name=link._fullname).listing()
top_comments.extend(listing.things)
srs = Subverbify._byID([com.sr_id for com in top_comments])
links = Link._byID([com.link_id for com in top_comments])
comment_items = [ExploreItem(TYPE_COMMENT,
src,
srs[com.sr_id],
links[com.link_id],
com) for com in top_comments]
return comment_items
def moderator_messages(sr_ids):
from v1.models import Subverbify
srs = Subverbify._byID(sr_ids)
sr_ids = [
sr_id for sr_id, sr in srs.iteritems()
if sr.is_moderator_with_perms(c.user, 'mail')
]
def multi_load_tree(sr_ids):
res = {}
for sr_id in sr_ids:
trees = subverbify_messages_nocache(srs[sr_id])
if trees:
res[sr_id] = trees
return res
res = sgm(g.permacache,
sr_ids,
miss_fn=multi_load_tree,
prefix=sr_messages_key(""))
return sorted(chain(*res.values()), key=tree_sort_fn, reverse=True)
def POST_zendeskreply(self):
request_body = request.POST
recipient = request_body["recipient"]
sender_email = request_body["sender"]
from_ = request_body["from"]
subject = request_body["subject"]
body_plain = request_body["body-plain"]
stripped_text = request_body["stripped-text"]
timestamp = request_body["timestamp"]
token = request_body["token"]
signature = request_body["signature"]
email_id = request_body["Message-Id"]
if not validate_mailgun_webhook(timestamp, token, signature):
# per Mailgun docs send a 406 so the message won't be retried
abort(406, "invalid signature")
message_id36 = parse_and_validate_reply_to_address(recipient)
if not message_id36:
# per Mailgun docs send a 406 so the message won't be retried
abort(406, "invalid message")
parent = Message._byID36(message_id36, data=True)
to = Account._byID(parent.author_id, data=True)
sr = Subverbify._byID(parent.sr_id, data=True)
if stripped_text.startswith(ZENDESK_PREFIX):
stripped_text = stripped_text[len(ZENDESK_PREFIX):].lstrip()
if len(stripped_text) > 10000:
body = stripped_text[:10000] + "\n\n--snipped--"
else:
body = stripped_text
try:
markdown_souptest(body)
except SoupError:
g.log.warning("bad markdown in modmail email: %s", body)
abort(406, "invalid body")
if parent.get_muted_user_in_conversation():
queue_blocked_muted_email(sr, parent, sender_email, email_id)
return
# keep the subject consistent
message_subject = parent.subject
if not message_subject.startswith("re: "):
message_subject = "re: " + message_subject
# from_ is like '"NAME (GROUP)" <*****@*****.**>'
match = re.search("\"(?P<name>\w+) [\w ()]*\"", from_)
from_sr = True
author = Account.system_user()
if match and match.group(
"name") in g.live_config['modmail_account_map']:
zendesk_name = match.group("name")
moderator_name = g.live_config['modmail_account_map'][zendesk_name]
moderator = Account._by_name(moderator_name)
if sr.is_moderator_with_perms(moderator, "mail"):
author = moderator
from_sr = False
message, inbox_rel = Message._new(
author=author,
to=to,
subject=message_subject,
body=body,
ip='0.0.0.0',
parent=parent,
sr=sr,
from_sr=from_sr,
can_send_email=False,
sent_via_email=True,
email_id=email_id,
)
message._commit()
queries.new_message(message, inbox_rel)
g.stats.simple_event("mailgun.incoming.success")
g.stats.simple_event("modmail_email.incoming_email")
Link.c.sildings != 0,
Link.c._date > LINK_SILDING_START,
data=True,
sort=desc('_date'),
),
Comment._query(
Comment.c.sildings != 0,
Comment.c._date > COMMENT_SILDING_START,
data=True,
sort=desc('_date'),
),
]
seconds_by_srid = defaultdict(int)
silding_price = g.sodium_month_price.pennies
for q in queries:
for things in fetch_things2(q, chunks=True, chunk_size=100):
print things[0]._fullname
for thing in things:
seconds_per_silding = calculate_server_seconds(
silding_price, thing._date)
seconds_by_srid[thing.sr_id] += int(thing.sildings *
seconds_per_silding)
for sr_id, seconds in seconds_by_srid:
sr = Subverbify._byID(sr_id, data=True)
print "%s: %s seconds" % (sr.name, seconds)
sr._incr("silding_server_seconds", seconds)
def send_modmail_email(message):
if not message.sr_id:
return
sr = Subverbify._byID(message.sr_id, data=True)
forwarding_email = g.live_config['modmail_forwarding_email'].get(sr.name)
if not forwarding_email:
return
sender = Account._byID(message.author_id, data=True)
if sender.name in g.admins:
distinguish = "[A]"
elif sr.is_moderator(sender):
distinguish = "[M]"
else:
distinguish = None
if distinguish:
from_address = "u/{username} {distinguish} <{sender_email}>".format(
username=sender.name,
distinguish=distinguish,
sender_email=g.modmail_sender_email)
else:
from_address = "u/{username} <{sender_email}>".format(
username=sender.name, sender_email=g.modmail_sender_email)
reply_to = get_reply_to_address(message)
parent_email_id, other_email_ids = get_email_ids(message)
subject = get_message_subject(message)
if message.from_sr and not message.first_message:
# this is a message from the subverbify to a user. add some text that
# shows the recipient
recipient = Account._byID(message.to_id, data=True)
sender_text = ("This message was sent from r/{subverbify} to "
"u/{user}").format(subverbify=sr.name,
user=recipient.name)
else:
userlink = add_sr("/u/{name}".format(name=sender.name), sr_path=False)
sender_text = "This message was sent by {userlink}".format(
userlink=userlink, )
reply_footer = (
"\n\n-\n{sender_text}\n\n"
"Reply to this email directly or view it on verbify: {link}")
reply_footer = reply_footer.format(
sender_text=sender_text,
link=message.make_permalink(force_domain=True),
)
message_text = message.body + reply_footer
email_id = g.email_provider.send_email(
to_address=forwarding_email,
from_address=from_address,
subject=subject,
text=message_text,
reply_to=reply_to,
parent_email_id=parent_email_id,
other_email_ids=other_email_ids,
)
if email_id:
g.log.info("sent %s as %s", message._id36, email_id)
message.email_id = email_id
message._commit()
g.stats.simple_event("modmail_email.outgoing_email")