def add_to_subverbify_query_q(link):
if g.shard_subverbify_query_queues:
subverbify_shard = link.sr_id % 10
queue_name = "subverbify_query_%s_q" % subverbify_shard
else:
queue_name = "subverbify_query_q"
amqp.add_item(queue_name, link._fullname)
def add_to_author_query_q(link):
if g.shard_author_query_queues:
author_shard = link.author_id % 10
queue_name = "author_query_%s_q" % author_shard
else:
queue_name = "author_query_q"
amqp.add_item(queue_name, link._fullname)
def queue_modmail_email(message):
amqp.add_item(
"modmail_email_q",
json.dumps({
"event": "new_message",
"message_id36": message._id36,
}),
)
def queue_blocked_muted_email(sr, parent, sender_email, incoming_email_id):
amqp.add_item(
"modmail_email_q",
json.dumps({
"event": "blocked_muted",
"subverbify_id36": sr._id36,
"parent_id36": parent._id36,
"sender_email": sender_email,
"incoming_email_id": incoming_email_id,
}),
)
def add_to_domain_query_q(link):
parsed = UrlParser(link.url)
if not parsed.domain_permutations():
# no valid domains found
return
if g.shard_domain_query_queues:
domain_shard = hash(parsed.hostname) % 10
queue_name = "domain_query_%s_q" % domain_shard
else:
queue_name = "domain_query_q"
amqp.add_item(queue_name, link._fullname)
def send_broadcast(namespace, type, payload):
"""Broadcast an object to all WebSocket listeners in a namespace.
The message type is used to differentiate between different kinds of
payloads that may be sent. The payload will be encoded as a JSON object
before being sent to the client.
"""
frame = {
"type": type,
"payload": payload,
}
amqp.add_item(routing_key=namespace,
body=json.dumps(frame),
exchange=_WEBSOCKET_EXCHANGE)
def _set_media(link, force=False, **kwargs):
sr = link.subverbify_slow
# Do not process thumbnails for quarantined subverbifys
if sr.quarantine:
return
if not link.is_self:
if not force and (link.has_thumbnail or link.media_object):
return
if not force and link.promoted:
return
scrape_url = _get_scrape_url(link)
if not scrape_url:
if link.preview_object:
# If the user edited out an image from a self post, we need to make
# sure to remove its metadata.
link.set_preview_object(None)
link._commit()
return
youtube_scraper = feature.is_enabled("youtube_scraper", subverbify=sr.name)
media = _scrape_media(scrape_url,
force=force,
use_youtube_scraper=youtube_scraper,
**kwargs)
if media and not link.promoted:
# While we want to add preview images to self posts for the new apps,
# let's not muck about with the old-style thumbnails in case that
# breaks assumptions.
if not link.is_self:
link.thumbnail_url = media.thumbnail_url
link.thumbnail_size = media.thumbnail_size
link.set_media_object(media.media_object)
link.set_secure_media_object(media.secure_media_object)
link.set_preview_object(media.preview_object)
link._commit()
hooks.get_hook("scraper.set_media").call(link=link)
if media.media_object or media.secure_media_object:
amqp.add_item("new_media_embed", link._fullname)
def cast_vote(user, thing, direction, **data):
"""Register a vote and queue it for processing."""
if not isinstance(thing, (Link, Comment)):
return
update_vote_lookups(user, thing, direction)
vote_data = {
"user_id": user._id,
"thing_fullname": thing._fullname,
"direction": direction,
"date": int(epoch_timestamp(datetime.now(g.tz))),
}
data['ip'] = getattr(request, "ip", None)
if data['ip'] is not None:
data['org'] = organization_by_ips(data['ip'])
vote_data['data'] = data
hooks.get_hook("vote.get_vote_data").call(
data=vote_data["data"],
user=user,
thing=thing,
request=request,
context=c,
)
# The vote event will actually be sent from an async queue processor, so
# we need to pull out the context data at this point
if not g.running_as_script:
vote_data["event_data"] = {
"context": Event.get_context_data(request, c),
"sensitive": Event.get_sensitive_context_data(request, c),
}
try:
vote_dump = json.dumps(vote_data)
except UnicodeDecodeError:
g.log.error("Got weird unicode in the vote data: %r", vote_data)
return
if isinstance(thing, Link):
queue = "vote_link_q"
elif isinstance(thing, Comment):
queue = "vote_comment_q"
amqp.add_item(queue, vote_dump)
def unspam(self,
things,
moderator_unbanned=True,
unbanner=None,
train_spam=True,
insert=True):
from v1.lib.db import queries
things = tup(things)
# We want to make unban-all moderately efficient, so when
# mass-unbanning, we're going to skip the code below on links that
# are already not banned. However, when someone manually clicks
# "approve" on an unbanned link, and there's just one, we want do
# want to run the code below. That way, the little green checkmark
# will have the right mouseover details, the reports will be
# cleared, etc.
if len(things) > 1:
things = [x for x in things if x._spam]
Report.accept(things, False)
for t in things:
ban_info = copy(getattr(t, 'ban_info', {}))
ban_info['unbanned_at'] = datetime.now(g.tz)
if unbanner:
ban_info['unbanner'] = unbanner
if ban_info.get('reset_used', None) == None:
ban_info['reset_used'] = False
else:
ban_info['reset_used'] = True
t.ban_info = ban_info
t._spam = False
if moderator_unbanned:
t.verdict = 'mod-approved'
else:
t.verdict = 'admin-approved'
t._commit()
if isinstance(t, Comment):
amqp.add_item("approved_comment", t._fullname)
elif isinstance(t, Link):
amqp.add_item("approved_link", t._fullname)
self.author_spammer(things, False)
self.set_last_sr_ban(things)
queries.unban(things, insert)
def backfill_deleted_accounts(resume_id=None):
del_accts = Account._query(Account.c._deleted == True, sort=desc('_date'))
if resume_id:
del_accts._filter(Account.c._id < resume_id)
for i, account in enumerate(progress(fetch_things2(del_accts))):
# Don't kill the rabbit! Wait for the relevant queues to calm down.
if i % 1000 == 0:
del_len = get_queue_length('del_account_q')
cs_len = get_queue_length('cloudsearch_changes')
while (del_len > 1000 or
cs_len > 10000):
sys.stderr.write(("CS: %d, DEL: %d" % (cs_len, del_len)) + "\n")
sys.stderr.flush()
time.sleep(1)
del_len = get_queue_length('del_account_q')
cs_len = get_queue_length('cloudsearch_changes')
amqp.add_item('account_deleted', account._fullname)
def delete(self, delete_message=None):
self.delete_message = delete_message
self.delete_time = datetime.now(g.tz)
self._deleted = True
self._commit()
#update caches
Account._by_name(self.name, allow_deleted=True, _update=True)
#we need to catch an exception here since it will have been
#recently deleted
try:
Account._by_name(self.name, _update=True)
except NotFound:
pass
# Mark this account for immediate cleanup tasks
amqp.add_item('account_deleted', self._fullname)
# schedule further cleanup after a possible recovery period
TryLater.schedule("account_deletion",
self._id36,
delay=timedelta(days=90))
def processor(msgs, chan):
events = []
test_events = []
for msg in msgs:
headers = msg.properties.get("application_headers", {})
truncatable_field = headers.get("truncatable_field")
event = PublishableEvent(msg.body, truncatable_field)
if msg.delivery_info["routing_key"] == "event_collector_test":
test_events.append(event)
else:
events.append(event)
to_publish = itertools.chain(
publisher.publish(events),
test_publisher.publish(test_events),
)
for response, sent in to_publish:
if response.ok:
g.log.info("Published %s events", len(sent))
else:
g.log.warning(
"Event send failed %s - %s",
response.status_code,
_get_reason(response),
)
g.log.warning("Response headers: %r", response.headers)
# if the events were too large, move them into a separate
# queue to get them out of here, since they'll always fail
if response.status_code == 413:
for event in sent:
amqp.add_item("event_collector_failed", event)
else:
response.raise_for_status()
def handle_register(controller,
form,
responder,
name,
email,
password,
rem=None,
newsletter_subscribe=False,
sponsor=False,
signature=None,
**kwargs):
def _event(error):
g.events.login_event('register_attempt',
error_msg=error,
user_name=request.urlvars.get('url_user'),
email=request.POST.get('email'),
remember_me=rem,
newsletter=newsletter_subscribe,
signature=signature,
request=request,
context=c)
if signature and not signature.is_valid():
_event(error="SIGNATURE")
abort(403)
if responder.has_errors('user', errors.USERNAME_TOO_SHORT):
_event(error='USERNAME_TOO_SHORT')
elif responder.has_errors('user', errors.USERNAME_INVALID_CHARACTERS):
_event(error='USERNAME_INVALID_CHARACTERS')
elif responder.has_errors('user', errors.USERNAME_TAKEN_DEL):
_event(error='USERNAME_TAKEN_DEL')
elif responder.has_errors('user', errors.USERNAME_TAKEN):
_event(error='USERNAME_TAKEN')
elif responder.has_errors('email', errors.BAD_EMAIL):
_event(error='BAD_EMAIL')
elif responder.has_errors('passwd', errors.SHORT_PASSWORD):
_event(error='SHORT_PASSWORD')
elif responder.has_errors('passwd', errors.BAD_PASSWORD):
# BAD_PASSWORD is set when SHORT_PASSWORD is set
_event(error='BAD_PASSWORD')
elif responder.has_errors('passwd2', errors.BAD_PASSWORD_MATCH):
_event(error='BAD_PASSWORD_MATCH')
elif responder.has_errors('ratelimit', errors.RATELIMIT):
_event(error='RATELIMIT')
elif (not g.disable_captcha
and responder.has_errors('captcha', errors.BAD_CAPTCHA)):
_event(error='BAD_CAPTCHA')
elif newsletter_subscribe and not email:
c.errors.add(errors.NEWSLETTER_NO_EMAIL, field="email")
form.has_errors("email", errors.NEWSLETTER_NO_EMAIL)
_event(error='NEWSLETTER_NO_EMAIL')
elif sponsor and not email:
c.errors.add(errors.SPONSOR_NO_EMAIL, field="email")
form.has_errors("email", errors.SPONSOR_NO_EMAIL)
_event(error='SPONSOR_NO_EMAIL')
else:
try:
user = register(name, password, request.ip)
except AccountExists:
c.errors.add(errors.USERNAME_TAKEN, field="user")
form.has_errors("user", errors.USERNAME_TAKEN)
_event(error='USERNAME_TAKEN')
return
VRatelimit.ratelimit(rate_ip=True, prefix="rate_register_")
# anything else we know (email, languages)?
if email:
user.set_email(email)
emailer.verify_email(user)
user.pref_lang = c.lang
user._commit()
amqp.add_item('new_account', user._fullname)
hooks.get_hook("account.registered").call(user=user)
reject = hooks.get_hook("account.spotcheck").call(account=user)
if any(reject):
_event(error='ACCOUNT_SPOTCHECK')
return
if newsletter_subscribe and email:
try:
newsletter.add_subscriber(email, source="register")
except newsletter.NewsletterError as e:
g.log.warning("Failed to subscribe: %r" % e)
controller._login(responder, user, rem)
_event(error=None)
def spam(self,
things,
auto=True,
moderator_banned=False,
banner=None,
date=None,
train_spam=True,
**kw):
from v1.lib.db import queries
all_things = tup(things)
new_things = [x for x in all_things if not x._spam]
Report.accept(all_things, True)
for t in all_things:
if getattr(t, "promoted", None) is not None:
g.log.debug("Refusing to mark promotion %r as spam" % t)
continue
if not t._spam and train_spam:
note = 'spam'
elif not t._spam and not train_spam:
note = 'remove not spam'
elif t._spam and not train_spam:
note = 'confirm spam'
elif t._spam and train_spam:
note = 'reinforce spam'
t._spam = True
if moderator_banned:
t.verdict = 'mod-removed'
elif not auto:
t.verdict = 'admin-removed'
ban_info = copy(getattr(t, 'ban_info', {}))
if isinstance(banner, dict):
ban_info['banner'] = banner[t._fullname]
else:
ban_info['banner'] = banner
ban_info.update(auto=auto,
moderator_banned=moderator_banned,
banned_at=date or datetime.now(g.tz),
**kw)
ban_info['note'] = note
t.ban_info = ban_info
t._commit()
if auto:
amqp.add_item("auto_removed", t._fullname)
if not auto:
self.author_spammer(new_things, True)
self.set_last_sr_ban(new_things)
queries.ban(all_things, filtered=auto)
for t in all_things:
if auto:
amqp.add_item("auto_removed", t._fullname)
if isinstance(t, Comment):
amqp.add_item("removed_comment", t._fullname)
elif isinstance(t, Link):
amqp.add_item("removed_link", t._fullname)