def click(self, ses_event: SesEvent) -> None:
# SES doesn't track the recipient that triggered this action, so process this
# only if the original email had a single recipient
if len(ses_event.mail.destination) == 1:
statsd.incr("email_address.ses_email.clicked")
email_address = self._email_address(ses_event.mail.destination[0])
email_address.mark_active()
def dispatch_notification(*notifications):
"""
Dispatches one or more notifications. Usage::
dispatch_notification(
MyNotification(document=doc, fragment=None),
MyOtherNotification(document=doc, fragment=frag)
)
This function performs a database commit to ensure notifications are available to
background jobs, so it must only be called when it's safe to commit.
"""
eventid = uuid4() # Create a single eventid
for notification in notifications:
if not isinstance(notification, Notification):
raise TypeError(f"Not a notification: {notification!r}")
if not notification.active:
raise TypeError(f"{notification!r} is marked inactive")
notification.eventid = eventid
notification.user = current_auth.user
if sum(_n.for_private_recipient for _n in notifications) not in (
0, # None are private
len(notifications), # Or all are private
):
raise TypeError(
"Mixed use of private and non-private notifications."
" Either all are private (no event tracking in links) or none are")
db.session.add_all(notifications)
db.session.commit()
dispatch_notification_job.queue(
eventid, [notification.id for notification in notifications])
for notification in notifications:
statsd.incr('notification.dispatch',
tags={'notification_type': notification.type})
def forget_email(email_hash):
with app.app_context():
email_address = EmailAddress.get(email_hash=email_hash)
if email_address.refcount() == 0:
app.logger.info("Forgetting email address with hash %s",
email_hash)
email_address.email = None
db.session.commit()
statsd.incr('email_address.forgotten')
def delivered(self, ses_event: SesEvent) -> None:
# Recipients here are strings and not structures. Unusual, but reflected in
# the documentation.
# https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-examples.html#event-publishing-retrieving-sns-send
statsd.incr("email_address.ses_email.delivered",
count=ses_event.delivery.recipients)
for sent in ses_event.delivery.recipients:
email_address = self._email_address(sent)
email_address.mark_sent()
def delayed(self, ses_event: SesEvent) -> None:
# Statistics for delayed recipients.
statsd.incr(
"email_address.ses_email.delayed",
count=len(ses_event.delivery_delay.delayed_recipients),
)
for failed in ses_event.delivery_delay.delayed_recipients:
email_address = self._email_address(failed.email)
email_address.mark_soft_fail()
def bounce(self, ses_event: SesEvent) -> None:
# Statistics for bounced recipients.
statsd.incr(
"email_address.ses_email.bounced",
count=len(ses_event.bounce.bounced_recipients),
)
# Process bounces
for bounced in ses_event.bounce.bounced_recipients:
email_address = self._email_address(bounced.email)
if ses_event.bounce.is_hard_bounce:
email_address.mark_hard_fail()
else:
email_address.mark_soft_fail()
def dispatch_transport_sms(user_notification, view):
if not user_notification.user.main_notification_preferences.by_transport(
'sms'):
# Cancel delivery if user's main switch is off. This was already checked, but
# the worker may be delayed and the user may have changed their preference.
user_notification.messageid_sms = 'cancelled'
return
user_notification.messageid_sms = sms.send(str(view.transport_for('sms')),
view.sms_with_unsubscribe())
statsd.incr(
'notification.transport',
tags={
'notification_type': user_notification.notification_type,
'transport': 'sms',
},
)
def complaint(self, ses_event: SesEvent) -> None:
# As per SES documentation, ISPs may not report the actual email addresses
# that filed the complaint. SES sends us the original recipients who are at
# the same domain, as a _maybe_ list. We respond to complaints by blocking their
# address from further use. Since this is a serious outcome, we can only do this
# when there was a single recipient to the original email.
# https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html#event-publishing-retrieving-sns-contents-complaint-object
if len(ses_event.complaint.complained_recipients) == 1:
for complained in ses_event.complaint.complained_recipients:
if ses_event.complaint.complaint_feedback_type == 'not-spam':
email_address = self._email_address(complained.email)
email_address.mark_active()
statsd.incr("email_address.ses_email.not_spam")
elif ses_event.complaint.complaint_feedback_type == 'abuse':
statsd.incr("email_address.ses_email.abuse")
EmailAddress.mark_blocked(complained.email)
else:
# TODO: Process 'auth-failure', 'fraud', 'other', 'virus'
pass
def dispatch_transport_email(user_notification, view):
if not user_notification.user.main_notification_preferences.by_transport(
'email'):
# Cancel delivery if user's main switch is off. This was already checked, but
# the worker may be delayed and the user may have changed their preference.
user_notification.messageid_email = 'cancelled'
return
address = view.transport_for('email')
subject = view.email_subject()
content = view.email_content()
attachments = view.email_attachments()
user_notification.messageid_email = email.send_email(
subject=subject,
to=[(user_notification.user.fullname, str(address))],
content=content,
attachments=attachments,
from_email=(view.email_from(),
'no-reply@' + app.config['DEFAULT_DOMAIN']),
headers={
'List-Id':
formataddr((
# formataddr can't handle lazy_gettext strings, so cast to regular
str(user_notification.notification.title),
user_notification.notification.type + '-notification.' +
app.config['DEFAULT_DOMAIN'],
)),
'List-Help':
f'<{url_for("notification_preferences")}>',
'List-Unsubscribe':
f'<{view.unsubscribe_url_email}>',
'List-Unsubscribe-Post':
'One-Click',
'List-Archive':
f'<{url_for("notifications")}>',
},
)
statsd.incr(
'notification.transport',
tags={
'notification_type': user_notification.notification_type,
'transport': 'email',
},
)
def dispatch_notification_job(eventid, notification_ids):
with app.app_context():
notifications = [
Notification.query.get((eventid, nid)) for nid in notification_ids
]
# Dispatch, creating batches of DISPATCH_BATCH_SIZE each
for notification in notifications:
for batch in (filterfalse(lambda x: x is None, unfiltered_batch)
for unfiltered_batch in zip_longest(
*[notification.dispatch()] * DISPATCH_BATCH_SIZE,
fillvalue=None)):
db.session.commit()
notification_ids = [
user_notification.identity for user_notification in batch
]
dispatch_user_notifications_job.queue(notification_ids)
statsd.incr(
'notification.recipient',
count=len(notification_ids),
tags={'notification_type': notification.type},
)
def send_email(
subject: str,
to: List[EmailRecipient],
content: str,
attachments: List[EmailAttachment] = None,
from_email: EmailRecipient = None,
headers: dict = None,
):
"""
Helper function to send an email.
:param str subject: Subject line of email message
:param list to: List of recipients. May contain (a) User objects, (b) tuple of
(name, email_address), or (c) a pre-formatted email address
:param str content: HTML content of the message (plain text is auto-generated)
:param list attachments: List of :class:`EmailAttachment` attachments
:param from_email: Email sender, same format as email recipient
:param dict headers: Optional extra email headers (for List-Unsubscribe, etc)
"""
# Parse recipients and convert as needed
to = [process_recipient(recipient) for recipient in to]
if from_email:
from_email = process_recipient(from_email)
body = html2text(content)
html = transform(content,
base_url=f'https://{app.config["DEFAULT_DOMAIN"]}/')
msg = EmailMultiAlternatives(
subject=subject,
to=to,
body=body,
from_email=from_email,
headers=headers,
alternatives=[(html, 'text/html')],
)
if attachments:
for attachment in attachments:
msg.attach(
content=attachment.content,
filename=attachment.filename,
mimetype=attachment.mimetype,
)
try:
# If an EmailAddress is blocked, this line will throw an exception
emails = [
EmailAddress.add(email)
for name, email in getaddresses(msg.recipients())
]
except EmailAddressBlockedError as e:
raise TransportRecipientError(e)
# FIXME: This won't raise an exception on delivery_state.HARD_FAIL. We need to do
# catch that, remove the recipient, and notify the user via the upcoming
# notification centre. (Raise a TransportRecipientError)
result = mail.send(msg)
# After sending, mark the address as having received an email and also update the statistics counters.
# Note that this will only track emails sent by *this app*. However SES events will track statistics
# across all apps and hence the difference between this counter and SES event counters will be emails
# sent by other apps.
statsd.incr("email_address.ses_email.sent", count=len(emails))
for ea in emails:
ea.mark_sent()
# FIXME: 'result' is a number. Why? We need message-id
return result
def process_ses_event():
"""
Processes SES Events from AWS.
The events are sent based on the configuration set of the outgoing email.
"""
# Register the fact that we got an SES event. If there are too many rejections, then it is a hack
# attempt.
statsd.incr('email_address.ses_event.received')
# Check for standard SNS headers and filter out, if they are not found.
for header in sns_headers:
if not request.headers.get(header):
statsd.incr('email_address.ses_event.rejected')
return {'status': 'error', 'error': 'not_json'}, 400
# Get the JSON message
message = request.get_json(force=True, silent=True)
if not message:
statsd.incr('email_address.ses_event.rejected')
return {'status': 'error', 'error': 'not_json'}, 400
# Validate the message
try:
validator.topics = app.config['SES_NOTIFICATION_TOPICS']
validator.check(message)
except SnsValidatorException as exc:
app.logger.info("SNS/SES event: %r", message)
statsd.incr('email_address.ses_event.rejected')
return {
'status': 'error',
'error': 'invalid_topic',
'message': exc.args
}, 400
# Message Type
m_type = message.get('Type')
# Subscription confirmation
if m_type == SnsNotificationType.SubscriptionConfirmation.value:
# We must confirm the subscription request
resp = requests.get(message['SubscribeURL'])
if resp.status_code != 200:
statsd.incr('email_address.ses_event.rejected')
return {'status': 'error', 'error': 'subscription_failed'}, 400
return {'status': 'ok', 'message': 'subscription_success'}
# Unsubscribe confirmation
if m_type == SnsNotificationType.UnsubscribeConfirmation.value:
# We don't want to unsubscribe, so request a resubscribe. Unsubscribe requests
# are typically in response to server errors. If an actual unsubscribe is
# required, this code must be disabled, or the server must be taken offline
resp = requests.get(message['SubscribeURL'])
if resp.status_code != 200:
statsd.incr('email_address.ses_event.rejected')
return {'status': 'error', 'error': 'resubscribe_failed'}, 400
return {'status': 'ok', 'message': 'resubscribe_success'}
# This is a Notification and we need to process it
if m_type == SnsNotificationType.Notification.value:
ses_event: SesEvent = SesEvent.from_json(message.get('Message'))
processor.process(ses_event)
db.session.commit()
return {'status': 'ok', 'message': 'notification_processed'}
# We'll only get here if there's a misconfiguration
statsd.incr('email_address.ses_event.rejected')
return {'status': 'error', 'error': 'unknown_message_type'}, 400
def validate_rate_limit(
resource, identifier, attempts, timeout, token=None, validator=None
):
"""
Confirm the rate limit has not been reached for the given string identifier, number
of attempts, and timeout period. Uses a simple limiter: once the number of attempts
is reached, no further attempts can be made for timeout seconds.
Aborts with HTTP 429 in case the limit has been reached.
:param str resource: Resource being rate limited
:param str identifier: Identifier for entity being rate limited
:param int attempts: Number of attempts allowed
:param int timeout: Duration in seconds to block after attempts are exhausted
:param str token: For advanced use, a token to check against for future calls
:param validator: A validator that receives token and previous token, and returns
two bools ``(count_this, retain_previous_token)``
For an example of how the token and validator are used, see
:func:`progressive_rate_limit_validator` and its users.
"""
statsd.set(
'rate_limit',
blake2b(identifier.encode(), digest_size=32).hexdigest(),
rate=1,
tags={'resource': resource},
)
cache_key = 'rate_limit/v1/%s/%s' % (resource, identifier)
cache_value = cache.get(cache_key)
if cache_value is None:
count, cache_token = None, None
statsd.incr('rate_limit', tags={'resource': resource, 'status_code': 201})
else:
count, cache_token = cache_value
if not count or not isinstance(count, int):
count = 0
if count >= attempts:
statsd.incr('rate_limit', tags={'resource': resource, 'status_code': 429})
abort(429)
if validator is not None:
do_increment, retain_token = validator(token, cache_token)
if retain_token:
token = cache_token
if do_increment:
current_app.logger.debug(
"Rate limit +1 (validated with %s, retain %r) for %s/%s",
cache_token,
retain_token,
resource,
identifier,
)
count += 1
statsd.incr('rate_limit', tags={'resource': resource, 'status_code': 200})
else:
current_app.logger.debug(
"Rate limit +0 (validated with %s, retain %r) for %s/%s",
cache_token,
retain_token,
resource,
identifier,
)
else:
current_app.logger.debug("Rate limit +1 for %s/%s", resource, identifier)
count += 1
statsd.incr('rate_limit', tags={'resource': resource, 'status_code': 200})
# Always set count, regardless of validator output
current_app.logger.debug(
"Setting rate limit usage for %s/%s to %s with token %s",
resource,
identifier,
count,
token,
)
cache.set(cache_key, (count, token), timeout=timeout)
