def click(self, ses_event: SesEvent) -> None: # SES doesn't track the recipient that triggered this action, so process this # only if the original email had a single recipient if len(ses_event.mail.destination) == 1: statsd.incr("email_address.ses_email.clicked") email_address = self._email_address(ses_event.mail.destination[0]) email_address.mark_active()
def dispatch_notification(*notifications): """ Dispatches one or more notifications. Usage:: dispatch_notification( MyNotification(document=doc, fragment=None), MyOtherNotification(document=doc, fragment=frag) ) This function performs a database commit to ensure notifications are available to background jobs, so it must only be called when it's safe to commit. """ eventid = uuid4() # Create a single eventid for notification in notifications: if not isinstance(notification, Notification): raise TypeError(f"Not a notification: {notification!r}") if not notification.active: raise TypeError(f"{notification!r} is marked inactive") notification.eventid = eventid notification.user = current_auth.user if sum(_n.for_private_recipient for _n in notifications) not in ( 0, # None are private len(notifications), # Or all are private ): raise TypeError( "Mixed use of private and non-private notifications." " Either all are private (no event tracking in links) or none are") db.session.add_all(notifications) db.session.commit() dispatch_notification_job.queue( eventid, [notification.id for notification in notifications]) for notification in notifications: statsd.incr('notification.dispatch', tags={'notification_type': notification.type})
def forget_email(email_hash): with app.app_context(): email_address = EmailAddress.get(email_hash=email_hash) if email_address.refcount() == 0: app.logger.info("Forgetting email address with hash %s", email_hash) email_address.email = None db.session.commit() statsd.incr('email_address.forgotten')
def delivered(self, ses_event: SesEvent) -> None: # Recipients here are strings and not structures. Unusual, but reflected in # the documentation. # https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-examples.html#event-publishing-retrieving-sns-send statsd.incr("email_address.ses_email.delivered", count=ses_event.delivery.recipients) for sent in ses_event.delivery.recipients: email_address = self._email_address(sent) email_address.mark_sent()
def delayed(self, ses_event: SesEvent) -> None: # Statistics for delayed recipients. statsd.incr( "email_address.ses_email.delayed", count=len(ses_event.delivery_delay.delayed_recipients), ) for failed in ses_event.delivery_delay.delayed_recipients: email_address = self._email_address(failed.email) email_address.mark_soft_fail()
def bounce(self, ses_event: SesEvent) -> None: # Statistics for bounced recipients. statsd.incr( "email_address.ses_email.bounced", count=len(ses_event.bounce.bounced_recipients), ) # Process bounces for bounced in ses_event.bounce.bounced_recipients: email_address = self._email_address(bounced.email) if ses_event.bounce.is_hard_bounce: email_address.mark_hard_fail() else: email_address.mark_soft_fail()
def dispatch_transport_sms(user_notification, view): if not user_notification.user.main_notification_preferences.by_transport( 'sms'): # Cancel delivery if user's main switch is off. This was already checked, but # the worker may be delayed and the user may have changed their preference. user_notification.messageid_sms = 'cancelled' return user_notification.messageid_sms = sms.send(str(view.transport_for('sms')), view.sms_with_unsubscribe()) statsd.incr( 'notification.transport', tags={ 'notification_type': user_notification.notification_type, 'transport': 'sms', }, )
def complaint(self, ses_event: SesEvent) -> None: # As per SES documentation, ISPs may not report the actual email addresses # that filed the complaint. SES sends us the original recipients who are at # the same domain, as a _maybe_ list. We respond to complaints by blocking their # address from further use. Since this is a serious outcome, we can only do this # when there was a single recipient to the original email. # https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html#event-publishing-retrieving-sns-contents-complaint-object if len(ses_event.complaint.complained_recipients) == 1: for complained in ses_event.complaint.complained_recipients: if ses_event.complaint.complaint_feedback_type == 'not-spam': email_address = self._email_address(complained.email) email_address.mark_active() statsd.incr("email_address.ses_email.not_spam") elif ses_event.complaint.complaint_feedback_type == 'abuse': statsd.incr("email_address.ses_email.abuse") EmailAddress.mark_blocked(complained.email) else: # TODO: Process 'auth-failure', 'fraud', 'other', 'virus' pass
def dispatch_transport_email(user_notification, view): if not user_notification.user.main_notification_preferences.by_transport( 'email'): # Cancel delivery if user's main switch is off. This was already checked, but # the worker may be delayed and the user may have changed their preference. user_notification.messageid_email = 'cancelled' return address = view.transport_for('email') subject = view.email_subject() content = view.email_content() attachments = view.email_attachments() user_notification.messageid_email = email.send_email( subject=subject, to=[(user_notification.user.fullname, str(address))], content=content, attachments=attachments, from_email=(view.email_from(), 'no-reply@' + app.config['DEFAULT_DOMAIN']), headers={ 'List-Id': formataddr(( # formataddr can't handle lazy_gettext strings, so cast to regular str(user_notification.notification.title), user_notification.notification.type + '-notification.' + app.config['DEFAULT_DOMAIN'], )), 'List-Help': f'<{url_for("notification_preferences")}>', 'List-Unsubscribe': f'<{view.unsubscribe_url_email}>', 'List-Unsubscribe-Post': 'One-Click', 'List-Archive': f'<{url_for("notifications")}>', }, ) statsd.incr( 'notification.transport', tags={ 'notification_type': user_notification.notification_type, 'transport': 'email', }, )
def dispatch_notification_job(eventid, notification_ids): with app.app_context(): notifications = [ Notification.query.get((eventid, nid)) for nid in notification_ids ] # Dispatch, creating batches of DISPATCH_BATCH_SIZE each for notification in notifications: for batch in (filterfalse(lambda x: x is None, unfiltered_batch) for unfiltered_batch in zip_longest( *[notification.dispatch()] * DISPATCH_BATCH_SIZE, fillvalue=None)): db.session.commit() notification_ids = [ user_notification.identity for user_notification in batch ] dispatch_user_notifications_job.queue(notification_ids) statsd.incr( 'notification.recipient', count=len(notification_ids), tags={'notification_type': notification.type}, )
def send_email( subject: str, to: List[EmailRecipient], content: str, attachments: List[EmailAttachment] = None, from_email: EmailRecipient = None, headers: dict = None, ): """ Helper function to send an email. :param str subject: Subject line of email message :param list to: List of recipients. May contain (a) User objects, (b) tuple of (name, email_address), or (c) a pre-formatted email address :param str content: HTML content of the message (plain text is auto-generated) :param list attachments: List of :class:`EmailAttachment` attachments :param from_email: Email sender, same format as email recipient :param dict headers: Optional extra email headers (for List-Unsubscribe, etc) """ # Parse recipients and convert as needed to = [process_recipient(recipient) for recipient in to] if from_email: from_email = process_recipient(from_email) body = html2text(content) html = transform(content, base_url=f'https://{app.config["DEFAULT_DOMAIN"]}/') msg = EmailMultiAlternatives( subject=subject, to=to, body=body, from_email=from_email, headers=headers, alternatives=[(html, 'text/html')], ) if attachments: for attachment in attachments: msg.attach( content=attachment.content, filename=attachment.filename, mimetype=attachment.mimetype, ) try: # If an EmailAddress is blocked, this line will throw an exception emails = [ EmailAddress.add(email) for name, email in getaddresses(msg.recipients()) ] except EmailAddressBlockedError as e: raise TransportRecipientError(e) # FIXME: This won't raise an exception on delivery_state.HARD_FAIL. We need to do # catch that, remove the recipient, and notify the user via the upcoming # notification centre. (Raise a TransportRecipientError) result = mail.send(msg) # After sending, mark the address as having received an email and also update the statistics counters. # Note that this will only track emails sent by *this app*. However SES events will track statistics # across all apps and hence the difference between this counter and SES event counters will be emails # sent by other apps. statsd.incr("email_address.ses_email.sent", count=len(emails)) for ea in emails: ea.mark_sent() # FIXME: 'result' is a number. Why? We need message-id return result
def process_ses_event(): """ Processes SES Events from AWS. The events are sent based on the configuration set of the outgoing email. """ # Register the fact that we got an SES event. If there are too many rejections, then it is a hack # attempt. statsd.incr('email_address.ses_event.received') # Check for standard SNS headers and filter out, if they are not found. for header in sns_headers: if not request.headers.get(header): statsd.incr('email_address.ses_event.rejected') return {'status': 'error', 'error': 'not_json'}, 400 # Get the JSON message message = request.get_json(force=True, silent=True) if not message: statsd.incr('email_address.ses_event.rejected') return {'status': 'error', 'error': 'not_json'}, 400 # Validate the message try: validator.topics = app.config['SES_NOTIFICATION_TOPICS'] validator.check(message) except SnsValidatorException as exc: app.logger.info("SNS/SES event: %r", message) statsd.incr('email_address.ses_event.rejected') return { 'status': 'error', 'error': 'invalid_topic', 'message': exc.args }, 400 # Message Type m_type = message.get('Type') # Subscription confirmation if m_type == SnsNotificationType.SubscriptionConfirmation.value: # We must confirm the subscription request resp = requests.get(message['SubscribeURL']) if resp.status_code != 200: statsd.incr('email_address.ses_event.rejected') return {'status': 'error', 'error': 'subscription_failed'}, 400 return {'status': 'ok', 'message': 'subscription_success'} # Unsubscribe confirmation if m_type == SnsNotificationType.UnsubscribeConfirmation.value: # We don't want to unsubscribe, so request a resubscribe. Unsubscribe requests # are typically in response to server errors. If an actual unsubscribe is # required, this code must be disabled, or the server must be taken offline resp = requests.get(message['SubscribeURL']) if resp.status_code != 200: statsd.incr('email_address.ses_event.rejected') return {'status': 'error', 'error': 'resubscribe_failed'}, 400 return {'status': 'ok', 'message': 'resubscribe_success'} # This is a Notification and we need to process it if m_type == SnsNotificationType.Notification.value: ses_event: SesEvent = SesEvent.from_json(message.get('Message')) processor.process(ses_event) db.session.commit() return {'status': 'ok', 'message': 'notification_processed'} # We'll only get here if there's a misconfiguration statsd.incr('email_address.ses_event.rejected') return {'status': 'error', 'error': 'unknown_message_type'}, 400
def validate_rate_limit( resource, identifier, attempts, timeout, token=None, validator=None ): """ Confirm the rate limit has not been reached for the given string identifier, number of attempts, and timeout period. Uses a simple limiter: once the number of attempts is reached, no further attempts can be made for timeout seconds. Aborts with HTTP 429 in case the limit has been reached. :param str resource: Resource being rate limited :param str identifier: Identifier for entity being rate limited :param int attempts: Number of attempts allowed :param int timeout: Duration in seconds to block after attempts are exhausted :param str token: For advanced use, a token to check against for future calls :param validator: A validator that receives token and previous token, and returns two bools ``(count_this, retain_previous_token)`` For an example of how the token and validator are used, see :func:`progressive_rate_limit_validator` and its users. """ statsd.set( 'rate_limit', blake2b(identifier.encode(), digest_size=32).hexdigest(), rate=1, tags={'resource': resource}, ) cache_key = 'rate_limit/v1/%s/%s' % (resource, identifier) cache_value = cache.get(cache_key) if cache_value is None: count, cache_token = None, None statsd.incr('rate_limit', tags={'resource': resource, 'status_code': 201}) else: count, cache_token = cache_value if not count or not isinstance(count, int): count = 0 if count >= attempts: statsd.incr('rate_limit', tags={'resource': resource, 'status_code': 429}) abort(429) if validator is not None: do_increment, retain_token = validator(token, cache_token) if retain_token: token = cache_token if do_increment: current_app.logger.debug( "Rate limit +1 (validated with %s, retain %r) for %s/%s", cache_token, retain_token, resource, identifier, ) count += 1 statsd.incr('rate_limit', tags={'resource': resource, 'status_code': 200}) else: current_app.logger.debug( "Rate limit +0 (validated with %s, retain %r) for %s/%s", cache_token, retain_token, resource, identifier, ) else: current_app.logger.debug("Rate limit +1 for %s/%s", resource, identifier) count += 1 statsd.incr('rate_limit', tags={'resource': resource, 'status_code': 200}) # Always set count, regardless of validator output current_app.logger.debug( "Setting rate limit usage for %s/%s to %s with token %s", resource, identifier, count, token, ) cache.set(cache_key, (count, token), timeout=timeout)