def test_scheduled_transient_bounce(self):
transient_message = self._get_message('scheduled_transient_bounce')
transient_email = '*****@*****.**'
self.assertEqual(
get_relevant_aws_meta(transient_message),
[
AwsMeta(
notification_type='Bounce',
main_type='Transient',
sub_type='General',
email='*****@*****.**',
reason='smtp; 554 4.4.7 Message expired: unable to deliver in 840 minutes.<421 4.4.0 Unable to lookup DNS for company.org>',
headers={
'returnPath': '*****@*****.**',
'from': ['*****@*****.**'],
'date': 'Fri, 31 Jul 2020 06:01:50 -0000',
'to': ['*****@*****.**'],
'messageId': '<redacted>',
'subject': 'Scheduled report from CommCare HQ'
},
timestamp=datetime.datetime(2020, 7, 31, 21, 8, 57, 723000,
tzinfo=datetime.timezone.utc),
destination=['*****@*****.**']
)
]
)
handle_email_sns_event(transient_message)
bounced_email = BouncedEmail.objects.filter(email=transient_email)
self.assertFalse(bounced_email.exists())
transient_info = TransientBounceEmail.objects.filter(email=transient_email)
self.assertTrue(transient_info.exists())
def test_sns_bounce_general(self):
sns_bounce_general = self._get_message('sns_bounce_general')
self.assertEqual(self.manager._get_aws_info(sns_bounce_general, 333), [
AwsMeta(notification_type='Bounce',
main_type='Permanent',
sub_type='General',
email='*****@*****.**',
reason="smtp; 550-5.1.1 The email account that you tried "
"to reach does not exist. Please try\n550-5.1.1 "
"double-checking the recipient's email address for "
"typos or\n550-5.1.1 unnecessary spaces. Learn more"
" at\n550 5.1.1 https://support.google.com/mail/?p="
"NoSuchUser h6si12061056qtp.98 - gsmtp",
headers={
'returnPath':
'*****@*****.**',
'from': ['*****@*****.**'],
'date': 'Tue, 28 Jan 2020 09:29:02 -0000',
'to': ['*****@*****.**'],
'messageId': '<redacted>',
'subject': 'Activate your CommCare project'
},
timestamp=datetime.datetime(
2020, 1, 28, 9, 29, 3, 30000, tzinfo=tzlocal()),
destination=['*****@*****.**'])
])
def test_sns_bounce_suppressed(self):
sns_bounce_suppressed = self._get_message('sns_bounce_suppressed')
self.assertEqual(
self.manager._get_aws_info(sns_bounce_suppressed, 333), [
AwsMeta(
notification_type='Bounce',
main_type='Permanent',
sub_type='Suppressed',
email='*****@*****.**',
reason='Amazon SES has suppressed sending to this address '
'because it has a recent history of bouncing as an '
'invalid address. For more information about how '
'to remove an address from the suppression list, '
'see the Amazon SES Developer Guide: '
'http://docs.aws.amazon.com/ses/latest/'
'DeveloperGuide/remove-from-suppressionlist.html ',
headers={
'from': ['*****@*****.**'],
'date': 'Tue, 28 Jan 2020 10:40:04 -0000',
'to': ['*****@*****.**'],
'messageId': '<redacted>',
'subject': 'Late'
},
timestamp=datetime.datetime(
2020, 1, 28, 10, 40, 4, 931000, tzinfo=tzlocal()),
destination=['*****@*****.**'])
])
def get_relevant_aws_meta(message_info):
"""
Creates a list of AwsMeta objects from the Message portion of an AWS
SNS Notification message.
:param message_info: (dict) the "Message" portion of an SNS notification
:return: (list) AwsMeta objects
"""
aws_info = []
mail_info = message_info.get('mail', {})
notification_type = message_info.get(
'notificationType') or message_info.get('eventType')
if notification_type == NotificationType.BOUNCE:
bounce_info = message_info['bounce']
for recipient in bounce_info['bouncedRecipients']:
aws_info.append(
AwsMeta(
notification_type=notification_type,
main_type=bounce_info['bounceType'],
sub_type=bounce_info['bounceSubType'],
timestamp=parse_datetime(bounce_info['timestamp']),
email=recipient['emailAddress'],
reason=recipient.get('diagnosticCode'),
headers=mail_info.get('commonHeaders', {}),
destination=mail_info.get('destination', []),
))
elif notification_type == NotificationType.COMPLAINT:
complaint_info = message_info['complaint']
for recipient in complaint_info['complainedRecipients']:
aws_info.append(
AwsMeta(
notification_type=notification_type,
main_type=message_info.get('complaintFeedbackType'),
sub_type=complaint_info.get('complaintSubType'),
timestamp=parse_datetime(complaint_info['timestamp']),
email=recipient['emailAddress'],
reason=None,
headers=mail_info.get('commonHeaders', {}),
destination=mail_info.get('destination', []),
))
return aws_info
def test_sns_bounce_complaint(self):
sns_complaint = self._get_message('sns_complaint')
self.assertEqual(self.manager._get_aws_info(sns_complaint, 333), [
AwsMeta(notification_type='Complaint',
main_type=None,
sub_type='',
email='*****@*****.**',
reason=None,
headers={},
timestamp=datetime.datetime(
2020, 1, 8, 8, 6, 45, tzinfo=tzlocal()),
destination=['*****@*****.**'])
])
def test_scheduled_suppressed_bounce(self):
bounce_message = self._get_message('scheduled_suppressed_bounce')
self.assertEqual(get_relevant_aws_meta(bounce_message), [
AwsMeta(
notification_type='Bounce',
main_type='Permanent',
sub_type='Suppressed',
email='*****@*****.**',
reason=
'Amazon SES has suppressed sending to this address because '
'it has a recent history of bouncing as an invalid address. '
'For more information about how to remove an address from the '
'suppression list, see the Amazon SES Developer Guide: '
'http://docs.aws.amazon.com/ses/latest/DeveloperGuide/remove-from-suppressionlist.html ',
headers={
'returnPath': '*****@*****.**',
'from': ['*****@*****.**'],
'date': 'Tue, 28 Jul 2020 00:28:28 -0000',
'to': ['*****@*****.**'],
'cc': ['*****@*****.**'],
'messageId': '<redacted@server-name>',
'subject': 'Invitation from John Doe to join CommCareHQ'
},
timestamp=datetime.datetime(2020,
7,
28,
0,
28,
28,
622000,
tzinfo=datetime.timezone.utc),
destination=[
'*****@*****.**', '*****@*****.**'
])
])
handle_email_sns_event(bounce_message)
bounced_email = BouncedEmail.objects.filter(
email='*****@*****.**')
self.assertTrue(bounced_email.exists())
permanent_meta = PermanentBounceMeta.objects.filter(
bounced_email=bounced_email.first())
self.assertTrue(permanent_meta.exists())
self.assertEqual(permanent_meta.first().sub_type,
BounceSubType.SUPPRESSED)
def test_scheduled_general_bounce(self):
bounce_message = self._get_message('scheduled_general_bounce')
self.assertEqual(get_relevant_aws_meta(bounce_message), [
AwsMeta(
notification_type='Bounce',
main_type='Permanent',
sub_type='General',
email='*****@*****.**',
reason=
'smtp; 550 5.4.1 Recipient address rejected: Access denied. AS(redacted) [redacted.outlook.com]',
headers={
'returnPath': '*****@*****.**',
'from': ['*****@*****.**'],
'date': 'Fri, 31 Jul 2020 20:09:55 -0000',
'to': ['*****@*****.**'],
'cc': ['*****@*****.**'],
'messageId': '<redacted@server-name>',
'subject': 'Invitation from John Doe to join CommCareHQ'
},
timestamp=datetime.datetime(2020,
7,
31,
20,
9,
56,
637000,
tzinfo=datetime.timezone.utc),
destination=[
'*****@*****.**',
'*****@*****.**'
])
])
handle_email_sns_event(bounce_message)
bounced_email = BouncedEmail.objects.filter(
email='*****@*****.**')
self.assertTrue(bounced_email.exists())
permanent_meta = PermanentBounceMeta.objects.filter(
bounced_email=bounced_email.first())
self.assertTrue(permanent_meta.exists())
self.assertEqual(permanent_meta.first().sub_type,
BounceSubType.GENERAL)
def test_sns_bounce_transient(self):
sns_bounce_transient = self._get_message('sns_bounce_transient')
self.assertEqual(self.manager._get_aws_info(
sns_bounce_transient, 333), [
AwsMeta(
notification_type='Bounce',
main_type='Transient',
sub_type='General',
email='*****@*****.**',
reason=None,
headers={
'returnPath':
'*****@*****.**',
'from': ['*****@*****.**'],
'date': 'Tue, 28 Jan 2020 13:00:27 -0000',
'to': ['*****@*****.**'],
'messageId': '<redacted>',
'subject': 'Scheduled report from CommCare HQ'
},
timestamp=datetime.datetime(
2020, 1, 28, 13, 0, 35, tzinfo=tzlocal()),
destination=['*****@*****.**'])
])
def _get_aws_info(self, message, uid):
"""
This gets the most important details from AWS SNS notifications for
Complaints and Bounces.
Relevant AWS Dcoumentation:
https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-examples.html
:param message: raw message
:param uid: SMTP uid of message
:return: list of AwsMeta objects
"""
aws_info = []
base_info = self._get_message_json(message)
if base_info.get('notificationType'):
message_info = base_info
else:
message_info = base_info.get('Message')
if isinstance(message_info, str):
message_info = json.loads(message_info)
if not message_info:
return aws_info
mail_info = message_info.get('mail', {})
if message_info['notificationType'] == NotificationType.BOUNCE:
bounce_info = message_info['bounce']
for recipient in bounce_info['bouncedRecipients']:
aws_info.append(
AwsMeta(
notification_type=message_info['notificationType'],
main_type=bounce_info['bounceType'],
sub_type=bounce_info['bounceSubType'],
timestamp=parse_datetime(bounce_info['timestamp']),
email=recipient['emailAddress'],
reason=recipient.get('diagnosticCode'),
headers=mail_info.get('commonHeaders', {}),
destination=mail_info.get('destination', []),
))
elif message_info['notificationType'] == NotificationType.COMPLAINT:
complaint_info = message_info['complaint']
for recipient in complaint_info['complainedRecipients']:
aws_info.append(
AwsMeta(
notification_type=message_info['notificationType'],
main_type=message_info.get('complaintFeedbackType'),
sub_type=complaint_info.get('complaintSubType'),
timestamp=parse_datetime(complaint_info['timestamp']),
email=recipient['emailAddress'],
reason=None,
headers=mail_info.get('commonHeaders', {}),
destination=mail_info.get('destination', []),
))
else:
metrics_counter(
'commcare.bounced_email_manager.unknown_notification_type')
self._label_problem_email(
uid, extra_labels=['UnknownAWSNotificationType'])
_bounced_email_soft_assert(
False, f'[{settings.SERVER_ENVIRONMENT}] '
f'Unknown AWS Notification Type sent to Inbox. ')
return aws_info
def test_scheduled_complaint(self):
complaint_message = self._get_message('scheduled_complaint')
self.assertEqual(
get_relevant_aws_meta(complaint_message),
[
AwsMeta(
notification_type='Complaint',
main_type=None,
sub_type=None,
email='*****@*****.**',
reason=None,
headers={
'returnPath': '*****@*****.**',
'from': ['*****@*****.**'],
'date': 'Thu, 16 Jul 2020 03:02:22 -0000',
'to': ['*****@*****.**'],
'cc': ['*****@*****.**'],
'messageId': '<redacted@server-name>',
'subject': 'Invitation from Alice Doe to join CommCareHQ'
},
timestamp=datetime.datetime(2020, 7, 16, 3, 16, 55, 129000,
tzinfo=datetime.timezone.utc),
destination=[
'*****@*****.**',
'*****@*****.**'
]
),
AwsMeta(
notification_type='Complaint',
main_type=None,
sub_type=None,
email='*****@*****.**',
reason=None,
headers={
'returnPath': '*****@*****.**',
'from': ['*****@*****.**'],
'date': 'Thu, 16 Jul 2020 03:02:22 -0000',
'to': ['*****@*****.**'],
'cc': ['*****@*****.**'],
'messageId': '<redacted@server-name>',
'subject': 'Invitation from Alice Doe to join CommCareHQ'
},
timestamp=datetime.datetime(2020, 7, 16, 3, 16, 55, 129000,
tzinfo=datetime.timezone.utc),
destination=[
'*****@*****.**',
'*****@*****.**'
]
),
]
)
handle_email_sns_event(complaint_message)
first_bounced_email = BouncedEmail.objects.filter(email='*****@*****.**')
second_bounced_email = BouncedEmail.objects.filter(email='*****@*****.**')
self.assertTrue(first_bounced_email.exists())
self.assertTrue(second_bounced_email.exists())
self.assertTrue(
ComplaintBounceMeta.objects.filter(
bounced_email=first_bounced_email.first()
).exists()
)
self.assertTrue(
ComplaintBounceMeta.objects.filter(
bounced_email=second_bounced_email.first()
).exists()
)