def test_notice(self):
# Test that the notice in the first subpart is correct.
send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
message = items[0].msg
notice = message.get_payload(0)
self.assertEqual(notice.get_content_type(), 'text/plain')
# The interesting bits are the parts that have been interpolated into
# the message. For now the best we can do is know that the
# interpolation values appear in the message.
self.assertMultiLineEqual(
notice.get_payload(), """\
This is a probe message. You can ignore this message.
The [email protected] mailing list has received a number of bounces
from you, indicating that there may be a problem delivering messages
to [email protected]. A sample is attached below. Please examine this
message to make sure there are no problems with your email address.
You may want to check with your mail administrator for more help.
You don't need to do anything to remain an enabled member of the
mailing list.
If you have any questions or problems, you can contact the mailing
list owner at
[email protected]
""")
def test_probe_is_multipart(self):
# The probe is a multipart/mixed with two subparts.
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
self.assertEqual(message.get_content_type(), 'multipart/mixed')
self.assertTrue(message.is_multipart())
self.assertEqual(len(message.get_payload()), 2)
def test_probe_is_multipart(self):
# The probe is a multipart/mixed with two subparts.
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
self.assertEqual(message.get_content_type(), 'multipart/mixed')
self.assertTrue(message.is_multipart())
self.assertEqual(len(message.get_payload()), 2)
def test_probe_sends_one_message(self):
# send_probe() places one message in the virgin queue.
items = get_queue_messages('virgin')
self.assertEqual(len(items), 0)
send_probe(self._member, self._msg)
items = get_queue_messages('virgin')
self.assertEqual(len(items), 1)
def test_probe_sends_one_message(self):
# send_probe() places one message in the virgin queue.
items = get_queue_messages('virgin')
self.assertEqual(len(items), 0)
send_probe(self._member, self._msg)
items = get_queue_messages('virgin')
self.assertEqual(len(items), 1)
def test_verp_probe_bounce(self):
# A VERP probe bounced. The primary difference here is that the
# registered bounce event will have a different context. The
# Message-Id will be different too, because of the way we're
# simulating the probe bounce.
#
# Start be simulating a probe bounce.
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
bounce = message_from_string("""\
To: {0}
From: [email protected]
Message-Id: <second>
""".format(message['From']))
self._bounceq.enqueue(bounce, self._msgdata)
self._runner.run()
self.assertEqual(len(get_queue_messages('bounces')), 0)
events = list(self._processor.events)
self.assertEqual(len(events), 1)
self.assertEqual(events[0].email, '*****@*****.**')
self.assertEqual(events[0].list_id, 'test.example.com')
self.assertEqual(events[0].message_id, '<second>')
self.assertEqual(events[0].context, BounceContext.probe)
self.assertEqual(events[0].processed, False)
def test_verp_probe_bounce(self):
# A VERP probe bounced. The primary difference here is that the
# registered bounce event will have a different context. The
# Message-Id will be different too, because of the way we're
# simulating the probe bounce.
#
# Start be simulating a probe bounce.
send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
message = items[0].msg
bounce = message_from_string("""\
To: {0}
From: [email protected]
Message-Id: <second>
""".format(message['From']))
self._bounceq.enqueue(bounce, self._msgdata)
self._runner.run()
get_queue_messages('bounces', expected_count=0)
events = list(self._processor.events)
self.assertEqual(len(events), 1)
self.assertEqual(events[0].email, '*****@*****.**')
self.assertEqual(events[0].list_id, 'test.example.com')
self.assertEqual(events[0].message_id, '<second>')
self.assertEqual(events[0].context, BounceContext.probe)
self.assertEqual(events[0].processed, False)
def test_subject_with_member_nonenglish(self):
# Test that members with non-English preferred language get a Subject
# header in the expected language.
send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
self.assertEqual(
items[0].msg['subject'].encode(),
'=?utf-8?q?ailing-may_ist-lay_Test_obe-pray_essage-may?=')
def test_subject_with_member_nonenglish(self):
# Test that members with non-English preferred language get a Subject
# header in the expected language.
send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
self.assertEqual(
items[0].msg['subject'].encode(),
'=?utf-8?q?ailing-may_ist-lay_Test_obe-pray_essage-may?=')
def test_subject_with_member_nonenglish(self):
# Test that members with non-English preferred language get a Subject
# header in the expected language.
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
self.assertEqual(
message['Subject'],
'=?utf-8?q?ailing-may_ist-lay_Test_obe-pray_essage-may?=')
def test_probe_notice_with_member_nonenglish(self):
# Test that a member with non-English preferred language gets the
# probe message in their language.
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
notice = message.get_payload(0).get_payload()
self.assertMultiLineEqual(notice, """\
blah blah blah [email protected] [email protected]
http://example.com/[email protected] [email protected]""")
def test_probe_notice_with_member_nonenglish(self):
# Test that a member with non-English preferred language gets the
# probe message in their language.
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
notice = message.get_payload(0).get_payload()
self.assertMultiLineEqual(notice, """\
blah blah blah [email protected] [email protected]
http://example.com/[email protected] [email protected]""")
def test_probe_contains_original(self):
# Show that send_probe() places a properly formatted message in the
# virgin queue.
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
rfc822 = message.get_payload(1)
self.assertEqual(rfc822.get_content_type(), 'message/rfc822')
self.assertTrue(rfc822.is_multipart())
self.assertEqual(len(rfc822.get_payload()), 1)
self.assertEqual(rfc822.get_payload(0).as_string(),
self._msg.as_string())
def test_probe_contains_original(self):
# Show that send_probe() places a properly formatted message in the
# virgin queue.
send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
rfc822 = items[0].msg.get_payload(1)
self.assertEqual(rfc822.get_content_type(), 'message/rfc822')
self.assertTrue(rfc822.is_multipart())
self.assertEqual(len(rfc822.get_payload()), 1)
self.assertEqual(
rfc822.get_payload(0).as_string(), self._msg.as_string())
def test_probe_notice_with_member_nonenglish(self):
# Test that a member with non-English preferred language gets the
# probe message in their language.
send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
message = items[0].msg
notice = message.get_payload(0).get_payload(decode=True)
notice = notice.decode(self._member.preferred_language.charset)
self.assertMultiLineEqual(
notice, """\
blah blah blah [email protected] [email protected]
[email protected]
""")
def test_send_probe_non_ascii(self):
# Send a pobe from an English language list to a user with non-ascii
# preferred language.
language_manager = getUtility(ILanguageManager)
self._mlist.preferred_language = language_manager.get('en')
# French charset is utf-8, but testing has it as latin-1
french = language_manager.get('fr')
french.charset = 'utf-8'
self._member.address.preferences.preferred_language = french
send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
msg = items[0].msg
self.assertIn(b'*****@*****.**',
msg.get_payload()[0].get_payload(decode=True))
def test_notice(self):
# Test that the notice in the first subpart is correct.
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
notice = message.get_payload(0)
self.assertEqual(notice.get_content_type(), 'text/plain')
# The interesting bits are the parts that have been interpolated into
# the message. For now the best we can do is know that the
# interpolation values appear in the message. When Python 2.7 is our
# minimum requirement, we can use assertRegexpMatches().
body = notice.get_payload()
self.assertTrue('*****@*****.**' in body)
self.assertTrue('*****@*****.**' in body)
self.assertTrue('http://example.com/[email protected]' in body)
self.assertTrue('*****@*****.**' in body)
def test_notice(self):
# Test that the notice in the first subpart is correct.
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
notice = message.get_payload(0)
self.assertEqual(notice.get_content_type(), 'text/plain')
# The interesting bits are the parts that have been interpolated into
# the message. For now the best we can do is know that the
# interpolation values appear in the message. When Python 2.7 is our
# minimum requirement, we can use assertRegexpMatches().
body = notice.get_payload()
self.assertTrue('*****@*****.**' in body)
self.assertTrue('*****@*****.**' in body)
self.assertTrue('http://example.com/[email protected]' in body)
self.assertTrue('*****@*****.**' in body)
def test_headers(self):
# Check the headers of the outer message.
token = send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
self.assertEqual(message['from'],
'test-bounces+{0}@example.com'.format(token))
self.assertEqual(message['to'], '*****@*****.**')
self.assertEqual(message['subject'], 'Test mailing list probe message')
def test_headers(self):
# Check the headers of the outer message.
token = send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
self.assertEqual(message['from'],
'test-bounces+{0}@example.com'.format(token))
self.assertEqual(message['to'], '*****@*****.**')
self.assertEqual(message['subject'], 'Test mailing list probe message')
def test_token(self):
# Show that send_probe() returns a proper token, and that the token
# corresponds to a record in the pending database.
token = send_probe(self._member, self._msg)
pendable = getUtility(IPendings).confirm(token)
self.assertEqual(len(pendable.items()), 2)
self.assertEqual(set(pendable.keys()),
set(['member_id', 'message_id']))
# member_ids are pended as unicodes.
self.assertEqual(uuid.UUID(hex=pendable['member_id']),
self._member.member_id)
self.assertEqual(pendable['message_id'], '<first>')
def test_token(self):
# Show that send_probe() returns a proper token, and that the token
# corresponds to a record in the pending database.
token = send_probe(self._member, self._msg)
pendable = getUtility(IPendings).confirm(token)
self.assertEqual(len(pendable.items()), 3)
self.assertEqual(set(pendable.keys()),
set(['member_id', 'message_id', 'type']))
# member_ids are pended as unicodes.
self.assertEqual(uuid.UUID(hex=pendable['member_id']),
self._member.member_id)
self.assertEqual(pendable['message_id'], '<first>')
def test_confirmed_probe_failure(self):
# This time, a probe also fails, but for some reason the probe token
# has already been confirmed and no longer exists in the database.
anne = getUtility(IUserManager).create_address('*****@*****.**')
member = self._mlist.subscribe(anne, MemberRole.member)
token = send_probe(member, self._msg)
getUtility(IPendings).confirm(token)
msgdata = dict(probe_token=token)
permanent_failures.append('*****@*****.**')
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 0)
def test_probe_temporary_failure(self):
# This time, a probe also fails, but the failures are temporary so
# they are not registered.
anne = getUtility(IUserManager).create_address('*****@*****.**')
member = self._mlist.subscribe(anne, MemberRole.member)
token = send_probe(member, self._msg)
getUtility(IPendings).confirm(token)
msgdata = dict(probe_token=token)
temporary_failures.append('*****@*****.**')
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 0)
def test_probe_temporary_failure(self):
# This time, a probe also fails, but the failures are temporary so
# they are not registered.
anne = getUtility(IUserManager).create_address('*****@*****.**')
member = self._mlist.subscribe(anne, MemberRole.member)
token = send_probe(member, self._msg)
getUtility(IPendings).confirm(token)
msgdata = dict(probe_token=token)
temporary_failures.append('*****@*****.**')
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 0)
def test_confirmed_probe_failure(self):
# This time, a probe also fails, but for some reason the probe token
# has already been confirmed and no longer exists in the database.
anne = getUtility(IUserManager).create_address('*****@*****.**')
member = self._mlist.subscribe(anne, MemberRole.member)
token = send_probe(member, self._msg)
getUtility(IPendings).confirm(token)
msgdata = dict(probe_token=token)
permanent_failures.append('*****@*****.**')
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 0)
def test_send_probe_to_user(self):
# Can we send probe to member subscribed as a user.
self._member = subscribe(self._mlist,
'Bart',
email='*****@*****.**',
as_user=True)
token = send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
message = items[0].msg
self.assertEqual(message['from'],
'test-bounces+{0}@example.com'.format(token))
self.assertEqual(message['to'], '*****@*****.**')
self.assertEqual(message['subject'], 'Test mailing list probe message')
def test_get_addresses(self):
# Be able to extract the probed address from the pending database
# based on the token in a probe bounce.
token = send_probe(self._member, self._msg)
# Simulate a bounce of the message in the virgin queue.
message = get_queue_messages('virgin')[0].msg
bounce = mfs("""\
To: {0}
From: [email protected]
""".format(message['From']))
addresses = ProbeVERP().get_verp(self._mlist, bounce)
self.assertEqual(addresses, set(['*****@*****.**']))
# The pendable is no longer in the database.
self.assertEqual(getUtility(IPendings).confirm(token), None)
def test_get_addresses(self):
# Be able to extract the probed address from the pending database
# based on the token in a probe bounce.
token = send_probe(self._member, self._msg)
# Simulate a bounce of the message in the virgin queue.
message = get_queue_messages('virgin')[0].msg
bounce = mfs("""\
To: {0}
From: [email protected]
""".format(message['From']))
addresses = ProbeVERP().get_verp(self._mlist, bounce)
self.assertEqual(addresses, set(['*****@*****.**']))
# The pendable is no longer in the database.
self.assertEqual(getUtility(IPendings).confirm(token), None)
def test_probe_failure(self):
# When a probe message fails during SMTP, a bounce event is recorded
# with the proper bounce context.
anne = getUtility(IUserManager).create_address('*****@*****.**')
member = self._mlist.subscribe(anne, MemberRole.member)
token = send_probe(member, self._msg)
msgdata = dict(probe_token=token)
permanent_failures.append('*****@*****.**')
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 1)
event = events[0]
self.assertEqual(event.list_id, 'test.example.com')
self.assertEqual(event.email, '*****@*****.**')
self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
self.assertEqual(event.message_id, '<first>')
self.assertEqual(event.context, BounceContext.probe)
self.assertFalse(event.processed)
def test_probe_failure(self):
# When a probe message fails during SMTP, a bounce event is recorded
# with the proper bounce context.
anne = getUtility(IUserManager).create_address('*****@*****.**')
member = self._mlist.subscribe(anne, MemberRole.member)
token = send_probe(member, self._msg)
msgdata = dict(probe_token=token)
permanent_failures.append('*****@*****.**')
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 1)
event = events[0]
self.assertEqual(event.list_id, 'test.example.com')
self.assertEqual(event.email, '*****@*****.**')
self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
self.assertEqual(event.message_id, '<first>')
self.assertEqual(event.context, BounceContext.probe)
self.assertFalse(event.processed)
def test_no_precedence_header(self):
# Probe messages should not have a Precedence header (LP: #808821).
send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
self.assertIsNone(items[0].msg['precedence'])
def test_send_probe_resets_bounce_score(self):
# Sending a probe should reset bounce_score so every subsequent bounce
# doesn't send another probe.
self._member.bounce_score = 5
send_probe(self._member, self._msg)
self.assertEqual(self._member.bounce_score, 0)
def test_no_precedence_header(self):
# Probe messages should not have a Precedence header (LP: #808821).
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
self.assertEqual(message['precedence'], None)
def test_no_precedence_header(self):
# Probe messages should not have a Precedence header (LP: #808821).
send_probe(self._member, self._msg)
message = get_queue_messages('virgin')[0].msg
self.assertEqual(message['precedence'], None)
def test_probe_sends_one_message(self):
# send_probe() places one message in the virgin queue. We start out
# with no messages in the queue.
get_queue_messages('virgin', expected_count=0)
send_probe(self._member, self._msg)
get_queue_messages('virgin', expected_count=1)
def test_send_probe_missing_required_params(self):
with self.assertRaises(ValueError):
send_probe(self._member)
def process_event(self, store, event):
"""See `IBounceProcessor`."""
list_manager = getUtility(IListManager)
mlist = list_manager.get(event.list_id)
if mlist is None:
# List was removed before the bounce is processed.
event.processed = True
# This needs an explicit commit because of the raise.
config.db.commit()
raise InvalidBounceEvent(
'Bounce for non-existent list {}'.format(event.list_id))
member = mlist.members.get_member(event.email)
if member is None:
event.processed = True
# This needs an explicit commit because of the raise.
config.db.commit()
raise InvalidBounceEvent(
'Email {} is not a subcriber of {}'.format(
event.email, mlist.list_id))
# If this is a probe bounce, that we are sent before to check for this
# Mailbox, we just disable the delivery for this member.
if event.context == BounceContext.probe:
log.info('Probe bounce received for member %s on list %s.',
event.email, mlist.list_id)
event.processed = True
self._disable_delivery(mlist, member, event)
return
# Looks like this is a regular bounce event, we need to process it
# in the follow order:
# 0. If the member is already disabled by bounce, we ignore this
# event.
# 1. If the date of the bounce is same as the previous bounce, we
# ignore this event.
# 2. Check if the bounce info is valid, bump the bounce_score and
# update the last bounce info.
# 3. If the bounce info is stale, reset the bounce score with
# this new value.
# 4. If the bounce_score is greater than threshold after the above,
# a) Send a VERP probe, if configured to do so
# b) Disable membership otherwise and notify the user and
# warnings.
if member.preferences.delivery_status == DeliveryStatus.by_bounces:
log.info('Residual bounce received for member %s on list %s.',
event.email, mlist.list_id)
event.processed = True
return
if (member.last_bounce_received is not None and
member.last_bounce_received.date() == event.timestamp.date()):
# Update the timestamp of the last bounce received.
member.last_bounce_received = event.timestamp
event.processed = True
log.info('Member %s already scored a bounce on list %s today.',
event.email, mlist.list_id)
return
if member.last_bounce_received is None or (
member.last_bounce_received <
event.timestamp - mlist.bounce_info_stale_after):
# Reset the bounce score to 1, for the current bounce that we got.
member.bounce_score = 1
else:
# Update the bounce score to reflect this bounce.
member.bounce_score += 1
# Update the last received time for the bounce.
member.last_bounce_received = event.timestamp
log.info('Member %s on list %s, bounce score = %d.', event.email,
mlist.list_id, member.bounce_score)
# Now, we are done updating. Let's see if the threshold is reached and
# disable based on that.
if member.bounce_score >= mlist.bounce_score_threshold:
# Save bounce_score because sending probe resets it.
saved_bounce_score = member.bounce_score
if as_boolean(config.mta.verp_probes):
send_probe(member, message_id=event.message_id)
action = 'sending probe'
else:
self._disable_delivery(mlist, member, event)
action = 'disabling delivery'
log.info(
'Member %s on list %s, bounce score %d >= threshold %d, %s.',
event.email, mlist.list_id, saved_bounce_score,
mlist.bounce_score_threshold, action)
event.processed = True
def test_no_precedence_header(self):
# Probe messages should not have a Precedence header (LP: #808821).
send_probe(self._member, self._msg)
items = get_queue_messages('virgin', expected_count=1)
self.assertIsNone(items[0].msg['precedence'])
def test_probe_sends_one_message(self):
# send_probe() places one message in the virgin queue. We start out
# with no messages in the queue.
get_queue_messages('virgin', expected_count=0)
send_probe(self._member, self._msg)
get_queue_messages('virgin', expected_count=1)