def test_nonverp_detectable_nonfatal_bounce(self):
# Here's a bounce that is not VERPd, but which has a bouncing address
# that can be parsed from a known bounce format. The bounce is
# non-fatal so no bounce event is registered and the bounce is not
# reported as unrecognized.
self._mlist.forward_unrecognized_bounces_to = (
UnrecognizedBounceDisposition.site_owner)
dsn = message_from_string("""\
From: [email protected]
To: [email protected]
Message-Id: <first>
Content-Type: multipart/report; report-type=delivery-status; boundary=AAA
MIME-Version: 1.0
--AAA
Content-Type: message/delivery-status
Action: delayed
Original-Recipient: rfc822; [email protected]
--AAA--
""")
self._bounceq.enqueue(dsn, self._msgdata)
mark = LogFileMark('mailman.bounce')
self._runner.run()
get_queue_messages('bounces', expected_count=0)
events = list(self._processor.events)
self.assertEqual(len(events), 0)
# There should be nothing in the 'virgin' queue.
get_queue_messages('virgin', expected_count=0)
# There should be log event in the log file.
log_lines = mark.read().splitlines()
self.assertTrue(len(log_lines) > 0)
def test_hold_chain(self):
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: A message
Message-ID: <ant>
MIME-Version: 1.0
A message body.
""")
msgdata = dict(moderation_reasons=[
'TEST-REASON-1',
'TEST-REASON-2',
])
logfile = LogFileMark('mailman.vette')
process_chain(self._mlist, msg, msgdata, start_chain='hold')
messages = get_queue_messages('virgin', expected_count=2)
payloads = {}
for item in messages:
if item.msg['to'] == '*****@*****.**':
part = item.msg.get_payload(0)
payloads['owner'] = part.get_payload().splitlines()
elif item.msg['To'] == '*****@*****.**':
payloads['sender'] = item.msg.get_payload().splitlines()
else:
self.fail('Unexpected message: %s' % item.msg)
self.assertIn(' TEST-REASON-1', payloads['owner'])
self.assertIn(' TEST-REASON-2', payloads['owner'])
self.assertIn(' TEST-REASON-1', payloads['sender'])
self.assertIn(' TEST-REASON-2', payloads['sender'])
logged = logfile.read()
self.assertIn('TEST-REASON-1', logged)
self.assertIn('TEST-REASON-2', logged)
def test_digest_messages(self):
# In LP: #1130697, the digest runner creates MIME digests using the
# stdlib MIMEMutlipart class, however this class does not have the
# extended attributes we require (e.g. .sender). The fix is to use a
# subclass of MIMEMultipart and our own Message subclass; this adds
# back the required attributes. (LP: #1130696)
#
# Start by creating the raw ingredients for the digests. This also
# runs the digest runner, thus producing the digest messages into the
# virgin queue.
make_digest_messages(self._mlist)
# Run the virgin queue processor, which runs the cook-headers and
# to-outgoing handlers. This should produce no error.
error_log = LogFileMark('mailman.error')
runner = make_testable_runner(VirginRunner, 'virgin')
runner.run()
error_text = error_log.read()
self.assertEqual(len(error_text), 0, error_text)
self.assertEqual(len(get_queue_messages('shunt')), 0)
messages = get_queue_messages('out')
self.assertEqual(len(messages), 2)
# Which one is the MIME digest?
mime_digest = None
for bag in messages:
if bag.msg.get_content_type() == 'multipart/mixed':
assert mime_digest is None, 'Found two MIME digests'
mime_digest = bag.msg
# The cook-headers handler ran.
self.assertIn('x-mailman-version', mime_digest)
self.assertEqual(mime_digest['precedence'], 'list')
# The list's -request address is the original sender.
self.assertEqual(bag.msgdata['original_sender'],
'*****@*****.**')
def test_digest_messages(self):
# In LP: #1130697, the digest runner creates MIME digests using the
# stdlib MIMEMutlipart class, however this class does not have the
# extended attributes we require (e.g. .sender). The fix is to use a
# subclass of MIMEMultipart and our own Message subclass; this adds
# back the required attributes. (LP: #1130696)
#
# Start by creating the raw ingredients for the digests. This also
# runs the digest runner, thus producing the digest messages into the
# virgin queue.
make_digest_messages(self._mlist)
# Run the virgin queue processor, which runs the cook-headers and
# to-outgoing handlers. This should produce no error.
error_log = LogFileMark('mailman.error')
runner = make_testable_runner(VirginRunner, 'virgin')
runner.run()
error_text = error_log.read()
self.assertEqual(len(error_text), 0, error_text)
self.assertEqual(len(get_queue_messages('shunt')), 0)
messages = get_queue_messages('out')
self.assertEqual(len(messages), 2)
# Which one is the MIME digest?
mime_digest = None
for bag in messages:
if bag.msg.get_content_type() == 'multipart/mixed':
assert mime_digest is None, 'Found two MIME digests'
mime_digest = bag.msg
# The cook-headers handler ran.
self.assertIn('x-mailman-version', mime_digest)
self.assertEqual(mime_digest['precedence'], 'list')
# The list's -request address is the original sender.
self.assertEqual(bag.msgdata['original_sender'],
'*****@*****.**')
def test_non_ascii_message(self):
msg = Message()
msg['From'] = '*****@*****.**'
msg['To'] = '*****@*****.**'
msg['Content-Type'] = 'multipart/mixed'
msg.attach(MIMEText('message with non-ascii chars: \xc3\xa9',
'plain', 'utf-8'))
mbox = digest_mbox(self._mlist)
mbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf')
mbox.add(msg.as_string())
self._digestq.enqueue(
msg,
listname=self._mlist.fqdn_listname,
digest_path=mbox_path,
volume=1, digest_number=1)
# Use any error logs as the error message if the test fails.
error_log = LogFileMark('mailman.error')
self._runner.run()
# The runner will send the file to the shunt queue on exception.
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
# There are two messages in the virgin queue: the digest as plain-text
# and as multipart.
messages = get_queue_messages('virgin')
self.assertEqual(len(messages), 2)
self.assertEqual(
sorted(item.msg.get_content_type() for item in messages),
['multipart/mixed', 'text/plain'])
for item in messages:
self.assertEqual(item.msg['subject'],
'Test Digest, Vol 1, Issue 1')
def test_discard_no_reasons(self):
# The log message contains n/a if no moderation reasons.
msgdata = {}
log_file = LogFileMark('mailman.vette')
process_chain(self._mlist, self._msg, msgdata, start_chain='discard')
log_entry = log_file.read()
self.assertIn('DISCARD: <*****@*****.**>', log_entry)
self.assertIn('[n/a]', log_entry)
def test_discard_reasons(self):
# The log message must contain the moderation reasons.
msgdata = dict(moderation_reasons=['TEST-REASON-1', 'TEST-REASON-2'])
log_file = LogFileMark('mailman.vette')
process_chain(self._mlist, self._msg, msgdata, start_chain='discard')
log_entry = log_file.read()
self.assertIn('DISCARD: <*****@*****.**>', log_entry)
self.assertIn('TEST-REASON-1', log_entry)
self.assertIn('TEST-REASON-2', log_entry)
def test_uheader_multiline(self):
# Multiline headers should be truncated (GL#273).
mark = LogFileMark('mailman.error')
header = cook_headers.uheader(self._mlist, 'A multiline\ndescription',
'X-Header')
self.assertEqual(header.encode(), 'A multiline [...]')
log_messages = mark.read()
self.assertIn('Header X-Header contains a newline, truncating it',
log_messages)
def test_bad_group(self):
self.mlist.linked_newsgroup = 'other.group'
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp():
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(len(lines), 2)
self.assertTrue(lines[0].endswith('NNTP error for list '
'[email protected]:'))
self.assertEqual(lines[1], 'No such group: other.group')
def test_up_to_date(self):
self.mlist.usenet_watermark = 3
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp():
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(self.mlist.usenet_watermark, 3)
self.assertEqual(len(lines), 3)
self.assertTrue(lines[0].endswith('[email protected]: [1..3]'))
self.assertTrue(lines[1].endswith('nothing new for list '
'*****@*****.**'))
self.assertTrue(lines[2].endswith('[email protected] watermark: 3'))
def test_catchup_only(self):
self.mlist.usenet_watermark = None
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp():
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(self.mlist.usenet_watermark, 3)
self.assertEqual(len(lines), 3)
self.assertTrue(lines[0].endswith('[email protected]: [1..3]'))
self.assertTrue(lines[1].endswith('[email protected] '
'caught up to article 3'))
self.assertTrue(lines[2].endswith('[email protected] watermark: 3'))
def test_bad_nntp_connect(self):
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp(fail=1):
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(len(lines), 4)
self.assertTrue(lines[0].endswith('error opening connection '
'to nntp_host: news.example.com'))
self.assertEqual(lines[1], 'Bad call to NNTP')
self.assertTrue(lines[2].endswith('NNTP error for list '
'[email protected]:'))
self.assertEqual(lines[3], 'Bad call to NNTP')
def test_article_exception(self):
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp(fail=2):
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(len(lines), 5)
self.assertTrue(lines[0].endswith('[email protected]: [1..3]'))
self.assertTrue(lines[1].endswith('gating [email protected] '
'articles [1..3]'))
self.assertTrue(lines[2].endswith('NNTP error for list '
'[email protected]: 2'))
self.assertEqual(lines[3], 'Bad call to article')
self.assertTrue(lines[4].endswith('[email protected] watermark: 3'))
def test_email_parser_exception(self):
mark = LogFileMark('mailman.fromusenet')
with get_email_exception():
with get_nntplib_nntp():
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(len(lines), 5)
self.assertTrue(lines[0].endswith('[email protected]: [1..3]'))
self.assertTrue(lines[1].endswith('gating [email protected] '
'articles [1..3]'))
self.assertTrue(lines[2].endswith('email package exception for '
'my.group:2'))
self.assertEqual(lines[3], 'Bad message')
self.assertTrue(lines[4].endswith('[email protected] watermark: 3'))
def test_broken_archiver(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
mark = LogFileMark('mailman.archiver')
self._archiveq.enqueue(self._msg, {},
listid=self._mlist.list_id,
received_time=now())
IListArchiverSet(self._mlist).get('broken').is_enabled = True
self._runner.run()
# The archiver is broken, so there are no messages on the file system,
# but there is a log message and the message was not shunted.
log_messages = mark.read()
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot archive message', log_messages)
get_queue_messages('shunt', expected_count=0)
def test_non_ascii_message(self):
msg = Message()
msg['From'] = '*****@*****.**'
msg['To'] = '*****@*****.**'
msg['Content-Type'] = 'multipart/mixed'
msg.attach(MIMEText('message with non-ascii chars: \xc3\xa9',
'plain', 'utf-8'))
mbox = digest_mbox(self._mlist)
mbox.add(msg.as_string())
# Use any error logs as the error message if the test fails.
error_log = LogFileMark('mailman.error')
make_digest_messages(self._mlist, msg)
# The runner will send the file to the shunt queue on exception.
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
self._check_virgin_queue()
def test_broken_archiver(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
mark = LogFileMark('mailman.archiver')
self._archiveq.enqueue(
self._msg, {},
listid=self._mlist.list_id,
received_time=now())
IListArchiverSet(self._mlist).get('broken').is_enabled = True
self._runner.run()
# The archiver is broken, so there are no messages on the file system,
# but there is a log message and the message was not shunted.
log_messages = mark.read()
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot archive message', log_messages)
get_queue_messages('shunt', expected_count=0)
def test_broken_permalink(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
site_dir = os.path.join(config.TEMPLATE_DIR, 'site', 'en')
os.makedirs(site_dir)
footer_path = os.path.join(site_dir, 'myfooter.txt')
with open(footer_path, 'w', encoding='utf-8') as fp:
print('${broken_url}', file=fp)
self._mlist.footer_uri = 'mailman:///myfooter.txt'
self._mlist.preferred_language = 'en'
mark = LogFileMark('mailman.archiver')
decorate.process(self._mlist, self._msg, {})
log_messages = mark.read()
self.assertNotIn('http:', self._msg.as_string())
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get permalink', log_messages)
def test_disable_delivery_already_disabled(self):
# Attempting to disable delivery for an already disabled member does
# nothing.
self._mlist.send_welcome_message = False
member = self._subscribe_and_add_bounce_event('*****@*****.**')
events = list(self._processor.events)
self.assertEqual(len(events), 1)
member.total_warnings_sent = 3
member.last_warning_sent = now() - timedelta(days=2)
member.preferences.delivery_status = DeliveryStatus.by_bounces
mark = LogFileMark('mailman.bounce')
self._processor._disable_delivery(self._mlist, member, events[0])
self.assertEqual(mark.read(), '')
self.assertEqual(member.total_warnings_sent, 3)
self.assertEqual(member.last_warning_sent, now() - timedelta(days=2))
get_queue_messages('virgin', expected_count=0)
def test_broken_permalink(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
site_dir = os.path.join(config.TEMPLATE_DIR, 'site', 'en')
os.makedirs(site_dir)
footer_path = os.path.join(site_dir, 'myfooter.txt')
with open(footer_path, 'w', encoding='utf-8') as fp:
print('${broken_url}', file=fp)
self._mlist.footer_uri = 'mailman:///myfooter.txt'
self._mlist.preferred_language = 'en'
mark = LogFileMark('mailman.archiver')
decorate.process(self._mlist, self._msg, {})
log_messages = mark.read()
self.assertNotIn('http:', self._msg.as_string())
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get permalink', log_messages)
def test_events_bounce_already_disabled(self):
# A bounce received for an already disabled member is only logged.
anne = self._subscribe_and_add_bounce_event('*****@*****.**',
subscribe=False,
create=False)
self._mlist.bounce_score_threshold = 3
anne.bounce_score = 3
anne.preferences.delivery_status = DeliveryStatus.by_bounces
anne.total_warnings_sent = 1
anne.last_warning_sent = now() - timedelta(days=3)
mark = LogFileMark('mailman.bounce')
self._runner.run()
get_queue_messages('virgin', expected_count=0)
self.assertEqual(anne.total_warnings_sent, 1)
self.assertIn(
'Residual bounce received for member [email protected] '
'on list test.example.com.', mark.read())
def test_hold_chain(self):
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: A message
Message-ID: <ant>
MIME-Version: 1.0
A message body.
""")
msgdata = dict(moderation_reasons=[
'TEST-REASON-1',
'TEST-REASON-2',
('TEST-{}-REASON-{}', 'FORMAT', 3),
])
logfile = LogFileMark('mailman.vette')
process_chain(self._mlist, msg, msgdata, start_chain='hold')
messages = get_queue_messages('virgin', expected_count=2)
payloads = {}
for item in messages:
if item.msg['to'] == '*****@*****.**':
part = item.msg.get_payload(0)
payloads['owner'] = part.get_payload().splitlines()
elif item.msg['To'] == '*****@*****.**':
payloads['sender'] = item.msg.get_payload().splitlines()
else:
self.fail('Unexpected message: %s' % item.msg)
self.assertIn(' TEST-REASON-1', payloads['owner'])
self.assertIn(' TEST-REASON-2', payloads['owner'])
self.assertIn(' TEST-FORMAT-REASON-3', payloads['owner'])
self.assertIn(' TEST-REASON-1', payloads['sender'])
self.assertIn(' TEST-REASON-2', payloads['sender'])
self.assertIn(' TEST-FORMAT-REASON-3', payloads['sender'])
logged = logfile.read()
self.assertIn('TEST-REASON-1', logged)
self.assertIn('TEST-REASON-2', logged)
self.assertIn('TEST-FORMAT-REASON-3', logged)
# Check the reason passed to hold_message().
requests = IListRequests(self._mlist)
self.assertEqual(requests.count_of(RequestType.held_message), 1)
request = requests.of_type(RequestType.held_message)[0]
key, data = requests.get_request(request.id)
self.assertEqual(
data.get('_mod_reason'),
'TEST-REASON-1; TEST-REASON-2; TEST-FORMAT-REASON-3')
def test_post_only_one_of_three(self):
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp():
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(self.mlist.usenet_watermark, 3)
self.assertEqual(len(lines), 4)
self.assertTrue(lines[0].endswith('[email protected]: [1..3]'))
self.assertTrue(lines[1].endswith('gating [email protected] '
'articles [1..3]'))
self.assertTrue(lines[2].endswith('posted to list [email protected]:'
' 2'))
self.assertTrue(lines[3].endswith('[email protected] watermark: 3'))
items = get_queue_messages('in', expected_count=1)
msg = items[0].msg
msgdata = items[0].msgdata
self.assertTrue(msgdata.get('fromusenet', False))
self.assertEqual(msg.get('message-id', ''), '<*****@*****.**>')
def test_utf7_message_with_inline_ascii_sig(self):
# The test message is bigger than 1K. Set the threshold bigger to
# avoid double processing in make_digest_messages.
self._mlist.digest_size_threshold = 5
# Subscribe some users receiving digests.
anne = subscribe(self._mlist, 'Anne')
anne.preferences.delivery_mode = DeliveryMode.mime_digests
bart = subscribe(self._mlist, 'Bart')
bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
with open_binary('mailman.runners.tests.data',
'ascii_in_utf7.eml') as fp:
msg = message_from_binary_file(fp, Message)
# Use any error logs as the error message if the test fails.
error_log = LogFileMark('mailman.error')
make_digest_messages(self._mlist, msg)
# The runner will send the file to the shunt queue on exception.
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
self._check_virgin_queue()
def test_broken_archiver(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
config.push('archiver', """
[archiver.broken]
class: {}.BrokenArchiver
enable: yes
""".format(BrokenArchiver.__module__))
self.addCleanup(config.pop, 'archiver')
mark = LogFileMark('mailman.archiver')
rfc_2369.process(self._mlist, self._msg, {})
log_messages = mark.read()
# Because .list_url() was broken, there will be no List-Archive header.
self.assertIsNone(self._msg.get('list-archive'))
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get list URL', log_messages)
# Because .permalink() was broken, there will be no Archived-At header.
self.assertIsNone(self._msg.get('archived-at'))
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get permalink', log_messages)
def test_non_ascii_message(self):
# Subscribe some users receiving digests.
anne = subscribe(self._mlist, 'Anne')
anne.preferences.delivery_mode = DeliveryMode.mime_digests
bart = subscribe(self._mlist, 'Bart')
bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
msg = Message()
msg['From'] = '*****@*****.**'
msg['To'] = '*****@*****.**'
msg['Content-Type'] = 'multipart/mixed'
msg.attach(MIMEText('message with non-ascii chars: \xc3\xa9',
'plain', 'utf-8'))
mbox = digest_mbox(self._mlist)
mbox.add(msg.as_string())
# Use any error logs as the error message if the test fails.
error_log = LogFileMark('mailman.error')
make_digest_messages(self._mlist, msg)
# The runner will send the file to the shunt queue on exception.
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
self._check_virgin_queue()
def test_non_ascii_message(self):
# Subscribe some users receiving digests.
anne = subscribe(self._mlist, 'Anne')
anne.preferences.delivery_mode = DeliveryMode.mime_digests
bart = subscribe(self._mlist, 'Bart')
bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
msg = Message()
msg['From'] = '*****@*****.**'
msg['To'] = '*****@*****.**'
msg['Content-Type'] = 'multipart/mixed'
msg.attach(MIMEText('message with non-ascii chars: \xc3\xa9',
'plain', 'utf-8'))
mbox = digest_mbox(self._mlist)
mbox.add(msg.as_string())
# Use any error logs as the error message if the test fails.
error_log = LogFileMark('mailman.error')
make_digest_messages(self._mlist, msg)
# The runner will send the file to the shunt queue on exception.
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
self._check_virgin_queue()
def test_broken_archiver(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
config.push('archiver', """
[archiver.broken]
class: {}.BrokenArchiver
enable: yes
""".format(BrokenArchiver.__module__))
self.addCleanup(config.pop, 'archiver')
mark = LogFileMark('mailman.archiver')
rfc_2369.process(self._mlist, self._msg, {})
log_messages = mark.read()
# Because .list_url() was broken, there will be no List-Archive header.
self.assertIsNone(self._msg.get('list-archive'))
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get list URL', log_messages)
# Because .permalink() was broken, there will be no Archived-At header.
self.assertIsNone(self._msg.get('archived-at'))
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get permalink', log_messages)
def test_log_exception_in_finish(self):
# If something bad happens in .finish(), the traceback should get
# logged. LP: #1165589.
msg = mfs("""\
From: [email protected]
To: [email protected]
Message-ID: <ant>
""")
switchboard = config.switchboards['shunt']
# Enqueue the message.
filebase = switchboard.enqueue(msg)
error_log = LogFileMark('mailman.error')
msg, data = switchboard.dequeue(filebase)
# Now, cause .finish() to throw an exception.
with patch('mailman.core.switchboard.os.rename',
side_effect=OSError('Oops!')):
switchboard.finish(filebase, preserve=True)
traceback = error_log.read().splitlines()
self.assertEqual(traceback[1], 'Traceback (most recent call last):')
self.assertEqual(traceback[-1], 'OSError: Oops!')
def test_log_exception_in_finish(self):
# If something bad happens in .finish(), the traceback should get
# logged. LP: #1165589.
msg = mfs("""\
From: [email protected]
To: [email protected]
Message-ID: <ant>
""")
switchboard = config.switchboards['shunt']
# Enqueue the message.
filebase = switchboard.enqueue(msg)
error_log = LogFileMark('mailman.error')
msg, data = switchboard.dequeue(filebase)
# Now, cause .finish() to throw an exception.
with patch('mailman.core.switchboard.os.rename',
side_effect=OSError('Oops!')):
switchboard.finish(filebase, preserve=True)
traceback = error_log.read().splitlines()
self.assertEqual(traceback[1], 'Traceback (most recent call last):')
self.assertEqual(traceback[-1], 'OSError: Oops!')
def test_digest_messages(self):
# In LP: #1130697, the digest runner creates MIME digests using the
# stdlib MIMEMutlipart class, however this class does not have the
# extended attributes we require (e.g. .sender). The fix is to use a
# subclass of MIMEMultipart and our own Message subclass; this adds
# back the required attributes. (LP: #1130696)
self._mlist.send_welcome_message = False
# Subscribe some users receiving digests.
anne = subscribe(self._mlist, 'Anne')
anne.preferences.delivery_mode = DeliveryMode.mime_digests
bart = subscribe(self._mlist, 'Bart')
bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
# Start by creating the raw ingredients for the digests. This also
# runs the digest runner, thus producing the digest messages into the
# virgin queue.
make_digest_messages(self._mlist)
# Run the virgin queue processor, which runs the cook-headers and
# to-outgoing handlers. This should produce no error.
error_log = LogFileMark('mailman.error')
runner = make_testable_runner(VirginRunner, 'virgin')
runner.run()
error_text = error_log.read()
self.assertEqual(len(error_text), 0, error_text)
get_queue_messages('shunt', expected_count=0)
items = get_queue_messages('out', expected_count=2)
# Which one is the MIME digest?
mime_digest = None
for item in items:
if item.msg.get_content_type() == 'multipart/mixed':
assert mime_digest is None, 'Found two MIME digests'
mime_digest = item.msg
# The cook-headers handler ran.
self.assertIn('x-mailman-version', mime_digest)
self.assertEqual(mime_digest['precedence'], 'list')
# The list's -request address is the original sender.
self.assertEqual(item.msgdata['original_sender'],
'*****@*****.**')
def test_digest_messages(self):
# In LP: #1130697, the digest runner creates MIME digests using the
# stdlib MIMEMutlipart class, however this class does not have the
# extended attributes we require (e.g. .sender). The fix is to use a
# subclass of MIMEMultipart and our own Message subclass; this adds
# back the required attributes. (LP: #1130696)
self._mlist.send_welcome_message = False
# Subscribe some users receiving digests.
anne = subscribe(self._mlist, 'Anne')
anne.preferences.delivery_mode = DeliveryMode.mime_digests
bart = subscribe(self._mlist, 'Bart')
bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
# Start by creating the raw ingredients for the digests. This also
# runs the digest runner, thus producing the digest messages into the
# virgin queue.
make_digest_messages(self._mlist)
# Run the virgin queue processor, which runs the cook-headers and
# to-outgoing handlers. This should produce no error.
error_log = LogFileMark('mailman.error')
runner = make_testable_runner(VirginRunner, 'virgin')
runner.run()
error_text = error_log.read()
self.assertEqual(len(error_text), 0, error_text)
get_queue_messages('shunt', expected_count=0)
items = get_queue_messages('out', expected_count=2)
# Which one is the MIME digest?
mime_digest = None
for item in items:
if item.msg.get_content_type() == 'multipart/mixed':
assert mime_digest is None, 'Found two MIME digests'
mime_digest = item.msg
# The cook-headers handler ran.
self.assertIn('x-mailman-version', mime_digest)
self.assertEqual(mime_digest['precedence'], 'list')
# The list's -request address is the original sender.
self.assertEqual(item.msgdata['original_sender'],
'*****@*****.**')
def test_header_matches(self):
# This test contail real cases of header_filter_rules
self._pckdict['header_filter_rules'] = [
('X\\-Spam\\-Status\\: Yes.*', 3, False),
('^X-Spam-Status: Yes\r\n\r\n', 2, False),
('^X-Spam-Level: \\*\\*\\*.*$', 3, False),
('^X-Spam-Level:.\\*\\*\r\n^X-Spam:.\\Yes', 3, False),
('Subject: \\[SPAM\\].*', 3, False),
('^Subject: .*loan.*', 3, False),
('Original-Received: from *linkedin.com*\r\n', 3, False),
('X-Git-Module: rhq.*git', 6, False),
('Approved: verysecretpassword', 6, False),
('^Subject: dev-\r\n^Subject: staging-', 3, False),
('from: .*[email protected]\r\nfrom: .*@jw-express.com',
2, False),
('^Received: from smtp-.*\\.fedoraproject\\.org\r\n'
'^Received: from mx.*\\.redhat.com\r\n'
'^Resent-date:\r\n'
'^Resent-from:\r\n'
'^Resent-Message-ID:\r\n'
'^Resent-to:\r\n'
'^Subject: [^mtv]\r\n',
7, False),
('^Received: from fedorahosted\\.org.*by fedorahosted\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'hosted.*\\.fedoraproject\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'fedoraproject\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'fedorahosted\\.org',
6, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(
[(hm.header, hm.pattern, hm.chain)
for hm in self._mlist.header_matches], [
('x-spam-status', 'Yes.*', 'discard'),
('x-spam-status', 'Yes', 'reject'),
('x-spam-level', '\\*\\*\\*.*$', 'discard'),
('x-spam-level', '\\*\\*', 'discard'),
('x-spam', '\\Yes', 'discard'),
('subject', '\\[SPAM\\].*', 'discard'),
('subject', '.*loan.*', 'discard'),
('original-received', 'from *linkedin.com*', 'discard'),
('x-git-module', 'rhq.*git', 'accept'),
('approved', 'verysecretpassword', 'accept'),
('subject', 'dev-', 'discard'),
('subject', 'staging-', 'discard'),
('from', '.*[email protected]', 'reject'),
('from', '.*@jw-express.com', 'reject'),
('received', 'from smtp-.*\\.fedoraproject\\.org', 'hold'),
('received', 'from mx.*\\.redhat.com', 'hold'),
('resent-date', '.*', 'hold'),
('resent-from', '.*', 'hold'),
('resent-message-id', '.*', 'hold'),
('resent-to', '.*', 'hold'),
('subject', '[^mtv]', 'hold'),
('received', 'from fedorahosted\\.org.*by fedorahosted\\.org',
'accept'),
('received',
'from hosted.*\\.fedoraproject.org.*by '
'hosted.*\\.fedoraproject\\.org', 'accept'),
('received',
'from hosted.*\\.fedoraproject.org.*by '
'fedoraproject\\.org', 'accept'),
('received',
'from hosted.*\\.fedoraproject.org.*by '
'fedorahosted\\.org', 'accept'),
])
loglines = error_log.read().strip()
self.assertEqual(len(loglines), 0)
def test_non_ascii_in_ascii_part(self):
# Subscribe some users receiving digests.
anne = subscribe(self._mlist, 'Anne')
anne.preferences.delivery_mode = DeliveryMode.mime_digests
bart = subscribe(self._mlist, 'Bart')
bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
msg = message_from_bytes(
b"""\
From: [email protected]
To: [email protected]
Subject: Non-ascii in ascii message
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="abcxyz"
--abcxyz
Content-Type: text/plain; charset=us-ascii
Content-Transfer-Encoding: 8bit
Don\xe2\x80\x99t try this at home.
--abcxyz--
""", Message)
# Use any error logs as the error message if the test fails.
error_log = LogFileMark('mailman.error')
make_digest_messages(self._mlist, msg)
# The runner will send the file to the shunt queue on exception.
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
items = self._check_virgin_queue()
self.assertEqual(
self._get_plain_body(items), b"""\
Send Test mailing list submissions to
\[email protected]
To subscribe or unsubscribe via email, send a message with subject or
body 'help' to
\[email protected]
You can reach the person managing the list at
\[email protected]
When replying, please edit your Subject line so it is more specific
than "Re: Contents of Test digest..."
Today\'s Topics:
1. Non-ascii in ascii message ([email protected])
----------------------------------------------------------------------
Message: 1
From: [email protected]
Subject: Non-ascii in ascii message
To: [email protected]
Content-Type: multipart/mixed; boundary="abcxyz"
Don\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbdt try this at home.
------------------------------
Subject: Digest Footer
_______________________________________________
Test mailing list -- [email protected]
To unsubscribe send an email to [email protected]
------------------------------
End of Test Digest, Vol 1, Issue 1
**********************************
""")
def test_header_matches(self):
# This test containes real cases of header_filter_rules.
self._pckdict['header_filter_rules'] = [
('X\\-Spam\\-Status\\: Yes.*', 3, False),
('^X-Spam-Status: Yes\r\n\r\n', 2, False),
('^X-Spam-Level: \\*\\*\\*.*$', 3, False),
('^X-Spam-Level:.\\*\\*\r\n^X-Spam:.Yes', 3, False),
('Subject: \\[SPAM\\].*', 3, False),
('^Subject: .*loan.*', 3, False),
('Original-Received: from *linkedin.com*\r\n', 3, False),
('X-Git-Module: rhq.*git', 6, False),
('Approved: verysecretpassword', 6, False),
('^Subject: dev-\r\n^Subject: staging-', 3, False),
('from: .*[email protected]\r\nfrom: .*@jw-express.com', 2,
False),
('^Subject:.*\\Wwas:\\W', 3, False),
('^Received: from smtp-.*\\.fedoraproject\\.org\r\n'
'^Received: from mx.*\\.redhat.com\r\n'
'^Resent-date:\r\n'
'^Resent-from:\r\n'
'^Resent-Message-ID:\r\n'
'^Resent-to:\r\n'
'^Subject: [^mtv]\r\n', 7, False),
('^Received: from fedorahosted\\.org.*by fedorahosted\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'hosted.*\\.fedoraproject\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'fedoraproject\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'fedorahosted\\.org', 6, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(
[(hm.header, hm.pattern, hm.chain)
for hm in self._mlist.header_matches], [
('x-spam-status', 'Yes.*', 'discard'),
('x-spam-status', 'Yes', 'reject'),
('x-spam-level', '\\*\\*\\*.*$', 'discard'),
('x-spam-level', '\\*\\*', 'discard'),
('x-spam', 'Yes', 'discard'),
('subject', '\\[SPAM\\].*', 'discard'),
('subject', '.*loan.*', 'discard'),
('original-received', 'from *linkedin.com*', 'discard'),
('x-git-module', 'rhq.*git', 'accept'),
('approved', 'verysecretpassword', 'accept'),
('subject', 'dev-', 'discard'),
('subject', 'staging-', 'discard'),
('from', '.*[email protected]', 'reject'),
('from', '.*@jw-express.com', 'reject'),
('subject', '\\Wwas:\\W', 'discard'),
('received', 'from smtp-.*\\.fedoraproject\\.org', 'hold'),
('received', 'from mx.*\\.redhat.com', 'hold'),
('resent-date', '.*', 'hold'),
('resent-from', '.*', 'hold'),
('resent-message-id', '.*', 'hold'),
('resent-to', '.*', 'hold'),
('subject', '[^mtv]', 'hold'),
('received', 'from fedorahosted\\.org.*by fedorahosted\\.org',
'accept'),
('received', 'from hosted.*\\.fedoraproject.org.*by '
'hosted.*\\.fedoraproject\\.org', 'accept'),
('received', 'from hosted.*\\.fedoraproject.org.*by '
'fedoraproject\\.org', 'accept'),
('received', 'from hosted.*\\.fedoraproject.org.*by '
'fedorahosted\\.org', 'accept'),
])
loglines = error_log.read().strip()
self.assertEqual(len(loglines), 0)
