def test_bad_configuration_line(self):
# Take a mark on the error log file.
mark = LogFileMark('mailman.error')
# A bad value in [antispam]header_checks should just get ignored, but
# with an error message logged.
chain = config.chains['header-match']
# The links are created dynamically; the rule names will all start
# with the same prefix, but have a variable suffix. The actions will
# all be to jump to the named chain. Do these checks now, while we
# collect other useful information.
post_checks = []
saw_any_rule = False
for link in chain.get_links(self._mlist, Message(), {}):
if link.rule.name == 'any':
saw_any_rule = True
self.assertEqual(link.action, LinkAction.jump)
elif saw_any_rule:
raise AssertionError("'any' rule was not last")
else:
self.assertEqual(link.rule.name[:13], 'header-match-')
self.assertEqual(link.action, LinkAction.defer)
post_checks.append((link.rule.header, link.rule.pattern))
self.assertListEqual(post_checks, [
('Foo', 'foo'),
('Bar', 'bar'),
])
# Check the error log.
self.assertEqual(mark.readline()[-77:-1],
'Configuration error: [antispam]header_checks '
'contains bogus line: A-bad-line')
def test_nonverp_detectable_nonfatal_bounce(self):
# Here's a bounce that is not VERPd, but which has a bouncing address
# that can be parsed from a known bounce format. The bounce is
# non-fatal so no bounce event is registered and the bounce is not
# reported as unrecognized.
self._mlist.forward_unrecognized_bounces_to = (
UnrecognizedBounceDisposition.site_owner)
dsn = message_from_string("""\
From: [email protected]
To: [email protected]
Message-Id: <first>
Content-Type: multipart/report; report-type=delivery-status; boundary=AAA
MIME-Version: 1.0
--AAA
Content-Type: message/delivery-status
Action: delayed
Original-Recipient: rfc822; [email protected]
--AAA--
""")
self._bounceq.enqueue(dsn, self._msgdata)
mark = LogFileMark('mailman.bounce')
self._runner.run()
get_queue_messages('bounces', expected_count=0)
events = list(self._processor.events)
self.assertEqual(len(events), 0)
# There should be nothing in the 'virgin' queue.
get_queue_messages('virgin', expected_count=0)
# There should be log event in the log file.
log_lines = mark.read().splitlines()
self.assertTrue(len(log_lines) > 0)
def test_no_detectable_bounce_addresses(self):
# A bounce message was received, but no addresses could be detected.
# A message will be logged in the bounce log though, and the message
# can be forwarded to someone who can do something about it.
self._mlist.forward_unrecognized_bounces_to = (
UnrecognizedBounceDisposition.site_owner)
bogus = message_from_string("""\
From: [email protected]
To: [email protected]
Message-Id: <third>
""")
self._bounceq.enqueue(bogus, self._msgdata)
mark = LogFileMark('mailman.bounce')
self._runner.run()
self.assertEqual(len(get_queue_messages('bounces')), 0)
events = list(self._processor.events)
self.assertEqual(len(events), 0)
line = mark.readline()
self.assertEqual(
line[-51:-1],
'Bounce message w/no discernable addresses: <third>')
# Here's the forwarded message to the site owners.
forwards = get_queue_messages('virgin')
self.assertEqual(len(forwards), 1)
self.assertEqual(forwards[0].msg['to'], '*****@*****.**')
def test_archive_lock_used(self):
# Test that locking the maildir when adding works as a failure here
# could mean we lose mail.
lock_file = os.path.join(
config.LOCK_DIR, '{0}-maildir.lock'.format(
self._mlist.fqdn_listname))
with Lock(lock_file):
# Acquire the archiver lock, then make sure the archiver logs the
# fact that it could not acquire the lock.
archive_thread = threading.Thread(
target=Prototype.archive_message,
args=(self._mlist, self._msg))
mark = LogFileMark('mailman.error')
archive_thread.run()
# Test that the archiver output the correct error.
line = mark.readline()
# XXX 2012-03-15 BAW: we really should remove timestamp prefixes
# from the loggers when under test.
self.assertTrue(line.endswith(
'Unable to acquire prototype archiver lock for {0}, '
'discarding: {1}\n'.format(
self._mlist.fqdn_listname,
self._msg.get('message-id'))))
# Check that the message didn't get archived.
created_files = self._find(config.ARCHIVE_DIR)
self.assertEqual(self._expected_dir_structure, created_files)
def test_bad_configuration_line(self):
# Take a mark on the error log file.
mark = LogFileMark('mailman.error')
# A bad value in [antispam]header_checks should just get ignored, but
# with an error message logged.
chain = config.chains['header-match']
# The links are created dynamically; the rule names will all start
# with the same prefix, but have a variable suffix. The actions will
# all be to jump to the named chain. Do these checks now, while we
# collect other useful information.
post_checks = []
saw_any_rule = False
for link in chain.get_links(self._mlist, Message(), {}):
if link.rule.name == 'any':
saw_any_rule = True
self.assertEqual(link.action, LinkAction.jump)
elif saw_any_rule:
raise AssertionError("'any' rule was not last")
else:
self.assertEqual(link.rule.name[:13], 'header-match-')
self.assertEqual(link.action, LinkAction.defer)
post_checks.append((link.rule.header, link.rule.pattern))
self.assertListEqual(post_checks, [
('Foo', 'foo'),
('Bar', 'bar'),
])
# Check the error log.
self.assertEqual(mark.readline()[-77:-1],
'Configuration error: [antispam]header_checks '
'contains bogus line: A-bad-line')
def test_connect_with_socket_failure(self, class_mock):
self._nntpq.enqueue(self._msg, {}, listid='test.example.com')
mark = LogFileMark('mailman.error')
self._runner.run()
log_message = mark.readline()[:-1]
self.assertTrue(
log_message.endswith('NNTP socket error for [email protected]'))
def test_no_detectable_bounce_addresses(self):
# A bounce message was received, but no addresses could be detected.
# A message will be logged in the bounce log though, and the message
# can be forwarded to someone who can do something about it.
self._mlist.forward_unrecognized_bounces_to = (
UnrecognizedBounceDisposition.site_owner)
bogus = message_from_string("""\
From: [email protected]
To: [email protected]
Message-Id: <third>
""")
self._bounceq.enqueue(bogus, self._msgdata)
mark = LogFileMark('mailman.bounce')
self._runner.run()
get_queue_messages('bounces', expected_count=0)
events = list(self._processor.events)
self.assertEqual(len(events), 0)
line = mark.readline()
self.assertEqual(
line[-51:-1],
'Bounce message w/no discernable addresses: <third>')
# Here's the forwarded message to the site owners.
items = get_queue_messages('virgin', expected_count=1)
self.assertEqual(items[0].msg['to'], '*****@*****.**')
def test_non_ascii_message(self):
msg = Message()
msg['From'] = '*****@*****.**'
msg['To'] = '*****@*****.**'
msg['Content-Type'] = 'multipart/mixed'
msg.attach(MIMEText('message with non-ascii chars: \xc3\xa9',
'plain', 'utf-8'))
mbox = digest_mbox(self._mlist)
mbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf')
mbox.add(msg.as_string())
self._digestq.enqueue(
msg,
listname=self._mlist.fqdn_listname,
digest_path=mbox_path,
volume=1, digest_number=1)
# Use any error logs as the error message if the test fails.
error_log = LogFileMark('mailman.error')
self._runner.run()
# The runner will send the file to the shunt queue on exception.
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
# There are two messages in the virgin queue: the digest as plain-text
# and as multipart.
messages = get_queue_messages('virgin')
self.assertEqual(len(messages), 2)
self.assertEqual(
sorted(item.msg.get_content_type() for item in messages),
['multipart/mixed', 'text/plain'])
for item in messages:
self.assertEqual(item.msg['subject'],
'Test Digest, Vol 1, Issue 1')
def test_connect_with_socket_failure(self, class_mock):
self._nntpq.enqueue(self._msg, {}, listname='*****@*****.**')
mark = LogFileMark('mailman.error')
self._runner.run()
log_message = mark.readline()[:-1]
self.assertTrue(log_message.endswith(
'NNTP socket error for [email protected]'))
def test_hold_chain(self):
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: A message
Message-ID: <ant>
MIME-Version: 1.0
A message body.
""")
msgdata = dict(moderation_reasons=[
'TEST-REASON-1',
'TEST-REASON-2',
])
logfile = LogFileMark('mailman.vette')
process_chain(self._mlist, msg, msgdata, start_chain='hold')
messages = get_queue_messages('virgin', expected_count=2)
payloads = {}
for item in messages:
if item.msg['to'] == '*****@*****.**':
part = item.msg.get_payload(0)
payloads['owner'] = part.get_payload().splitlines()
elif item.msg['To'] == '*****@*****.**':
payloads['sender'] = item.msg.get_payload().splitlines()
else:
self.fail('Unexpected message: %s' % item.msg)
self.assertIn(' TEST-REASON-1', payloads['owner'])
self.assertIn(' TEST-REASON-2', payloads['owner'])
self.assertIn(' TEST-REASON-1', payloads['sender'])
self.assertIn(' TEST-REASON-2', payloads['sender'])
logged = logfile.read()
self.assertIn('TEST-REASON-1', logged)
self.assertIn('TEST-REASON-2', logged)
def test_archive_lock_used(self):
# Test that locking the maildir when adding works as a failure here
# could mean we lose mail.
lock_file = os.path.join(
config.LOCK_DIR, '{0}-maildir.lock'.format(
self._mlist.fqdn_listname))
with Lock(lock_file):
# Acquire the archiver lock, then make sure the archiver logs the
# fact that it could not acquire the lock.
archive_thread = threading.Thread(
target=Prototype.archive_message,
args=(self._mlist, self._msg))
mark = LogFileMark('mailman.error')
archive_thread.run()
# Test that the archiver output the correct error.
line = mark.readline()
# XXX 2012-03-15 BAW: we really should remove timestamp prefixes
# from the loggers when under test.
self.assertTrue(line.endswith(
'Unable to acquire prototype archiver lock for {0}, '
'discarding: {1}\n'.format(
self._mlist.fqdn_listname,
self._msg.get('message-id'))))
# Check that the message didn't get archived.
created_files = self._find(config.ARCHIVE_DIR)
self.assertEqual(self._expected_dir_structure, created_files)
def test_bad_regexp(self):
# Take a mark on the error log file.
mark = LogFileMark('mailman.error')
# A bad regexp in header_checks should just get ignored, but
# with an error message logged.
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Foo', '+a bad regexp', 'reject')
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: A message
Message-ID: <ant>
Foo: foo
MIME-Version: 1.0
A message body.
""")
msgdata = {}
# This event subscriber records the event that occurs when the message
# is processed by the header-match chain.
events = []
with event_subscribers(events.append):
process(self._mlist, msg, msgdata, start_chain='header-match')
self.assertEqual(len(events), 0)
# Check the error log.
self.assertEqual(mark.readline()[-89:-1],
"Invalid regexp '+a bad regexp' in header_matches "
'for test.example.com: nothing to repeat')
def test_digest_messages(self):
# In LP: #1130697, the digest runner creates MIME digests using the
# stdlib MIMEMutlipart class, however this class does not have the
# extended attributes we require (e.g. .sender). The fix is to use a
# subclass of MIMEMultipart and our own Message subclass; this adds
# back the required attributes. (LP: #1130696)
#
# Start by creating the raw ingredients for the digests. This also
# runs the digest runner, thus producing the digest messages into the
# virgin queue.
make_digest_messages(self._mlist)
# Run the virgin queue processor, which runs the cook-headers and
# to-outgoing handlers. This should produce no error.
error_log = LogFileMark('mailman.error')
runner = make_testable_runner(VirginRunner, 'virgin')
runner.run()
error_text = error_log.read()
self.assertEqual(len(error_text), 0, error_text)
self.assertEqual(len(get_queue_messages('shunt')), 0)
messages = get_queue_messages('out')
self.assertEqual(len(messages), 2)
# Which one is the MIME digest?
mime_digest = None
for bag in messages:
if bag.msg.get_content_type() == 'multipart/mixed':
assert mime_digest is None, 'Found two MIME digests'
mime_digest = bag.msg
# The cook-headers handler ran.
self.assertIn('x-mailman-version', mime_digest)
self.assertEqual(mime_digest['precedence'], 'list')
# The list's -request address is the original sender.
self.assertEqual(bag.msgdata['original_sender'],
'*****@*****.**')
def test_digest_messages(self):
# In LP: #1130697, the digest runner creates MIME digests using the
# stdlib MIMEMutlipart class, however this class does not have the
# extended attributes we require (e.g. .sender). The fix is to use a
# subclass of MIMEMultipart and our own Message subclass; this adds
# back the required attributes. (LP: #1130696)
#
# Start by creating the raw ingredients for the digests. This also
# runs the digest runner, thus producing the digest messages into the
# virgin queue.
make_digest_messages(self._mlist)
# Run the virgin queue processor, which runs the cook-headers and
# to-outgoing handlers. This should produce no error.
error_log = LogFileMark('mailman.error')
runner = make_testable_runner(VirginRunner, 'virgin')
runner.run()
error_text = error_log.read()
self.assertEqual(len(error_text), 0, error_text)
self.assertEqual(len(get_queue_messages('shunt')), 0)
messages = get_queue_messages('out')
self.assertEqual(len(messages), 2)
# Which one is the MIME digest?
mime_digest = None
for bag in messages:
if bag.msg.get_content_type() == 'multipart/mixed':
assert mime_digest is None, 'Found two MIME digests'
mime_digest = bag.msg
# The cook-headers handler ran.
self.assertIn('x-mailman-version', mime_digest)
self.assertEqual(mime_digest['precedence'], 'list')
# The list's -request address is the original sender.
self.assertEqual(bag.msgdata['original_sender'],
'*****@*****.**')
def test_discard_no_reasons(self):
# The log message contains n/a if no moderation reasons.
msgdata = {}
log_file = LogFileMark('mailman.vette')
process_chain(self._mlist, self._msg, msgdata, start_chain='discard')
log_entry = log_file.read()
self.assertIn('DISCARD: <*****@*****.**>', log_entry)
self.assertIn('[n/a]', log_entry)
def test_discarding_pipeline(self):
# If a handler in the pipeline raises DiscardMessage, the message will
# be thrown away, but with a log message.
mark = LogFileMark('mailman.vette')
process(self._mlist, self._msg, {}, 'test-discarding')
line = mark.readline()[:-1]
self.assertTrue(line.endswith(
'<ant> discarded by "test-discarding" pipeline handler '
'"discarding": by test handler'))
def test_discarding_pipeline(self):
# If a handler in the pipeline raises DiscardMessage, the message will
# be thrown away, but with a log message.
mark = LogFileMark('mailman.vette')
process(self._mlist, self._msg, {}, 'test-discarding')
line = mark.readline()[:-1]
self.assertTrue(line.endswith(
'<ant> discarded by "test-discarding" pipeline handler '
'"discarding": by test handler'))
def test_uheader_multiline(self):
# Multiline headers should be truncated (GL#273).
mark = LogFileMark('mailman.error')
header = cook_headers.uheader(self._mlist, 'A multiline\ndescription',
'X-Header')
self.assertEqual(header.encode(), 'A multiline [...]')
log_messages = mark.read()
self.assertIn('Header X-Header contains a newline, truncating it',
log_messages)
def test_discard_reasons(self):
# The log message must contain the moderation reasons.
msgdata = dict(moderation_reasons=['TEST-REASON-1', 'TEST-REASON-2'])
log_file = LogFileMark('mailman.vette')
process_chain(self._mlist, self._msg, msgdata, start_chain='discard')
log_entry = log_file.read()
self.assertIn('DISCARD: <*****@*****.**>', log_entry)
self.assertIn('TEST-REASON-1', log_entry)
self.assertIn('TEST-REASON-2', log_entry)
def test_header_matches_header_only(self):
# Check that an empty pattern is skipped.
self._pckdict['header_filter_rules'] = [
('SomeHeaderName', 3, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Unsupported header_filter_rules pattern',
error_log.readline())
def test_header_matches_header_only(self):
# Check that an empty pattern is skipped.
self._pckdict['header_filter_rules'] = [
('SomeHeaderName', 3, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Unsupported header_filter_rules pattern',
error_log.readline())
def test_header_matches_invalid_re(self):
# Check that an invalid regular expression pattern is skipped.
self._pckdict['header_filter_rules'] = [
('SomeHeaderName: *invalid-re', 3, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Skipping header_filter rule because of an invalid '
'regular expression', error_log.readline())
def test_bad_group(self):
self.mlist.linked_newsgroup = 'other.group'
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp():
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(len(lines), 2)
self.assertTrue(lines[0].endswith('NNTP error for list '
'[email protected]:'))
self.assertEqual(lines[1], 'No such group: other.group')
def test_header_matches_anything(self):
# Check that a wild card header pattern is skipped.
self._pckdict['header_filter_rules'] = [
('.*', 7, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Unsupported header_filter_rules pattern',
error_log.readline())
def test_header_matches_invalid_re(self):
# Check that an invalid regular expression pattern is skipped.
self._pckdict['header_filter_rules'] = [
('SomeHeaderName: *invalid-re', 3, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Skipping header_filter rule because of an invalid '
'regular expression', error_log.readline())
def test_header_matches_anything(self):
# Check that a wild card header pattern is skipped.
self._pckdict['header_filter_rules'] = [
('.*', 7, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Unsupported header_filter_rules pattern',
error_log.readline())
def test_maybe_forward_discard(self):
# When forward_unrecognized_bounces_to is set to discard, no bounce
# messages are forwarded.
self._mlist.forward_unrecognized_bounces_to = (
UnrecognizedBounceDisposition.discard)
# The only artifact of this call is a log file entry.
mark = LogFileMark('mailman.bounce')
maybe_forward(self._mlist, self._msg)
get_queue_messages('virgin', expected_count=0)
line = mark.readline()
self.assertEqual(line[-40:-1],
'Discarding unrecognized bounce: <first>')
def test_bad_nntp_connect(self):
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp(fail=1):
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(len(lines), 4)
self.assertTrue(lines[0].endswith('error opening connection '
'to nntp_host: news.example.com'))
self.assertEqual(lines[1], 'Bad call to NNTP')
self.assertTrue(lines[2].endswith('NNTP error for list '
'[email protected]:'))
self.assertEqual(lines[3], 'Bad call to NNTP')
def test_catchup_only(self):
self.mlist.usenet_watermark = None
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp():
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(self.mlist.usenet_watermark, 3)
self.assertEqual(len(lines), 3)
self.assertTrue(lines[0].endswith('[email protected]: [1..3]'))
self.assertTrue(lines[1].endswith('[email protected] '
'caught up to article 3'))
self.assertTrue(lines[2].endswith('[email protected] watermark: 3'))
def test_up_to_date(self):
self.mlist.usenet_watermark = 3
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp():
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(self.mlist.usenet_watermark, 3)
self.assertEqual(len(lines), 3)
self.assertTrue(lines[0].endswith('[email protected]: [1..3]'))
self.assertTrue(lines[1].endswith('nothing new for list '
'*****@*****.**'))
self.assertTrue(lines[2].endswith('[email protected] watermark: 3'))
def test_rejecting_pipeline(self):
# If a handler in the pipeline raises DiscardMessage, the message will
# be thrown away, but with a log message.
mark = LogFileMark('mailman.vette')
process(self._mlist, self._msg, {}, 'test-rejecting')
line = mark.readline()[:-1]
self.assertTrue(line.endswith(
'<ant> rejected by "test-rejecting" pipeline handler '
'"rejecting": by test handler'))
# In the rejection case, the original message will also be in the
# virgin queue.
items = get_queue_messages('virgin', expected_count=1)
self.assertEqual(str(items[0].msg['subject']), 'a test')
def test_maybe_forward_discard(self):
# When forward_unrecognized_bounces_to is set to discard, no bounce
# messages are forwarded.
self._mlist.forward_unrecognized_bounces_to = (
UnrecognizedBounceDisposition.discard)
# The only artifact of this call is a log file entry.
mark = LogFileMark('mailman.bounce')
maybe_forward(self._mlist, self._msg)
get_queue_messages('virgin', expected_count=0)
line = mark.readline()
self.assertEqual(
line[-40:-1],
'Discarding unrecognized bounce: <first>')
def test_header_matches_duplicate(self):
# Check that duplicate patterns don't cause tracebacks.
self._pckdict['header_filter_rules'] = [
('SomeHeaderName: test-pattern', 3, False),
('SomeHeaderName: test-pattern', 2, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual([(hm.header, hm.pattern, hm.chain)
for hm in self._mlist.header_matches],
[('someheadername', 'test-pattern', 'discard')])
self.assertIn('Skipping duplicate header_filter rule',
error_log.readline())
def test_article_exception(self):
mark = LogFileMark('mailman.fromusenet')
with get_nntplib_nntp(fail=2):
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(len(lines), 5)
self.assertTrue(lines[0].endswith('[email protected]: [1..3]'))
self.assertTrue(lines[1].endswith('gating [email protected] '
'articles [1..3]'))
self.assertTrue(lines[2].endswith('NNTP error for list '
'[email protected]: 2'))
self.assertEqual(lines[3], 'Bad call to article')
self.assertTrue(lines[4].endswith('[email protected] watermark: 3'))
def test_bad_config_listname_chars(self):
mark = LogFileMark('mailman.error')
# This list create should succeed but log an error
mlist = create_list('*****@*****.**')
# Check the error log.
self.assertRegex(
mark.readline(),
r'^.*Bad config\.mailman\.listname_chars setting: '
r'\[a-z0-9-\+\\]: '
'(unterminated character set|'
'unexpected end of regular expression)$')
# Check that the list was actually created.
self.assertIs(os.path.isdir(mlist.data_path), True)
def test_rejecting_pipeline(self):
# If a handler in the pipeline raises DiscardMessage, the message will
# be thrown away, but with a log message.
mark = LogFileMark('mailman.vette')
process(self._mlist, self._msg, {}, 'test-rejecting')
line = mark.readline()[:-1]
self.assertTrue(
line.endswith(
'<ant> rejected by "test-rejecting" pipeline handler '
'"rejecting": by test handler'))
# In the rejection case, the original message will also be in the
# virgin queue.
items = get_queue_messages('virgin', expected_count=1)
self.assertEqual(str(items[0].msg['subject']), 'a test')
def test_email_parser_exception(self):
mark = LogFileMark('mailman.fromusenet')
with get_email_exception():
with get_nntplib_nntp():
self._command.invoke(gatenews)
lines = mark.read().splitlines()
self.assertEqual(len(lines), 5)
self.assertTrue(lines[0].endswith('[email protected]: [1..3]'))
self.assertTrue(lines[1].endswith('gating [email protected] '
'articles [1..3]'))
self.assertTrue(lines[2].endswith('email package exception for '
'my.group:2'))
self.assertEqual(lines[3], 'Bad message')
self.assertTrue(lines[4].endswith('[email protected] watermark: 3'))
def test_error_with_numeric_port(self):
# Test the code path where a socket.error is raised in the delivery
# function, and the MTA port is set to zero. The only real effect of
# that is a log message. Start by opening the error log and reading
# the current file position.
mark = LogFileMark('mailman.error')
self._outq.enqueue(self._msg, {}, listid='test.example.com')
with configuration('mta', smtp_port=2112):
self._runner.run()
line = mark.readline()
# The log line will contain a variable timestamp, the PID, and a
# trailing newline. Ignore these.
self.assertEqual(
line[-53:-1],
'Cannot connect to SMTP server localhost on port 2112')
def test_get_moderator_approval_log_on_hold(self):
# When the subscription is held for moderator approval, a message is
# logged.
mark = LogFileMark('mailman.subscribe')
self._mlist.subscription_policy = SubscriptionPolicy.moderate
anne = self._user_manager.create_address(self._anne)
workflow = SubscriptionWorkflow(self._mlist, anne,
pre_verified=True,
pre_confirmed=True)
# Consume the entire state machine.
list(workflow)
self.assertIn(
'[email protected]: held subscription request from [email protected]',
mark.readline()
)
def test_non_ascii_message(self):
msg = Message()
msg['From'] = '*****@*****.**'
msg['To'] = '*****@*****.**'
msg['Content-Type'] = 'multipart/mixed'
msg.attach(MIMEText('message with non-ascii chars: \xc3\xa9',
'plain', 'utf-8'))
mbox = digest_mbox(self._mlist)
mbox.add(msg.as_string())
# Use any error logs as the error message if the test fails.
error_log = LogFileMark('mailman.error')
make_digest_messages(self._mlist, msg)
# The runner will send the file to the shunt queue on exception.
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
self._check_virgin_queue()
def test_header_matches_duplicate(self):
# Check that duplicate patterns don't cause tracebacks.
self._pckdict['header_filter_rules'] = [
('SomeHeaderName: test-pattern', 3, False),
('SomeHeaderName: test-pattern', 2, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(
[(hm.header, hm.pattern, hm.chain)
for hm in self._mlist.header_matches],
[('someheadername', 'test-pattern', 'discard')]
)
self.assertIn('Skipping duplicate header_filter rule',
error_log.readline())
def test_error_with_numeric_port(self):
# Test the code path where a socket.error is raised in the delivery
# function, and the MTA port is set to zero. The only real effect of
# that is a log message. Start by opening the error log and reading
# the current file position.
mark = LogFileMark('mailman.error')
self._outq.enqueue(self._msg, {}, listid='test.example.com')
with configuration('mta', smtp_port=2112):
self._runner.run()
line = mark.readline()
# The log line will contain a variable timestamp, the PID, and a
# trailing newline. Ignore these.
self.assertEqual(
line[-53:-1],
'Cannot connect to SMTP server localhost on port 2112')
def test_get_moderator_approval_log_on_hold(self):
# When the subscription is held for moderator approval, a message is
# logged.
mark = LogFileMark('mailman.subscribe')
self._mlist.subscription_policy = SubscriptionPolicy.moderate
anne = self._user_manager.create_address(self._anne)
workflow = SubscriptionWorkflow(self._mlist,
anne,
pre_verified=True,
pre_confirmed=True)
# Consume the entire state machine.
list(workflow)
self.assertIn(
'[email protected]: held subscription request from [email protected]',
mark.readline())
def test_broken_archiver(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
mark = LogFileMark('mailman.archiver')
self._archiveq.enqueue(self._msg, {},
listid=self._mlist.list_id,
received_time=now())
IListArchiverSet(self._mlist).get('broken').is_enabled = True
self._runner.run()
# The archiver is broken, so there are no messages on the file system,
# but there is a log message and the message was not shunted.
log_messages = mark.read()
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot archive message', log_messages)
get_queue_messages('shunt', expected_count=0)
def test_connect_with_other_failure(self, class_mock):
# In this failure mode, the message stays queued, so we can only run
# the nntp runner once.
def once(runner):
# I.e. stop immediately, since the queue will not be empty.
return True
runner = make_testable_runner(nntp.NNTPRunner, 'nntp', predicate=once)
self._nntpq.enqueue(self._msg, {}, listid='test.example.com')
mark = LogFileMark('mailman.error')
runner.run()
log_message = mark.readline()[:-1]
self.assertTrue(log_message.endswith(
'NNTP unexpected exception for [email protected]'))
items = get_queue_messages('nntp', expected_count=1)
self.assertEqual(items[0].msgdata['listid'], 'test.example.com')
self.assertEqual(items[0].msg['subject'], 'A newsgroup posting')
def test_header_matches_unsupported_action(self):
# Check that unsupported actions are skipped.
for action_num in (1, 4, 5):
self._pckdict['header_filter_rules'] = [
('HeaderName: test-re', action_num, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(self._mlist.header_matches, [])
self.assertIn('Unsupported header_filter_rules action',
error_log.readline())
# Avoid a useless warning.
for member in self._mlist.members.members:
member.unsubscribe()
for member in self._mlist.owners.members:
member.unsubscribe()
def test_broken_archiver(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
mark = LogFileMark('mailman.archiver')
self._archiveq.enqueue(
self._msg, {},
listid=self._mlist.list_id,
received_time=now())
IListArchiverSet(self._mlist).get('broken').is_enabled = True
self._runner.run()
# The archiver is broken, so there are no messages on the file system,
# but there is a log message and the message was not shunted.
log_messages = mark.read()
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot archive message', log_messages)
get_queue_messages('shunt', expected_count=0)
def test_broken_permalink(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
site_dir = os.path.join(config.TEMPLATE_DIR, 'site', 'en')
os.makedirs(site_dir)
footer_path = os.path.join(site_dir, 'myfooter.txt')
with open(footer_path, 'w', encoding='utf-8') as fp:
print('${broken_url}', file=fp)
self._mlist.footer_uri = 'mailman:///myfooter.txt'
self._mlist.preferred_language = 'en'
mark = LogFileMark('mailman.archiver')
decorate.process(self._mlist, self._msg, {})
log_messages = mark.read()
self.assertNotIn('http:', self._msg.as_string())
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get permalink', log_messages)
def test_broken_permalink(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
site_dir = os.path.join(config.TEMPLATE_DIR, 'site', 'en')
os.makedirs(site_dir)
footer_path = os.path.join(site_dir, 'myfooter.txt')
with open(footer_path, 'w', encoding='utf-8') as fp:
print('${broken_url}', file=fp)
self._mlist.footer_uri = 'mailman:///myfooter.txt'
self._mlist.preferred_language = 'en'
mark = LogFileMark('mailman.archiver')
decorate.process(self._mlist, self._msg, {})
log_messages = mark.read()
self.assertNotIn('http:', self._msg.as_string())
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get permalink', log_messages)
def test_bad_action(self):
# This should never happen, but what if it does?
# FilterAction.accept, FilterAction.hold, and FilterAction.defer are
# not valid. They are treated as discard actions, but the problem is
# also logged.
for action in (FilterAction.accept,
FilterAction.hold,
FilterAction.defer):
self._mlist.filter_action = action
mark = LogFileMark('mailman.error')
with self.assertRaises(errors.DiscardMessage) as cm:
mime_delete.dispose(self._mlist, self._msg, {}, 'bad action')
self.assertEqual(cm.exception.message, 'bad action')
line = mark.readline()[:-1]
self.assertTrue(line.endswith(
'{0} invalid FilterAction: [email protected]. '
'Treating as discard'.format(action.name)))
def test_non_ascii_message(self):
# Subscribe some users receiving digests.
anne = subscribe(self._mlist, 'Anne')
anne.preferences.delivery_mode = DeliveryMode.mime_digests
bart = subscribe(self._mlist, 'Bart')
bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
msg = Message()
msg['From'] = '*****@*****.**'
msg['To'] = '*****@*****.**'
msg['Content-Type'] = 'multipart/mixed'
msg.attach(MIMEText('message with non-ascii chars: \xc3\xa9',
'plain', 'utf-8'))
mbox = digest_mbox(self._mlist)
mbox.add(msg.as_string())
# Use any error logs as the error message if the test fails.
error_log = LogFileMark('mailman.error')
make_digest_messages(self._mlist, msg)
# The runner will send the file to the shunt queue on exception.
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
self._check_virgin_queue()
def test_broken_archiver(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
config.push('archiver', """
[archiver.broken]
class: {}.BrokenArchiver
enable: yes
""".format(BrokenArchiver.__module__))
self.addCleanup(config.pop, 'archiver')
mark = LogFileMark('mailman.archiver')
rfc_2369.process(self._mlist, self._msg, {})
log_messages = mark.read()
# Because .list_url() was broken, there will be no List-Archive header.
self.assertIsNone(self._msg.get('list-archive'))
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get list URL', log_messages)
# Because .permalink() was broken, there will be no Archived-At header.
self.assertIsNone(self._msg.get('archived-at'))
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot get permalink', log_messages)
def test_log_exception_in_finish(self):
# If something bad happens in .finish(), the traceback should get
# logged. LP: #1165589.
msg = mfs("""\
From: [email protected]
To: [email protected]
Message-ID: <ant>
""")
switchboard = config.switchboards['shunt']
# Enqueue the message.
filebase = switchboard.enqueue(msg)
error_log = LogFileMark('mailman.error')
msg, data = switchboard.dequeue(filebase)
# Now, cause .finish() to throw an exception.
with patch('mailman.core.switchboard.os.rename',
side_effect=OSError('Oops!')):
switchboard.finish(filebase, preserve=True)
traceback = error_log.read().splitlines()
self.assertEqual(traceback[1], 'Traceback (most recent call last):')
self.assertEqual(traceback[-1], 'OSError: Oops!')
def test_digest_messages(self):
# In LP: #1130697, the digest runner creates MIME digests using the
# stdlib MIMEMutlipart class, however this class does not have the
# extended attributes we require (e.g. .sender). The fix is to use a
# subclass of MIMEMultipart and our own Message subclass; this adds
# back the required attributes. (LP: #1130696)
self._mlist.send_welcome_message = False
# Subscribe some users receiving digests.
anne = subscribe(self._mlist, 'Anne')
anne.preferences.delivery_mode = DeliveryMode.mime_digests
bart = subscribe(self._mlist, 'Bart')
bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
# Start by creating the raw ingredients for the digests. This also
# runs the digest runner, thus producing the digest messages into the
# virgin queue.
make_digest_messages(self._mlist)
# Run the virgin queue processor, which runs the cook-headers and
# to-outgoing handlers. This should produce no error.
error_log = LogFileMark('mailman.error')
runner = make_testable_runner(VirginRunner, 'virgin')
runner.run()
error_text = error_log.read()
self.assertEqual(len(error_text), 0, error_text)
get_queue_messages('shunt', expected_count=0)
items = get_queue_messages('out', expected_count=2)
# Which one is the MIME digest?
mime_digest = None
for item in items:
if item.msg.get_content_type() == 'multipart/mixed':
assert mime_digest is None, 'Found two MIME digests'
mime_digest = item.msg
# The cook-headers handler ran.
self.assertIn('x-mailman-version', mime_digest)
self.assertEqual(mime_digest['precedence'], 'list')
# The list's -request address is the original sender.
self.assertEqual(item.msgdata['original_sender'],
'*****@*****.**')
def test_no_progress_on_retries_with_expired_retry_period(self):
# We've had temporary failures with no progress, and the retry period
# has expired. In that case, a log entry is written and message is
# discarded. There's nothing more that can be done.
temporary_failures.append('*****@*****.**')
temporary_failures.append('*****@*****.**')
retry_period = as_timedelta(config.mta.delivery_retry_period)
deliver_until = datetime(2005, 8, 1, 7, 49, 23) + retry_period
msgdata = dict(last_recip_count=2,
deliver_until=deliver_until)
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
# Before the runner runs, several days pass.
factory.fast_forward(retry_period.days + 1)
mark = LogFileMark('mailman.smtp')
self._runner.run()
# There should be no message in the retry or outgoing queues.
get_queue_messages('retry', expected_count=0)
get_queue_messages('out', expected_count=0)
# There should be a log message in the smtp log indicating that the
# message has been discarded.
line = mark.readline()
self.assertEqual(
line[-63:-1],
'Discarding message with persistent temporary failures: <first>')
def test_header_matches(self):
# This test contail real cases of header_filter_rules
self._pckdict['header_filter_rules'] = [
('X\\-Spam\\-Status\\: Yes.*', 3, False),
('^X-Spam-Status: Yes\r\n\r\n', 2, False),
('^X-Spam-Level: \\*\\*\\*.*$', 3, False),
('^X-Spam-Level:.\\*\\*\r\n^X-Spam:.\\Yes', 3, False),
('Subject: \\[SPAM\\].*', 3, False),
('^Subject: .*loan.*', 3, False),
('Original-Received: from *linkedin.com*\r\n', 3, False),
('X-Git-Module: rhq.*git', 6, False),
('Approved: verysecretpassword', 6, False),
('^Subject: dev-\r\n^Subject: staging-', 3, False),
('from: .*[email protected]\r\nfrom: .*@jw-express.com',
2, False),
('^Received: from smtp-.*\\.fedoraproject\\.org\r\n'
'^Received: from mx.*\\.redhat.com\r\n'
'^Resent-date:\r\n'
'^Resent-from:\r\n'
'^Resent-Message-ID:\r\n'
'^Resent-to:\r\n'
'^Subject: [^mtv]\r\n',
7, False),
('^Received: from fedorahosted\\.org.*by fedorahosted\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'hosted.*\\.fedoraproject\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'fedoraproject\\.org\r\n'
'^Received: from hosted.*\\.fedoraproject.org.*by '
'fedorahosted\\.org',
6, False),
]
error_log = LogFileMark('mailman.error')
self._import()
self.assertListEqual(
[(hm.header, hm.pattern, hm.chain)
for hm in self._mlist.header_matches], [
('x-spam-status', 'Yes.*', 'discard'),
('x-spam-status', 'Yes', 'reject'),
('x-spam-level', '\\*\\*\\*.*$', 'discard'),
('x-spam-level', '\\*\\*', 'discard'),
('x-spam', '\\Yes', 'discard'),
('subject', '\\[SPAM\\].*', 'discard'),
('subject', '.*loan.*', 'discard'),
('original-received', 'from *linkedin.com*', 'discard'),
('x-git-module', 'rhq.*git', 'accept'),
('approved', 'verysecretpassword', 'accept'),
('subject', 'dev-', 'discard'),
('subject', 'staging-', 'discard'),
('from', '.*[email protected]', 'reject'),
('from', '.*@jw-express.com', 'reject'),
('received', 'from smtp-.*\\.fedoraproject\\.org', 'hold'),
('received', 'from mx.*\\.redhat.com', 'hold'),
('resent-date', '.*', 'hold'),
('resent-from', '.*', 'hold'),
('resent-message-id', '.*', 'hold'),
('resent-to', '.*', 'hold'),
('subject', '[^mtv]', 'hold'),
('received', 'from fedorahosted\\.org.*by fedorahosted\\.org',
'accept'),
('received',
'from hosted.*\\.fedoraproject.org.*by '
'hosted.*\\.fedoraproject\\.org', 'accept'),
('received',
'from hosted.*\\.fedoraproject.org.*by '
'fedoraproject\\.org', 'accept'),
('received',
'from hosted.*\\.fedoraproject.org.*by '
'fedorahosted\\.org', 'accept'),
])
loglines = error_log.read().strip()
self.assertEqual(len(loglines), 0)
