def test_site_disabled(self):
# Here the system configuration enables all the archivers in time for
# the archive set to be created with all list archivers enabled. But
# then the site-wide archiver gets disabled, so the list specific
# archiver will also be disabled.
archiver_set = IListArchiverSet(self._mlist)
archiver = archiver_set.get('prototype')
self.assertTrue(archiver.is_enabled)
# Disable the site-wide archiver.
config.push('enable prototype', """\
[archiver.prototype]
enable: no
""")
self.assertFalse(archiver.is_enabled)
config.pop('enable prototype')
def test_site_disabled(self):
# Here the system configuration enables all the archivers in time for
# the archive set to be created with all list archivers enabled. But
# then the site-wide archiver gets disabled, so the list specific
# archiver will also be disabled.
archiver_set = IListArchiverSet(self._mlist)
archiver = archiver_set.get('prototype')
self.assertTrue(archiver.is_enabled)
# Disable the site-wide archiver.
config.push('enable prototype', """\
[archiver.prototype]
enable: no
""")
self.assertFalse(archiver.is_enabled)
config.pop('enable prototype')
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._now = now()
# Enable just the dummy archiver.
config.push(
'dummy', """
[archiver.dummy]
class: mailman.runners.tests.test_archiver.DummyArchiver
enable: no
[archiver.prototype]
enable: no
[archiver.mhonarc]
enable: no
[archiver.mail_archive]
enable: no
""")
self._archiveq = config.switchboards['archive']
self._msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: My first post
Message-ID: <first>
Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB
First post!
""")
self._runner = make_testable_runner(ArchiveRunner)
IListArchiverSet(self._mlist).get('dummy').is_enabled = True
def on_get(self, request, response):
"""Get all the archiver statuses."""
archiver_set = IListArchiverSet(self._mlist)
resource = {
archiver.name: archiver.is_enabled
for archiver in archiver_set.archivers
}
okay(response, etag(resource))
def test_disable_all_list_archivers(self):
# Let's disable all the archivers for the mailing list, but not the
# global archivers. No messages will get archived.
for archiver in IListArchiverSet(self._mlist).archivers:
archiver.is_enabled = False
config.db.store.commit()
self._archiveq.enqueue(self._msg, {}, listid=self._mlist.list_id)
self._runner.run()
self.assertEqual(os.listdir(config.MESSAGES_DIR), [])
def test_enable_list_archiver(self):
# When the system configuration file disables an archiver site-wide,
# the list-specific mailing list will get initialized as not enabled.
# Create the archiver set on the fly so that it doesn't get
# initialized with a configuration that enables the prototype archiver.
archiver_set = IListArchiverSet(self._mlist)
archiver = archiver_set.get('prototype')
self.assertFalse(archiver.is_enabled)
# Enable both the list archiver and the system archiver.
archiver.is_enabled = True
config.push('enable prototype', """\
[archiver.prototype]
enable: yes
""")
# Get the IListArchiver again.
archiver = archiver_set.get('prototype')
self.assertTrue(archiver.is_enabled)
config.pop('enable prototype')
def test_enable_list_archiver(self):
# When the system configuration file disables an archiver site-wide,
# the list-specific mailing list will get initialized as not enabled.
# Create the archiver set on the fly so that it doesn't get
# initialized with a configuration that enables the prototype archiver.
archiver_set = IListArchiverSet(self._mlist)
archiver = archiver_set.get('prototype')
self.assertFalse(archiver.is_enabled)
# Enable both the list archiver and the system archiver.
archiver.is_enabled = True
config.push('enable prototype', """\
[archiver.prototype]
enable: yes
""")
# Get the IListArchiver again.
archiver = archiver_set.get('prototype')
self.assertTrue(archiver.is_enabled)
config.pop('enable prototype')
def test_delete_list_with_list_archiver_set(self):
# Ensure that mailing lists with archiver sets can be deleted. In
# issue #115, this fails under PostgreSQL, but not SQLite.
mlist = create_list('*****@*****.**')
# We don't keep a reference to this archiver set just because it makes
# pyflakes unhappy. It doesn't change the outcome.
IListArchiverSet(mlist)
list_manager = getUtility(IListManager)
list_manager.delete(mlist)
self.assertIsNone(list_manager.get('*****@*****.**'))
class TestListArchiver(unittest.TestCase):
layer = ConfigLayer
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._set = IListArchiverSet(self._mlist)
def test_list_archivers(self):
# Find the set of archivers registered for this mailing list.
self.assertEqual(['mail-archive', 'mhonarc', 'prototype'],
sorted(archiver.name
for archiver in self._set.archivers))
def test_get_archiver(self):
# Use .get() to see if a mailing list has an archiver.
archiver = self._set.get('mhonarc')
self.assertEqual(archiver.name, 'mhonarc')
self.assertTrue(archiver.is_enabled)
self.assertEqual(archiver.mailing_list, self._mlist)
self.assertEqual(archiver.system_archiver.name, 'mhonarc')
def test_get_archiver_no_such(self):
# Using .get() on a non-existing name returns None.
self.assertIsNone(self._set.get('no-such-archiver'))
def test_site_disabled(self):
# Here the system configuration enables all the archivers in time for
# the archive set to be created with all list archivers enabled. But
# then the site-wide archiver gets disabled, so the list specific
# archiver will also be disabled.
archiver_set = IListArchiverSet(self._mlist)
archiver = archiver_set.get('mhonarc')
self.assertTrue(archiver.is_enabled)
# Disable the site-wide archiver.
config.push(
'enable mhonarc', """\
[archiver.mhonarc]
enable: no
""")
self.assertFalse(archiver.is_enabled)
config.pop('enable mhonarc')
class TestListArchiver(unittest.TestCase):
layer = ConfigLayer
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._set = IListArchiverSet(self._mlist)
def test_list_archivers(self):
# Find the set of archivers registered for this mailing list.
self.assertEqual(
['mail-archive', 'mhonarc', 'prototype'],
sorted(archiver.name for archiver in self._set.archivers))
def test_get_archiver(self):
# Use .get() to see if a mailing list has an archiver.
archiver = self._set.get('prototype')
self.assertEqual(archiver.name, 'prototype')
self.assertTrue(archiver.is_enabled)
self.assertEqual(archiver.mailing_list, self._mlist)
self.assertEqual(archiver.system_archiver.name, 'prototype')
def test_get_archiver_no_such(self):
# Using .get() on a non-existing name returns None.
self.assertIsNone(self._set.get('no-such-archiver'))
def test_site_disabled(self):
# Here the system configuration enables all the archivers in time for
# the archive set to be created with all list archivers enabled. But
# then the site-wide archiver gets disabled, so the list specific
# archiver will also be disabled.
archiver_set = IListArchiverSet(self._mlist)
archiver = archiver_set.get('prototype')
self.assertTrue(archiver.is_enabled)
# Disable the site-wide archiver.
config.push('enable prototype', """\
[archiver.prototype]
enable: no
""")
self.assertFalse(archiver.is_enabled)
config.pop('enable prototype')
class ArchiverGetterSetter(GetterSetter):
"""Resource for updating archiver statuses."""
def __init__(self, mlist):
super().__init__()
self._archiver_set = IListArchiverSet(mlist)
def put(self, mlist, attribute, value):
# attribute will contain the (bytes) name of the archiver that is
# getting a new status. value will be the representation of the new
# boolean status.
archiver = self._archiver_set.get(attribute)
assert archiver is not None, attribute
archiver.is_enabled = as_boolean(value)
def patch_put(self, request, response, is_optional):
archiver_set = IListArchiverSet(self._mlist)
kws = {archiver.name: ArchiverGetterSetter(self._mlist)
for archiver in archiver_set.archivers}
if is_optional:
# For a PATCH, all attributes are optional.
kws['_optional'] = kws.keys()
try:
Validator(**kws).update(self._mlist, request)
except ValueError as error:
bad_request(response, str(error))
else:
no_content(response)
class ArchiverGetterSetter(GetterSetter):
"""Resource for updating archiver statuses."""
def __init__(self, mlist):
super().__init__()
self._archiver_set = IListArchiverSet(mlist)
def put(self, mlist, attribute, value):
# attribute will contain the (bytes) name of the archiver that is
# getting a new status. value will be the representation of the new
# boolean status.
archiver = self._archiver_set.get(attribute)
assert archiver is not None, attribute
archiver.is_enabled = as_boolean(value)
class ArchiverGetterSetter(GetterSetter):
"""Resource for updating archiver statuses."""
def __init__(self, mlist):
super(ArchiverGetterSetter, self).__init__()
self._archiver_set = IListArchiverSet(mlist)
def put(self, mlist, attribute, value):
# attribute will contain the (bytes) name of the archiver that is
# getting a new status. value will be the representation of the new
# boolean status.
archiver = self._archiver_set.get(attribute)
if archiver is None:
raise ValueError('No such archiver: {}'.format(attribute))
archiver.is_enabled = as_boolean(value)
class ArchiverGetterSetter(GetterSetter):
"""Resource for updating archiver statuses."""
def __init__(self, mlist):
super(ArchiverGetterSetter, self).__init__()
self._archiver_set = IListArchiverSet(mlist)
def put(self, mlist, attribute, value):
# attribute will contain the (bytes) name of the archiver that is
# getting a new status. value will be the representation of the new
# boolean status.
archiver = self._archiver_set.get(attribute.decode('utf-8'))
if archiver is None:
raise ValueError('No such archiver: {}'.format(attribute))
archiver.is_enabled = as_boolean(value)
def test_broken_archiver(self):
# GL issue #208 - IArchive messages raise exceptions, breaking the
# rfc-2369 handler and shunting messages.
mark = LogFileMark('mailman.archiver')
self._archiveq.enqueue(self._msg, {},
listid=self._mlist.list_id,
received_time=now())
IListArchiverSet(self._mlist).get('broken').is_enabled = True
self._runner.run()
# The archiver is broken, so there are no messages on the file system,
# but there is a log message and the message was not shunted.
log_messages = mark.read()
self.assertIn('Exception in "broken" archiver', log_messages)
self.assertIn('RuntimeError: Cannot archive message', log_messages)
get_queue_messages('shunt', expected_count=0)
def _dispose(self, mlist, msg, msgdata):
received_time = msgdata.get('received_time', now(strip_tzinfo=False))
archiver_set = IListArchiverSet(mlist)
for archiver in archiver_set.archivers:
# The archiver is disabled if either the list-specific or
# site-wide archiver is disabled.
if not archiver.is_enabled:
continue
msg_copy = copy.deepcopy(msg)
if _should_clobber(msg, msgdata, archiver.name):
original_date = msg_copy['date']
del msg_copy['date']
del msg_copy['x-original-date']
msg_copy['Date'] = received_time.strftime(RFC822_DATE_FMT)
if original_date:
msg_copy['X-Original-Date'] = original_date
# A problem in one archiver should not prevent other archivers
# from running.
try:
archiver.system_archiver.archive_message(mlist, msg_copy)
except Exception:
log.exception('Broken archiver: %s' % archiver.name)
def __init__(self, mlist):
super().__init__()
self._archiver_set = IListArchiverSet(mlist)
def process(mlist, msg, msgdata):
"""Add the RFC 2369 List-* and related headers."""
# Some people really hate the List-* headers. It seems that the free
# version of Eudora (possibly on for some platforms) does not hide these
# headers by default, pissing off their users. Too bad. Fix the MUAs.
if not mlist.include_rfc2369_headers:
return
list_id = '{0.list_name}.{0.mail_host}'.format(mlist)
if mlist.description:
# Don't wrap the header since here we just want to get it properly RFC
# 2047 encoded.
i18ndesc = uheader(mlist, mlist.description, 'List-Id', maxlinelen=998)
listid_h = formataddr((str(i18ndesc), list_id))
else:
# Without a description, we need to ensure the MUST brackets.
listid_h = '<{}>'.format(list_id)
# No other agent should add a List-ID header except Mailman.
del msg['list-id']
msg['List-Id'] = listid_h
# For internally crafted messages, we also add a (nonstandard),
# "X-List-Administrivia: yes" header. For all others (i.e. those coming
# from list posts), we add a bunch of other RFC 2369 headers.
requestaddr = mlist.request_address
headers = []
# XXX reduced_list_headers used to suppress List-Help, List-Subject, and
# List-Unsubscribe from UserNotification. That doesn't seem to make sense
# any more, so always add those three headers (others will still be
# suppressed).
headers.extend((
('List-Help', '<mailto:{}?subject=help>'.format(requestaddr)),
('List-Unsubscribe', '<mailto:{}>'.format(mlist.leave_address)),
('List-Subscribe', '<mailto:{}>'.format(mlist.join_address)),
))
if not msgdata.get('reduced_list_headers'):
# List-Post: is controlled by a separate attribute, which is somewhat
# misnamed. RFC 2369 requires a value of NO if posting is not
# allowed, i.e. for an announce-only list.
list_post = ('<mailto:{}>'.format(mlist.posting_address)
if mlist.allow_list_posts
else 'NO')
headers.append(('List-Post', list_post))
# Add RFC 2369 and 5064 archiving headers, if archiving is enabled.
if mlist.archive_policy is not ArchivePolicy.never:
archiver_set = IListArchiverSet(mlist)
for archiver in archiver_set.archivers:
if not archiver.is_enabled:
continue
# Watch out for exceptions in the archiver plugin.
try:
archiver_url = archiver.system_archiver.list_url(mlist)
except Exception:
log.exception('Exception in "{}" archiver'.format(
archiver.system_archiver.name))
archiver_url = None
if archiver_url is not None:
headers.append(('List-Archive',
'<{}>'.format(archiver_url)))
try:
permalink = archiver.system_archiver.permalink(mlist, msg)
except Exception:
log.exception('Exception in "{}" archiver'.format(
archiver.system_archiver.name))
permalink = None
if permalink is not None:
headers.append(('Archived-At', '<{}>'.format(permalink)))
# XXX RFC 2369 also defines a List-Owner header which we are not currently
# supporting, but should.
#
# Some headers will appear more than once in the new set, e.g. the
# List-Archive and Archived-At headers. We want to delete any RFC 2369
# headers from the original message, but make sure to preserve all of the
# new headers we're adding. Go through the list of new headers twice,
# first removing any old ones, then adding all the new ones.
for h, v in headers:
del msg[h]
for h, v in sorted(headers):
# Wrap these lines if they are too long. 78 character width probably
# shouldn't be hardcoded, but is at least text-MUA friendly. The
# adding of 2 is for the colon-space separator.
if len(h) + 2 + len(v) > 78:
v = CONTINUATION.join(v.split(', '))
msg[h] = v
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._set = IListArchiverSet(self._mlist)
def process(mlist, msg, msgdata):
"""Decorate the message with headers and footers."""
# Digests and Mailman-craft messages should not get additional headers.
if msgdata.get('isdigest') or msgdata.get('nodecorate'):
return
# Kludge to not decorate mail for Mail-Archive.com.
if ('recipients' in msgdata and len(msgdata['recipients']) == 1
and list(msgdata['recipients'])[0] == MailArchive().recipient):
return
d = {}
member = msgdata.get('member')
if member is not None:
# Calculate the extra personalization dictionary.
# member.subscriber can be a User instance or an Address instance, and
# member.address can be None and so can member._user.preferred_address.
if member._address is not None:
_address = member._address
else:
_address = (member._user.preferred_address
or list(member._user.addresses)[0])
recipient = msgdata.get('recipient', _address.original_email)
d['member'] = formataddr((_address.display_name, _address.email))
d['user_email'] = recipient
d['user_delivered_to'] = _address.original_email
d['user_language'] = member.preferred_language.description
d['user_name'] = member.display_name
d['user_name_or_address'] = member.display_name or recipient
# For backward compatibility.
d['user_address'] = recipient
# Calculate the archiver permalink substitution variables. This provides
# the $<archive-name>_url placeholder for every enabled archiver.
for archiver in IListArchiverSet(mlist).archivers:
if archiver.is_enabled:
# Get the permalink of the message from the archiver. Watch out
# for exceptions in the archiver plugin.
try:
archive_url = archiver.system_archiver.permalink(mlist, msg)
except Exception:
alog.exception('Exception in "{}" archiver'.format(
archiver.system_archiver.name))
archive_url = None
if archive_url is not None:
placeholder = '{}_url'.format(archiver.system_archiver.name)
d[placeholder] = archive_url
# These strings are descriptive for the log file and shouldn't be i18n'd
d.update(msgdata.get('decoration-data', {}))
header = decorate('list:member:regular:header', mlist, d)
footer = decorate('list:member:regular:footer', mlist, d)
# Escape hatch if both the footer and header are empty or None.
if len(header) == 0 and len(footer) == 0:
return
# Be MIME smart here. We only attach the header and footer by
# concatenation when the message is a non-multipart of type text/plain.
# Otherwise, if it is not a multipart, we make it a multipart, and then we
# add the header and footer as text/plain parts.
#
# BJG: In addition, only add the footer if the message's character set
# matches the charset of the list's preferred language. This is a
# suboptimal solution, and should be solved by allowing a list to have
# multiple headers/footers, for each language the list supports.
#
# Also, if the list's preferred charset is us-ascii, we can always
# safely add the header/footer to a plain text message since all
# charsets Mailman supports are strict supersets of us-ascii --
# no, UTF-16 emails are not supported yet.
#
# TK: Message with 'charset=' cause trouble. So, instead of
# mgs.get_content_charset('us-ascii') ...
mcset = msg.get_content_charset() or 'us-ascii'
lcset = mlist.preferred_language.charset
msgtype = msg.get_content_type()
# BAW: If the charsets don't match, should we add the header and footer by
# MIME multipart chroming the message?
wrap = True
if not msg.is_multipart() and msgtype == 'text/plain':
# Save the RFC-3676 format parameters.
format_param = msg.get_param('format')
delsp = msg.get_param('delsp')
# Save 'Content-Transfer-Encoding' header in case decoration fails.
cte = msg.get('content-transfer-encoding')
# header/footer is now in unicode.
try:
oldpayload = msg.get_payload(decode=True).decode(mcset)
del msg['content-transfer-encoding']
frontsep = endsep = ''
if len(header) > 0 and not header.endswith('\n'):
frontsep = '\n'
if len(footer) > 0 and not oldpayload.endswith('\n'):
endsep = '\n'
payload = header + frontsep + oldpayload + endsep + footer
# When setting the payload for the message, try various charset
# encodings until one does not produce a UnicodeError. We'll try
# charsets in this order: the list's charset, the message's
# charset, then utf-8. It's okay if some of these are duplicates.
for cset in (lcset, mcset, 'utf-8'):
try:
msg.set_payload(payload.encode(cset), cset)
except UnicodeError:
pass
else:
if format_param:
msg.set_param('format', format_param)
if delsp:
msg.set_param('delsp', delsp)
wrap = False
break
except (LookupError, UnicodeError):
if cte:
# Restore the original c-t-e.
del msg['content-transfer-encoding']
msg['Content-Transfer-Encoding'] = cte
elif msg.get_content_type() == 'multipart/mixed':
# The next easiest thing to do is just prepend the header and append
# the footer as additional subparts
payload = msg.get_payload()
if not isinstance(payload, list):
payload = [payload]
if len(footer) > 0:
mimeftr = MIMEText(footer.encode(lcset, errors='replace'), 'plain',
lcset)
mimeftr['Content-Disposition'] = 'inline'
payload.append(mimeftr)
if len(header) > 0:
mimehdr = MIMEText(header.encode(lcset, errors='replace'), 'plain',
lcset)
mimehdr['Content-Disposition'] = 'inline'
payload.insert(0, mimehdr)
msg.set_payload(payload)
wrap = False
# If we couldn't add the header or footer in a less intrusive way, we can
# at least do it by MIME encapsulation. We want to keep as much of the
# outer chrome as possible.
if not wrap:
return
# Because of the way Message objects are passed around to process(), we
# need to play tricks with the outer message -- i.e. the outer one must
# remain the same instance. So we're going to create a clone of the outer
# message, with all the header chrome intact, then copy the payload to it.
# This will give us a clone of the original message, and it will form the
# basis of the interior, wrapped Message.
inner = Message()
# Which headers to copy? Let's just do the Content-* headers
for h, v in msg.items():
if h.lower().startswith('content-'):
inner[h] = v
inner.set_payload(msg.get_payload())
# For completeness
inner.set_unixfrom(msg.get_unixfrom())
inner.preamble = msg.preamble
inner.epilogue = msg.epilogue
# Don't copy get_charset, as this might be None, even if
# get_content_charset isn't. However, do make sure there is a default
# content-type, even if the original message was not MIME.
inner.set_default_type(msg.get_default_type())
# BAW: HACK ALERT.
if hasattr(msg, '__version__'):
inner.__version__ = msg.__version__
# Now, play games with the outer message to make it contain three
# subparts: the header (if any), the wrapped message, and the footer (if
# any).
payload = [inner]
if len(header) > 0:
mimehdr = MIMEText(header.encode(lcset, errors='replace'), 'plain',
lcset)
mimehdr['Content-Disposition'] = 'inline'
payload.insert(0, mimehdr)
if len(footer) > 0:
mimeftr = MIMEText(footer.encode(lcset, errors='replace'), 'plain',
lcset)
mimeftr['Content-Disposition'] = 'inline'
payload.append(mimeftr)
msg.set_payload(payload)
del msg['content-type']
del msg['content-transfer-encoding']
del msg['content-disposition']
msg['Content-Type'] = 'multipart/mixed'
def __init__(self, mlist):
super().__init__()
self._archiver_set = IListArchiverSet(mlist)
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._set = IListArchiverSet(self._mlist)