def test_site_disabled(self): # Here the system configuration enables all the archivers in time for # the archive set to be created with all list archivers enabled. But # then the site-wide archiver gets disabled, so the list specific # archiver will also be disabled. archiver_set = IListArchiverSet(self._mlist) archiver = archiver_set.get('prototype') self.assertTrue(archiver.is_enabled) # Disable the site-wide archiver. config.push('enable prototype', """\ [archiver.prototype] enable: no """) self.assertFalse(archiver.is_enabled) config.pop('enable prototype')
def setUp(self): self._mlist = create_list('*****@*****.**') self._now = now() # Enable just the dummy archiver. config.push( 'dummy', """ [archiver.dummy] class: mailman.runners.tests.test_archiver.DummyArchiver enable: no [archiver.prototype] enable: no [archiver.mhonarc] enable: no [archiver.mail_archive] enable: no """) self._archiveq = config.switchboards['archive'] self._msg = mfs("""\ From: [email protected] To: [email protected] Subject: My first post Message-ID: <first> Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB First post! """) self._runner = make_testable_runner(ArchiveRunner) IListArchiverSet(self._mlist).get('dummy').is_enabled = True
def on_get(self, request, response): """Get all the archiver statuses.""" archiver_set = IListArchiverSet(self._mlist) resource = { archiver.name: archiver.is_enabled for archiver in archiver_set.archivers } okay(response, etag(resource))
def test_disable_all_list_archivers(self): # Let's disable all the archivers for the mailing list, but not the # global archivers. No messages will get archived. for archiver in IListArchiverSet(self._mlist).archivers: archiver.is_enabled = False config.db.store.commit() self._archiveq.enqueue(self._msg, {}, listid=self._mlist.list_id) self._runner.run() self.assertEqual(os.listdir(config.MESSAGES_DIR), [])
def test_enable_list_archiver(self): # When the system configuration file disables an archiver site-wide, # the list-specific mailing list will get initialized as not enabled. # Create the archiver set on the fly so that it doesn't get # initialized with a configuration that enables the prototype archiver. archiver_set = IListArchiverSet(self._mlist) archiver = archiver_set.get('prototype') self.assertFalse(archiver.is_enabled) # Enable both the list archiver and the system archiver. archiver.is_enabled = True config.push('enable prototype', """\ [archiver.prototype] enable: yes """) # Get the IListArchiver again. archiver = archiver_set.get('prototype') self.assertTrue(archiver.is_enabled) config.pop('enable prototype')
def test_delete_list_with_list_archiver_set(self): # Ensure that mailing lists with archiver sets can be deleted. In # issue #115, this fails under PostgreSQL, but not SQLite. mlist = create_list('*****@*****.**') # We don't keep a reference to this archiver set just because it makes # pyflakes unhappy. It doesn't change the outcome. IListArchiverSet(mlist) list_manager = getUtility(IListManager) list_manager.delete(mlist) self.assertIsNone(list_manager.get('*****@*****.**'))
class TestListArchiver(unittest.TestCase): layer = ConfigLayer def setUp(self): self._mlist = create_list('*****@*****.**') self._set = IListArchiverSet(self._mlist) def test_list_archivers(self): # Find the set of archivers registered for this mailing list. self.assertEqual(['mail-archive', 'mhonarc', 'prototype'], sorted(archiver.name for archiver in self._set.archivers)) def test_get_archiver(self): # Use .get() to see if a mailing list has an archiver. archiver = self._set.get('mhonarc') self.assertEqual(archiver.name, 'mhonarc') self.assertTrue(archiver.is_enabled) self.assertEqual(archiver.mailing_list, self._mlist) self.assertEqual(archiver.system_archiver.name, 'mhonarc') def test_get_archiver_no_such(self): # Using .get() on a non-existing name returns None. self.assertIsNone(self._set.get('no-such-archiver')) def test_site_disabled(self): # Here the system configuration enables all the archivers in time for # the archive set to be created with all list archivers enabled. But # then the site-wide archiver gets disabled, so the list specific # archiver will also be disabled. archiver_set = IListArchiverSet(self._mlist) archiver = archiver_set.get('mhonarc') self.assertTrue(archiver.is_enabled) # Disable the site-wide archiver. config.push( 'enable mhonarc', """\ [archiver.mhonarc] enable: no """) self.assertFalse(archiver.is_enabled) config.pop('enable mhonarc')
class TestListArchiver(unittest.TestCase): layer = ConfigLayer def setUp(self): self._mlist = create_list('*****@*****.**') self._set = IListArchiverSet(self._mlist) def test_list_archivers(self): # Find the set of archivers registered for this mailing list. self.assertEqual( ['mail-archive', 'mhonarc', 'prototype'], sorted(archiver.name for archiver in self._set.archivers)) def test_get_archiver(self): # Use .get() to see if a mailing list has an archiver. archiver = self._set.get('prototype') self.assertEqual(archiver.name, 'prototype') self.assertTrue(archiver.is_enabled) self.assertEqual(archiver.mailing_list, self._mlist) self.assertEqual(archiver.system_archiver.name, 'prototype') def test_get_archiver_no_such(self): # Using .get() on a non-existing name returns None. self.assertIsNone(self._set.get('no-such-archiver')) def test_site_disabled(self): # Here the system configuration enables all the archivers in time for # the archive set to be created with all list archivers enabled. But # then the site-wide archiver gets disabled, so the list specific # archiver will also be disabled. archiver_set = IListArchiverSet(self._mlist) archiver = archiver_set.get('prototype') self.assertTrue(archiver.is_enabled) # Disable the site-wide archiver. config.push('enable prototype', """\ [archiver.prototype] enable: no """) self.assertFalse(archiver.is_enabled) config.pop('enable prototype')
class ArchiverGetterSetter(GetterSetter): """Resource for updating archiver statuses.""" def __init__(self, mlist): super().__init__() self._archiver_set = IListArchiverSet(mlist) def put(self, mlist, attribute, value): # attribute will contain the (bytes) name of the archiver that is # getting a new status. value will be the representation of the new # boolean status. archiver = self._archiver_set.get(attribute) assert archiver is not None, attribute archiver.is_enabled = as_boolean(value)
def patch_put(self, request, response, is_optional): archiver_set = IListArchiverSet(self._mlist) kws = {archiver.name: ArchiverGetterSetter(self._mlist) for archiver in archiver_set.archivers} if is_optional: # For a PATCH, all attributes are optional. kws['_optional'] = kws.keys() try: Validator(**kws).update(self._mlist, request) except ValueError as error: bad_request(response, str(error)) else: no_content(response)
class ArchiverGetterSetter(GetterSetter): """Resource for updating archiver statuses.""" def __init__(self, mlist): super(ArchiverGetterSetter, self).__init__() self._archiver_set = IListArchiverSet(mlist) def put(self, mlist, attribute, value): # attribute will contain the (bytes) name of the archiver that is # getting a new status. value will be the representation of the new # boolean status. archiver = self._archiver_set.get(attribute) if archiver is None: raise ValueError('No such archiver: {}'.format(attribute)) archiver.is_enabled = as_boolean(value)
class ArchiverGetterSetter(GetterSetter): """Resource for updating archiver statuses.""" def __init__(self, mlist): super(ArchiverGetterSetter, self).__init__() self._archiver_set = IListArchiverSet(mlist) def put(self, mlist, attribute, value): # attribute will contain the (bytes) name of the archiver that is # getting a new status. value will be the representation of the new # boolean status. archiver = self._archiver_set.get(attribute.decode('utf-8')) if archiver is None: raise ValueError('No such archiver: {}'.format(attribute)) archiver.is_enabled = as_boolean(value)
def test_broken_archiver(self): # GL issue #208 - IArchive messages raise exceptions, breaking the # rfc-2369 handler and shunting messages. mark = LogFileMark('mailman.archiver') self._archiveq.enqueue(self._msg, {}, listid=self._mlist.list_id, received_time=now()) IListArchiverSet(self._mlist).get('broken').is_enabled = True self._runner.run() # The archiver is broken, so there are no messages on the file system, # but there is a log message and the message was not shunted. log_messages = mark.read() self.assertIn('Exception in "broken" archiver', log_messages) self.assertIn('RuntimeError: Cannot archive message', log_messages) get_queue_messages('shunt', expected_count=0)
def _dispose(self, mlist, msg, msgdata): received_time = msgdata.get('received_time', now(strip_tzinfo=False)) archiver_set = IListArchiverSet(mlist) for archiver in archiver_set.archivers: # The archiver is disabled if either the list-specific or # site-wide archiver is disabled. if not archiver.is_enabled: continue msg_copy = copy.deepcopy(msg) if _should_clobber(msg, msgdata, archiver.name): original_date = msg_copy['date'] del msg_copy['date'] del msg_copy['x-original-date'] msg_copy['Date'] = received_time.strftime(RFC822_DATE_FMT) if original_date: msg_copy['X-Original-Date'] = original_date # A problem in one archiver should not prevent other archivers # from running. try: archiver.system_archiver.archive_message(mlist, msg_copy) except Exception: log.exception('Broken archiver: %s' % archiver.name)
def __init__(self, mlist): super().__init__() self._archiver_set = IListArchiverSet(mlist)
def process(mlist, msg, msgdata): """Add the RFC 2369 List-* and related headers.""" # Some people really hate the List-* headers. It seems that the free # version of Eudora (possibly on for some platforms) does not hide these # headers by default, pissing off their users. Too bad. Fix the MUAs. if not mlist.include_rfc2369_headers: return list_id = '{0.list_name}.{0.mail_host}'.format(mlist) if mlist.description: # Don't wrap the header since here we just want to get it properly RFC # 2047 encoded. i18ndesc = uheader(mlist, mlist.description, 'List-Id', maxlinelen=998) listid_h = formataddr((str(i18ndesc), list_id)) else: # Without a description, we need to ensure the MUST brackets. listid_h = '<{}>'.format(list_id) # No other agent should add a List-ID header except Mailman. del msg['list-id'] msg['List-Id'] = listid_h # For internally crafted messages, we also add a (nonstandard), # "X-List-Administrivia: yes" header. For all others (i.e. those coming # from list posts), we add a bunch of other RFC 2369 headers. requestaddr = mlist.request_address headers = [] # XXX reduced_list_headers used to suppress List-Help, List-Subject, and # List-Unsubscribe from UserNotification. That doesn't seem to make sense # any more, so always add those three headers (others will still be # suppressed). headers.extend(( ('List-Help', '<mailto:{}?subject=help>'.format(requestaddr)), ('List-Unsubscribe', '<mailto:{}>'.format(mlist.leave_address)), ('List-Subscribe', '<mailto:{}>'.format(mlist.join_address)), )) if not msgdata.get('reduced_list_headers'): # List-Post: is controlled by a separate attribute, which is somewhat # misnamed. RFC 2369 requires a value of NO if posting is not # allowed, i.e. for an announce-only list. list_post = ('<mailto:{}>'.format(mlist.posting_address) if mlist.allow_list_posts else 'NO') headers.append(('List-Post', list_post)) # Add RFC 2369 and 5064 archiving headers, if archiving is enabled. if mlist.archive_policy is not ArchivePolicy.never: archiver_set = IListArchiverSet(mlist) for archiver in archiver_set.archivers: if not archiver.is_enabled: continue # Watch out for exceptions in the archiver plugin. try: archiver_url = archiver.system_archiver.list_url(mlist) except Exception: log.exception('Exception in "{}" archiver'.format( archiver.system_archiver.name)) archiver_url = None if archiver_url is not None: headers.append(('List-Archive', '<{}>'.format(archiver_url))) try: permalink = archiver.system_archiver.permalink(mlist, msg) except Exception: log.exception('Exception in "{}" archiver'.format( archiver.system_archiver.name)) permalink = None if permalink is not None: headers.append(('Archived-At', '<{}>'.format(permalink))) # XXX RFC 2369 also defines a List-Owner header which we are not currently # supporting, but should. # # Some headers will appear more than once in the new set, e.g. the # List-Archive and Archived-At headers. We want to delete any RFC 2369 # headers from the original message, but make sure to preserve all of the # new headers we're adding. Go through the list of new headers twice, # first removing any old ones, then adding all the new ones. for h, v in headers: del msg[h] for h, v in sorted(headers): # Wrap these lines if they are too long. 78 character width probably # shouldn't be hardcoded, but is at least text-MUA friendly. The # adding of 2 is for the colon-space separator. if len(h) + 2 + len(v) > 78: v = CONTINUATION.join(v.split(', ')) msg[h] = v
def setUp(self): self._mlist = create_list('*****@*****.**') self._set = IListArchiverSet(self._mlist)
def process(mlist, msg, msgdata): """Decorate the message with headers and footers.""" # Digests and Mailman-craft messages should not get additional headers. if msgdata.get('isdigest') or msgdata.get('nodecorate'): return # Kludge to not decorate mail for Mail-Archive.com. if ('recipients' in msgdata and len(msgdata['recipients']) == 1 and list(msgdata['recipients'])[0] == MailArchive().recipient): return d = {} member = msgdata.get('member') if member is not None: # Calculate the extra personalization dictionary. # member.subscriber can be a User instance or an Address instance, and # member.address can be None and so can member._user.preferred_address. if member._address is not None: _address = member._address else: _address = (member._user.preferred_address or list(member._user.addresses)[0]) recipient = msgdata.get('recipient', _address.original_email) d['member'] = formataddr((_address.display_name, _address.email)) d['user_email'] = recipient d['user_delivered_to'] = _address.original_email d['user_language'] = member.preferred_language.description d['user_name'] = member.display_name d['user_name_or_address'] = member.display_name or recipient # For backward compatibility. d['user_address'] = recipient # Calculate the archiver permalink substitution variables. This provides # the $<archive-name>_url placeholder for every enabled archiver. for archiver in IListArchiverSet(mlist).archivers: if archiver.is_enabled: # Get the permalink of the message from the archiver. Watch out # for exceptions in the archiver plugin. try: archive_url = archiver.system_archiver.permalink(mlist, msg) except Exception: alog.exception('Exception in "{}" archiver'.format( archiver.system_archiver.name)) archive_url = None if archive_url is not None: placeholder = '{}_url'.format(archiver.system_archiver.name) d[placeholder] = archive_url # These strings are descriptive for the log file and shouldn't be i18n'd d.update(msgdata.get('decoration-data', {})) header = decorate('list:member:regular:header', mlist, d) footer = decorate('list:member:regular:footer', mlist, d) # Escape hatch if both the footer and header are empty or None. if len(header) == 0 and len(footer) == 0: return # Be MIME smart here. We only attach the header and footer by # concatenation when the message is a non-multipart of type text/plain. # Otherwise, if it is not a multipart, we make it a multipart, and then we # add the header and footer as text/plain parts. # # BJG: In addition, only add the footer if the message's character set # matches the charset of the list's preferred language. This is a # suboptimal solution, and should be solved by allowing a list to have # multiple headers/footers, for each language the list supports. # # Also, if the list's preferred charset is us-ascii, we can always # safely add the header/footer to a plain text message since all # charsets Mailman supports are strict supersets of us-ascii -- # no, UTF-16 emails are not supported yet. # # TK: Message with 'charset=' cause trouble. So, instead of # mgs.get_content_charset('us-ascii') ... mcset = msg.get_content_charset() or 'us-ascii' lcset = mlist.preferred_language.charset msgtype = msg.get_content_type() # BAW: If the charsets don't match, should we add the header and footer by # MIME multipart chroming the message? wrap = True if not msg.is_multipart() and msgtype == 'text/plain': # Save the RFC-3676 format parameters. format_param = msg.get_param('format') delsp = msg.get_param('delsp') # Save 'Content-Transfer-Encoding' header in case decoration fails. cte = msg.get('content-transfer-encoding') # header/footer is now in unicode. try: oldpayload = msg.get_payload(decode=True).decode(mcset) del msg['content-transfer-encoding'] frontsep = endsep = '' if len(header) > 0 and not header.endswith('\n'): frontsep = '\n' if len(footer) > 0 and not oldpayload.endswith('\n'): endsep = '\n' payload = header + frontsep + oldpayload + endsep + footer # When setting the payload for the message, try various charset # encodings until one does not produce a UnicodeError. We'll try # charsets in this order: the list's charset, the message's # charset, then utf-8. It's okay if some of these are duplicates. for cset in (lcset, mcset, 'utf-8'): try: msg.set_payload(payload.encode(cset), cset) except UnicodeError: pass else: if format_param: msg.set_param('format', format_param) if delsp: msg.set_param('delsp', delsp) wrap = False break except (LookupError, UnicodeError): if cte: # Restore the original c-t-e. del msg['content-transfer-encoding'] msg['Content-Transfer-Encoding'] = cte elif msg.get_content_type() == 'multipart/mixed': # The next easiest thing to do is just prepend the header and append # the footer as additional subparts payload = msg.get_payload() if not isinstance(payload, list): payload = [payload] if len(footer) > 0: mimeftr = MIMEText(footer.encode(lcset, errors='replace'), 'plain', lcset) mimeftr['Content-Disposition'] = 'inline' payload.append(mimeftr) if len(header) > 0: mimehdr = MIMEText(header.encode(lcset, errors='replace'), 'plain', lcset) mimehdr['Content-Disposition'] = 'inline' payload.insert(0, mimehdr) msg.set_payload(payload) wrap = False # If we couldn't add the header or footer in a less intrusive way, we can # at least do it by MIME encapsulation. We want to keep as much of the # outer chrome as possible. if not wrap: return # Because of the way Message objects are passed around to process(), we # need to play tricks with the outer message -- i.e. the outer one must # remain the same instance. So we're going to create a clone of the outer # message, with all the header chrome intact, then copy the payload to it. # This will give us a clone of the original message, and it will form the # basis of the interior, wrapped Message. inner = Message() # Which headers to copy? Let's just do the Content-* headers for h, v in msg.items(): if h.lower().startswith('content-'): inner[h] = v inner.set_payload(msg.get_payload()) # For completeness inner.set_unixfrom(msg.get_unixfrom()) inner.preamble = msg.preamble inner.epilogue = msg.epilogue # Don't copy get_charset, as this might be None, even if # get_content_charset isn't. However, do make sure there is a default # content-type, even if the original message was not MIME. inner.set_default_type(msg.get_default_type()) # BAW: HACK ALERT. if hasattr(msg, '__version__'): inner.__version__ = msg.__version__ # Now, play games with the outer message to make it contain three # subparts: the header (if any), the wrapped message, and the footer (if # any). payload = [inner] if len(header) > 0: mimehdr = MIMEText(header.encode(lcset, errors='replace'), 'plain', lcset) mimehdr['Content-Disposition'] = 'inline' payload.insert(0, mimehdr) if len(footer) > 0: mimeftr = MIMEText(footer.encode(lcset, errors='replace'), 'plain', lcset) mimeftr['Content-Disposition'] = 'inline' payload.append(mimeftr) msg.set_payload(payload) del msg['content-type'] del msg['content-transfer-encoding'] del msg['content-disposition'] msg['Content-Type'] = 'multipart/mixed'