Exemple #1
0
def process(mlist, msg, msgdata):
    interval = mm_cfg.VERP_DELIVERY_INTERVAL
    # Should we VERP this message?  If personalization is enabled for this
    # list and VERP_PERSONALIZED_DELIVERIES is true, then yes we VERP it.
    # Also, if personalization is /not/ enabled, but VERP_DELIVERY_INTERVAL is
    # set (and we've hit this interval), then again, this message should be
    # VERPed. Otherwise, no.
    #
    # Note that the verp flag may already be set, e.g. by mailpasswds using
    # VERP_PASSWORD_REMINDERS.  Preserve any existing verp flag.
    if msgdata.has_key("verp"):
        pass
    elif mlist.personalize:
        if mm_cfg.VERP_PERSONALIZED_DELIVERIES:
            msgdata["verp"] = 1
    elif interval == 0:
        # Never VERP
        pass
    elif interval == 1:
        # VERP every time
        msgdata["verp"] = 1
    else:
        # VERP every `inteval' number of times
        msgdata["verp"] = not int(mlist.post_id) % interval
    # And now drop the message in qfiles/out
    outq = get_switchboard(mm_cfg.OUTQUEUE_DIR)
    outq.enqueue(msg, msgdata, listname=mlist.internal_name())
Exemple #2
0
def process(mlist, msg, msgdata):
    interval = mm_cfg.VERP_DELIVERY_INTERVAL
    # Should we VERP this message?  If personalization is enabled for this
    # list and VERP_PERSONALIZED_DELIVERIES is true, then yes we VERP it.
    # Also, if personalization is /not/ enabled, but VERP_DELIVERY_INTERVAL is
    # set (and we've hit this interval), then again, this message should be
    # VERPed. Otherwise, no.
    #
    # Note that the verp flag may already be set, e.g. by mailpasswds using
    # VERP_PASSWORD_REMINDERS.  Preserve any existing verp flag.
    if msgdata.has_key('verp'):
        pass
    elif mlist.personalize:
        if mm_cfg.VERP_PERSONALIZED_DELIVERIES:
            msgdata['verp'] = 1
    elif interval == 0:
        # Never VERP
        pass
    elif interval == 1:
        # VERP every time
        msgdata['verp'] = 1
    else:
        # VERP every `inteval' number of times
        msgdata['verp'] = not int(mlist.post_id) % interval
    # And now drop the message in qfiles/out
    outq = get_switchboard(mm_cfg.OUTQUEUE_DIR)
    outq.enqueue(msg, msgdata, listname=mlist.internal_name())
Exemple #3
0
 def _enqueue(self, mlist, **_kws):
     # Not imported at module scope to avoid import loop
     from Mailman.Queue.sbcache import get_switchboard
     virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
     # The message metadata better have a `recip' attribute
     virginq.enqueue(self,
                     listname = mlist.internal_name(),
                     recips = self.recips,
                     nodecorate = 1,
                     reduced_list_headers = 1,
                     **_kws)
 def _enqueue(self, mlist, **_kws):
     # Not imported at module scope to avoid import loop
     from Mailman.Queue.sbcache import get_switchboard
     virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
     # The message metadata better have a `recip' attribute
     virginq.enqueue(self,
                     listname=mlist.internal_name(),
                     recips=self.recips,
                     nodecorate=1,
                     reduced_list_headers=1,
                     **_kws)
Exemple #5
0
def inject(listname, msg, recips=None, qdir=None):
    if qdir is None:
        qdir = mm_cfg.INQUEUE_DIR
    queue = get_switchboard(qdir)
    kws = {'listname'  : listname,
           'tolist'    : 1,
           '_plaintext': 1,
           }
    if recips:
        kws['recips'] = recips
    queue.enqueue(msg, **kws)
Exemple #6
0
def inject(listname, msg, recips=None, qdir=None):
    if qdir is None:
        qdir = mm_cfg.INQUEUE_DIR
    queue = get_switchboard(qdir)
    kws = {
        'listname': listname,
        'tolist': 1,
        '_plaintext': 1,
    }
    if recips:
        kws['recips'] = recips
    queue.enqueue(msg, **kws)
Exemple #7
0
def process(mlist, msg, msgdata):
    # short circuits
    if msgdata.get('isdigest') or not mlist.archive:
        return
    # Common practice seems to favor "X-No-Archive: yes".  No other value for
    # this header seems to make sense, so we'll just test for it's presence.
    # I'm keeping "X-Archive: no" for backwards compatibility.
    if msg.has_key('x-no-archive') or msg.get('x-archive', '').lower() == 'no':
        return
    # Send the message to the archiver queue
    archq = get_switchboard(mm_cfg.ARCHQUEUE_DIR)
    # Send the message to the queue
    archq.enqueue(msg, msgdata)
def process(mlist, msg, msgdata):
    # short circuits
    if msgdata.get('isdigest') or not mlist.archive:
        return
    # Common practice seems to favor "X-No-Archive: yes".  No other value for
    # this header seems to make sense, so we'll just test for it's presence.
    # I'm keeping "X-Archive: no" for backwards compatibility.
    if 'x-no-archive' in msg or msg.get('x-archive', '').lower() == 'no':
        return
    # Send the message to the archiver queue
    archq = get_switchboard(mm_cfg.ARCHQUEUE_DIR)
    # Send the message to the queue
    archq.enqueue(msg, msgdata)
Exemple #9
0
def handle_proxy_error(error, msg=None, msgdata=None):
    """Log the error and enqueue the message if needed.

    :param error: The error to log.
    :param msg: An optional Mailman.Message to re-enqueue.
    :param msgdata: The message data to enque with the message.
    :raise DiscardMessage: When a message is enqueued.
    """
    if isinstance(error, (xmlrpclib.ProtocolError, socket.error)):
        log_exception('Cannot talk to Launchpad:\n%s', error)
    else:
        log_exception('Launchpad exception: %s', error)
    if msg is not None:
        queue = get_switchboard(mm_cfg.INQUEUE_DIR)
        queue.enqueue(msg, msgdata)
        raise Errors.DiscardMessage
Exemple #10
0
def process(mlist, msg, msgdata):
    # short circuits
    if not mlist.gateway_to_news or msgdata.get("isdigest") or msgdata.get("fromusenet"):
        return
    # sanity checks
    error = []
    if not mlist.linked_newsgroup:
        error.append("no newsgroup")
    if not mlist.nntp_host:
        error.append("no NNTP host")
    if error:
        syslog("error", "NNTP gateway improperly configured: %s", COMMASPACE.join(error))
        return
    # Put the message in the news runner's queue
    newsq = get_switchboard(mm_cfg.NEWSQUEUE_DIR)
    newsq.enqueue(msg, msgdata, listname=mlist.internal_name())
def handle_proxy_error(error, msg=None, msgdata=None):
    """Log the error and enqueue the message if needed.

    :param error: The error to log.
    :param msg: An optional Mailman.Message to re-enqueue.
    :param msgdata: The message data to enque with the message.
    :raise DiscardMessage: When a message is enqueued.
    """
    if isinstance(error, (xmlrpclib.ProtocolError, socket.error)):
        log_exception("Cannot talk to Launchpad:\n%s", error)
    else:
        log_exception("Launchpad exception: %s", error)
    if msg is not None:
        queue = get_switchboard(mm_cfg.INQUEUE_DIR)
        queue.enqueue(msg, msgdata)
        raise Errors.DiscardMessage
def remove(mlist, cgi=False):
    listname = mlist.internal_name()
    fieldsz = len(listname) + len('-unsubscribe')
    if cgi:
        # If a list is being removed via the CGI, the best we can do is send
        # an email message to mailman-owner requesting that the appropriate
        # aliases be deleted.
        sfp = StringIO()
        print(_("""\
The mailing list `%(listname)s' has been removed via the through-the-web
interface.  In order to complete the de-activation of this mailing list, the
appropriate /etc/aliases (or equivalent) file must be updated.  The program
`newaliases' may also have to be run.

Here are the entries in the /etc/aliases file that should be removed:
"""),
              file=sfp)
        outfp = sfp
    else:
        print(
            C_("""
To finish removing your mailing list, you must edit your /etc/aliases (or
equivalent) file by removing the following lines, and possibly running the
`newaliases' program:

## %(listname)s mailing list"""))
        outfp = sys.stdout
    # Common path
    for k, v in makealiases(listname):
        print(k + ':', ((fieldsz - len(k)) * ' '), v, file=outfp)
    # If we're using the command line interface, we're done.  For ttw, we need
    # to actually send the message to mailman-owner now.
    if not cgi:
        print(file=outfp)
        return
    siteowner = Utils.get_site_email(extra='owner')
    # Should this be sent in the site list's preferred language?
    msg = Message.UserNotification(
        siteowner, siteowner,
        _('Mailing list removal request for list %(listname)s'),
        sfp.getvalue(), mm_cfg.DEFAULT_SERVER_LANGUAGE)
    msg['Date'] = email.utils.formatdate(localtime=1)
    outq = get_switchboard(mm_cfg.OUTQUEUE_DIR)
    outq.enqueue(msg, recips=[siteowner], nodecorate=1)
Exemple #13
0
def process(mlist, msg, msgdata):
    # short circuits
    if not mlist.gateway_to_news or \
           msgdata.get('isdigest') or \
           msgdata.get('fromusenet'):
        return
    # sanity checks
    error = []
    if not mlist.linked_newsgroup:
        error.append('no newsgroup')
    if not mlist.nntp_host:
        error.append('no NNTP host')
    if error:
        syslog('error', 'NNTP gateway improperly configured: %s',
               COMMASPACE.join(error))
        return
    # Put the message in the news runner's queue
    newsq = get_switchboard(mm_cfg.NEWSQUEUE_DIR)
    newsq.enqueue(msg, msgdata, listname=mlist.internal_name())
Exemple #14
0
def remove(mlist, cgi=False):
    listname = mlist.internal_name()
    fieldsz = len(listname) + len('-unsubscribe')
    if cgi:
        # If a list is being removed via the CGI, the best we can do is send
        # an email message to mailman-owner requesting that the appropriate
        # aliases be deleted.
        sfp = StringIO()
        print >> sfp, _("""\
The mailing list `%(listname)s' has been removed via the through-the-web
interface.  In order to complete the de-activation of this mailing list, the
appropriate /etc/aliases (or equivalent) file must be updated.  The program
`newaliases' may also have to be run.

Here are the entries in the /etc/aliases file that should be removed:
""")
        outfp = sfp
    else:
        print _("""
To finish removing your mailing list, you must edit your /etc/aliases (or
equivalent) file by removing the following lines, and possibly running the
`newaliases' program:

## %(listname)s mailing list""")
        outfp = sys.stdout
    # Common path
    for k, v in makealiases(listname):
        print >> outfp, k + ':', ((fieldsz - len(k)) * ' '), v
    # If we're using the command line interface, we're done.  For ttw, we need
    # to actually send the message to mailman-owner now.
    if not cgi:
        print >> outfp
        return
    siteowner = Utils.get_site_email(extra='owner')
    # Should this be sent in the site list's preferred language?
    msg = Message.UserNotification(
        siteowner, siteowner,
        _('Mailing list removal request for list %(listname)s'),
        sfp.getvalue(), mm_cfg.DEFAULT_SERVER_LANGUAGE)
    msg['Date'] = email.Utils.formatdate(localtime=1)
    outq = get_switchboard(mm_cfg.OUTQUEUE_DIR)
    outq.enqueue(msg, recips=[siteowner], nodecorate=1)
Exemple #15
0
def dispose(mlist, msg, msgdata, why):
    # filter_action == 0 just discards, see below
    if mlist.filter_action == 1:
        # Bounce the message to the original author
        raise Errors.RejectMessage, why
    if mlist.filter_action == 2:
        # Forward it on to the list owner
        listname = mlist.internal_name()
        mlist.ForwardMessage(
            msg,
            text=_("""\
The attached message matched the %(listname)s mailing list's content filtering
rules and was prevented from being forwarded on to the list membership.  You
are receiving the only remaining copy of the discarded message.

"""),
            subject=_('Content filtered message notification'))
    if mlist.filter_action == 3 and \
           mm_cfg.OWNERS_CAN_PRESERVE_FILTERED_MESSAGES:
        badq = get_switchboard(mm_cfg.BADQUEUE_DIR)
        badq.enqueue(msg, msgdata)
    # Most cases also discard the message
    raise Errors.DiscardMessage
Exemple #16
0
def dispose(mlist, msg, msgdata, why):
    # filter_action == 0 just discards, see below
    if mlist.filter_action == 1:
        # Bounce the message to the original author
        raise Errors.RejectMessage, why
    if mlist.filter_action == 2:
        # Forward it on to the list owner
        listname = mlist.internal_name()
        mlist.ForwardMessage(
            msg,
            text=_("""\
The attached message matched the %(listname)s mailing list's content filtering
rules and was prevented from being forwarded on to the list membership.  You
are receiving the only remaining copy of the discarded message.

"""),
            subject=_('Content filtered message notification'))
    if mlist.filter_action == 3 and \
           mm_cfg.OWNERS_CAN_PRESERVE_FILTERED_MESSAGES:
        badq = get_switchboard(mm_cfg.BADQUEUE_DIR)
        badq.enqueue(msg, msgdata)
    # Most cases also discard the message
    raise Errors.DiscardMessage
Exemple #17
0
 def _dispose(self, mlist, msg, msgdata):
     # Make sure we have the most up-to-date state
     mlist.Load()
     outq = get_switchboard(mm_cfg.OUTQUEUE_DIR)
     # There are a few possibilities here:
     #
     # - the message could have been VERP'd in which case, we know exactly
     #   who the message was destined for.  That make our job easy.
     # - the message could have been originally destined for a list owner,
     #   but a list owner address itself bounced.  That's bad, and for now
     #   we'll simply attempt to deliver the message to the site list
     #   owner.
     #   Note that this means that automated bounce processing doesn't work
     #   for the site list.  Because we can't reliably tell to what address
     #   a non-VERP'd bounce was originally sent, we have to treat all
     #   bounces sent to the site list as potential list owner bounces.
     # - the list owner could have set list-bounces (or list-admin) as the
     #   owner address.  That's really bad as it results in a loop of ever
     #   growing unrecognized bounce messages.  We detect this based on the
     #   fact that this message itself will be from the site bounces
     #   address.  We then send this to the site list owner instead.
     # Notices to list-owner have their envelope sender and From: set to
     # the site-bounces address.  Check if this is this a bounce for a
     # message to a list owner, coming to site-bounces, or a looping
     # message sent directly to the -bounces address.  We have to do these
     # cases separately, because sending to site-owner will reset the
     # envelope sender.
     # Is this a site list bounce?
     if (mlist.internal_name().lower() == mm_cfg.MAILMAN_SITE_LIST.lower()):
         # Send it on to the site owners, but craft the envelope sender to
         # be the -loop detection address, so if /they/ bounce, we won't
         # get stuck in a bounce loop.
         outq.enqueue(
             msg,
             msgdata,
             recips=mlist.owner,
             envsender=Utils.get_site_email(extra='loop'),
             nodecorate=1,
         )
         return
     # Is this a possible looping message sent directly to a list-bounces
     # address other than the site list?
     # Check From: because unix_from might be VERP'd.
     # Also, check the From: that Message.OwnerNotification uses.
     if (msg.get('from') == Utils.get_site_email(mlist.host_name,
                                                 'bounces')):
         # Just send it to the sitelist-owner address.  If that bounces
         # we'll handle it above.
         outq.enqueue(
             msg,
             msgdata,
             recips=[Utils.get_site_email(extra='owner')],
             envsender=Utils.get_site_email(extra='loop'),
             nodecorate=1,
         )
         return
     # List isn't doing bounce processing?
     if not mlist.bounce_processing:
         return
     # Try VERP detection first, since it's quick and easy
     addrs = verp_bounce(mlist, msg)
     if addrs:
         # We have an address, but check if the message is non-fatal.
         if BouncerAPI.ScanMessages(mlist, msg) is BouncerAPI.Stop:
             return
     else:
         # See if this was a probe message.
         token = verp_probe(mlist, msg)
         if token:
             self._probe_bounce(mlist, token)
             return
         # That didn't give us anything useful, so try the old fashion
         # bounce matching modules.
         addrs = BouncerAPI.ScanMessages(mlist, msg)
         if addrs is BouncerAPI.Stop:
             # This is a recognized, non-fatal notice. Ignore it.
             return
     # If that still didn't return us any useful addresses, then send it on
     # or discard it.
     addrs = filter(None, addrs)
     if not addrs:
         syslog('bounce',
                '%s: bounce message w/no discernable addresses: %s',
                mlist.internal_name(), msg.get('message-id', 'n/a'))
         maybe_forward(mlist, msg)
         return
     # BAW: It's possible that there are None's in the list of addresses,
     # although I'm unsure how that could happen.  Possibly ScanMessages()
     # can let None's sneak through.  In any event, this will kill them.
     # addrs = filter(None, addrs)
     # MAS above filter moved up so we don't try to queue an empty list.
     self._queue_bounces(mlist.internal_name(), addrs, msg)
 def assertIsEnqueued(self, msg):
     """Assert the message was appended to the incoming queue."""
     switchboard = get_switchboard(mm_cfg.INQUEUE_DIR)
     file_path = switchboard.files()[-1]
     queued_msg, queued_msg_data = switchboard.dequeue(file_path)
     self.assertEqual(msg['message-id'], queued_msg['message-id'])
 def _oneloop(self):
     # Refresh this each time through the list.  BAW: could be too
     # expensive.
     listnames = Utils.list_names()
     # Cruise through all the files currently in the new/ directory
     try:
         files = os.listdir(self._dir)
     except OSError as e:
         if e.errno != errno.ENOENT: raise
         # Nothing's been delivered yet
         return 0
     for file in files:
         srcname = os.path.join(self._dir, file)
         dstname = os.path.join(self._cur, file + ':1,P')
         xdstname = os.path.join(self._cur, file + ':1,X')
         try:
             os.rename(srcname, dstname)
         except OSError as e:
             if e.errno == errno.ENOENT:
                 # Some other MaildirRunner beat us to it
                 continue
             syslog('error', 'Could not rename maildir file: %s', srcname)
             raise
         # Now open, read, parse, and enqueue this message
         try:
             fp = open(dstname)
             try:
                 msg = self._parser.parse(fp)
             finally:
                 fp.close()
             # Now we need to figure out which queue of which list this
             # message was destined for.  See verp_bounce() in
             # BounceRunner.py for why we do things this way.
             vals = []
             for header in ('delivered-to', 'envelope-to', 'apparently-to'):
                 vals.extend(msg.get_all(header, []))
             for field in vals:
                 to = parseaddr(field)[1]
                 if not to:
                     continue
                 mo = lre.match(to)
                 if not mo:
                     # This isn't an address we care about
                     continue
                 listname, subq = mo.group('listname', 'subq')
                 if listname in listnames:
                     break
             else:
                 # As far as we can tell, this message isn't destined for
                 # any list on the system.  What to do?
                 syslog('error', 'Message apparently not for any list: %s',
                        xdstname)
                 os.rename(dstname, xdstname)
                 continue
             # BAW: blech, hardcoded
             msgdata = {'listname': listname}
             # -admin is deprecated
             if subq in ('bounces', 'admin'):
                 queue = get_switchboard(mm_cfg.BOUNCEQUEUE_DIR)
             elif subq == 'confirm':
                 msgdata['toconfirm'] = 1
                 queue = get_switchboard(mm_cfg.CMDQUEUE_DIR)
             elif subq in ('join', 'subscribe'):
                 msgdata['tojoin'] = 1
                 queue = get_switchboard(mm_cfg.CMDQUEUE_DIR)
             elif subq in ('leave', 'unsubscribe'):
                 msgdata['toleave'] = 1
                 queue = get_switchboard(mm_cfg.CMDQUEUE_DIR)
             elif subq == 'owner':
                 msgdata.update({
                     'toowner': 1,
                     'envsender': Utils.get_site_email(extra='bounces'),
                     'pipeline': mm_cfg.OWNER_PIPELINE,
                     })
                 queue = get_switchboard(mm_cfg.INQUEUE_DIR)
             elif subq is None:
                 msgdata['tolist'] = 1
                 queue = get_switchboard(mm_cfg.INQUEUE_DIR)
             elif subq == 'request':
                 msgdata['torequest'] = 1
                 queue = get_switchboard(mm_cfg.CMDQUEUE_DIR)
             else:
                 syslog('error', 'Unknown sub-queue: %s', subq)
                 os.rename(dstname, xdstname)
                 continue
             queue.enqueue(msg, msgdata)
             os.unlink(dstname)
         except Exception as e:
             os.rename(dstname, xdstname)
             syslog('error', str(e))
def send_i18n_digests(mlist, mboxfp):
    mbox = Mailbox(mboxfp)
    # Prepare common information (first lang/charset)
    lang = mlist.preferred_language
    lcset = Utils.GetCharSet(lang)
    lcset_out = Charset(lcset).output_charset or lcset
    # Common Information (contd)
    realname = mlist.real_name
    volume = mlist.volume
    issue = mlist.next_digest_number
    digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d')
    digestsubj = Header(digestid, lcset, header_name='Subject')
    # Set things up for the MIME digest.  Only headers not added by
    # CookHeaders need be added here.
    # Date/Message-ID should be added here also.
    mimemsg = Message.Message()
    mimemsg['Content-Type'] = 'multipart/mixed'
    mimemsg['MIME-Version'] = '1.0'
    mimemsg['From'] = mlist.GetRequestEmail()
    mimemsg['Subject'] = digestsubj
    mimemsg['To'] = mlist.GetListEmail()
    mimemsg['Reply-To'] = mlist.GetListEmail()
    mimemsg['Date'] = formatdate(localtime=1)
    mimemsg['Message-ID'] = Utils.unique_message_id(mlist)
    # Set things up for the rfc1153 digest
    plainmsg = StringIO()
    rfc1153msg = Message.Message()
    rfc1153msg['From'] = mlist.GetRequestEmail()
    rfc1153msg['Subject'] = digestsubj
    rfc1153msg['To'] = mlist.GetListEmail()
    rfc1153msg['Reply-To'] = mlist.GetListEmail()
    rfc1153msg['Date'] = formatdate(localtime=1)
    rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist)
    separator70 = '-' * 70
    separator30 = '-' * 30
    # In the rfc1153 digest, the masthead contains the digest boilerplate plus
    # any digest header.  In the MIME digests, the masthead and digest header
    # are separate MIME subobjects.  In either case, it's the first thing in
    # the digest, and we can calculate it now, so go ahead and add it now.
    mastheadtxt = Utils.maketext(
        'masthead.txt',
        {'real_name' :        mlist.real_name,
         'got_list_email':    mlist.GetListEmail(),
         'got_listinfo_url':  mlist.GetScriptURL('listinfo', absolute=1),
         'got_request_email': mlist.GetRequestEmail(),
         'got_owner_email':   mlist.GetOwnerEmail(),
         }, mlist=mlist)
    # MIME
    masthead = MIMEText(mastheadtxt, _charset=lcset)
    masthead['Content-Description'] = digestid
    mimemsg.attach(masthead)
    # RFC 1153
    print >> plainmsg, mastheadtxt
    print >> plainmsg
    # Now add the optional digest header but only if more than whitespace.
    if re.sub('\s', '', mlist.digest_header):
        headertxt = decorate(mlist, mlist.digest_header, _('digest header'))
        # MIME
        header = MIMEText(headertxt, _charset=lcset)
        header['Content-Description'] = _('Digest Header')
        mimemsg.attach(header)
        # RFC 1153
        print >> plainmsg, headertxt
        print >> plainmsg
    # Now we have to cruise through all the messages accumulated in the
    # mailbox file.  We can't add these messages to the plainmsg and mimemsg
    # yet, because we first have to calculate the table of contents
    # (i.e. grok out all the Subjects).  Store the messages in a list until
    # we're ready for them.
    #
    # Meanwhile prepare things for the table of contents
    toc = StringIO()
    print >> toc, _("Today's Topics:\n")
    # Now cruise through all the messages in the mailbox of digest messages,
    # building the MIME payload and core of the RFC 1153 digest.  We'll also
    # accumulate Subject: headers and authors for the table-of-contents.
    messages = []
    msgcount = 0
    msg = mbox.next()
    while msg is not None:
        if msg == '':
            # It was an unparseable message
            msg = mbox.next()
            continue
        msgcount += 1
        messages.append(msg)
        # Get the Subject header
        msgsubj = msg.get('subject', _('(no subject)'))
        subject = Utils.oneline(msgsubj, lcset)
        # Don't include the redundant subject prefix in the toc
        mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix),
                      subject, re.IGNORECASE)
        if mo:
            subject = subject[:mo.start(2)] + subject[mo.end(2):]
        username = ''
        addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)])
        # Take only the first author we find
        if isinstance(addresses, ListType) and addresses:
            username = addresses[0][0]
            if not username:
                username = addresses[0][1]
        if username:
            username = '******' % username
        # Put count and Wrap the toc subject line
        wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65)
        slines = wrapped.split('\n')
        # See if the user's name can fit on the last line
        if len(slines[-1]) + len(username) > 70:
            slines.append(username)
        else:
            slines[-1] += username
        # Add this subject to the accumulating topics
        first = True
        for line in slines:
            if first:
                print >> toc, ' ', line
                first = False
            else:
                print >> toc, '     ', line.lstrip()
        # We do not want all the headers of the original message to leak
        # through in the digest messages.  For this phase, we'll leave the
        # same set of headers in both digests, i.e. those required in RFC 1153
        # plus a couple of other useful ones.  We also need to reorder the
        # headers according to RFC 1153.  Later, we'll strip out headers for
        # for the specific MIME or plain digests.
        keeper = {}
        all_keepers = {}
        for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS +
                       mm_cfg.PLAIN_DIGEST_KEEP_HEADERS):
            all_keepers[header] = True
        all_keepers = all_keepers.keys()
        for keep in all_keepers:
            keeper[keep] = msg.get_all(keep, [])
        # Now remove all unkempt headers :)
        for header in msg.keys():
            del msg[header]
        # And add back the kept header in the RFC 1153 designated order
        for keep in all_keepers:
            for field in keeper[keep]:
                msg[keep] = field
        # And a bit of extra stuff
        msg['Message'] = `msgcount`
        # Get the next message in the digest mailbox
        msg = mbox.next()
    # Now we're finished with all the messages in the digest.  First do some
    # sanity checking and then on to adding the toc.
    if msgcount == 0:
        # Why did we even get here?
        return
    toctext = to_cset_out(toc.getvalue(), lcset)
    # MIME
    tocpart = MIMEText(toctext, _charset=lcset)
    tocpart['Content-Description']= _("Today's Topics (%(msgcount)d messages)")
    mimemsg.attach(tocpart)
    # RFC 1153
    print >> plainmsg, toctext
    print >> plainmsg
    # For RFC 1153 digests, we now need the standard separator
    print >> plainmsg, separator70
    print >> plainmsg
    # Now go through and add each message
    mimedigest = MIMEBase('multipart', 'digest')
    mimemsg.attach(mimedigest)
    first = True
    for msg in messages:
        # MIME.  Make a copy of the message object since the rfc1153
        # processing scrubs out attachments.
        mimedigest.attach(MIMEMessage(copy.deepcopy(msg)))
        # rfc1153
        if first:
            first = False
        else:
            print >> plainmsg, separator30
            print >> plainmsg
        # Use Mailman.Handlers.Scrubber.process() to get plain text
        try:
            msg = scrubber(mlist, msg)
        except Errors.DiscardMessage:
            print >> plainmsg, _('[Message discarded by content filter]')
            continue
        # Honor the default setting
        for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS:
            if msg[h]:
                uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset)))
                uh = '\n\t'.join(uh.split('\n'))
                print >> plainmsg, uh
        print >> plainmsg
        # If decoded payload is empty, this may be multipart message.
        # -- just stringfy it.
        payload = msg.get_payload(decode=True) \
                  or msg.as_string().split('\n\n',1)[1]
        mcset = msg.get_content_charset('')
        if mcset and mcset <> lcset and mcset <> lcset_out:
            try:
                payload = unicode(payload, mcset, 'replace'
                          ).encode(lcset, 'replace')
            except (UnicodeError, LookupError):
                # TK: Message has something unknown charset.
                #     _out means charset in 'outer world'.
                payload = unicode(payload, lcset_out, 'replace'
                          ).encode(lcset, 'replace')
        print >> plainmsg, payload
        if not payload.endswith('\n'):
            print >> plainmsg
    # Now add the footer but only if more than whitespace.
    if re.sub('\s', '', mlist.digest_footer):
        footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
        # MIME
        footer = MIMEText(footertxt, _charset=lcset)
        footer['Content-Description'] = _('Digest Footer')
        mimemsg.attach(footer)
        # RFC 1153
        # MAS: There is no real place for the digest_footer in an RFC 1153
        # compliant digest, so add it as an additional message with
        # Subject: Digest Footer
        print >> plainmsg, separator30
        print >> plainmsg
        print >> plainmsg, 'Subject: ' + _('Digest Footer')
        print >> plainmsg
        print >> plainmsg, footertxt
        print >> plainmsg
        print >> plainmsg, separator30
        print >> plainmsg
    # Do the last bit of stuff for each digest type
    signoff = _('End of ') + digestid
    # MIME
    # BAW: This stuff is outside the normal MIME goo, and it's what the old
    # MIME digester did.  No one seemed to complain, probably because you
    # won't see it in an MUA that can't display the raw message.  We've never
    # got complaints before, but if we do, just wax this.  It's primarily
    # included for (marginally useful) backwards compatibility.
    mimemsg.postamble = signoff
    # rfc1153
    print >> plainmsg, signoff
    print >> plainmsg, '*' * len(signoff)
    # Do our final bit of housekeeping, and then send each message to the
    # outgoing queue for delivery.
    mlist.next_digest_number += 1
    virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
    # Calculate the recipients lists
    plainrecips = []
    mimerecips = []
    drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys()
    for user in mlist.getMemberCPAddresses(drecips):
        # user might be None if someone who toggled off digest delivery
        # subsequently unsubscribed from the mailing list.  Also, filter out
        # folks who have disabled delivery.
        if user is None or mlist.getDeliveryStatus(user) <> ENABLED:
            continue
        # Otherwise, decide whether they get MIME or RFC 1153 digests
        if mlist.getMemberOption(user, mm_cfg.DisableMime):
            plainrecips.append(user)
        else:
            mimerecips.append(user)
    # Zap this since we're now delivering the last digest to these folks.
    mlist.one_last_digest.clear()
    # MIME
    virginq.enqueue(mimemsg,
                    recips=mimerecips,
                    listname=mlist.internal_name(),
                    isdigest=True)
    # RFC 1153
    rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset)
    virginq.enqueue(rfc1153msg,
                    recips=plainrecips,
                    listname=mlist.internal_name(),
                    isdigest=True)
Exemple #21
0
class MaildirRunner(Runner):
    # This class is much different than most runners because it pulls files
    # of a different format than what scripts/post and friends leaves.  The
    # files this runner reads are just single message files as dropped into
    # the directory by the MTA.  This runner will read the file, and enqueue
    # it in the expected qfiles directory for normal processing.
    def __init__(self, slice=None, numslices=1):
        # Don't call the base class constructor, but build enough of the
        # underlying attributes to use the base class's implementation.
        self._stop = 0
        self._dir = os.path.join(mm_cfg.MAILDIR_DIR, 'new')
        self._cur = os.path.join(mm_cfg.MAILDIR_DIR, 'cur')
        self._parser = Parser(Message)

    def _oneloop(self):
        # Refresh this each time through the list.  BAW: could be too
        # expensive.
        listnames = Utils.list_names()
        # Cruise through all the files currently in the new/ directory
        try:
            files = os.listdir(self._dir)
        except OSError, e:
            if e.errno <> errno.ENOENT: raise
            # Nothing's been delivered yet
            return 0
        for file in files:
            srcname = os.path.join(self._dir, file)
            dstname = os.path.join(self._cur, file + ':1,P')
            xdstname = os.path.join(self._cur, file + ':1,X')
            try:
                os.rename(srcname, dstname)
            except OSError, e:
                if e.errno == errno.ENOENT:
                    # Some other MaildirRunner beat us to it
                    continue
                syslog('error', 'Could not rename maildir file: %s', srcname)
                raise
            # Now open, read, parse, and enqueue this message
            try:
                fp = open(dstname)
                try:
                    msg = self._parser.parse(fp)
                finally:
                    fp.close()
                # Now we need to figure out which queue of which list this
                # message was destined for.  See verp_bounce() in
                # BounceRunner.py for why we do things this way.
                vals = []
                for header in ('delivered-to', 'envelope-to', 'apparently-to'):
                    vals.extend(msg.get_all(header, []))
                for field in vals:
                    to = parseaddr(field)[1]
                    if not to:
                        continue
                    mo = lre.match(to)
                    if not mo:
                        # This isn't an address we care about
                        continue
                    listname, subq = mo.group('listname', 'subq')
                    if listname in listnames:
                        break
                else:
                    # As far as we can tell, this message isn't destined for
                    # any list on the system.  What to do?
                    syslog('error', 'Message apparently not for any list: %s',
                           xdstname)
                    os.rename(dstname, xdstname)
                    continue
                # BAW: blech, hardcoded
                msgdata = {'listname': listname}
                # -admin is deprecated
                if subq in ('bounces', 'admin'):
                    queue = get_switchboard(mm_cfg.BOUNCEQUEUE_DIR)
                elif subq == 'confirm':
                    msgdata['toconfirm'] = 1
                    queue = get_switchboard(mm_cfg.CMDQUEUE_DIR)
                elif subq in ('join', 'subscribe'):
                    msgdata['tojoin'] = 1
                    queue = get_switchboard(mm_cfg.CMDQUEUE_DIR)
                elif subq in ('leave', 'unsubscribe'):
                    msgdata['toleave'] = 1
                    queue = get_switchboard(mm_cfg.CMDQUEUE_DIR)
                elif subq == 'owner':
                    msgdata.update({
                        'toowner': 1,
                        'envsender': Utils.get_site_email(extra='bounces'),
                        'pipeline': mm_cfg.OWNER_PIPELINE,
                        })
                    queue = get_switchboard(mm_cfg.INQUEUE_DIR)
                elif subq is None:
                    msgdata['tolist'] = 1
                    queue = get_switchboard(mm_cfg.INQUEUE_DIR)
                elif subq == 'request':
                    msgdata['torequest'] = 1
                    queue = get_switchboard(mm_cfg.CMDQUEUE_DIR)
                else:
                    syslog('error', 'Unknown sub-queue: %s', subq)
                    os.rename(dstname, xdstname)
                    continue
                queue.enqueue(msg, msgdata)
                os.unlink(dstname)
            except Exception, e:
                os.rename(dstname, xdstname)
                syslog('error', str(e))
Exemple #22
0
 def assertIsEnqueued(self, msg):
     """Assert the message was appended to the incoming queue."""
     switchboard = get_switchboard(mm_cfg.INQUEUE_DIR)
     file_path = switchboard.files()[-1]
     queued_msg, queued_msg_data = switchboard.dequeue(file_path)
     self.assertEqual(msg['message-id'], queued_msg['message-id'])
Exemple #23
0
     msgdata['adminapproved'] = 1
     # Calculate a new filebase for the approved message, otherwise
     # delivery errors will cause duplicates.
     try:
         del msgdata['filebase']
     except KeyError:
         pass
     # Queue the file for delivery by qrunner.  Trying to deliver the
     # message directly here can lead to a huge delay in web
     # turnaround.  Log the moderation and add a header.
     msg['X-Mailman-Approved-At'] = email.Utils.formatdate(localtime=1)
     syslog('vette', 'held message approved, message-id: %s',
            msg.get('message-id', 'n/a'))
     # Stick the message back in the incoming queue for further
     # processing.
     inq = get_switchboard(mm_cfg.INQUEUE_DIR)
     inq.enqueue(msg, _metadata=msgdata)
 elif value == mm_cfg.REJECT:
     # Rejected
     rejection = 'Refused'
     lang = self.getMemberLanguage(sender)
     subject = Utils.oneline(subject, Utils.GetCharSet(lang))
     self.__refuse(_('Posting of your message titled "%(subject)s"'),
                   sender, comment or _('[No reason given]'),
                   lang=lang)
 else:
     assert value == mm_cfg.DISCARD
     # Discarded
     rejection = 'Discarded'
 # Forward the message
 if forward and addr:
   def __handlepost(self, record, value, comment, preserve, forward, addr):
       # For backwards compatibility with pre 2.0beta3
       ptime, sender, subject, reason, filename, msgdata = record
       path = os.path.join(mm_cfg.DATA_DIR, filename)
       # Handle message preservation
       if preserve:
           parts = os.path.split(path)[1].split(DASH)
           parts[0] = 'spam'
           spamfile = DASH.join(parts)
           # Preserve the message as plain text, not as a pickle
           try:
               fp = open(path)
           except IOError as e:
               if e.errno != errno.ENOENT: raise
               return LOST
           try:
               if path.endswith('.pck'):
                   msg = pickle.load(fp)
               else:
                   assert path.endswith('.txt'), '%s not .pck or .txt' % path
                   msg = fp.read()
           finally:
               fp.close()
           # Save the plain text to a .msg file, not a .pck file
           outpath = os.path.join(mm_cfg.SPAM_DIR, spamfile)
           head, ext = os.path.splitext(outpath)
           outpath = head + '.msg'
           outfp = open(outpath, 'wb')
           try:
               if path.endswith('.pck'):
                   g = Generator(outfp)
                   g.flatten(msg, 1)
               else:
                   outfp.write(msg)
           finally:
               outfp.close()
       # Now handle updates to the database
       rejection = None
       fp = None
       msg = None
       status = REMOVE
       if value == mm_cfg.DEFER:
           # Defer
           status = DEFER
       elif value == mm_cfg.APPROVE:
           # Approved.
           try:
               msg = readMessage(path)
           except IOError as e:
               if e.errno != errno.ENOENT: raise
               return LOST
           msg = readMessage(path)
           msgdata['approved'] = 1
           # adminapproved is used by the Emergency handler
           msgdata['adminapproved'] = 1
           # Calculate a new filebase for the approved message, otherwise
           # delivery errors will cause duplicates.
           try:
               del msgdata['filebase']
           except KeyError:
               pass
           # Queue the file for delivery by qrunner.  Trying to deliver the
           # message directly here can lead to a huge delay in web
           # turnaround.  Log the moderation and add a header.
           msg['X-Mailman-Approved-At'] = email.utils.formatdate(localtime=1)
           syslog('vette', '%s: held message approved, message-id: %s',
                  self.internal_name(), msg.get('message-id', 'n/a'))
           # Stick the message back in the incoming queue for further
           # processing.
           inq = get_switchboard(mm_cfg.INQUEUE_DIR)
           inq.enqueue(msg, _metadata=msgdata)
       elif value == mm_cfg.REJECT:
           # Rejected
           rejection = 'Refused'
           lang = self.getMemberLanguage(sender)
           subject = Utils.oneline(subject, Utils.GetCharSet(lang))
           self.__refuse(_('Posting of your message titled "%(subject)s"'),
                         sender,
                         comment or _('[No reason given]'),
                         lang=lang)
       else:
           assert value == mm_cfg.DISCARD
           # Discarded
           rejection = 'Discarded'
       # Forward the message
       if forward and addr:
           # If we've approved the message, we need to be sure to craft a
           # completely unique second message for the forwarding operation,
           # since we don't want to share any state or information with the
           # normal delivery.
           try:
               copy = readMessage(path)
           except IOError as e:
               if e.errno != errno.ENOENT: raise
               raise Errors.LostHeldMessage(path)
           # It's possible the addr is a comma separated list of addresses.
           addrs = getaddresses([addr])
           if len(addrs) == 1:
               realname, addr = addrs[0]
               # If the address getting the forwarded message is a member of
               # the list, we want the headers of the outer message to be
               # encoded in their language.  Otherwise it'll be the preferred
               # language of the mailing list.
               lang = self.getMemberLanguage(addr)
           else:
               # Throw away the realnames
               addr = [a for realname, a in addrs]
               # Which member language do we attempt to use?  We could use
               # the first match or the first address, but in the face of
               # ambiguity, let's just use the list's preferred language
               lang = self.preferred_language
           otrans = i18n.get_translation()
           i18n.set_language(lang)
           try:
               fmsg = Message.UserNotification(
                   addr,
                   self.GetBouncesEmail(),
                   _('Forward of moderated message'),
                   lang=lang)
           finally:
               i18n.set_translation(otrans)
           fmsg.set_type('message/rfc822')
           fmsg.attach(copy)
           fmsg.send(self)
       # Log the rejection
       if rejection:
           note = '''%(listname)s: %(rejection)s posting:
tFrom: %(sender)s
tSubject: %(subject)s''' % {
               'listname': self.internal_name(),
               'rejection': rejection,
               'sender': str(sender).replace('%', '%%'),
               'subject': str(subject).replace('%', '%%'),
           }
           if comment:
               note += '\n\tReason: ' + comment.replace('%', '%%')
           syslog('vette', note)
       # Always unlink the file containing the message text.  It's not
       # necessary anymore, regardless of the disposition of the message.
       if status != DEFER:
           try:
               os.unlink(path)
           except OSError as e:
               if e.errno != errno.ENOENT: raise
               # We lost the message text file.  Clean up our housekeeping
               # and inform of this status.
               return LOST
       return status
Exemple #25
0
     msgdata['adminapproved'] = 1
     # Calculate a new filebase for the approved message, otherwise
     # delivery errors will cause duplicates.
     try:
         del msgdata['filebase']
     except KeyError:
         pass
     # Queue the file for delivery by qrunner.  Trying to deliver the
     # message directly here can lead to a huge delay in web
     # turnaround.  Log the moderation and add a header.
     msg['X-Mailman-Approved-At'] = email.Utils.formatdate(localtime=1)
     syslog('vette', '%s: held message approved, message-id: %s',
            self.internal_name(), msg.get('message-id', 'n/a'))
     # Stick the message back in the incoming queue for further
     # processing.
     inq = get_switchboard(mm_cfg.INQUEUE_DIR)
     inq.enqueue(msg, _metadata=msgdata)
 elif value == mm_cfg.REJECT:
     # Rejected
     rejection = 'Refused'
     lang = self.getMemberLanguage(sender)
     subject = Utils.oneline(subject, Utils.GetCharSet(lang))
     self.__refuse(_('Posting of your message titled "%(subject)s"'),
                   sender,
                   comment or _('[No reason given]'),
                   lang=lang)
 else:
     assert value == mm_cfg.DISCARD
     # Discarded
     rejection = 'Discarded'
 # Forward the message
Exemple #26
0
 def _dispose(self, mlist, msg, msgdata):
     # Make sure we have the most up-to-date state
     mlist.Load()
     outq = get_switchboard(mm_cfg.OUTQUEUE_DIR)
     # There are a few possibilities here:
     #
     # - the message could have been VERP'd in which case, we know exactly
     #   who the message was destined for.  That make our job easy.
     # - the message could have been originally destined for a list owner,
     #   but a list owner address itself bounced.  That's bad, and for now
     #   we'll simply attempt to deliver the message to the site list
     #   owner.
     #   Note that this means that automated bounce processing doesn't work
     #   for the site list.  Because we can't reliably tell to what address
     #   a non-VERP'd bounce was originally sent, we have to treat all
     #   bounces sent to the site list as potential list owner bounces.
     # - the list owner could have set list-bounces (or list-admin) as the
     #   owner address.  That's really bad as it results in a loop of ever
     #   growing unrecognized bounce messages.  We detect this based on the
     #   fact that this message itself will be from the site bounces
     #   address.  We then send this to the site list owner instead.
     # Notices to list-owner have their envelope sender and From: set to
     # the site-bounces address.  Check if this is this a bounce for a
     # message to a list owner, coming to site-bounces, or a looping
     # message sent directly to the -bounces address.  We have to do these
     # cases separately, because sending to site-owner will reset the
     # envelope sender.
     # Is this a site list bounce?
     if (mlist.internal_name().lower() ==
             mm_cfg.MAILMAN_SITE_LIST.lower()):
         # Send it on to the site owners, but craft the envelope sender to
         # be the -loop detection address, so if /they/ bounce, we won't
         # get stuck in a bounce loop.
         outq.enqueue(msg, msgdata,
                      recips=mlist.owner,
                      envsender=Utils.get_site_email(extra='loop'),
                      nodecorate=1,
                      )
         return
     # Is this a possible looping message sent directly to a list-bounces
     # address other than the site list?
     # Check From: because unix_from might be VERP'd.
     # Also, check the From: that Message.OwnerNotification uses.
     if (msg.get('from') ==
             Utils.get_site_email(mlist.host_name, 'bounces')):
         # Just send it to the sitelist-owner address.  If that bounces
         # we'll handle it above.
         outq.enqueue(msg, msgdata,
                      recips=[Utils.get_site_email(extra='owner')],
                      envsender=Utils.get_site_email(extra='loop'),
                      nodecorate=1,
                      )
         return
     # List isn't doing bounce processing?
     if not mlist.bounce_processing:
         return
     # Try VERP detection first, since it's quick and easy
     addrs = verp_bounce(mlist, msg)
     if addrs:
         # We have an address, but check if the message is non-fatal.
         if BouncerAPI.ScanMessages(mlist, msg) is BouncerAPI.Stop:
             return
     else:
         # See if this was a probe message.
         token = verp_probe(mlist, msg)
         if token:
             self._probe_bounce(mlist, token)
             return
         # That didn't give us anything useful, so try the old fashion
         # bounce matching modules.
         addrs = BouncerAPI.ScanMessages(mlist, msg)
         if addrs is BouncerAPI.Stop:
             # This is a recognized, non-fatal notice. Ignore it.
             return
     # If that still didn't return us any useful addresses, then send it on
     # or discard it.
     addrs = filter(None, addrs)
     if not addrs:
         syslog('bounce',
                '%s: bounce message w/no discernable addresses: %s',
                mlist.internal_name(),
                msg.get('message-id', 'n/a'))
         maybe_forward(mlist, msg)
         return
     # BAW: It's possible that there are None's in the list of addresses,
     # although I'm unsure how that could happen.  Possibly ScanMessages()
     # can let None's sneak through.  In any event, this will kill them.
     # addrs = filter(None, addrs)
     # MAS above filter moved up so we don't try to queue an empty list.
     self._queue_bounces(mlist.internal_name(), addrs, msg)
Exemple #27
0
def send_i18n_digests(mlist, mboxfp):
    mbox = Mailbox(mboxfp)
    # Prepare common information (first lang/charset)
    lang = mlist.preferred_language
    lcset = Utils.GetCharSet(lang)
    lcset_out = Charset(lcset).output_charset or lcset
    # Common Information (contd)
    realname = mlist.real_name
    volume = mlist.volume
    issue = mlist.next_digest_number
    digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d')
    digestsubj = Header(digestid, lcset, header_name='Subject')
    # Set things up for the MIME digest.  Only headers not added by
    # CookHeaders need be added here.
    # Date/Message-ID should be added here also.
    mimemsg = Message.Message()
    mimemsg['Content-Type'] = 'multipart/mixed'
    mimemsg['MIME-Version'] = '1.0'
    mimemsg['From'] = mlist.GetRequestEmail()
    mimemsg['Subject'] = digestsubj
    mimemsg['To'] = mlist.GetListEmail()
    mimemsg['Reply-To'] = mlist.GetListEmail()
    mimemsg['Date'] = formatdate(localtime=1)
    mimemsg['Message-ID'] = Utils.unique_message_id(mlist)
    # Set things up for the rfc1153 digest
    plainmsg = StringIO()
    rfc1153msg = Message.Message()
    rfc1153msg['From'] = mlist.GetRequestEmail()
    rfc1153msg['Subject'] = digestsubj
    rfc1153msg['To'] = mlist.GetListEmail()
    rfc1153msg['Reply-To'] = mlist.GetListEmail()
    rfc1153msg['Date'] = formatdate(localtime=1)
    rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist)
    separator70 = '-' * 70
    separator30 = '-' * 30
    # In the rfc1153 digest, the masthead contains the digest boilerplate plus
    # any digest header.  In the MIME digests, the masthead and digest header
    # are separate MIME subobjects.  In either case, it's the first thing in
    # the digest, and we can calculate it now, so go ahead and add it now.
    mastheadtxt = Utils.maketext(
        'masthead.txt', {
            'real_name': mlist.real_name,
            'got_list_email': mlist.GetListEmail(),
            'got_listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
            'got_request_email': mlist.GetRequestEmail(),
            'got_owner_email': mlist.GetOwnerEmail(),
        },
        mlist=mlist)
    # MIME
    masthead = MIMEText(mastheadtxt, _charset=lcset)
    masthead['Content-Description'] = digestid
    mimemsg.attach(masthead)
    # RFC 1153
    print >> plainmsg, mastheadtxt
    print >> plainmsg
    # Now add the optional digest header but only if more than whitespace.
    if re.sub('\s', '', mlist.digest_header):
        headertxt = decorate(mlist, mlist.digest_header, _('digest header'))
        # MIME
        header = MIMEText(headertxt, _charset=lcset)
        header['Content-Description'] = _('Digest Header')
        mimemsg.attach(header)
        # RFC 1153
        print >> plainmsg, headertxt
        print >> plainmsg
    # Now we have to cruise through all the messages accumulated in the
    # mailbox file.  We can't add these messages to the plainmsg and mimemsg
    # yet, because we first have to calculate the table of contents
    # (i.e. grok out all the Subjects).  Store the messages in a list until
    # we're ready for them.
    #
    # Meanwhile prepare things for the table of contents
    toc = StringIO()
    print >> toc, _("Today's Topics:\n")
    # Now cruise through all the messages in the mailbox of digest messages,
    # building the MIME payload and core of the RFC 1153 digest.  We'll also
    # accumulate Subject: headers and authors for the table-of-contents.
    messages = []
    msgcount = 0
    msg = mbox.next()
    while msg is not None:
        if msg == '':
            # It was an unparseable message
            msg = mbox.next()
            continue
        msgcount += 1
        messages.append(msg)
        # Get the Subject header
        msgsubj = msg.get('subject', _('(no subject)'))
        subject = Utils.oneline(msgsubj, lcset)
        # Don't include the redundant subject prefix in the toc
        mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix),
                      subject, re.IGNORECASE)
        if mo:
            subject = subject[:mo.start(2)] + subject[mo.end(2):]
        username = ''
        addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)])
        # Take only the first author we find
        if isinstance(addresses, ListType) and addresses:
            username = addresses[0][0]
            if not username:
                username = addresses[0][1]
        if username:
            username = '******' % username
        # Put count and Wrap the toc subject line
        wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65)
        slines = wrapped.split('\n')
        # See if the user's name can fit on the last line
        if len(slines[-1]) + len(username) > 70:
            slines.append(username)
        else:
            slines[-1] += username
        # Add this subject to the accumulating topics
        first = True
        for line in slines:
            if first:
                print >> toc, ' ', line
                first = False
            else:
                print >> toc, '     ', line.lstrip()
        # We do not want all the headers of the original message to leak
        # through in the digest messages.  For this phase, we'll leave the
        # same set of headers in both digests, i.e. those required in RFC 1153
        # plus a couple of other useful ones.  We also need to reorder the
        # headers according to RFC 1153.  Later, we'll strip out headers for
        # for the specific MIME or plain digests.
        keeper = {}
        all_keepers = {}
        for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS +
                       mm_cfg.PLAIN_DIGEST_KEEP_HEADERS):
            all_keepers[header] = True
        all_keepers = all_keepers.keys()
        for keep in all_keepers:
            keeper[keep] = msg.get_all(keep, [])
        # Now remove all unkempt headers :)
        for header in msg.keys():
            del msg[header]
        # And add back the kept header in the RFC 1153 designated order
        for keep in all_keepers:
            for field in keeper[keep]:
                msg[keep] = field
        # And a bit of extra stuff
        msg['Message'] = ` msgcount `
        # Get the next message in the digest mailbox
        msg = mbox.next()
    # Now we're finished with all the messages in the digest.  First do some
    # sanity checking and then on to adding the toc.
    if msgcount == 0:
        # Why did we even get here?
        return
    toctext = to_cset_out(toc.getvalue(), lcset)
    # MIME
    tocpart = MIMEText(toctext, _charset=lcset)
    tocpart['Content-Description'] = _(
        "Today's Topics (%(msgcount)d messages)")
    mimemsg.attach(tocpart)
    # RFC 1153
    print >> plainmsg, toctext
    print >> plainmsg
    # For RFC 1153 digests, we now need the standard separator
    print >> plainmsg, separator70
    print >> plainmsg
    # Now go through and add each message
    mimedigest = MIMEBase('multipart', 'digest')
    mimemsg.attach(mimedigest)
    first = True
    for msg in messages:
        # MIME.  Make a copy of the message object since the rfc1153
        # processing scrubs out attachments.
        mimedigest.attach(MIMEMessage(copy.deepcopy(msg)))
        # rfc1153
        if first:
            first = False
        else:
            print >> plainmsg, separator30
            print >> plainmsg
        # Use Mailman.Handlers.Scrubber.process() to get plain text
        try:
            msg = scrubber(mlist, msg)
        except Errors.DiscardMessage:
            print >> plainmsg, _('[Message discarded by content filter]')
            continue
        # Honor the default setting
        for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS:
            if msg[h]:
                uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset)))
                uh = '\n\t'.join(uh.split('\n'))
                print >> plainmsg, uh
        print >> plainmsg
        # If decoded payload is empty, this may be multipart message.
        # -- just stringfy it.
        payload = msg.get_payload(decode=True) \
                  or msg.as_string().split('\n\n',1)[1]
        mcset = msg.get_content_charset('')
        if mcset and mcset <> lcset and mcset <> lcset_out:
            try:
                payload = unicode(payload, mcset,
                                  'replace').encode(lcset, 'replace')
            except (UnicodeError, LookupError):
                # TK: Message has something unknown charset.
                #     _out means charset in 'outer world'.
                payload = unicode(payload, lcset_out,
                                  'replace').encode(lcset, 'replace')
        print >> plainmsg, payload
        if not payload.endswith('\n'):
            print >> plainmsg
    # Now add the footer but only if more than whitespace.
    if re.sub('\s', '', mlist.digest_footer):
        footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
        # MIME
        footer = MIMEText(footertxt, _charset=lcset)
        footer['Content-Description'] = _('Digest Footer')
        mimemsg.attach(footer)
        # RFC 1153
        # MAS: There is no real place for the digest_footer in an RFC 1153
        # compliant digest, so add it as an additional message with
        # Subject: Digest Footer
        print >> plainmsg, separator30
        print >> plainmsg
        print >> plainmsg, 'Subject: ' + _('Digest Footer')
        print >> plainmsg
        print >> plainmsg, footertxt
        print >> plainmsg
        print >> plainmsg, separator30
        print >> plainmsg
    # Do the last bit of stuff for each digest type
    signoff = _('End of ') + digestid
    # MIME
    # BAW: This stuff is outside the normal MIME goo, and it's what the old
    # MIME digester did.  No one seemed to complain, probably because you
    # won't see it in an MUA that can't display the raw message.  We've never
    # got complaints before, but if we do, just wax this.  It's primarily
    # included for (marginally useful) backwards compatibility.
    mimemsg.postamble = signoff
    # rfc1153
    print >> plainmsg, signoff
    print >> plainmsg, '*' * len(signoff)
    # Do our final bit of housekeeping, and then send each message to the
    # outgoing queue for delivery.
    mlist.next_digest_number += 1
    virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
    # Calculate the recipients lists
    plainrecips = []
    mimerecips = []
    drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys()
    for user in mlist.getMemberCPAddresses(drecips):
        # user might be None if someone who toggled off digest delivery
        # subsequently unsubscribed from the mailing list.  Also, filter out
        # folks who have disabled delivery.
        if user is None or mlist.getDeliveryStatus(user) <> ENABLED:
            continue
        # Otherwise, decide whether they get MIME or RFC 1153 digests
        if mlist.getMemberOption(user, mm_cfg.DisableMime):
            plainrecips.append(user)
        else:
            mimerecips.append(user)
    # Zap this since we're now delivering the last digest to these folks.
    mlist.one_last_digest.clear()
    # MIME
    virginq.enqueue(mimemsg,
                    recips=mimerecips,
                    listname=mlist.internal_name(),
                    isdigest=True)
    # RFC 1153
    rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset)
    virginq.enqueue(rfc1153msg,
                    recips=plainrecips,
                    listname=mlist.internal_name(),
                    isdigest=True)