def calculate_attachments_dir(mlist, msg, msgdata): # Calculate the directory that attachments for this message will go # under. To avoid inode limitations, the scheme will be: # archives/private/<listname>/attachments/YYYYMMDD/<msgid-hash>/<files> # Start by calculating the date-based and msgid-hash components. fmt = "%Y%m%d" datestr = msg.get("Date") if datestr: now = parsedate(datestr) else: now = time.gmtime(msgdata.get("received_time", time.time())) datedir = safe_strftime(fmt, now) if not datedir: datestr = msgdata.get("X-List-Received-Date") if datestr: datedir = safe_strftime(fmt, datestr) if not datedir: # What next? Unixfrom, I guess. parts = msg.get_unixfrom().split() try: month = { "Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6, "Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12, }.get(parts[3], 0) day = int(parts[4]) year = int(parts[6]) except (IndexError, ValueError): # Best we can do I think month = day = year = 0 datedir = "%04d%02d%02d" % (year, month, day) assert datedir if mm_cfg.SCRUBBER_ADD_PAYLOAD_HASH_FILENAME: return os.path.join("attachments", datedir) else: # As for the msgid hash, we'll base this part on the Message-ID: so that # all attachments for the same message end up in the same directory (we'll # uniquify the filenames in that directory as needed). We use the first 2 # and last 2 bytes of the SHA1 hash of the message id as the basis of the # directory name. Clashes here don't really matter too much, and that # still gives us a 32-bit space to work with. msgid = msg["message-id"] if msgid is None: msgid = msg["Message-ID"] = Utils.unique_message_id(mlist) # We assume that the message id actually /is/ unique! digest = sha_new(msgid).hexdigest() return os.path.join("attachments", datedir, digest[:4] + digest[-4:])
def calculate_attachments_dir(mlist, msg, msgdata): # Calculate the directory that attachments for this message will go # under. To avoid inode limitations, the scheme will be: # archives/private/<listname>/attachments/YYYYMMDD/<msgid-hash>/<files> # Start by calculating the date-based and msgid-hash components. fmt = '%Y%m%d' datestr = msg.get('Date') if datestr: now = parsedate(datestr) else: now = time.gmtime(msgdata.get('received_time', time.time())) datedir = safe_strftime(fmt, now) if not datedir: datestr = msgdata.get('X-List-Received-Date') if datestr: datedir = safe_strftime(fmt, datestr) if not datedir: # What next? Unixfrom, I guess. parts = msg.get_unixfrom().split() try: month = { 'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12, }.get(parts[3], 0) day = int(parts[4]) year = int(parts[6]) except (IndexError, ValueError): # Best we can do I think month = day = year = 0 datedir = '%04d%02d%02d' % (year, month, day) assert datedir # As for the msgid hash, we'll base this part on the Message-ID: so that # all attachments for the same message end up in the same directory (we'll # uniquify the filenames in that directory as needed). We use the first 2 # and last 2 bytes of the SHA1 hash of the message id as the basis of the # directory name. Clashes here don't really matter too much, and that # still gives us a 32-bit space to work with. msgid = msg['message-id'] if msgid is None: msgid = msg['Message-ID'] = Utils.unique_message_id(mlist) # We assume that the message id actually /is/ unique! digest = sha_new(msgid).hexdigest() # hash disabled to handle file duplicate over mutiple email. #return os.path.join('attachments', datedir, digest[:4] + digest[-4:]) return os.path.join('attachments', datedir)
def process(mlist, msg, msgdata): # This is the negation of we're wrapping because dmarc_moderation_action # is wrap this message or from_is_list applies and is wrap. if not (msgdata.get('from_is_list') == 2 or (mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)): # Now see if we need to add a From:, Reply-To: or Cc: without wrapping. # See comments in CookHeaders.change_header for why we do this here. a_h = msgdata.get('add_header') if a_h: if a_h.get('From'): del msg['from'] msg['From'] = a_h.get('From') if a_h.get('Reply-To'): del msg['reply-to'] msg['Reply-To'] = a_h.get('Reply-To') if a_h.get('Cc'): del msg['cc'] msg['Cc'] = a_h.get('Cc') return # There are various headers in msg that we don't want, so we basically # make a copy of the msg, then delete almost everything and set/copy # what we want. omsg = copy.deepcopy(msg) for key in list(msg.keys()): if key.lower() not in KEEPERS: del msg[key] msg['MIME-Version'] = '1.0' msg['Message-ID'] = Utils.unique_message_id(mlist) # Add the headers from CookHeaders. for k, v in list(msgdata['add_header'].items()): msg[k] = v # Are we including dmarc_wrapped_message_text? I.e., do we have text and # are we wrapping because of dmarc_moderation_action? if mlist.dmarc_wrapped_message_text and msgdata.get('from_is_list') == 2: part1 = text.MIMEText(Utils.wrap(mlist.dmarc_wrapped_message_text), 'plain', Utils.GetCharSet(mlist.preferred_language)) part1['Content-Disposition'] = 'inline' part2 = message.MIMEMessage(omsg) part2['Content-Disposition'] = 'inline' msg['Content-Type'] = 'multipart/mixed' msg.set_payload([part1, part2]) else: msg['Content-Type'] = 'message/rfc822' msg['Content-Disposition'] = 'inline' msg.set_payload([omsg])
def process(mlist, msg, msgdata): # This is the negation of we're wrapping because dmarc_moderation_action # is wrap this message or from_is_list applies and is wrap. if not (msgdata.get('from_is_list') == 2 or (mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)): # Now see if we need to add a From:, Reply-To: or Cc: without wrapping. # See comments in CookHeaders.change_header for why we do this here. a_h = msgdata.get('add_header') if a_h: if a_h.get('From'): del msg['from'] msg['From'] = a_h.get('From') if a_h.get('Reply-To'): del msg['reply-to'] msg['Reply-To'] = a_h.get('Reply-To') if a_h.get('Cc'): del msg['cc'] msg['Cc'] = a_h.get('Cc') return # There are various headers in msg that we don't want, so we basically # make a copy of the msg, then delete almost everything and set/copy # what we want. omsg = copy.deepcopy(msg) for key in msg.keys(): if key.lower() not in KEEPERS: del msg[key] msg['MIME-Version'] = '1.0' msg['Message-ID'] = Utils.unique_message_id(mlist) # Add the headers from CookHeaders. for k, v in msgdata['add_header'].items(): msg[k] = v # Are we including dmarc_wrapped_message_text? I.e., do we have text and # are we wrapping because of dmarc_moderation_action? if mlist.dmarc_wrapped_message_text and msgdata.get('from_is_list') == 2: part1 = MIMEText(Utils.wrap(mlist.dmarc_wrapped_message_text), 'plain', Utils.GetCharSet(mlist.preferred_language)) part1['Content-Disposition'] = 'inline' part2 = MIMEMessage(omsg) part2['Content-Disposition'] = 'inline' msg['Content-Type'] = 'multipart/mixed' msg.set_payload([part1, part2]) else: msg['Content-Type'] = 'message/rfc822' msg['Content-Disposition'] = 'inline' msg.set_payload([omsg])
def calculate_attachments_dir(mlist, msg, msgdata): # Calculate the directory that attachments for this message will go # under. To avoid inode limitations, the scheme will be: # archives/private/<listname>/attachments/YYYYMMDD/<msgid-hash>/<files> # Start by calculating the date-based and msgid-hash components. fmt = '%Y%m%d' datestr = msg.get('Date') if datestr: now = parsedate(datestr) else: now = time.gmtime(msgdata.get('received_time', time.time())) datedir = safe_strftime(fmt, now) if not datedir: datestr = msgdata.get('X-List-Received-Date') if datestr: datedir = safe_strftime(fmt, datestr) if not datedir: # What next? Unixfrom, I guess. parts = msg.get_unixfrom().split() try: month = {'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 'Aug':8, 'Sep':9, 'Oct':10, 'Nov':11, 'Dec':12, }.get(parts[3], 0) day = int(parts[4]) year = int(parts[6]) except (IndexError, ValueError): # Best we can do I think month = day = year = 0 datedir = '%04d%02d%02d' % (year, month, day) assert datedir # As for the msgid hash, we'll base this part on the Message-ID: so that # all attachments for the same message end up in the same directory (we'll # uniquify the filenames in that directory as needed). We use the first 2 # and last 2 bytes of the SHA1 hash of the message id as the basis of the # directory name. Clashes here don't really matter too much, and that # still gives us a 32-bit space to work with. msgid = msg['message-id'] if msgid is None: msgid = msg['Message-ID'] = Utils.unique_message_id(mlist) # We assume that the message id actually /is/ unique! digest = sha_new(msgid).hexdigest() return os.path.join('attachments', datedir, digest[:4] + digest[-4:])
def send(self, mlist, noprecedence=False, **_kws): """Sends the message by enqueuing it to the `virgin' queue. This is used for all internally crafted messages. """ # Since we're crafting the message from whole cloth, let's make sure # this message has a Message-ID. Yes, the MTA would give us one, but # this is useful for logging to logs/smtp. if 'message-id' not in self: self['Message-ID'] = Utils.unique_message_id(mlist) # Ditto for Date: which is required by RFC 2822 if 'date' not in self: self['Date'] = email.utils.formatdate(localtime=1) # UserNotifications are typically for admin messages, and for messages # other than list explosions. Send these out as Precedence: bulk, but # don't override an existing Precedence: header. # Also, if the message is To: the list-owner address, set Precedence: # list. See note below in OwnerNotification. if not ('precedence' in self or noprecedence): if self.get('to') == mlist.GetOwnerEmail(): self['Precedence'] = 'list' else: self['Precedence'] = 'bulk' self._enqueue(mlist, **_kws)
def send_i18n_digests(mlist, mboxfp): mbox = Mailbox(mboxfp) # Prepare common information (first lang/charset) lang = mlist.preferred_language lcset = Utils.GetCharSet(lang) lcset_out = Charset(lcset).output_charset or lcset # Common Information (contd) realname = mlist.real_name volume = mlist.volume issue = mlist.next_digest_number digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d') digestsubj = Header(digestid, lcset, header_name='Subject') # Set things up for the MIME digest. Only headers not added by # CookHeaders need be added here. # Date/Message-ID should be added here also. mimemsg = Message.Message() mimemsg['Content-Type'] = 'multipart/mixed' mimemsg['MIME-Version'] = '1.0' mimemsg['From'] = mlist.GetRequestEmail() mimemsg['Subject'] = digestsubj mimemsg['To'] = mlist.GetListEmail() mimemsg['Reply-To'] = mlist.GetListEmail() mimemsg['Date'] = formatdate(localtime=1) mimemsg['Message-ID'] = Utils.unique_message_id(mlist) # Set things up for the rfc1153 digest plainmsg = StringIO() rfc1153msg = Message.Message() rfc1153msg['From'] = mlist.GetRequestEmail() rfc1153msg['Subject'] = digestsubj rfc1153msg['To'] = mlist.GetListEmail() rfc1153msg['Reply-To'] = mlist.GetListEmail() rfc1153msg['Date'] = formatdate(localtime=1) rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist) separator70 = '-' * 70 separator30 = '-' * 30 # In the rfc1153 digest, the masthead contains the digest boilerplate plus # any digest header. In the MIME digests, the masthead and digest header # are separate MIME subobjects. In either case, it's the first thing in # the digest, and we can calculate it now, so go ahead and add it now. mastheadtxt = Utils.maketext( 'masthead.txt', {'real_name' : mlist.real_name, 'got_list_email': mlist.GetListEmail(), 'got_listinfo_url': mlist.GetScriptURL('listinfo', absolute=1), 'got_request_email': mlist.GetRequestEmail(), 'got_owner_email': mlist.GetOwnerEmail(), }, mlist=mlist) # MIME masthead = MIMEText(mastheadtxt, _charset=lcset) masthead['Content-Description'] = digestid mimemsg.attach(masthead) # RFC 1153 print >> plainmsg, mastheadtxt print >> plainmsg # Now add the optional digest header but only if more than whitespace. if re.sub('\s', '', mlist.digest_header): headertxt = decorate(mlist, mlist.digest_header, _('digest header')) # MIME header = MIMEText(headertxt, _charset=lcset) header['Content-Description'] = _('Digest Header') mimemsg.attach(header) # RFC 1153 print >> plainmsg, headertxt print >> plainmsg # Now we have to cruise through all the messages accumulated in the # mailbox file. We can't add these messages to the plainmsg and mimemsg # yet, because we first have to calculate the table of contents # (i.e. grok out all the Subjects). Store the messages in a list until # we're ready for them. # # Meanwhile prepare things for the table of contents toc = StringIO() print >> toc, _("Today's Topics:\n") # Now cruise through all the messages in the mailbox of digest messages, # building the MIME payload and core of the RFC 1153 digest. We'll also # accumulate Subject: headers and authors for the table-of-contents. messages = [] msgcount = 0 msg = mbox.next() while msg is not None: if msg == '': # It was an unparseable message msg = mbox.next() continue msgcount += 1 messages.append(msg) # Get the Subject header msgsubj = msg.get('subject', _('(no subject)')) subject = Utils.oneline(msgsubj, lcset) # Don't include the redundant subject prefix in the toc mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix), subject, re.IGNORECASE) if mo: subject = subject[:mo.start(2)] + subject[mo.end(2):] username = '' addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)]) # Take only the first author we find if isinstance(addresses, ListType) and addresses: username = addresses[0][0] if not username: username = addresses[0][1] if username: username = '******' % username # Put count and Wrap the toc subject line wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65) slines = wrapped.split('\n') # See if the user's name can fit on the last line if len(slines[-1]) + len(username) > 70: slines.append(username) else: slines[-1] += username # Add this subject to the accumulating topics first = True for line in slines: if first: print >> toc, ' ', line first = False else: print >> toc, ' ', line.lstrip() # We do not want all the headers of the original message to leak # through in the digest messages. For this phase, we'll leave the # same set of headers in both digests, i.e. those required in RFC 1153 # plus a couple of other useful ones. We also need to reorder the # headers according to RFC 1153. Later, we'll strip out headers for # for the specific MIME or plain digests. keeper = {} all_keepers = {} for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS + mm_cfg.PLAIN_DIGEST_KEEP_HEADERS): all_keepers[header] = True all_keepers = all_keepers.keys() for keep in all_keepers: keeper[keep] = msg.get_all(keep, []) # Now remove all unkempt headers :) for header in msg.keys(): del msg[header] # And add back the kept header in the RFC 1153 designated order for keep in all_keepers: for field in keeper[keep]: msg[keep] = field # And a bit of extra stuff msg['Message'] = `msgcount` # Get the next message in the digest mailbox msg = mbox.next() # Now we're finished with all the messages in the digest. First do some # sanity checking and then on to adding the toc. if msgcount == 0: # Why did we even get here? return toctext = to_cset_out(toc.getvalue(), lcset) # MIME tocpart = MIMEText(toctext, _charset=lcset) tocpart['Content-Description']= _("Today's Topics (%(msgcount)d messages)") mimemsg.attach(tocpart) # RFC 1153 print >> plainmsg, toctext print >> plainmsg # For RFC 1153 digests, we now need the standard separator print >> plainmsg, separator70 print >> plainmsg # Now go through and add each message mimedigest = MIMEBase('multipart', 'digest') mimemsg.attach(mimedigest) first = True for msg in messages: # MIME. Make a copy of the message object since the rfc1153 # processing scrubs out attachments. mimedigest.attach(MIMEMessage(copy.deepcopy(msg))) # rfc1153 if first: first = False else: print >> plainmsg, separator30 print >> plainmsg # Use Mailman.Handlers.Scrubber.process() to get plain text try: msg = scrubber(mlist, msg) except Errors.DiscardMessage: print >> plainmsg, _('[Message discarded by content filter]') continue # Honor the default setting for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS: if msg[h]: uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset))) uh = '\n\t'.join(uh.split('\n')) print >> plainmsg, uh print >> plainmsg # If decoded payload is empty, this may be multipart message. # -- just stringfy it. payload = msg.get_payload(decode=True) \ or msg.as_string().split('\n\n',1)[1] mcset = msg.get_content_charset('') if mcset and mcset <> lcset and mcset <> lcset_out: try: payload = unicode(payload, mcset, 'replace' ).encode(lcset, 'replace') except (UnicodeError, LookupError): # TK: Message has something unknown charset. # _out means charset in 'outer world'. payload = unicode(payload, lcset_out, 'replace' ).encode(lcset, 'replace') print >> plainmsg, payload if not payload.endswith('\n'): print >> plainmsg # Now add the footer but only if more than whitespace. if re.sub('\s', '', mlist.digest_footer): footertxt = decorate(mlist, mlist.digest_footer, _('digest footer')) # MIME footer = MIMEText(footertxt, _charset=lcset) footer['Content-Description'] = _('Digest Footer') mimemsg.attach(footer) # RFC 1153 # MAS: There is no real place for the digest_footer in an RFC 1153 # compliant digest, so add it as an additional message with # Subject: Digest Footer print >> plainmsg, separator30 print >> plainmsg print >> plainmsg, 'Subject: ' + _('Digest Footer') print >> plainmsg print >> plainmsg, footertxt print >> plainmsg print >> plainmsg, separator30 print >> plainmsg # Do the last bit of stuff for each digest type signoff = _('End of ') + digestid # MIME # BAW: This stuff is outside the normal MIME goo, and it's what the old # MIME digester did. No one seemed to complain, probably because you # won't see it in an MUA that can't display the raw message. We've never # got complaints before, but if we do, just wax this. It's primarily # included for (marginally useful) backwards compatibility. mimemsg.postamble = signoff # rfc1153 print >> plainmsg, signoff print >> plainmsg, '*' * len(signoff) # Do our final bit of housekeeping, and then send each message to the # outgoing queue for delivery. mlist.next_digest_number += 1 virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR) # Calculate the recipients lists plainrecips = [] mimerecips = [] drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys() for user in mlist.getMemberCPAddresses(drecips): # user might be None if someone who toggled off digest delivery # subsequently unsubscribed from the mailing list. Also, filter out # folks who have disabled delivery. if user is None or mlist.getDeliveryStatus(user) <> ENABLED: continue # Otherwise, decide whether they get MIME or RFC 1153 digests if mlist.getMemberOption(user, mm_cfg.DisableMime): plainrecips.append(user) else: mimerecips.append(user) # Zap this since we're now delivering the last digest to these folks. mlist.one_last_digest.clear() # MIME virginq.enqueue(mimemsg, recips=mimerecips, listname=mlist.internal_name(), isdigest=True) # RFC 1153 rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset) virginq.enqueue(rfc1153msg, recips=plainrecips, listname=mlist.internal_name(), isdigest=True)
def prepare_message(mlist, msg, msgdata): # If the newsgroup is moderated, we need to add this header for the Usenet # software to accept the posting, and not forward it on to the n.g.'s # moderation address. The posting would not have gotten here if it hadn't # already been approved. 1 == open list, mod n.g., 2 == moderated if mlist.news_moderation in (1, 2): del msg['approved'] msg['Approved'] = mlist.GetListEmail() # Should we restore the original, non-prefixed subject for gatewayed # messages? TK: We use stripped_subject (prefix stripped) which was # crafted in CookHeaders.py to ensure prefix was stripped from the subject # came from mailing list user. stripped_subject = msgdata.get('stripped_subject') \ or msgdata.get('origsubj') if not mlist.news_prefix_subject_too and stripped_subject is not None: del msg['subject'] msg['Subject'] = stripped_subject # Make sure we have a non-blank subject. if not msg.get('subject', ''): del msg['subject'] msg['Subject'] = '(no subject)' # Add the appropriate Newsgroups: header if msg['newsgroups'] is not None: # This message is gated from our list to it's associated usnet group. # If it has a Newsgroups: header mentioning other groups, it's not # up to us to post it to those groups. del msg['newsgroups'] msg['Newsgroups'] = mlist.linked_newsgroup # Note: We need to be sure two messages aren't ever sent to the same list # in the same process, since message ids need to be unique. Further, if # messages are crossposted to two Usenet-gated mailing lists, they each # need to have unique message ids or the nntpd will only accept one of # them. The solution here is to substitute any existing message-id that # isn't ours with one of ours, so we need to parse it to be sure we're not # looping. # # We also add the original Message-ID: to References: to try to help with # threading issues and create another header for documentation. # # Our Message-ID format is <mailman.secs.pid.listname@hostname> msgid = msg['message-id'] hackmsgid = True if msgid: mo = mcre.search(msgid) if mo: lname, hname = mo.group('listname', 'hostname') if lname == mlist.internal_name() and hname == mlist.host_name: hackmsgid = False if hackmsgid: del msg['message-id'] msg['Message-ID'] = Utils.unique_message_id(mlist) if msgid: msg['X-Mailman-Original-Message-ID'] = msgid refs = msg['references'] del msg['references'] if not refs: refs = msg.get('in-reply-to', '') else: msg['X-Mailman-Original-References'] = refs if refs: msg['References'] = '\n '.join([refs, msgid]) else: msg['References'] = msgid # Lines: is useful if msg['Lines'] is None: # BAW: is there a better way? count = len(list(email.iterators.body_line_iterator(msg))) msg['Lines'] = str(count) # Massage the message headers by remove some and rewriting others. This # woon't completely sanitize the message, but it will eliminate the bulk # of the rejections based on message headers. The NNTP server may still # reject the message because of other problems. for header in mm_cfg.NNTP_REMOVE_HEADERS: del msg[header] for header, rewrite in mm_cfg.NNTP_REWRITE_DUPLICATE_HEADERS: values = msg.get_all(header, []) if len(values) < 2: # We only care about duplicates continue del msg[header] # But keep the first one... msg[header] = values[0] for v in values[1:]: msg[rewrite] = v # Mark this message as prepared in case it has to be requeued msgdata['prepped'] = True
def prepare_message(mlist, msg, msgdata): # If the newsgroup is moderated, we need to add this header for the Usenet # software to accept the posting, and not forward it on to the n.g.'s # moderation address. The posting would not have gotten here if it hadn't # already been approved. 1 == open list, mod n.g., 2 == moderated if mlist.news_moderation in (1, 2): del msg['approved'] msg['Approved'] = mlist.GetListEmail() # Should we restore the original, non-prefixed subject for gatewayed # messages? TK: We use stripped_subject (prefix stripped) which was # crafted in CookHeaders.py to ensure prefix was stripped from the subject # came from mailing list user. stripped_subject = msgdata.get('stripped_subject') \ or msgdata.get('origsubj') if not mlist.news_prefix_subject_too and stripped_subject is not None: del msg['subject'] msg['subject'] = stripped_subject # Add the appropriate Newsgroups: header ngheader = msg['newsgroups'] if ngheader is not None: # See if the Newsgroups: header already contains our linked_newsgroup. # If so, don't add it again. If not, append our linked_newsgroup to # the end of the header list ngroups = [s.strip() for s in ngheader.split(',')] if mlist.linked_newsgroup not in ngroups: ngroups.append(mlist.linked_newsgroup) # Subtitute our new header for the old one. del msg['newsgroups'] msg['Newsgroups'] = COMMASPACE.join(ngroups) else: # Newsgroups: isn't in the message msg['Newsgroups'] = mlist.linked_newsgroup # Note: We need to be sure two messages aren't ever sent to the same list # in the same process, since message ids need to be unique. Further, if # messages are crossposted to two Usenet-gated mailing lists, they each # need to have unique message ids or the nntpd will only accept one of # them. The solution here is to substitute any existing message-id that # isn't ours with one of ours, so we need to parse it to be sure we're not # looping. # # Our Message-ID format is <mailman.secs.pid.listname@hostname> msgid = msg['message-id'] hackmsgid = True if msgid: mo = mcre.search(msgid) if mo: lname, hname = mo.group('listname', 'hostname') if lname == mlist.internal_name() and hname == mlist.host_name: hackmsgid = False if hackmsgid: del msg['message-id'] msg['Message-ID'] = Utils.unique_message_id(mlist) # Lines: is useful if msg['Lines'] is None: # BAW: is there a better way? count = len(list(email.Iterators.body_line_iterator(msg))) msg['Lines'] = str(count) # Massage the message headers by remove some and rewriting others. This # woon't completely sanitize the message, but it will eliminate the bulk # of the rejections based on message headers. The NNTP server may still # reject the message because of other problems. for header in mm_cfg.NNTP_REMOVE_HEADERS: del msg[header] for header, rewrite in mm_cfg.NNTP_REWRITE_DUPLICATE_HEADERS: values = msg.get_all(header, []) if len(values) < 2: # We only care about duplicates continue del msg[header] # But keep the first one... msg[header] = values[0] for v in values[1:]: msg[rewrite] = v # Mark this message as prepared in case it has to be requeued msgdata['prepped'] = True
def hold_for_approval(mlist, msg, msgdata, exc): # BAW: This should really be tied into the email confirmation system so # that the message can be approved or denied via email as well as the # web. # # XXX We use the weird type(type) construct below because in Python 2.1, # type is a function not a type and so can't be used as the second # argument in isinstance(). However, in Python 2.5, exceptions are # new-style classes and so are not of ClassType. # FIXME pzv if isinstance(exc, type) or isinstance(exc, type(type)): # Go ahead and instantiate it now. exc = exc() listname = mlist.real_name sender = msgdata.get('sender', msg.get_sender()) usersubject = msg.get('subject') charset = Utils.GetCharSet(mlist.preferred_language) if usersubject: usersubject = Utils.oneline(usersubject, charset) else: usersubject = _('(no subject)') message_id = msg.get('message-id', 'n/a') owneraddr = mlist.GetOwnerEmail() adminaddr = mlist.GetBouncesEmail() requestaddr = mlist.GetRequestEmail() # We need to send both the reason and the rejection notice through the # translator again, because of the games we play above reason = Utils.wrap(exc.reason_notice()) if isinstance(exc, NonMemberPost) and mlist.nonmember_rejection_notice: msgdata['rejection_notice'] = Utils.wrap( mlist.nonmember_rejection_notice.replace('%(listowner)s', owneraddr)) else: msgdata['rejection_notice'] = Utils.wrap(exc.rejection_notice(mlist)) id = mlist.HoldMessage(msg, reason, msgdata) # Now we need to craft and send a message to the list admin so they can # deal with the held message. d = { 'listname': listname, 'hostname': mlist.host_name, 'reason': _(reason), 'sender': sender, 'subject': usersubject, 'admindb_url': mlist.GetScriptURL('admindb', absolute=1), } # We may want to send a notification to the original sender too fromusenet = msgdata.get('fromusenet') # Since we're sending two messages, which may potentially be in different # languages (the user's preferred and the list's preferred for the admin), # we need to play some i18n games here. Since the current language # context ought to be set up for the user, let's craft his message first. cookie = mlist.pend_new(Pending.HELD_MESSAGE, id) if not fromusenet and ackp(msg) and mlist.respond_to_post_requests and \ mlist.autorespondToSender(sender, mlist.getMemberLanguage(sender)): # Get a confirmation cookie d['confirmurl'] = '%s/%s' % (mlist.GetScriptURL('confirm', absolute=1), cookie) lang = msgdata.get('lang', mlist.getMemberLanguage(sender)) subject = _('Your message to %(listname)s awaits moderator approval') text = Utils.maketext('postheld.txt', d, lang=lang, mlist=mlist) nmsg = Message.UserNotification(sender, owneraddr, subject, text, lang) nmsg.send(mlist) # Now the message for the list owners. Be sure to include the list # moderators in this message. This one should appear to come from # <list>-owner since we really don't need to do bounce processing on it. if mlist.admin_immed_notify: # Now let's temporarily set the language context to that which the # admin is expecting. otranslation = i18n.get_translation() i18n.set_language(mlist.preferred_language) try: lang = mlist.preferred_language charset = Utils.GetCharSet(lang) # We need to regenerate or re-translate a few values in d d['reason'] = _(reason) d['subject'] = usersubject # craft the admin notification message and deliver it subject = _('%(listname)s post from %(sender)s requires approval') nmsg = Message.UserNotification(owneraddr, owneraddr, subject, lang=lang) nmsg.set_type('multipart/mixed') text = MIMEText(Utils.maketext('postauth.txt', d, raw=1, mlist=mlist), _charset=charset) dmsg = MIMEText(Utils.wrap( _("""\ If you reply to this message, keeping the Subject: header intact, Mailman will discard the held message. Do this if the message is spam. If you reply to this message and include an Approved: header with the list password in it, the message will be approved for posting to the list. The Approved: header can also appear in the first line of the body of the reply.""")), _charset=Utils.GetCharSet(lang)) dmsg['Subject'] = 'confirm ' + cookie dmsg['Sender'] = requestaddr dmsg['From'] = requestaddr dmsg['Date'] = email.utils.formatdate(localtime=True) dmsg['Message-ID'] = Utils.unique_message_id(mlist) nmsg.attach(text) nmsg.attach(message.MIMEMessage(msg)) nmsg.attach(message.MIMEMessage(dmsg)) nmsg.send(mlist, **{'tomoderators': 1}) finally: i18n.set_translation(otranslation) # Log the held message syslog('vette', '%s post from %s held, message-id=%s: %s', listname, sender, message_id, reason) # raise the specific MessageHeld exception to exit out of the message # delivery pipeline raise exc
def send_i18n_digests(mlist, mboxfp): mbox = Mailbox(mboxfp) # Prepare common information (first lang/charset) lang = mlist.preferred_language lcset = Utils.GetCharSet(lang) lcset_out = Charset(lcset).output_charset or lcset # Common Information (contd) realname = mlist.real_name volume = mlist.volume issue = mlist.next_digest_number digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d') digestsubj = Header(digestid, lcset, header_name='Subject') # Set things up for the MIME digest. Only headers not added by # CookHeaders need be added here. # Date/Message-ID should be added here also. mimemsg = Message.Message() mimemsg['Content-Type'] = 'multipart/mixed' mimemsg['MIME-Version'] = '1.0' mimemsg['From'] = mlist.GetRequestEmail() mimemsg['Subject'] = digestsubj mimemsg['To'] = mlist.GetListEmail() mimemsg['Reply-To'] = mlist.GetListEmail() mimemsg['Date'] = formatdate(localtime=1) mimemsg['Message-ID'] = Utils.unique_message_id(mlist) # Set things up for the rfc1153 digest plainmsg = StringIO() rfc1153msg = Message.Message() rfc1153msg['From'] = mlist.GetRequestEmail() rfc1153msg['Subject'] = digestsubj rfc1153msg['To'] = mlist.GetListEmail() rfc1153msg['Reply-To'] = mlist.GetListEmail() rfc1153msg['Date'] = formatdate(localtime=1) rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist) separator70 = '-' * 70 separator30 = '-' * 30 # In the rfc1153 digest, the masthead contains the digest boilerplate plus # any digest header. In the MIME digests, the masthead and digest header # are separate MIME subobjects. In either case, it's the first thing in # the digest, and we can calculate it now, so go ahead and add it now. mastheadtxt = Utils.maketext( 'masthead.txt', { 'real_name': mlist.real_name, 'got_list_email': mlist.GetListEmail(), 'got_listinfo_url': mlist.GetScriptURL('listinfo', absolute=1), 'got_request_email': mlist.GetRequestEmail(), 'got_owner_email': mlist.GetOwnerEmail(), }, mlist=mlist) # MIME masthead = MIMEText(mastheadtxt, _charset=lcset) masthead['Content-Description'] = digestid mimemsg.attach(masthead) # RFC 1153 print >> plainmsg, mastheadtxt print >> plainmsg # Now add the optional digest header but only if more than whitespace. if re.sub('\s', '', mlist.digest_header): headertxt = decorate(mlist, mlist.digest_header, _('digest header')) # MIME header = MIMEText(headertxt, _charset=lcset) header['Content-Description'] = _('Digest Header') mimemsg.attach(header) # RFC 1153 print >> plainmsg, headertxt print >> plainmsg # Now we have to cruise through all the messages accumulated in the # mailbox file. We can't add these messages to the plainmsg and mimemsg # yet, because we first have to calculate the table of contents # (i.e. grok out all the Subjects). Store the messages in a list until # we're ready for them. # # Meanwhile prepare things for the table of contents toc = StringIO() print >> toc, _("Today's Topics:\n") # Now cruise through all the messages in the mailbox of digest messages, # building the MIME payload and core of the RFC 1153 digest. We'll also # accumulate Subject: headers and authors for the table-of-contents. messages = [] msgcount = 0 msg = mbox.next() while msg is not None: if msg == '': # It was an unparseable message msg = mbox.next() continue msgcount += 1 messages.append(msg) # Get the Subject header msgsubj = msg.get('subject', _('(no subject)')) subject = Utils.oneline(msgsubj, lcset) # Don't include the redundant subject prefix in the toc mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix), subject, re.IGNORECASE) if mo: subject = subject[:mo.start(2)] + subject[mo.end(2):] username = '' addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)]) # Take only the first author we find if isinstance(addresses, ListType) and addresses: username = addresses[0][0] if not username: username = addresses[0][1] if username: username = '******' % username # Put count and Wrap the toc subject line wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65) slines = wrapped.split('\n') # See if the user's name can fit on the last line if len(slines[-1]) + len(username) > 70: slines.append(username) else: slines[-1] += username # Add this subject to the accumulating topics first = True for line in slines: if first: print >> toc, ' ', line first = False else: print >> toc, ' ', line.lstrip() # We do not want all the headers of the original message to leak # through in the digest messages. For this phase, we'll leave the # same set of headers in both digests, i.e. those required in RFC 1153 # plus a couple of other useful ones. We also need to reorder the # headers according to RFC 1153. Later, we'll strip out headers for # for the specific MIME or plain digests. keeper = {} all_keepers = {} for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS + mm_cfg.PLAIN_DIGEST_KEEP_HEADERS): all_keepers[header] = True all_keepers = all_keepers.keys() for keep in all_keepers: keeper[keep] = msg.get_all(keep, []) # Now remove all unkempt headers :) for header in msg.keys(): del msg[header] # And add back the kept header in the RFC 1153 designated order for keep in all_keepers: for field in keeper[keep]: msg[keep] = field # And a bit of extra stuff msg['Message'] = ` msgcount ` # Get the next message in the digest mailbox msg = mbox.next() # Now we're finished with all the messages in the digest. First do some # sanity checking and then on to adding the toc. if msgcount == 0: # Why did we even get here? return toctext = to_cset_out(toc.getvalue(), lcset) # MIME tocpart = MIMEText(toctext, _charset=lcset) tocpart['Content-Description'] = _( "Today's Topics (%(msgcount)d messages)") mimemsg.attach(tocpart) # RFC 1153 print >> plainmsg, toctext print >> plainmsg # For RFC 1153 digests, we now need the standard separator print >> plainmsg, separator70 print >> plainmsg # Now go through and add each message mimedigest = MIMEBase('multipart', 'digest') mimemsg.attach(mimedigest) first = True for msg in messages: # MIME. Make a copy of the message object since the rfc1153 # processing scrubs out attachments. mimedigest.attach(MIMEMessage(copy.deepcopy(msg))) # rfc1153 if first: first = False else: print >> plainmsg, separator30 print >> plainmsg # Use Mailman.Handlers.Scrubber.process() to get plain text try: msg = scrubber(mlist, msg) except Errors.DiscardMessage: print >> plainmsg, _('[Message discarded by content filter]') continue # Honor the default setting for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS: if msg[h]: uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset))) uh = '\n\t'.join(uh.split('\n')) print >> plainmsg, uh print >> plainmsg # If decoded payload is empty, this may be multipart message. # -- just stringfy it. payload = msg.get_payload(decode=True) \ or msg.as_string().split('\n\n',1)[1] mcset = msg.get_content_charset('') if mcset and mcset <> lcset and mcset <> lcset_out: try: payload = unicode(payload, mcset, 'replace').encode(lcset, 'replace') except (UnicodeError, LookupError): # TK: Message has something unknown charset. # _out means charset in 'outer world'. payload = unicode(payload, lcset_out, 'replace').encode(lcset, 'replace') print >> plainmsg, payload if not payload.endswith('\n'): print >> plainmsg # Now add the footer but only if more than whitespace. if re.sub('\s', '', mlist.digest_footer): footertxt = decorate(mlist, mlist.digest_footer, _('digest footer')) # MIME footer = MIMEText(footertxt, _charset=lcset) footer['Content-Description'] = _('Digest Footer') mimemsg.attach(footer) # RFC 1153 # MAS: There is no real place for the digest_footer in an RFC 1153 # compliant digest, so add it as an additional message with # Subject: Digest Footer print >> plainmsg, separator30 print >> plainmsg print >> plainmsg, 'Subject: ' + _('Digest Footer') print >> plainmsg print >> plainmsg, footertxt print >> plainmsg print >> plainmsg, separator30 print >> plainmsg # Do the last bit of stuff for each digest type signoff = _('End of ') + digestid # MIME # BAW: This stuff is outside the normal MIME goo, and it's what the old # MIME digester did. No one seemed to complain, probably because you # won't see it in an MUA that can't display the raw message. We've never # got complaints before, but if we do, just wax this. It's primarily # included for (marginally useful) backwards compatibility. mimemsg.postamble = signoff # rfc1153 print >> plainmsg, signoff print >> plainmsg, '*' * len(signoff) # Do our final bit of housekeeping, and then send each message to the # outgoing queue for delivery. mlist.next_digest_number += 1 virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR) # Calculate the recipients lists plainrecips = [] mimerecips = [] drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys() for user in mlist.getMemberCPAddresses(drecips): # user might be None if someone who toggled off digest delivery # subsequently unsubscribed from the mailing list. Also, filter out # folks who have disabled delivery. if user is None or mlist.getDeliveryStatus(user) <> ENABLED: continue # Otherwise, decide whether they get MIME or RFC 1153 digests if mlist.getMemberOption(user, mm_cfg.DisableMime): plainrecips.append(user) else: mimerecips.append(user) # Zap this since we're now delivering the last digest to these folks. mlist.one_last_digest.clear() # MIME virginq.enqueue(mimemsg, recips=mimerecips, listname=mlist.internal_name(), isdigest=True) # RFC 1153 rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset) virginq.enqueue(rfc1153msg, recips=plainrecips, listname=mlist.internal_name(), isdigest=True)