def calculate_attachments_dir(mlist, msg, msgdata):
# Calculate the directory that attachments for this message will go
# under. To avoid inode limitations, the scheme will be:
# archives/private/<listname>/attachments/YYYYMMDD/<msgid-hash>/<files>
# Start by calculating the date-based and msgid-hash components.
fmt = "%Y%m%d"
datestr = msg.get("Date")
if datestr:
now = parsedate(datestr)
else:
now = time.gmtime(msgdata.get("received_time", time.time()))
datedir = safe_strftime(fmt, now)
if not datedir:
datestr = msgdata.get("X-List-Received-Date")
if datestr:
datedir = safe_strftime(fmt, datestr)
if not datedir:
# What next? Unixfrom, I guess.
parts = msg.get_unixfrom().split()
try:
month = {
"Jan": 1,
"Feb": 2,
"Mar": 3,
"Apr": 4,
"May": 5,
"Jun": 6,
"Jul": 7,
"Aug": 8,
"Sep": 9,
"Oct": 10,
"Nov": 11,
"Dec": 12,
}.get(parts[3], 0)
day = int(parts[4])
year = int(parts[6])
except (IndexError, ValueError):
# Best we can do I think
month = day = year = 0
datedir = "%04d%02d%02d" % (year, month, day)
assert datedir
if mm_cfg.SCRUBBER_ADD_PAYLOAD_HASH_FILENAME:
return os.path.join("attachments", datedir)
else:
# As for the msgid hash, we'll base this part on the Message-ID: so that
# all attachments for the same message end up in the same directory (we'll
# uniquify the filenames in that directory as needed). We use the first 2
# and last 2 bytes of the SHA1 hash of the message id as the basis of the
# directory name. Clashes here don't really matter too much, and that
# still gives us a 32-bit space to work with.
msgid = msg["message-id"]
if msgid is None:
msgid = msg["Message-ID"] = Utils.unique_message_id(mlist)
# We assume that the message id actually /is/ unique!
digest = sha_new(msgid).hexdigest()
return os.path.join("attachments", datedir, digest[:4] + digest[-4:])
def calculate_attachments_dir(mlist, msg, msgdata):
# Calculate the directory that attachments for this message will go
# under. To avoid inode limitations, the scheme will be:
# archives/private/<listname>/attachments/YYYYMMDD/<msgid-hash>/<files>
# Start by calculating the date-based and msgid-hash components.
fmt = '%Y%m%d'
datestr = msg.get('Date')
if datestr:
now = parsedate(datestr)
else:
now = time.gmtime(msgdata.get('received_time', time.time()))
datedir = safe_strftime(fmt, now)
if not datedir:
datestr = msgdata.get('X-List-Received-Date')
if datestr:
datedir = safe_strftime(fmt, datestr)
if not datedir:
# What next? Unixfrom, I guess.
parts = msg.get_unixfrom().split()
try:
month = {
'Jan': 1,
'Feb': 2,
'Mar': 3,
'Apr': 4,
'May': 5,
'Jun': 6,
'Jul': 7,
'Aug': 8,
'Sep': 9,
'Oct': 10,
'Nov': 11,
'Dec': 12,
}.get(parts[3], 0)
day = int(parts[4])
year = int(parts[6])
except (IndexError, ValueError):
# Best we can do I think
month = day = year = 0
datedir = '%04d%02d%02d' % (year, month, day)
assert datedir
# As for the msgid hash, we'll base this part on the Message-ID: so that
# all attachments for the same message end up in the same directory (we'll
# uniquify the filenames in that directory as needed). We use the first 2
# and last 2 bytes of the SHA1 hash of the message id as the basis of the
# directory name. Clashes here don't really matter too much, and that
# still gives us a 32-bit space to work with.
msgid = msg['message-id']
if msgid is None:
msgid = msg['Message-ID'] = Utils.unique_message_id(mlist)
# We assume that the message id actually /is/ unique!
digest = sha_new(msgid).hexdigest()
# hash disabled to handle file duplicate over mutiple email.
#return os.path.join('attachments', datedir, digest[:4] + digest[-4:])
return os.path.join('attachments', datedir)
def process(mlist, msg, msgdata):
# This is the negation of we're wrapping because dmarc_moderation_action
# is wrap this message or from_is_list applies and is wrap.
if not (msgdata.get('from_is_list') == 2 or
(mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)):
# Now see if we need to add a From:, Reply-To: or Cc: without wrapping.
# See comments in CookHeaders.change_header for why we do this here.
a_h = msgdata.get('add_header')
if a_h:
if a_h.get('From'):
del msg['from']
msg['From'] = a_h.get('From')
if a_h.get('Reply-To'):
del msg['reply-to']
msg['Reply-To'] = a_h.get('Reply-To')
if a_h.get('Cc'):
del msg['cc']
msg['Cc'] = a_h.get('Cc')
return
# There are various headers in msg that we don't want, so we basically
# make a copy of the msg, then delete almost everything and set/copy
# what we want.
omsg = copy.deepcopy(msg)
for key in list(msg.keys()):
if key.lower() not in KEEPERS:
del msg[key]
msg['MIME-Version'] = '1.0'
msg['Message-ID'] = Utils.unique_message_id(mlist)
# Add the headers from CookHeaders.
for k, v in list(msgdata['add_header'].items()):
msg[k] = v
# Are we including dmarc_wrapped_message_text? I.e., do we have text and
# are we wrapping because of dmarc_moderation_action?
if mlist.dmarc_wrapped_message_text and msgdata.get('from_is_list') == 2:
part1 = text.MIMEText(Utils.wrap(mlist.dmarc_wrapped_message_text),
'plain',
Utils.GetCharSet(mlist.preferred_language))
part1['Content-Disposition'] = 'inline'
part2 = message.MIMEMessage(omsg)
part2['Content-Disposition'] = 'inline'
msg['Content-Type'] = 'multipart/mixed'
msg.set_payload([part1, part2])
else:
msg['Content-Type'] = 'message/rfc822'
msg['Content-Disposition'] = 'inline'
msg.set_payload([omsg])
def process(mlist, msg, msgdata):
# This is the negation of we're wrapping because dmarc_moderation_action
# is wrap this message or from_is_list applies and is wrap.
if not (msgdata.get('from_is_list') == 2 or
(mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)):
# Now see if we need to add a From:, Reply-To: or Cc: without wrapping.
# See comments in CookHeaders.change_header for why we do this here.
a_h = msgdata.get('add_header')
if a_h:
if a_h.get('From'):
del msg['from']
msg['From'] = a_h.get('From')
if a_h.get('Reply-To'):
del msg['reply-to']
msg['Reply-To'] = a_h.get('Reply-To')
if a_h.get('Cc'):
del msg['cc']
msg['Cc'] = a_h.get('Cc')
return
# There are various headers in msg that we don't want, so we basically
# make a copy of the msg, then delete almost everything and set/copy
# what we want.
omsg = copy.deepcopy(msg)
for key in msg.keys():
if key.lower() not in KEEPERS:
del msg[key]
msg['MIME-Version'] = '1.0'
msg['Message-ID'] = Utils.unique_message_id(mlist)
# Add the headers from CookHeaders.
for k, v in msgdata['add_header'].items():
msg[k] = v
# Are we including dmarc_wrapped_message_text? I.e., do we have text and
# are we wrapping because of dmarc_moderation_action?
if mlist.dmarc_wrapped_message_text and msgdata.get('from_is_list') == 2:
part1 = MIMEText(Utils.wrap(mlist.dmarc_wrapped_message_text),
'plain',
Utils.GetCharSet(mlist.preferred_language))
part1['Content-Disposition'] = 'inline'
part2 = MIMEMessage(omsg)
part2['Content-Disposition'] = 'inline'
msg['Content-Type'] = 'multipart/mixed'
msg.set_payload([part1, part2])
else:
msg['Content-Type'] = 'message/rfc822'
msg['Content-Disposition'] = 'inline'
msg.set_payload([omsg])
def calculate_attachments_dir(mlist, msg, msgdata):
# Calculate the directory that attachments for this message will go
# under. To avoid inode limitations, the scheme will be:
# archives/private/<listname>/attachments/YYYYMMDD/<msgid-hash>/<files>
# Start by calculating the date-based and msgid-hash components.
fmt = '%Y%m%d'
datestr = msg.get('Date')
if datestr:
now = parsedate(datestr)
else:
now = time.gmtime(msgdata.get('received_time', time.time()))
datedir = safe_strftime(fmt, now)
if not datedir:
datestr = msgdata.get('X-List-Received-Date')
if datestr:
datedir = safe_strftime(fmt, datestr)
if not datedir:
# What next? Unixfrom, I guess.
parts = msg.get_unixfrom().split()
try:
month = {'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6,
'Jul':7, 'Aug':8, 'Sep':9, 'Oct':10, 'Nov':11, 'Dec':12,
}.get(parts[3], 0)
day = int(parts[4])
year = int(parts[6])
except (IndexError, ValueError):
# Best we can do I think
month = day = year = 0
datedir = '%04d%02d%02d' % (year, month, day)
assert datedir
# As for the msgid hash, we'll base this part on the Message-ID: so that
# all attachments for the same message end up in the same directory (we'll
# uniquify the filenames in that directory as needed). We use the first 2
# and last 2 bytes of the SHA1 hash of the message id as the basis of the
# directory name. Clashes here don't really matter too much, and that
# still gives us a 32-bit space to work with.
msgid = msg['message-id']
if msgid is None:
msgid = msg['Message-ID'] = Utils.unique_message_id(mlist)
# We assume that the message id actually /is/ unique!
digest = sha_new(msgid).hexdigest()
return os.path.join('attachments', datedir, digest[:4] + digest[-4:])
def send(self, mlist, noprecedence=False, **_kws):
"""Sends the message by enqueuing it to the `virgin' queue.
This is used for all internally crafted messages.
"""
# Since we're crafting the message from whole cloth, let's make sure
# this message has a Message-ID. Yes, the MTA would give us one, but
# this is useful for logging to logs/smtp.
if 'message-id' not in self:
self['Message-ID'] = Utils.unique_message_id(mlist)
# Ditto for Date: which is required by RFC 2822
if 'date' not in self:
self['Date'] = email.utils.formatdate(localtime=1)
# UserNotifications are typically for admin messages, and for messages
# other than list explosions. Send these out as Precedence: bulk, but
# don't override an existing Precedence: header.
# Also, if the message is To: the list-owner address, set Precedence:
# list. See note below in OwnerNotification.
if not ('precedence' in self or noprecedence):
if self.get('to') == mlist.GetOwnerEmail():
self['Precedence'] = 'list'
else:
self['Precedence'] = 'bulk'
self._enqueue(mlist, **_kws)
def send_i18n_digests(mlist, mboxfp):
mbox = Mailbox(mboxfp)
# Prepare common information (first lang/charset)
lang = mlist.preferred_language
lcset = Utils.GetCharSet(lang)
lcset_out = Charset(lcset).output_charset or lcset
# Common Information (contd)
realname = mlist.real_name
volume = mlist.volume
issue = mlist.next_digest_number
digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d')
digestsubj = Header(digestid, lcset, header_name='Subject')
# Set things up for the MIME digest. Only headers not added by
# CookHeaders need be added here.
# Date/Message-ID should be added here also.
mimemsg = Message.Message()
mimemsg['Content-Type'] = 'multipart/mixed'
mimemsg['MIME-Version'] = '1.0'
mimemsg['From'] = mlist.GetRequestEmail()
mimemsg['Subject'] = digestsubj
mimemsg['To'] = mlist.GetListEmail()
mimemsg['Reply-To'] = mlist.GetListEmail()
mimemsg['Date'] = formatdate(localtime=1)
mimemsg['Message-ID'] = Utils.unique_message_id(mlist)
# Set things up for the rfc1153 digest
plainmsg = StringIO()
rfc1153msg = Message.Message()
rfc1153msg['From'] = mlist.GetRequestEmail()
rfc1153msg['Subject'] = digestsubj
rfc1153msg['To'] = mlist.GetListEmail()
rfc1153msg['Reply-To'] = mlist.GetListEmail()
rfc1153msg['Date'] = formatdate(localtime=1)
rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist)
separator70 = '-' * 70
separator30 = '-' * 30
# In the rfc1153 digest, the masthead contains the digest boilerplate plus
# any digest header. In the MIME digests, the masthead and digest header
# are separate MIME subobjects. In either case, it's the first thing in
# the digest, and we can calculate it now, so go ahead and add it now.
mastheadtxt = Utils.maketext(
'masthead.txt',
{'real_name' : mlist.real_name,
'got_list_email': mlist.GetListEmail(),
'got_listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
'got_request_email': mlist.GetRequestEmail(),
'got_owner_email': mlist.GetOwnerEmail(),
}, mlist=mlist)
# MIME
masthead = MIMEText(mastheadtxt, _charset=lcset)
masthead['Content-Description'] = digestid
mimemsg.attach(masthead)
# RFC 1153
print >> plainmsg, mastheadtxt
print >> plainmsg
# Now add the optional digest header but only if more than whitespace.
if re.sub('\s', '', mlist.digest_header):
headertxt = decorate(mlist, mlist.digest_header, _('digest header'))
# MIME
header = MIMEText(headertxt, _charset=lcset)
header['Content-Description'] = _('Digest Header')
mimemsg.attach(header)
# RFC 1153
print >> plainmsg, headertxt
print >> plainmsg
# Now we have to cruise through all the messages accumulated in the
# mailbox file. We can't add these messages to the plainmsg and mimemsg
# yet, because we first have to calculate the table of contents
# (i.e. grok out all the Subjects). Store the messages in a list until
# we're ready for them.
#
# Meanwhile prepare things for the table of contents
toc = StringIO()
print >> toc, _("Today's Topics:\n")
# Now cruise through all the messages in the mailbox of digest messages,
# building the MIME payload and core of the RFC 1153 digest. We'll also
# accumulate Subject: headers and authors for the table-of-contents.
messages = []
msgcount = 0
msg = mbox.next()
while msg is not None:
if msg == '':
# It was an unparseable message
msg = mbox.next()
continue
msgcount += 1
messages.append(msg)
# Get the Subject header
msgsubj = msg.get('subject', _('(no subject)'))
subject = Utils.oneline(msgsubj, lcset)
# Don't include the redundant subject prefix in the toc
mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix),
subject, re.IGNORECASE)
if mo:
subject = subject[:mo.start(2)] + subject[mo.end(2):]
username = ''
addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)])
# Take only the first author we find
if isinstance(addresses, ListType) and addresses:
username = addresses[0][0]
if not username:
username = addresses[0][1]
if username:
username = '******' % username
# Put count and Wrap the toc subject line
wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65)
slines = wrapped.split('\n')
# See if the user's name can fit on the last line
if len(slines[-1]) + len(username) > 70:
slines.append(username)
else:
slines[-1] += username
# Add this subject to the accumulating topics
first = True
for line in slines:
if first:
print >> toc, ' ', line
first = False
else:
print >> toc, ' ', line.lstrip()
# We do not want all the headers of the original message to leak
# through in the digest messages. For this phase, we'll leave the
# same set of headers in both digests, i.e. those required in RFC 1153
# plus a couple of other useful ones. We also need to reorder the
# headers according to RFC 1153. Later, we'll strip out headers for
# for the specific MIME or plain digests.
keeper = {}
all_keepers = {}
for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS +
mm_cfg.PLAIN_DIGEST_KEEP_HEADERS):
all_keepers[header] = True
all_keepers = all_keepers.keys()
for keep in all_keepers:
keeper[keep] = msg.get_all(keep, [])
# Now remove all unkempt headers :)
for header in msg.keys():
del msg[header]
# And add back the kept header in the RFC 1153 designated order
for keep in all_keepers:
for field in keeper[keep]:
msg[keep] = field
# And a bit of extra stuff
msg['Message'] = `msgcount`
# Get the next message in the digest mailbox
msg = mbox.next()
# Now we're finished with all the messages in the digest. First do some
# sanity checking and then on to adding the toc.
if msgcount == 0:
# Why did we even get here?
return
toctext = to_cset_out(toc.getvalue(), lcset)
# MIME
tocpart = MIMEText(toctext, _charset=lcset)
tocpart['Content-Description']= _("Today's Topics (%(msgcount)d messages)")
mimemsg.attach(tocpart)
# RFC 1153
print >> plainmsg, toctext
print >> plainmsg
# For RFC 1153 digests, we now need the standard separator
print >> plainmsg, separator70
print >> plainmsg
# Now go through and add each message
mimedigest = MIMEBase('multipart', 'digest')
mimemsg.attach(mimedigest)
first = True
for msg in messages:
# MIME. Make a copy of the message object since the rfc1153
# processing scrubs out attachments.
mimedigest.attach(MIMEMessage(copy.deepcopy(msg)))
# rfc1153
if first:
first = False
else:
print >> plainmsg, separator30
print >> plainmsg
# Use Mailman.Handlers.Scrubber.process() to get plain text
try:
msg = scrubber(mlist, msg)
except Errors.DiscardMessage:
print >> plainmsg, _('[Message discarded by content filter]')
continue
# Honor the default setting
for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS:
if msg[h]:
uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset)))
uh = '\n\t'.join(uh.split('\n'))
print >> plainmsg, uh
print >> plainmsg
# If decoded payload is empty, this may be multipart message.
# -- just stringfy it.
payload = msg.get_payload(decode=True) \
or msg.as_string().split('\n\n',1)[1]
mcset = msg.get_content_charset('')
if mcset and mcset <> lcset and mcset <> lcset_out:
try:
payload = unicode(payload, mcset, 'replace'
).encode(lcset, 'replace')
except (UnicodeError, LookupError):
# TK: Message has something unknown charset.
# _out means charset in 'outer world'.
payload = unicode(payload, lcset_out, 'replace'
).encode(lcset, 'replace')
print >> plainmsg, payload
if not payload.endswith('\n'):
print >> plainmsg
# Now add the footer but only if more than whitespace.
if re.sub('\s', '', mlist.digest_footer):
footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
# MIME
footer = MIMEText(footertxt, _charset=lcset)
footer['Content-Description'] = _('Digest Footer')
mimemsg.attach(footer)
# RFC 1153
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print >> plainmsg, separator30
print >> plainmsg
print >> plainmsg, 'Subject: ' + _('Digest Footer')
print >> plainmsg
print >> plainmsg, footertxt
print >> plainmsg
print >> plainmsg, separator30
print >> plainmsg
# Do the last bit of stuff for each digest type
signoff = _('End of ') + digestid
# MIME
# BAW: This stuff is outside the normal MIME goo, and it's what the old
# MIME digester did. No one seemed to complain, probably because you
# won't see it in an MUA that can't display the raw message. We've never
# got complaints before, but if we do, just wax this. It's primarily
# included for (marginally useful) backwards compatibility.
mimemsg.postamble = signoff
# rfc1153
print >> plainmsg, signoff
print >> plainmsg, '*' * len(signoff)
# Do our final bit of housekeeping, and then send each message to the
# outgoing queue for delivery.
mlist.next_digest_number += 1
virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
# Calculate the recipients lists
plainrecips = []
mimerecips = []
drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys()
for user in mlist.getMemberCPAddresses(drecips):
# user might be None if someone who toggled off digest delivery
# subsequently unsubscribed from the mailing list. Also, filter out
# folks who have disabled delivery.
if user is None or mlist.getDeliveryStatus(user) <> ENABLED:
continue
# Otherwise, decide whether they get MIME or RFC 1153 digests
if mlist.getMemberOption(user, mm_cfg.DisableMime):
plainrecips.append(user)
else:
mimerecips.append(user)
# Zap this since we're now delivering the last digest to these folks.
mlist.one_last_digest.clear()
# MIME
virginq.enqueue(mimemsg,
recips=mimerecips,
listname=mlist.internal_name(),
isdigest=True)
# RFC 1153
rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset)
virginq.enqueue(rfc1153msg,
recips=plainrecips,
listname=mlist.internal_name(),
isdigest=True)
def prepare_message(mlist, msg, msgdata):
# If the newsgroup is moderated, we need to add this header for the Usenet
# software to accept the posting, and not forward it on to the n.g.'s
# moderation address. The posting would not have gotten here if it hadn't
# already been approved. 1 == open list, mod n.g., 2 == moderated
if mlist.news_moderation in (1, 2):
del msg['approved']
msg['Approved'] = mlist.GetListEmail()
# Should we restore the original, non-prefixed subject for gatewayed
# messages? TK: We use stripped_subject (prefix stripped) which was
# crafted in CookHeaders.py to ensure prefix was stripped from the subject
# came from mailing list user.
stripped_subject = msgdata.get('stripped_subject') \
or msgdata.get('origsubj')
if not mlist.news_prefix_subject_too and stripped_subject is not None:
del msg['subject']
msg['Subject'] = stripped_subject
# Make sure we have a non-blank subject.
if not msg.get('subject', ''):
del msg['subject']
msg['Subject'] = '(no subject)'
# Add the appropriate Newsgroups: header
if msg['newsgroups'] is not None:
# This message is gated from our list to it's associated usnet group.
# If it has a Newsgroups: header mentioning other groups, it's not
# up to us to post it to those groups.
del msg['newsgroups']
msg['Newsgroups'] = mlist.linked_newsgroup
# Note: We need to be sure two messages aren't ever sent to the same list
# in the same process, since message ids need to be unique. Further, if
# messages are crossposted to two Usenet-gated mailing lists, they each
# need to have unique message ids or the nntpd will only accept one of
# them. The solution here is to substitute any existing message-id that
# isn't ours with one of ours, so we need to parse it to be sure we're not
# looping.
#
# We also add the original Message-ID: to References: to try to help with
# threading issues and create another header for documentation.
#
# Our Message-ID format is <mailman.secs.pid.listname@hostname>
msgid = msg['message-id']
hackmsgid = True
if msgid:
mo = mcre.search(msgid)
if mo:
lname, hname = mo.group('listname', 'hostname')
if lname == mlist.internal_name() and hname == mlist.host_name:
hackmsgid = False
if hackmsgid:
del msg['message-id']
msg['Message-ID'] = Utils.unique_message_id(mlist)
if msgid:
msg['X-Mailman-Original-Message-ID'] = msgid
refs = msg['references']
del msg['references']
if not refs:
refs = msg.get('in-reply-to', '')
else:
msg['X-Mailman-Original-References'] = refs
if refs:
msg['References'] = '\n '.join([refs, msgid])
else:
msg['References'] = msgid
# Lines: is useful
if msg['Lines'] is None:
# BAW: is there a better way?
count = len(list(email.iterators.body_line_iterator(msg)))
msg['Lines'] = str(count)
# Massage the message headers by remove some and rewriting others. This
# woon't completely sanitize the message, but it will eliminate the bulk
# of the rejections based on message headers. The NNTP server may still
# reject the message because of other problems.
for header in mm_cfg.NNTP_REMOVE_HEADERS:
del msg[header]
for header, rewrite in mm_cfg.NNTP_REWRITE_DUPLICATE_HEADERS:
values = msg.get_all(header, [])
if len(values) < 2:
# We only care about duplicates
continue
del msg[header]
# But keep the first one...
msg[header] = values[0]
for v in values[1:]:
msg[rewrite] = v
# Mark this message as prepared in case it has to be requeued
msgdata['prepped'] = True
def prepare_message(mlist, msg, msgdata):
# If the newsgroup is moderated, we need to add this header for the Usenet
# software to accept the posting, and not forward it on to the n.g.'s
# moderation address. The posting would not have gotten here if it hadn't
# already been approved. 1 == open list, mod n.g., 2 == moderated
if mlist.news_moderation in (1, 2):
del msg['approved']
msg['Approved'] = mlist.GetListEmail()
# Should we restore the original, non-prefixed subject for gatewayed
# messages? TK: We use stripped_subject (prefix stripped) which was
# crafted in CookHeaders.py to ensure prefix was stripped from the subject
# came from mailing list user.
stripped_subject = msgdata.get('stripped_subject') \
or msgdata.get('origsubj')
if not mlist.news_prefix_subject_too and stripped_subject is not None:
del msg['subject']
msg['subject'] = stripped_subject
# Add the appropriate Newsgroups: header
ngheader = msg['newsgroups']
if ngheader is not None:
# See if the Newsgroups: header already contains our linked_newsgroup.
# If so, don't add it again. If not, append our linked_newsgroup to
# the end of the header list
ngroups = [s.strip() for s in ngheader.split(',')]
if mlist.linked_newsgroup not in ngroups:
ngroups.append(mlist.linked_newsgroup)
# Subtitute our new header for the old one.
del msg['newsgroups']
msg['Newsgroups'] = COMMASPACE.join(ngroups)
else:
# Newsgroups: isn't in the message
msg['Newsgroups'] = mlist.linked_newsgroup
# Note: We need to be sure two messages aren't ever sent to the same list
# in the same process, since message ids need to be unique. Further, if
# messages are crossposted to two Usenet-gated mailing lists, they each
# need to have unique message ids or the nntpd will only accept one of
# them. The solution here is to substitute any existing message-id that
# isn't ours with one of ours, so we need to parse it to be sure we're not
# looping.
#
# Our Message-ID format is <mailman.secs.pid.listname@hostname>
msgid = msg['message-id']
hackmsgid = True
if msgid:
mo = mcre.search(msgid)
if mo:
lname, hname = mo.group('listname', 'hostname')
if lname == mlist.internal_name() and hname == mlist.host_name:
hackmsgid = False
if hackmsgid:
del msg['message-id']
msg['Message-ID'] = Utils.unique_message_id(mlist)
# Lines: is useful
if msg['Lines'] is None:
# BAW: is there a better way?
count = len(list(email.Iterators.body_line_iterator(msg)))
msg['Lines'] = str(count)
# Massage the message headers by remove some and rewriting others. This
# woon't completely sanitize the message, but it will eliminate the bulk
# of the rejections based on message headers. The NNTP server may still
# reject the message because of other problems.
for header in mm_cfg.NNTP_REMOVE_HEADERS:
del msg[header]
for header, rewrite in mm_cfg.NNTP_REWRITE_DUPLICATE_HEADERS:
values = msg.get_all(header, [])
if len(values) < 2:
# We only care about duplicates
continue
del msg[header]
# But keep the first one...
msg[header] = values[0]
for v in values[1:]:
msg[rewrite] = v
# Mark this message as prepared in case it has to be requeued
msgdata['prepped'] = True
def hold_for_approval(mlist, msg, msgdata, exc):
# BAW: This should really be tied into the email confirmation system so
# that the message can be approved or denied via email as well as the
# web.
#
# XXX We use the weird type(type) construct below because in Python 2.1,
# type is a function not a type and so can't be used as the second
# argument in isinstance(). However, in Python 2.5, exceptions are
# new-style classes and so are not of ClassType.
# FIXME pzv
if isinstance(exc, type) or isinstance(exc, type(type)):
# Go ahead and instantiate it now.
exc = exc()
listname = mlist.real_name
sender = msgdata.get('sender', msg.get_sender())
usersubject = msg.get('subject')
charset = Utils.GetCharSet(mlist.preferred_language)
if usersubject:
usersubject = Utils.oneline(usersubject, charset)
else:
usersubject = _('(no subject)')
message_id = msg.get('message-id', 'n/a')
owneraddr = mlist.GetOwnerEmail()
adminaddr = mlist.GetBouncesEmail()
requestaddr = mlist.GetRequestEmail()
# We need to send both the reason and the rejection notice through the
# translator again, because of the games we play above
reason = Utils.wrap(exc.reason_notice())
if isinstance(exc, NonMemberPost) and mlist.nonmember_rejection_notice:
msgdata['rejection_notice'] = Utils.wrap(
mlist.nonmember_rejection_notice.replace('%(listowner)s',
owneraddr))
else:
msgdata['rejection_notice'] = Utils.wrap(exc.rejection_notice(mlist))
id = mlist.HoldMessage(msg, reason, msgdata)
# Now we need to craft and send a message to the list admin so they can
# deal with the held message.
d = {
'listname': listname,
'hostname': mlist.host_name,
'reason': _(reason),
'sender': sender,
'subject': usersubject,
'admindb_url': mlist.GetScriptURL('admindb', absolute=1),
}
# We may want to send a notification to the original sender too
fromusenet = msgdata.get('fromusenet')
# Since we're sending two messages, which may potentially be in different
# languages (the user's preferred and the list's preferred for the admin),
# we need to play some i18n games here. Since the current language
# context ought to be set up for the user, let's craft his message first.
cookie = mlist.pend_new(Pending.HELD_MESSAGE, id)
if not fromusenet and ackp(msg) and mlist.respond_to_post_requests and \
mlist.autorespondToSender(sender, mlist.getMemberLanguage(sender)):
# Get a confirmation cookie
d['confirmurl'] = '%s/%s' % (mlist.GetScriptURL('confirm',
absolute=1), cookie)
lang = msgdata.get('lang', mlist.getMemberLanguage(sender))
subject = _('Your message to %(listname)s awaits moderator approval')
text = Utils.maketext('postheld.txt', d, lang=lang, mlist=mlist)
nmsg = Message.UserNotification(sender, owneraddr, subject, text, lang)
nmsg.send(mlist)
# Now the message for the list owners. Be sure to include the list
# moderators in this message. This one should appear to come from
# <list>-owner since we really don't need to do bounce processing on it.
if mlist.admin_immed_notify:
# Now let's temporarily set the language context to that which the
# admin is expecting.
otranslation = i18n.get_translation()
i18n.set_language(mlist.preferred_language)
try:
lang = mlist.preferred_language
charset = Utils.GetCharSet(lang)
# We need to regenerate or re-translate a few values in d
d['reason'] = _(reason)
d['subject'] = usersubject
# craft the admin notification message and deliver it
subject = _('%(listname)s post from %(sender)s requires approval')
nmsg = Message.UserNotification(owneraddr,
owneraddr,
subject,
lang=lang)
nmsg.set_type('multipart/mixed')
text = MIMEText(Utils.maketext('postauth.txt',
d,
raw=1,
mlist=mlist),
_charset=charset)
dmsg = MIMEText(Utils.wrap(
_("""\
If you reply to this message, keeping the Subject: header intact, Mailman will
discard the held message. Do this if the message is spam. If you reply to
this message and include an Approved: header with the list password in it, the
message will be approved for posting to the list. The Approved: header can
also appear in the first line of the body of the reply.""")),
_charset=Utils.GetCharSet(lang))
dmsg['Subject'] = 'confirm ' + cookie
dmsg['Sender'] = requestaddr
dmsg['From'] = requestaddr
dmsg['Date'] = email.utils.formatdate(localtime=True)
dmsg['Message-ID'] = Utils.unique_message_id(mlist)
nmsg.attach(text)
nmsg.attach(message.MIMEMessage(msg))
nmsg.attach(message.MIMEMessage(dmsg))
nmsg.send(mlist, **{'tomoderators': 1})
finally:
i18n.set_translation(otranslation)
# Log the held message
syslog('vette', '%s post from %s held, message-id=%s: %s', listname,
sender, message_id, reason)
# raise the specific MessageHeld exception to exit out of the message
# delivery pipeline
raise exc
def send_i18n_digests(mlist, mboxfp):
mbox = Mailbox(mboxfp)
# Prepare common information (first lang/charset)
lang = mlist.preferred_language
lcset = Utils.GetCharSet(lang)
lcset_out = Charset(lcset).output_charset or lcset
# Common Information (contd)
realname = mlist.real_name
volume = mlist.volume
issue = mlist.next_digest_number
digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d')
digestsubj = Header(digestid, lcset, header_name='Subject')
# Set things up for the MIME digest. Only headers not added by
# CookHeaders need be added here.
# Date/Message-ID should be added here also.
mimemsg = Message.Message()
mimemsg['Content-Type'] = 'multipart/mixed'
mimemsg['MIME-Version'] = '1.0'
mimemsg['From'] = mlist.GetRequestEmail()
mimemsg['Subject'] = digestsubj
mimemsg['To'] = mlist.GetListEmail()
mimemsg['Reply-To'] = mlist.GetListEmail()
mimemsg['Date'] = formatdate(localtime=1)
mimemsg['Message-ID'] = Utils.unique_message_id(mlist)
# Set things up for the rfc1153 digest
plainmsg = StringIO()
rfc1153msg = Message.Message()
rfc1153msg['From'] = mlist.GetRequestEmail()
rfc1153msg['Subject'] = digestsubj
rfc1153msg['To'] = mlist.GetListEmail()
rfc1153msg['Reply-To'] = mlist.GetListEmail()
rfc1153msg['Date'] = formatdate(localtime=1)
rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist)
separator70 = '-' * 70
separator30 = '-' * 30
# In the rfc1153 digest, the masthead contains the digest boilerplate plus
# any digest header. In the MIME digests, the masthead and digest header
# are separate MIME subobjects. In either case, it's the first thing in
# the digest, and we can calculate it now, so go ahead and add it now.
mastheadtxt = Utils.maketext(
'masthead.txt', {
'real_name': mlist.real_name,
'got_list_email': mlist.GetListEmail(),
'got_listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
'got_request_email': mlist.GetRequestEmail(),
'got_owner_email': mlist.GetOwnerEmail(),
},
mlist=mlist)
# MIME
masthead = MIMEText(mastheadtxt, _charset=lcset)
masthead['Content-Description'] = digestid
mimemsg.attach(masthead)
# RFC 1153
print >> plainmsg, mastheadtxt
print >> plainmsg
# Now add the optional digest header but only if more than whitespace.
if re.sub('\s', '', mlist.digest_header):
headertxt = decorate(mlist, mlist.digest_header, _('digest header'))
# MIME
header = MIMEText(headertxt, _charset=lcset)
header['Content-Description'] = _('Digest Header')
mimemsg.attach(header)
# RFC 1153
print >> plainmsg, headertxt
print >> plainmsg
# Now we have to cruise through all the messages accumulated in the
# mailbox file. We can't add these messages to the plainmsg and mimemsg
# yet, because we first have to calculate the table of contents
# (i.e. grok out all the Subjects). Store the messages in a list until
# we're ready for them.
#
# Meanwhile prepare things for the table of contents
toc = StringIO()
print >> toc, _("Today's Topics:\n")
# Now cruise through all the messages in the mailbox of digest messages,
# building the MIME payload and core of the RFC 1153 digest. We'll also
# accumulate Subject: headers and authors for the table-of-contents.
messages = []
msgcount = 0
msg = mbox.next()
while msg is not None:
if msg == '':
# It was an unparseable message
msg = mbox.next()
continue
msgcount += 1
messages.append(msg)
# Get the Subject header
msgsubj = msg.get('subject', _('(no subject)'))
subject = Utils.oneline(msgsubj, lcset)
# Don't include the redundant subject prefix in the toc
mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix),
subject, re.IGNORECASE)
if mo:
subject = subject[:mo.start(2)] + subject[mo.end(2):]
username = ''
addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)])
# Take only the first author we find
if isinstance(addresses, ListType) and addresses:
username = addresses[0][0]
if not username:
username = addresses[0][1]
if username:
username = '******' % username
# Put count and Wrap the toc subject line
wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65)
slines = wrapped.split('\n')
# See if the user's name can fit on the last line
if len(slines[-1]) + len(username) > 70:
slines.append(username)
else:
slines[-1] += username
# Add this subject to the accumulating topics
first = True
for line in slines:
if first:
print >> toc, ' ', line
first = False
else:
print >> toc, ' ', line.lstrip()
# We do not want all the headers of the original message to leak
# through in the digest messages. For this phase, we'll leave the
# same set of headers in both digests, i.e. those required in RFC 1153
# plus a couple of other useful ones. We also need to reorder the
# headers according to RFC 1153. Later, we'll strip out headers for
# for the specific MIME or plain digests.
keeper = {}
all_keepers = {}
for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS +
mm_cfg.PLAIN_DIGEST_KEEP_HEADERS):
all_keepers[header] = True
all_keepers = all_keepers.keys()
for keep in all_keepers:
keeper[keep] = msg.get_all(keep, [])
# Now remove all unkempt headers :)
for header in msg.keys():
del msg[header]
# And add back the kept header in the RFC 1153 designated order
for keep in all_keepers:
for field in keeper[keep]:
msg[keep] = field
# And a bit of extra stuff
msg['Message'] = ` msgcount `
# Get the next message in the digest mailbox
msg = mbox.next()
# Now we're finished with all the messages in the digest. First do some
# sanity checking and then on to adding the toc.
if msgcount == 0:
# Why did we even get here?
return
toctext = to_cset_out(toc.getvalue(), lcset)
# MIME
tocpart = MIMEText(toctext, _charset=lcset)
tocpart['Content-Description'] = _(
"Today's Topics (%(msgcount)d messages)")
mimemsg.attach(tocpart)
# RFC 1153
print >> plainmsg, toctext
print >> plainmsg
# For RFC 1153 digests, we now need the standard separator
print >> plainmsg, separator70
print >> plainmsg
# Now go through and add each message
mimedigest = MIMEBase('multipart', 'digest')
mimemsg.attach(mimedigest)
first = True
for msg in messages:
# MIME. Make a copy of the message object since the rfc1153
# processing scrubs out attachments.
mimedigest.attach(MIMEMessage(copy.deepcopy(msg)))
# rfc1153
if first:
first = False
else:
print >> plainmsg, separator30
print >> plainmsg
# Use Mailman.Handlers.Scrubber.process() to get plain text
try:
msg = scrubber(mlist, msg)
except Errors.DiscardMessage:
print >> plainmsg, _('[Message discarded by content filter]')
continue
# Honor the default setting
for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS:
if msg[h]:
uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset)))
uh = '\n\t'.join(uh.split('\n'))
print >> plainmsg, uh
print >> plainmsg
# If decoded payload is empty, this may be multipart message.
# -- just stringfy it.
payload = msg.get_payload(decode=True) \
or msg.as_string().split('\n\n',1)[1]
mcset = msg.get_content_charset('')
if mcset and mcset <> lcset and mcset <> lcset_out:
try:
payload = unicode(payload, mcset,
'replace').encode(lcset, 'replace')
except (UnicodeError, LookupError):
# TK: Message has something unknown charset.
# _out means charset in 'outer world'.
payload = unicode(payload, lcset_out,
'replace').encode(lcset, 'replace')
print >> plainmsg, payload
if not payload.endswith('\n'):
print >> plainmsg
# Now add the footer but only if more than whitespace.
if re.sub('\s', '', mlist.digest_footer):
footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
# MIME
footer = MIMEText(footertxt, _charset=lcset)
footer['Content-Description'] = _('Digest Footer')
mimemsg.attach(footer)
# RFC 1153
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print >> plainmsg, separator30
print >> plainmsg
print >> plainmsg, 'Subject: ' + _('Digest Footer')
print >> plainmsg
print >> plainmsg, footertxt
print >> plainmsg
print >> plainmsg, separator30
print >> plainmsg
# Do the last bit of stuff for each digest type
signoff = _('End of ') + digestid
# MIME
# BAW: This stuff is outside the normal MIME goo, and it's what the old
# MIME digester did. No one seemed to complain, probably because you
# won't see it in an MUA that can't display the raw message. We've never
# got complaints before, but if we do, just wax this. It's primarily
# included for (marginally useful) backwards compatibility.
mimemsg.postamble = signoff
# rfc1153
print >> plainmsg, signoff
print >> plainmsg, '*' * len(signoff)
# Do our final bit of housekeeping, and then send each message to the
# outgoing queue for delivery.
mlist.next_digest_number += 1
virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
# Calculate the recipients lists
plainrecips = []
mimerecips = []
drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys()
for user in mlist.getMemberCPAddresses(drecips):
# user might be None if someone who toggled off digest delivery
# subsequently unsubscribed from the mailing list. Also, filter out
# folks who have disabled delivery.
if user is None or mlist.getDeliveryStatus(user) <> ENABLED:
continue
# Otherwise, decide whether they get MIME or RFC 1153 digests
if mlist.getMemberOption(user, mm_cfg.DisableMime):
plainrecips.append(user)
else:
mimerecips.append(user)
# Zap this since we're now delivering the last digest to these folks.
mlist.one_last_digest.clear()
# MIME
virginq.enqueue(mimemsg,
recips=mimerecips,
listname=mlist.internal_name(),
isdigest=True)
# RFC 1153
rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset)
virginq.enqueue(rfc1153msg,
recips=plainrecips,
listname=mlist.internal_name(),
isdigest=True)
