def processUnixMailbox(self, input, articleClass = Article):
mbox = Mailbox(input)
while 1:
m = mbox.next()
if not m:
break
a = articleClass(m, self.sequence)
self.sequence = self.sequence + 1
self.add_article(a)
def process(mlist, msg, msgdata):
# Short circuit non-digestable lists.
if not mlist.digestable or msgdata.get('isdigest'):
return
mboxfile = os.path.join(mlist.fullpath(), 'digest.mbox')
omask = os.umask(007)
try:
mboxfp = open(mboxfile, 'a+')
finally:
os.umask(omask)
mbox = Mailbox(mboxfp)
mbox.AppendMessage(msg)
# Calculate the current size of the accumulation file. This will not tell
# us exactly how big the MIME, rfc1153, or any other generated digest
# message will be, but it's the most easily available metric to decide
# whether the size threshold has been reached.
mboxfp.flush()
size = os.path.getsize(mboxfile)
if (mlist.digest_size_threshhold > 0
and size / 1024.0 >= mlist.digest_size_threshhold):
# This is a bit of a kludge to get the mbox file moved to the digest
# queue directory.
try:
# Enclose in try/except here because a error in send_digest() can
# silently stop regular delivery. Unsuccessful digest delivery
# should be tried again by cron and the site administrator will be
# notified of any error explicitly by the cron error message.
mboxfp.seek(0)
send_digests(mlist, mboxfp)
os.unlink(mboxfile)
except Exception, errmsg:
# Bare except is generally prohibited in Mailman, but we can't
# forecast what exceptions can occur here.
syslog('error', 'send_digests() failed: %s', errmsg)
s = StringIO()
traceback.print_exc(file=s)
syslog('error', s.getvalue())
def send_i18n_digests(mlist, mboxfp):
mbox = Mailbox(mboxfp)
# Prepare common information (first lang/charset)
lang = mlist.preferred_language
lcset = Utils.GetCharSet(lang)
lcset_out = Charset(lcset).output_charset or lcset
# Common Information (contd)
realname = mlist.real_name
volume = mlist.volume
issue = mlist.next_digest_number
digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d')
digestsubj = Header(digestid, lcset, header_name='Subject')
# Set things up for the MIME digest. Only headers not added by
# CookHeaders need be added here.
# Date/Message-ID should be added here also.
mimemsg = Message.Message()
mimemsg['Content-Type'] = 'multipart/mixed'
mimemsg['MIME-Version'] = '1.0'
mimemsg['From'] = mlist.GetRequestEmail()
mimemsg['Subject'] = digestsubj
mimemsg['To'] = mlist.GetListEmail()
mimemsg['Reply-To'] = mlist.GetListEmail()
mimemsg['Date'] = formatdate(localtime=1)
mimemsg['Message-ID'] = Utils.unique_message_id(mlist)
# Set things up for the rfc1153 digest
plainmsg = StringIO()
rfc1153msg = Message.Message()
rfc1153msg['From'] = mlist.GetRequestEmail()
rfc1153msg['Subject'] = digestsubj
rfc1153msg['To'] = mlist.GetListEmail()
rfc1153msg['Reply-To'] = mlist.GetListEmail()
rfc1153msg['Date'] = formatdate(localtime=1)
rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist)
separator70 = '-' * 70
separator30 = '-' * 30
# In the rfc1153 digest, the masthead contains the digest boilerplate plus
# any digest header. In the MIME digests, the masthead and digest header
# are separate MIME subobjects. In either case, it's the first thing in
# the digest, and we can calculate it now, so go ahead and add it now.
mastheadtxt = Utils.maketext(
'masthead.txt',
{'real_name' : mlist.real_name,
'got_list_email': mlist.GetListEmail(),
'got_listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
'got_request_email': mlist.GetRequestEmail(),
'got_owner_email': mlist.GetOwnerEmail(),
}, mlist=mlist)
# MIME
masthead = MIMEText(mastheadtxt, _charset=lcset)
masthead['Content-Description'] = digestid
mimemsg.attach(masthead)
# RFC 1153
print >> plainmsg, mastheadtxt
print >> plainmsg
# Now add the optional digest header but only if more than whitespace.
if re.sub('\s', '', mlist.digest_header):
headertxt = decorate(mlist, mlist.digest_header, _('digest header'))
# MIME
header = MIMEText(headertxt, _charset=lcset)
header['Content-Description'] = _('Digest Header')
mimemsg.attach(header)
# RFC 1153
print >> plainmsg, headertxt
print >> plainmsg
# Now we have to cruise through all the messages accumulated in the
# mailbox file. We can't add these messages to the plainmsg and mimemsg
# yet, because we first have to calculate the table of contents
# (i.e. grok out all the Subjects). Store the messages in a list until
# we're ready for them.
#
# Meanwhile prepare things for the table of contents
toc = StringIO()
print >> toc, _("Today's Topics:\n")
# Now cruise through all the messages in the mailbox of digest messages,
# building the MIME payload and core of the RFC 1153 digest. We'll also
# accumulate Subject: headers and authors for the table-of-contents.
messages = []
msgcount = 0
msg = mbox.next()
while msg is not None:
if msg == '':
# It was an unparseable message
msg = mbox.next()
continue
msgcount += 1
messages.append(msg)
# Get the Subject header
msgsubj = msg.get('subject', _('(no subject)'))
subject = Utils.oneline(msgsubj, lcset)
# Don't include the redundant subject prefix in the toc
mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix),
subject, re.IGNORECASE)
if mo:
subject = subject[:mo.start(2)] + subject[mo.end(2):]
username = ''
addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)])
# Take only the first author we find
if isinstance(addresses, ListType) and addresses:
username = addresses[0][0]
if not username:
username = addresses[0][1]
if username:
username = '******' % username
# Put count and Wrap the toc subject line
wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65)
slines = wrapped.split('\n')
# See if the user's name can fit on the last line
if len(slines[-1]) + len(username) > 70:
slines.append(username)
else:
slines[-1] += username
# Add this subject to the accumulating topics
first = True
for line in slines:
if first:
print >> toc, ' ', line
first = False
else:
print >> toc, ' ', line.lstrip()
# We do not want all the headers of the original message to leak
# through in the digest messages. For this phase, we'll leave the
# same set of headers in both digests, i.e. those required in RFC 1153
# plus a couple of other useful ones. We also need to reorder the
# headers according to RFC 1153. Later, we'll strip out headers for
# for the specific MIME or plain digests.
keeper = {}
all_keepers = {}
for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS +
mm_cfg.PLAIN_DIGEST_KEEP_HEADERS):
all_keepers[header] = True
all_keepers = all_keepers.keys()
for keep in all_keepers:
keeper[keep] = msg.get_all(keep, [])
# Now remove all unkempt headers :)
for header in msg.keys():
del msg[header]
# And add back the kept header in the RFC 1153 designated order
for keep in all_keepers:
for field in keeper[keep]:
msg[keep] = field
# And a bit of extra stuff
msg['Message'] = `msgcount`
# Get the next message in the digest mailbox
msg = mbox.next()
# Now we're finished with all the messages in the digest. First do some
# sanity checking and then on to adding the toc.
if msgcount == 0:
# Why did we even get here?
return
toctext = to_cset_out(toc.getvalue(), lcset)
# MIME
tocpart = MIMEText(toctext, _charset=lcset)
tocpart['Content-Description']= _("Today's Topics (%(msgcount)d messages)")
mimemsg.attach(tocpart)
# RFC 1153
print >> plainmsg, toctext
print >> plainmsg
# For RFC 1153 digests, we now need the standard separator
print >> plainmsg, separator70
print >> plainmsg
# Now go through and add each message
mimedigest = MIMEBase('multipart', 'digest')
mimemsg.attach(mimedigest)
first = True
for msg in messages:
# MIME. Make a copy of the message object since the rfc1153
# processing scrubs out attachments.
mimedigest.attach(MIMEMessage(copy.deepcopy(msg)))
# rfc1153
if first:
first = False
else:
print >> plainmsg, separator30
print >> plainmsg
# Use Mailman.Handlers.Scrubber.process() to get plain text
try:
msg = scrubber(mlist, msg)
except Errors.DiscardMessage:
print >> plainmsg, _('[Message discarded by content filter]')
continue
# Honor the default setting
for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS:
if msg[h]:
uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset)))
uh = '\n\t'.join(uh.split('\n'))
print >> plainmsg, uh
print >> plainmsg
# If decoded payload is empty, this may be multipart message.
# -- just stringfy it.
payload = msg.get_payload(decode=True) \
or msg.as_string().split('\n\n',1)[1]
mcset = msg.get_content_charset('')
if mcset and mcset <> lcset and mcset <> lcset_out:
try:
payload = unicode(payload, mcset, 'replace'
).encode(lcset, 'replace')
except (UnicodeError, LookupError):
# TK: Message has something unknown charset.
# _out means charset in 'outer world'.
payload = unicode(payload, lcset_out, 'replace'
).encode(lcset, 'replace')
print >> plainmsg, payload
if not payload.endswith('\n'):
print >> plainmsg
# Now add the footer but only if more than whitespace.
if re.sub('\s', '', mlist.digest_footer):
footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
# MIME
footer = MIMEText(footertxt, _charset=lcset)
footer['Content-Description'] = _('Digest Footer')
mimemsg.attach(footer)
# RFC 1153
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print >> plainmsg, separator30
print >> plainmsg
print >> plainmsg, 'Subject: ' + _('Digest Footer')
print >> plainmsg
print >> plainmsg, footertxt
print >> plainmsg
print >> plainmsg, separator30
print >> plainmsg
# Do the last bit of stuff for each digest type
signoff = _('End of ') + digestid
# MIME
# BAW: This stuff is outside the normal MIME goo, and it's what the old
# MIME digester did. No one seemed to complain, probably because you
# won't see it in an MUA that can't display the raw message. We've never
# got complaints before, but if we do, just wax this. It's primarily
# included for (marginally useful) backwards compatibility.
mimemsg.postamble = signoff
# rfc1153
print >> plainmsg, signoff
print >> plainmsg, '*' * len(signoff)
# Do our final bit of housekeeping, and then send each message to the
# outgoing queue for delivery.
mlist.next_digest_number += 1
virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
# Calculate the recipients lists
plainrecips = []
mimerecips = []
drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys()
for user in mlist.getMemberCPAddresses(drecips):
# user might be None if someone who toggled off digest delivery
# subsequently unsubscribed from the mailing list. Also, filter out
# folks who have disabled delivery.
if user is None or mlist.getDeliveryStatus(user) <> ENABLED:
continue
# Otherwise, decide whether they get MIME or RFC 1153 digests
if mlist.getMemberOption(user, mm_cfg.DisableMime):
plainrecips.append(user)
else:
mimerecips.append(user)
# Zap this since we're now delivering the last digest to these folks.
mlist.one_last_digest.clear()
# MIME
virginq.enqueue(mimemsg,
recips=mimerecips,
listname=mlist.internal_name(),
isdigest=True)
# RFC 1153
rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset)
virginq.enqueue(rfc1153msg,
recips=plainrecips,
listname=mlist.internal_name(),
isdigest=True)
def send_i18n_digests(mlist, mboxfp):
mbox = Mailbox(mboxfp)
# Prepare common information (first lang/charset)
lang = mlist.preferred_language
lcset = Utils.GetCharSet(lang)
lcset_out = Charset(lcset).output_charset or lcset
# Common Information (contd)
realname = mlist.real_name
volume = mlist.volume
issue = mlist.next_digest_number
digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d')
digestsubj = Header(digestid, lcset, header_name='Subject')
# Set things up for the MIME digest. Only headers not added by
# CookHeaders need be added here.
# Date/Message-ID should be added here also.
mimemsg = Message.Message()
mimemsg['Content-Type'] = 'multipart/mixed'
mimemsg['MIME-Version'] = '1.0'
mimemsg['From'] = mlist.GetRequestEmail()
mimemsg['Subject'] = digestsubj
mimemsg['To'] = mlist.GetListEmail()
mimemsg['Reply-To'] = mlist.GetListEmail()
mimemsg['Date'] = formatdate(localtime=1)
mimemsg['Message-ID'] = Utils.unique_message_id(mlist)
# Set things up for the rfc1153 digest
plainmsg = StringIO()
rfc1153msg = Message.Message()
rfc1153msg['From'] = mlist.GetRequestEmail()
rfc1153msg['Subject'] = digestsubj
rfc1153msg['To'] = mlist.GetListEmail()
rfc1153msg['Reply-To'] = mlist.GetListEmail()
rfc1153msg['Date'] = formatdate(localtime=1)
rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist)
separator70 = '-' * 70
separator30 = '-' * 30
# In the rfc1153 digest, the masthead contains the digest boilerplate plus
# any digest header. In the MIME digests, the masthead and digest header
# are separate MIME subobjects. In either case, it's the first thing in
# the digest, and we can calculate it now, so go ahead and add it now.
mastheadtxt = Utils.maketext(
'masthead.txt', {
'real_name': mlist.real_name,
'got_list_email': mlist.GetListEmail(),
'got_listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
'got_request_email': mlist.GetRequestEmail(),
'got_owner_email': mlist.GetOwnerEmail(),
},
mlist=mlist)
# MIME
masthead = MIMEText(mastheadtxt, _charset=lcset)
masthead['Content-Description'] = digestid
mimemsg.attach(masthead)
# RFC 1153
print >> plainmsg, mastheadtxt
print >> plainmsg
# Now add the optional digest header but only if more than whitespace.
if re.sub('\s', '', mlist.digest_header):
headertxt = decorate(mlist, mlist.digest_header, _('digest header'))
# MIME
header = MIMEText(headertxt, _charset=lcset)
header['Content-Description'] = _('Digest Header')
mimemsg.attach(header)
# RFC 1153
print >> plainmsg, headertxt
print >> plainmsg
# Now we have to cruise through all the messages accumulated in the
# mailbox file. We can't add these messages to the plainmsg and mimemsg
# yet, because we first have to calculate the table of contents
# (i.e. grok out all the Subjects). Store the messages in a list until
# we're ready for them.
#
# Meanwhile prepare things for the table of contents
toc = StringIO()
print >> toc, _("Today's Topics:\n")
# Now cruise through all the messages in the mailbox of digest messages,
# building the MIME payload and core of the RFC 1153 digest. We'll also
# accumulate Subject: headers and authors for the table-of-contents.
messages = []
msgcount = 0
msg = mbox.next()
while msg is not None:
if msg == '':
# It was an unparseable message
msg = mbox.next()
continue
msgcount += 1
messages.append(msg)
# Get the Subject header
msgsubj = msg.get('subject', _('(no subject)'))
subject = Utils.oneline(msgsubj, lcset)
# Don't include the redundant subject prefix in the toc
mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix),
subject, re.IGNORECASE)
if mo:
subject = subject[:mo.start(2)] + subject[mo.end(2):]
username = ''
addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)])
# Take only the first author we find
if isinstance(addresses, ListType) and addresses:
username = addresses[0][0]
if not username:
username = addresses[0][1]
if username:
username = '******' % username
# Put count and Wrap the toc subject line
wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65)
slines = wrapped.split('\n')
# See if the user's name can fit on the last line
if len(slines[-1]) + len(username) > 70:
slines.append(username)
else:
slines[-1] += username
# Add this subject to the accumulating topics
first = True
for line in slines:
if first:
print >> toc, ' ', line
first = False
else:
print >> toc, ' ', line.lstrip()
# We do not want all the headers of the original message to leak
# through in the digest messages. For this phase, we'll leave the
# same set of headers in both digests, i.e. those required in RFC 1153
# plus a couple of other useful ones. We also need to reorder the
# headers according to RFC 1153. Later, we'll strip out headers for
# for the specific MIME or plain digests.
keeper = {}
all_keepers = {}
for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS +
mm_cfg.PLAIN_DIGEST_KEEP_HEADERS):
all_keepers[header] = True
all_keepers = all_keepers.keys()
for keep in all_keepers:
keeper[keep] = msg.get_all(keep, [])
# Now remove all unkempt headers :)
for header in msg.keys():
del msg[header]
# And add back the kept header in the RFC 1153 designated order
for keep in all_keepers:
for field in keeper[keep]:
msg[keep] = field
# And a bit of extra stuff
msg['Message'] = ` msgcount `
# Get the next message in the digest mailbox
msg = mbox.next()
# Now we're finished with all the messages in the digest. First do some
# sanity checking and then on to adding the toc.
if msgcount == 0:
# Why did we even get here?
return
toctext = to_cset_out(toc.getvalue(), lcset)
# MIME
tocpart = MIMEText(toctext, _charset=lcset)
tocpart['Content-Description'] = _(
"Today's Topics (%(msgcount)d messages)")
mimemsg.attach(tocpart)
# RFC 1153
print >> plainmsg, toctext
print >> plainmsg
# For RFC 1153 digests, we now need the standard separator
print >> plainmsg, separator70
print >> plainmsg
# Now go through and add each message
mimedigest = MIMEBase('multipart', 'digest')
mimemsg.attach(mimedigest)
first = True
for msg in messages:
# MIME. Make a copy of the message object since the rfc1153
# processing scrubs out attachments.
mimedigest.attach(MIMEMessage(copy.deepcopy(msg)))
# rfc1153
if first:
first = False
else:
print >> plainmsg, separator30
print >> plainmsg
# Use Mailman.Handlers.Scrubber.process() to get plain text
try:
msg = scrubber(mlist, msg)
except Errors.DiscardMessage:
print >> plainmsg, _('[Message discarded by content filter]')
continue
# Honor the default setting
for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS:
if msg[h]:
uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset)))
uh = '\n\t'.join(uh.split('\n'))
print >> plainmsg, uh
print >> plainmsg
# If decoded payload is empty, this may be multipart message.
# -- just stringfy it.
payload = msg.get_payload(decode=True) \
or msg.as_string().split('\n\n',1)[1]
mcset = msg.get_content_charset('')
if mcset and mcset <> lcset and mcset <> lcset_out:
try:
payload = unicode(payload, mcset,
'replace').encode(lcset, 'replace')
except (UnicodeError, LookupError):
# TK: Message has something unknown charset.
# _out means charset in 'outer world'.
payload = unicode(payload, lcset_out,
'replace').encode(lcset, 'replace')
print >> plainmsg, payload
if not payload.endswith('\n'):
print >> plainmsg
# Now add the footer but only if more than whitespace.
if re.sub('\s', '', mlist.digest_footer):
footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
# MIME
footer = MIMEText(footertxt, _charset=lcset)
footer['Content-Description'] = _('Digest Footer')
mimemsg.attach(footer)
# RFC 1153
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print >> plainmsg, separator30
print >> plainmsg
print >> plainmsg, 'Subject: ' + _('Digest Footer')
print >> plainmsg
print >> plainmsg, footertxt
print >> plainmsg
print >> plainmsg, separator30
print >> plainmsg
# Do the last bit of stuff for each digest type
signoff = _('End of ') + digestid
# MIME
# BAW: This stuff is outside the normal MIME goo, and it's what the old
# MIME digester did. No one seemed to complain, probably because you
# won't see it in an MUA that can't display the raw message. We've never
# got complaints before, but if we do, just wax this. It's primarily
# included for (marginally useful) backwards compatibility.
mimemsg.postamble = signoff
# rfc1153
print >> plainmsg, signoff
print >> plainmsg, '*' * len(signoff)
# Do our final bit of housekeeping, and then send each message to the
# outgoing queue for delivery.
mlist.next_digest_number += 1
virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
# Calculate the recipients lists
plainrecips = []
mimerecips = []
drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys()
for user in mlist.getMemberCPAddresses(drecips):
# user might be None if someone who toggled off digest delivery
# subsequently unsubscribed from the mailing list. Also, filter out
# folks who have disabled delivery.
if user is None or mlist.getDeliveryStatus(user) <> ENABLED:
continue
# Otherwise, decide whether they get MIME or RFC 1153 digests
if mlist.getMemberOption(user, mm_cfg.DisableMime):
plainrecips.append(user)
else:
mimerecips.append(user)
# Zap this since we're now delivering the last digest to these folks.
mlist.one_last_digest.clear()
# MIME
virginq.enqueue(mimemsg,
recips=mimerecips,
listname=mlist.internal_name(),
isdigest=True)
# RFC 1153
rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset)
virginq.enqueue(rfc1153msg,
recips=plainrecips,
listname=mlist.internal_name(),
isdigest=True)