def scrub_msg822(self, part):
# submessage
submsg = part.get_payload(0)
omask = os.umask(002)
try:
url = save_attachment(self.mlist, part, self.dir)
finally:
os.umask(omask)
subject = submsg.get('subject', _('no subject'))
subject = Utils.oneline(subject, self.lcset)
date = submsg.get('date', _('no date'))
who = submsg.get('from', _('unknown sender'))
who = Utils.oneline(who, self.lcset)
size = len(str(submsg))
self.msgtexts.append(unicode(_("""\
An embedded message was scrubbed...
From: %(who)s
Subject: %(subject)s
Date: %(date)s
Size: %(size)s
URL: %(url)s
"""), self.lcset))
# Replace this part because subparts should not be walk()-ed.
del part['content-type']
part.set_payload('blah blah', 'us-ascii')
def process(mlist, msg, msgdata):
# Extract the sender's address and find them in the user database
sender = msgdata.get('original_sender', msg.get_sender())
try:
ack = mlist.getMemberOption(sender, mm_cfg.AcknowledgePosts)
if not ack:
return
except Errors.NotAMemberError:
return
# Okay, they want acknowledgement of their post. Give them their original
# subject. BAW: do we want to use the decoded header?
origsubj = msgdata.get('origsubj', msg.get('subject', _('(no subject)')))
# Get the user's preferred language
lang = msgdata.get('lang', mlist.getMemberLanguage(sender))
# Now get the acknowledgement template
realname = mlist.real_name
text = Utils.maketext(
'postack.txt', {
'subject': Utils.oneline(origsubj, Utils.GetCharSet(lang)),
'listname': realname,
'listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
'optionsurl': mlist.GetOptionsURL(sender, absolute=1),
},
lang=lang,
mlist=mlist,
raw=1)
# Craft the outgoing message, with all headers and attributes
# necessary for general delivery. Then enqueue it to the outgoing
# queue.
subject = _('%(realname)s post acknowledgement')
usermsg = Message.UserNotification(sender, mlist.GetBouncesEmail(),
subject, text, lang)
usermsg.send(mlist)
def process(mlist, msg, msgdata):
# Extract the sender's address and find them in the user database
sender = msgdata.get('original_sender', msg.get_sender())
try:
ack = mlist.getMemberOption(sender, mm_cfg.AcknowledgePosts)
if not ack:
return
except Errors.NotAMemberError:
return
# Okay, they want acknowledgement of their post. Give them their original
# subject. BAW: do we want to use the decoded header?
origsubj = msgdata.get('origsubj', msg.get('subject', _('(no subject)')))
# Get the user's preferred language
lang = msgdata.get('lang', mlist.getMemberLanguage(sender))
# Now get the acknowledgement template
realname = mlist.real_name
text = Utils.maketext(
'postack.txt',
{'subject' : Utils.oneline(origsubj, Utils.GetCharSet(lang)),
'listname' : realname,
'listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
'optionsurl' : mlist.GetOptionsURL(sender, absolute=1),
}, lang=lang, mlist=mlist, raw=1)
# Craft the outgoing message, with all headers and attributes
# necessary for general delivery. Then enqueue it to the outgoing
# queue.
subject = _('%(realname)s post acknowledgement')
usermsg = Message.UserNotification(sender, mlist.GetBouncesEmail(),
subject, text, lang)
usermsg.send(mlist)
def quote(s, is_header=False):
if is_header:
h = Utils.oneline(s, 'iso-8859-1')
else:
h = s
h = str('').join(re.split('[\x00-\x08\x0B-\x1f]+', h))
return Utils.uquote(
h.replace('&', '&').replace('>', '>').replace('<', '<'))
def process(mlist, msg, msgdata):
try:
confs_by_list = mm_cfg.REWRITE_FROM
except AttributeError:
return
if mlist.internal_name() in confs_by_list:
conf = confs_by_list[mlist.internal_name()]
elif '*' in confs_by_list:
conf = confs_by_list['*']
else:
return
from_name_fmt = conf.get('from_name', '%(from_name)s')
from_address_fmt = conf.get('from_address', '%(from_address)s')
save_original = conf.get('save_original')
d = SafeDict({'list_real_name': mlist.real_name,
'list_name': mlist.internal_name(),
'list_address': mlist.GetListEmail(),
'list_domain': mlist.host_name,
'list_desc': mlist.description,
'list_info': mlist.info,
})
lcset = Utils.GetCharSet(mlist.preferred_language)
from_name, from_address = parseaddr(msg.get('From'))
d['from_address'] = from_address
try:
d['from_local'], d['from_domain'] = re.split('@', from_address, 1)
except ValueError:
d['from_local'] = from_address
d['from_domain'] = ''
if from_name != '':
d['from_name'] = Utils.oneline(from_name, lcset)
else:
d['from_name'] = d['from_local']
try:
membername = mlist.getMemberName(from_address) or None
try:
d['from_membername'] = membername.encode(lcset)
except (AttributeError, UnicodeError):
d['from_membername'] = d['from_name']
except Errors.NotAMemberError:
d['from_membername'] = d['from_name']
from_name = from_name_fmt % d
from_address = from_address_fmt % d
if save_original:
msg[save_original] = msg['From']
del msg['From']
msg['From'] = formataddr((from_name, from_address))
def scrub_any(self, part):
# Images and MS Office files and all
payload = part.get_payload(decode=True)
ctype = part.get_content_type()
# XXX email 2.5 special care is omitted.
size = len(payload)
omask = os.umask(002)
try:
url = save_attachment(self.mlist, part, self.dir)
finally:
os.umask(omask)
desc = part.get('content-description', _('not available'))
desc = Utils.oneline(desc, self.lcset)
filename = part.get_filename(_('not available'))
filename = Utils.oneline(filename, self.lcset)
self.msgtexts.append(unicode(_("""\
A non-text attachment was scrubbed...
Name: %(filename)s
Type: %(ctype)s
Size: %(size)d bytes
Desc: %(desc)s
URL: %(url)s
"""), self.lcset))
def scrub_text(self, part):
# Plain text scrubber.
omask = os.umask(002)
try:
url = save_attachment(self.mlist, part, self.dir)
finally:
os.umask(omask)
filename = part.get_filename(_('not available'))
filename = Utils.oneline(filename, self.lcset)
self.msgtexts.append(unicode(_("""\
An embedded and charset-unspecified text was scrubbed...
Name: %(filename)s
URL: %(url)s
"""), self.lcset))
def quote(s, is_header=False):
if is_header:
h = Utils.oneline(s, 'utf-8')
else:
h = s
# Remove illegal XML characters
# Try to decode UTF-8, so that Utils.uquote can escape multibyte characters
# correctly.
try:
hclean = h.decode('utf-8')
hclean = u''.join(re.split(u'[\x00-\x08\x0B-\x1f]+', hclean))
except UnicodeDecodeError:
hclean = ''.join(re.split('[\x00-\x08\x0B-\x1f]+', h))
return Utils.uquote(hclean.replace('&', '&').replace('>', '>').replace('<', '<'))
def quote(s, is_header=False):
if is_header:
h = Utils.oneline(s, 'utf-8')
else:
h = s
# Remove illegal XML characters
# Try to decode UTF-8, so that Utils.uquote can escape multibyte characters
# correctly.
try:
hclean = h.decode('utf-8')
hclean = u''.join(re.split(u'[\x00-\x08\x0B-\x1f]+', hclean))
except UnicodeDecodeError:
hclean = ''.join(re.split('[\x00-\x08\x0B-\x1f]+', h))
return Utils.uquote(hclean.replace('&', '&').replace('>', '>').replace('<', '<'))
def process(mlist, msg, msgdata):
try:
confs_by_list = mm_cfg.ADD_HEADERS
except AttributeError:
return
if mlist.internal_name() in confs_by_list:
conf = confs_by_list[mlist.internal_name()]
elif '*' in confs_by_list:
conf = confs_by_list['*']
else:
return
d = SafeDict({'list_real_name': mlist.real_name,
'list_name': mlist.internal_name(),
'list_address': mlist.GetListEmail(),
'list_domain': mlist.host_name,
'list_desc': mlist.description,
'list_info': mlist.info,
'post_id': mlist.post_id,
})
lcset = Utils.GetCharSet(mlist.preferred_language)
d['from_header'] = msg.get('From')
from_name, from_address = parseaddr(d['from_header'])
d['from_address'] = from_address
try:
d['from_local'], d['from_domain'] = re.split('@', from_address, 1)
except ValueError:
d['from_local'] = from_address
d['from_domain'] = ''
if from_name != '':
d['from_name'] = Utils.oneline(from_name, lcset)
else:
d['from_name'] = d['from_local']
try:
membername = mlist.getMemberName(from_address) or None
try:
d['from_membername'] = membername.encode(lcset)
except (AttributeError, UnicodeError):
d['from_membername'] = d['from_name']
except Errors.NotAMemberError:
d['from_membername'] = d['from_name']
for name, value_fmt in conf.items():
value = value_fmt % d
change_header(name, value, mlist, msg, msgdata, delete=False)
def BounceMessage(self, msg, msgdata, e=None):
# Bounce a message back to the sender, with an error message if
# provided in the exception argument.
sender = msg.get_sender()
subject = msg.get("subject", _("(no subject)"))
subject = Utils.oneline(subject, Utils.GetCharSet(self.preferred_language))
if e is None:
notice = _("[No bounce details are available]")
else:
notice = _(e.notice())
# Currently we always craft bounces as MIME messages.
bmsg = Message.UserNotification(msg.get_sender(), self.GetOwnerEmail(), subject, lang=self.preferred_language)
# BAW: Be sure you set the type before trying to attach, or you'll get
# a MultipartConversionError.
bmsg.set_type("multipart/mixed")
txt = MIMEText(notice, _charset=Utils.GetCharSet(self.preferred_language))
bmsg.attach(txt)
bmsg.attach(MIMEMessage(msg))
bmsg.send(self)
def BounceMessage(self, msg, msgdata, e=None):
# Bounce a message back to the sender, with an error message if
# provided in the exception argument.
sender = msg.get_sender()
subject = msg.get('subject', _('(no subject)'))
subject = Utils.oneline(subject,
Utils.GetCharSet(self.preferred_language))
if e is None:
notice = _('[No bounce details are available]')
else:
notice = _(e.notice())
# Currently we always craft bounces as MIME messages.
bmsg = Message.UserNotification(msg.get_sender(),
self.GetOwnerEmail(),
subject,
lang=self.preferred_language)
# BAW: Be sure you set the type before trying to attach, or you'll get
# a MultipartConversionError.
bmsg.set_type('multipart/mixed')
txt = MIMEText(notice,
_charset=Utils.GetCharSet(self.preferred_language))
bmsg.attach(txt)
bmsg.attach(MIMEMessage(msg))
bmsg.send(self)
mcset = 'us-ascii'
lcset = Utils.GetCharSet(mlist.preferred_language)
if mcset <> lcset:
try:
body = unicode(body, mcset).encode(lcset, 'replace')
except (LookupError, UnicodeError, ValueError):
pass
hdrtxt = NL.join(['%s: %s' % (k, v) for k, v in msg.items()])
hdrtxt = Utils.websafe(hdrtxt)
# Okay, we've reconstituted the message just fine. Now for the fun part!
t = Table(cellspacing=0, cellpadding=0, width='100%')
t.AddRow([Bold(_('From:')), sender])
row, col = t.GetCurrentRowIndex(), t.GetCurrentCellIndex()
t.AddCellInfo(row, col-1, align='right')
t.AddRow([Bold(_('Subject:')),
Utils.websafe(Utils.oneline(subject, lcset))])
t.AddCellInfo(row+1, col-1, align='right')
t.AddRow([Bold(_('Reason:')), _(reason)])
t.AddCellInfo(row+2, col-1, align='right')
when = msgdata.get('received_time')
if when:
t.AddRow([Bold(_('Received:')), time.ctime(when)])
t.AddCellInfo(row+2, col-1, align='right')
# We can't use a RadioButtonArray here because horizontal placement can be
# confusing to the user and vertical placement takes up too much
# real-estate. This is a hack!
buttons = Table(cellspacing="5", cellpadding="0")
buttons.AddRow(map(lambda x, s=' '*5: s+x+s,
(_('Defer'), _('Approve'), _('Reject'), _('Discard'))))
buttons.AddRow([Center(RadioButton(id, mm_cfg.DEFER, 1)),
Center(RadioButton(id, mm_cfg.APPROVE, 0)),
def _decode(h):
if not h:
return h
return Utils.oneline(h, Utils.GetCharSet(mlist.preferred_language))
def process(mlist, msg, msgdata=None):
sanitize = mm_cfg.ARCHIVE_HTML_SANITIZER
outer = True
if msgdata is None:
msgdata = {}
if msgdata:
# msgdata is available if it is in GLOBAL_PIPELINE
# ie. not in digest or archiver
# check if the list owner want to scrub regular delivery
if not mlist.scrub_nondigest:
return
dir = calculate_attachments_dir(mlist, msg, msgdata)
charset = None
lcset = Utils.GetCharSet(mlist.preferred_language)
lcset_out = Charset(lcset).output_charset or lcset
# Now walk over all subparts of this message and scrub out various types
format = delsp = None
for part in msg.walk():
ctype = part.get_content_type()
# If the part is text/plain, we leave it alone
if ctype == 'text/plain':
# We need to choose a charset for the scrubbed message, so we'll
# arbitrarily pick the charset of the first text/plain part in the
# message.
# MAS: Also get the RFC 3676 stuff from this part. This seems to
# work OK for scrub_nondigest. It will also work as far as
# scrubbing messages for the archive is concerned, but pipermail
# doesn't pay any attention to the RFC 3676 parameters. The plain
# format digest is going to be a disaster in any case as some of
# messages will be format="flowed" and some not. ToDigest creates
# its own Content-Type: header for the plain digest which won't
# have RFC 3676 parameters. If the message Content-Type: headers
# are retained for display in the digest, the parameters will be
# there for information, but not for the MUA. This is the best we
# can do without having get_payload() process the parameters.
if charset is None:
charset = part.get_content_charset(lcset)
format = part.get_param('format')
delsp = part.get_param('delsp')
# TK: if part is attached then check charset and scrub if none
if part.get('content-disposition') and \
not part.get_content_charset():
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir)
finally:
os.umask(omask)
filename = part.get_filename(_('not available'))
filename = Utils.oneline(filename, lcset)
replace_payload_by_text(part, _("""\
An embedded and charset-unspecified text was scrubbed...
Name: %(filename)s
URL: %(url)s
"""), lcset)
elif ctype == 'text/html' and isinstance(sanitize, IntType):
if sanitize == 0:
if outer:
raise DiscardMessage
replace_payload_by_text(part,
_('HTML attachment scrubbed and removed'),
# Adding charset arg and removing content-type
# sets content-type to text/plain
lcset)
elif sanitize == 2:
# By leaving it alone, Pipermail will automatically escape it
pass
elif sanitize == 3:
# Pull it out as an attachment but leave it unescaped. This
# is dangerous, but perhaps useful for heavily moderated
# lists.
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir, filter_html=False)
finally:
os.umask(omask)
replace_payload_by_text(part, _("""\
An HTML attachment was scrubbed...
URL: %(url)s
"""), lcset)
else:
# HTML-escape it and store it as an attachment, but make it
# look a /little/ bit prettier. :(
payload = Utils.websafe(part.get_payload(decode=True))
# For whitespace in the margin, change spaces into
# non-breaking spaces, and tabs into 8 of those. Then use a
# mono-space font. Still looks hideous to me, but then I'd
# just as soon discard them.
def doreplace(s):
return s.replace(' ', ' ').replace('\t', ' '*8)
lines = [doreplace(s) for s in payload.split('\n')]
payload = '<tt>\n' + BR.join(lines) + '\n</tt>\n'
part.set_payload(payload)
# We're replacing the payload with the decoded payload so this
# will just get in the way.
del part['content-transfer-encoding']
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir, filter_html=False)
finally:
os.umask(omask)
replace_payload_by_text(part, _("""\
An HTML attachment was scrubbed...
URL: %(url)s
"""), lcset)
elif ctype == 'message/rfc822':
# This part contains a submessage, so it too needs scrubbing
submsg = part.get_payload(0)
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir)
finally:
os.umask(omask)
subject = submsg.get('subject', _('no subject'))
date = submsg.get('date', _('no date'))
who = submsg.get('from', _('unknown sender'))
size = len(str(submsg))
replace_payload_by_text(part, _("""\
An embedded message was scrubbed...
From: %(who)s
Subject: %(subject)s
Date: %(date)s
Size: %(size)s
URL: %(url)s
"""), lcset)
# If the message isn't a multipart, then we'll strip it out as an
# attachment that would have to be separately downloaded. Pipermail
# will transform the url into a hyperlink.
elif part.get_payload() and not part.is_multipart():
payload = part.get_payload(decode=True)
ctype = part.get_content_type()
# XXX Under email 2.5, it is possible that payload will be None.
# This can happen when you have a Content-Type: multipart/* with
# only one part and that part has two blank lines between the
# first boundary and the end boundary. In email 3.0 you end up
# with a string in the payload. I think in this case it's safe to
# ignore the part.
if payload is None:
continue
size = len(payload)
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir)
finally:
os.umask(omask)
desc = part.get('content-description', _('not available'))
desc = Utils.oneline(desc, lcset)
filename = part.get_filename(_('not available'))
filename = Utils.oneline(filename, lcset)
replace_payload_by_text(part, _("""\
A non-text attachment was scrubbed...
Name: %(filename)s
Type: %(ctype)s
Size: %(size)d bytes
Desc: %(desc)s
URL: %(url)s
"""), lcset)
outer = False
# We still have to sanitize multipart messages to flat text because
# Pipermail can't handle messages with list payloads. This is a kludge;
# def (n) clever hack ;).
if msg.is_multipart() and sanitize <> 2:
# By default we take the charset of the first text/plain part in the
# message, but if there was none, we'll use the list's preferred
# language's charset.
if not charset or charset == 'us-ascii':
charset = lcset_out
else:
# normalize to the output charset if input/output are different
charset = Charset(charset).output_charset or charset
# We now want to concatenate all the parts which have been scrubbed to
# text/plain, into a single text/plain payload. We need to make sure
# all the characters in the concatenated string are in the same
# encoding, so we'll use the 'replace' key in the coercion call.
# BAW: Martin's original patch suggested we might want to try
# generalizing to utf-8, and that's probably a good idea (eventually).
text = []
for part in msg.walk():
# TK: bug-id 1099138 and multipart
# MAS test payload - if part may fail if there are no headers.
if not part.get_payload() or part.is_multipart():
continue
# All parts should be scrubbed to text/plain by now.
partctype = part.get_content_type()
if partctype <> 'text/plain':
text.append(_('Skipped content of type %(partctype)s\n'))
continue
try:
t = part.get_payload(decode=True) or ''
# MAS: TypeError exception can occur if payload is None. This
# was observed with a message that contained an attached
# message/delivery-status part. Because of the special parsing
# of this type, this resulted in a text/plain sub-part with a
# null body. See bug 1430236.
except (binascii.Error, TypeError):
t = part.get_payload() or ''
# TK: get_content_charset() returns 'iso-2022-jp' for internally
# crafted (scrubbed) 'euc-jp' text part. So, first try
# get_charset(), then get_content_charset() for the parts
# which are already embeded in the incoming message.
partcharset = part.get_charset()
if partcharset:
partcharset = str(partcharset)
else:
partcharset = part.get_content_charset()
if partcharset and partcharset <> charset:
try:
t = unicode(t, partcharset, 'replace')
except (UnicodeError, LookupError, ValueError,
AssertionError):
# We can get here if partcharset is bogus in come way.
# Replace funny characters. We use errors='replace'
t = unicode(t, 'ascii', 'replace')
try:
# Should use HTML-Escape, or try generalizing to UTF-8
t = t.encode(charset, 'replace')
except (UnicodeError, LookupError, ValueError,
AssertionError):
# if the message charset is bogus, use the list's.
t = t.encode(lcset, 'replace')
# Separation is useful
if isinstance(t, StringType):
if not t.endswith('\n'):
t += '\n'
text.append(t)
# Now join the text and set the payload
sep = _('-------------- next part --------------\n')
# The i18n separator is in the list's charset. Coerce it to the
# message charset.
try:
s = unicode(sep, lcset, 'replace')
sep = s.encode(charset, 'replace')
except (UnicodeError, LookupError, ValueError,
AssertionError):
pass
replace_payload_by_text(msg, sep.join(text), charset)
if format:
msg.set_param('Format', format)
if delsp:
msg.set_param('DelSp', delsp)
return msg
def send_i18n_digests(mlist, mboxfp):
mbox = Mailbox(mboxfp)
# Prepare common information (first lang/charset)
lang = mlist.preferred_language
lcset = Utils.GetCharSet(lang)
lcset_out = Charset(lcset).output_charset or lcset
# Common Information (contd)
realname = mlist.real_name
volume = mlist.volume
issue = mlist.next_digest_number
digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d')
digestsubj = Header(digestid, lcset, header_name='Subject')
# Set things up for the MIME digest. Only headers not added by
# CookHeaders need be added here.
# Date/Message-ID should be added here also.
mimemsg = Message.Message()
mimemsg['Content-Type'] = 'multipart/mixed'
mimemsg['MIME-Version'] = '1.0'
mimemsg['From'] = mlist.GetRequestEmail()
mimemsg['Subject'] = digestsubj
mimemsg['To'] = mlist.GetListEmail()
mimemsg['Reply-To'] = mlist.GetListEmail()
mimemsg['Date'] = formatdate(localtime=1)
mimemsg['Message-ID'] = Utils.unique_message_id(mlist)
# Set things up for the rfc1153 digest
plainmsg = StringIO()
rfc1153msg = Message.Message()
rfc1153msg['From'] = mlist.GetRequestEmail()
rfc1153msg['Subject'] = digestsubj
rfc1153msg['To'] = mlist.GetListEmail()
rfc1153msg['Reply-To'] = mlist.GetListEmail()
rfc1153msg['Date'] = formatdate(localtime=1)
rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist)
separator70 = '-' * 70
separator30 = '-' * 30
# In the rfc1153 digest, the masthead contains the digest boilerplate plus
# any digest header. In the MIME digests, the masthead and digest header
# are separate MIME subobjects. In either case, it's the first thing in
# the digest, and we can calculate it now, so go ahead and add it now.
mastheadtxt = Utils.maketext(
'masthead.txt', {
'real_name': mlist.real_name,
'got_list_email': mlist.GetListEmail(),
'got_listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
'got_request_email': mlist.GetRequestEmail(),
'got_owner_email': mlist.GetOwnerEmail(),
},
mlist=mlist)
# MIME
masthead = MIMEText(mastheadtxt, _charset=lcset)
masthead['Content-Description'] = digestid
mimemsg.attach(masthead)
# RFC 1153
print >> plainmsg, mastheadtxt
print >> plainmsg
# Now add the optional digest header but only if more than whitespace.
if re.sub('\s', '', mlist.digest_header):
headertxt = decorate(mlist, mlist.digest_header, _('digest header'))
# MIME
header = MIMEText(headertxt, _charset=lcset)
header['Content-Description'] = _('Digest Header')
mimemsg.attach(header)
# RFC 1153
print >> plainmsg, headertxt
print >> plainmsg
# Now we have to cruise through all the messages accumulated in the
# mailbox file. We can't add these messages to the plainmsg and mimemsg
# yet, because we first have to calculate the table of contents
# (i.e. grok out all the Subjects). Store the messages in a list until
# we're ready for them.
#
# Meanwhile prepare things for the table of contents
toc = StringIO()
print >> toc, _("Today's Topics:\n")
# Now cruise through all the messages in the mailbox of digest messages,
# building the MIME payload and core of the RFC 1153 digest. We'll also
# accumulate Subject: headers and authors for the table-of-contents.
messages = []
msgcount = 0
msg = mbox.next()
while msg is not None:
if msg == '':
# It was an unparseable message
msg = mbox.next()
continue
msgcount += 1
messages.append(msg)
# Get the Subject header
msgsubj = msg.get('subject', _('(no subject)'))
subject = Utils.oneline(msgsubj, lcset)
# Don't include the redundant subject prefix in the toc
mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix),
subject, re.IGNORECASE)
if mo:
subject = subject[:mo.start(2)] + subject[mo.end(2):]
username = ''
addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)])
# Take only the first author we find
if isinstance(addresses, ListType) and addresses:
username = addresses[0][0]
if not username:
username = addresses[0][1]
if username:
username = '******' % username
# Put count and Wrap the toc subject line
wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65)
slines = wrapped.split('\n')
# See if the user's name can fit on the last line
if len(slines[-1]) + len(username) > 70:
slines.append(username)
else:
slines[-1] += username
# Add this subject to the accumulating topics
first = True
for line in slines:
if first:
print >> toc, ' ', line
first = False
else:
print >> toc, ' ', line.lstrip()
# We do not want all the headers of the original message to leak
# through in the digest messages. For this phase, we'll leave the
# same set of headers in both digests, i.e. those required in RFC 1153
# plus a couple of other useful ones. We also need to reorder the
# headers according to RFC 1153. Later, we'll strip out headers for
# for the specific MIME or plain digests.
keeper = {}
all_keepers = {}
for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS +
mm_cfg.PLAIN_DIGEST_KEEP_HEADERS):
all_keepers[header] = True
all_keepers = all_keepers.keys()
for keep in all_keepers:
keeper[keep] = msg.get_all(keep, [])
# Now remove all unkempt headers :)
for header in msg.keys():
del msg[header]
# And add back the kept header in the RFC 1153 designated order
for keep in all_keepers:
for field in keeper[keep]:
msg[keep] = field
# And a bit of extra stuff
msg['Message'] = ` msgcount `
# Get the next message in the digest mailbox
msg = mbox.next()
# Now we're finished with all the messages in the digest. First do some
# sanity checking and then on to adding the toc.
if msgcount == 0:
# Why did we even get here?
return
toctext = to_cset_out(toc.getvalue(), lcset)
# MIME
tocpart = MIMEText(toctext, _charset=lcset)
tocpart['Content-Description'] = _(
"Today's Topics (%(msgcount)d messages)")
mimemsg.attach(tocpart)
# RFC 1153
print >> plainmsg, toctext
print >> plainmsg
# For RFC 1153 digests, we now need the standard separator
print >> plainmsg, separator70
print >> plainmsg
# Now go through and add each message
mimedigest = MIMEBase('multipart', 'digest')
mimemsg.attach(mimedigest)
first = True
for msg in messages:
# MIME. Make a copy of the message object since the rfc1153
# processing scrubs out attachments.
mimedigest.attach(MIMEMessage(copy.deepcopy(msg)))
# rfc1153
if first:
first = False
else:
print >> plainmsg, separator30
print >> plainmsg
# Use Mailman.Handlers.Scrubber.process() to get plain text
try:
msg = scrubber(mlist, msg)
except Errors.DiscardMessage:
print >> plainmsg, _('[Message discarded by content filter]')
continue
# Honor the default setting
for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS:
if msg[h]:
uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset)))
uh = '\n\t'.join(uh.split('\n'))
print >> plainmsg, uh
print >> plainmsg
# If decoded payload is empty, this may be multipart message.
# -- just stringfy it.
payload = msg.get_payload(decode=True) \
or msg.as_string().split('\n\n',1)[1]
mcset = msg.get_content_charset('')
if mcset and mcset <> lcset and mcset <> lcset_out:
try:
payload = unicode(payload, mcset,
'replace').encode(lcset, 'replace')
except (UnicodeError, LookupError):
# TK: Message has something unknown charset.
# _out means charset in 'outer world'.
payload = unicode(payload, lcset_out,
'replace').encode(lcset, 'replace')
print >> plainmsg, payload
if not payload.endswith('\n'):
print >> plainmsg
# Now add the footer but only if more than whitespace.
if re.sub('\s', '', mlist.digest_footer):
footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
# MIME
footer = MIMEText(footertxt, _charset=lcset)
footer['Content-Description'] = _('Digest Footer')
mimemsg.attach(footer)
# RFC 1153
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print >> plainmsg, separator30
print >> plainmsg
print >> plainmsg, 'Subject: ' + _('Digest Footer')
print >> plainmsg
print >> plainmsg, footertxt
print >> plainmsg
print >> plainmsg, separator30
print >> plainmsg
# Do the last bit of stuff for each digest type
signoff = _('End of ') + digestid
# MIME
# BAW: This stuff is outside the normal MIME goo, and it's what the old
# MIME digester did. No one seemed to complain, probably because you
# won't see it in an MUA that can't display the raw message. We've never
# got complaints before, but if we do, just wax this. It's primarily
# included for (marginally useful) backwards compatibility.
mimemsg.postamble = signoff
# rfc1153
print >> plainmsg, signoff
print >> plainmsg, '*' * len(signoff)
# Do our final bit of housekeeping, and then send each message to the
# outgoing queue for delivery.
mlist.next_digest_number += 1
virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
# Calculate the recipients lists
plainrecips = []
mimerecips = []
drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys()
for user in mlist.getMemberCPAddresses(drecips):
# user might be None if someone who toggled off digest delivery
# subsequently unsubscribed from the mailing list. Also, filter out
# folks who have disabled delivery.
if user is None or mlist.getDeliveryStatus(user) <> ENABLED:
continue
# Otherwise, decide whether they get MIME or RFC 1153 digests
if mlist.getMemberOption(user, mm_cfg.DisableMime):
plainrecips.append(user)
else:
mimerecips.append(user)
# Zap this since we're now delivering the last digest to these folks.
mlist.one_last_digest.clear()
# MIME
virginq.enqueue(mimemsg,
recips=mimerecips,
listname=mlist.internal_name(),
isdigest=True)
# RFC 1153
rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset)
virginq.enqueue(rfc1153msg,
recips=plainrecips,
listname=mlist.internal_name(),
isdigest=True)
def save_attachment(mlist, msg, dir, filter_html=True):
fsdir = os.path.join(mlist.archive_dir(), dir)
makedirs(fsdir)
# Figure out the attachment type and get the decoded data
decodedpayload = msg.get_payload(decode=True)
# BAW: mimetypes ought to handle non-standard, but commonly found types,
# e.g. image/jpg (should be image/jpeg). For now we just store such
# things as application/octet-streams since that seems the safest.
ctype = msg.get_content_type()
# i18n file name is encoded
lcset = Utils.GetCharSet(mlist.preferred_language)
filename = Utils.oneline(msg.get_filename(''), lcset)
filename, fnext = os.path.splitext(filename)
# For safety, we should confirm this is valid ext for content-type
# but we can use fnext if we introduce fnext filtering
if mm_cfg.SCRUBBER_USE_ATTACHMENT_FILENAME_EXTENSION:
# HTML message doesn't have filename :-(
ext = fnext or guess_extension(ctype, fnext)
else:
ext = guess_extension(ctype, fnext)
if not ext:
# We don't know what it is, so assume it's just a shapeless
# application/octet-stream, unless the Content-Type: is
# message/rfc822, in which case we know we'll coerce the type to
# text/plain below.
if ctype == 'message/rfc822':
ext = '.txt'
else:
ext = '.bin'
# Allow only alphanumerics, dash, underscore, and dot
ext = sre.sub('', ext)
path = None
# We need a lock to calculate the next attachment number
lockfile = os.path.join(fsdir, 'attachments.lock')
lock = LockFile.LockFile(lockfile)
lock.lock()
try:
# Now base the filename on what's in the attachment, uniquifying it if
# necessary.
if not filename or mm_cfg.SCRUBBER_DONT_USE_ATTACHMENT_FILENAME:
filebase = 'attachment'
else:
# Sanitize the filename given in the message headers
parts = pre.split(filename)
filename = parts[-1]
# Strip off leading dots
filename = dre.sub('', filename)
# Allow only alphanumerics, dash, underscore, and dot
filename = sre.sub('', filename)
# If the filename's extension doesn't match the type we guessed,
# which one should we go with? For now, let's go with the one we
# guessed so attachments can't lie about their type. Also, if the
# filename /has/ no extension, then tack on the one we guessed.
# The extension was removed from the name above.
filebase = filename
# Now we're looking for a unique name for this file on the file
# system. If msgdir/filebase.ext isn't unique, we'll add a counter
# after filebase, e.g. msgdir/filebase-cnt.ext
counter = 0
extra = ''
while True:
path = os.path.join(fsdir, filebase + extra + ext)
# Generally it is not a good idea to test for file existance
# before just trying to create it, but the alternatives aren't
# wonderful (i.e. os.open(..., O_CREAT | O_EXCL) isn't
# NFS-safe). Besides, we have an exclusive lock now, so we're
# guaranteed that no other process will be racing with us.
if os.path.exists(path):
counter += 1
extra = '-%04d' % counter
else:
break
finally:
lock.unlock()
# `path' now contains the unique filename for the attachment. There's
# just one more step we need to do. If the part is text/html and
# ARCHIVE_HTML_SANITIZER is a string (which it must be or we wouldn't be
# here), then send the attachment through the filter program for
# sanitization
if filter_html and ctype == 'text/html':
base, ext = os.path.splitext(path)
tmppath = base + '-tmp' + ext
fp = open(tmppath, 'w')
try:
fp.write(decodedpayload)
fp.close()
cmd = mm_cfg.ARCHIVE_HTML_SANITIZER % {'filename' : tmppath}
progfp = os.popen(cmd, 'r')
decodedpayload = progfp.read()
status = progfp.close()
if status:
syslog('error',
'HTML sanitizer exited with non-zero status: %s',
status)
finally:
os.unlink(tmppath)
# BAW: Since we've now sanitized the document, it should be plain
# text. Blarg, we really want the sanitizer to tell us what the type
# if the return data is. :(
ext = '.txt'
path = base + '.txt'
# Is it a message/rfc822 attachment?
elif ctype == 'message/rfc822':
submsg = msg.get_payload()
# BAW: I'm sure we can eventually do better than this. :(
decodedpayload = Utils.websafe(str(submsg))
fp = open(path, 'w')
fp.write(decodedpayload)
fp.close()
# Now calculate the url
baseurl = mlist.GetBaseArchiveURL()
# Private archives will likely have a trailing slash. Normalize.
if baseurl[-1] <> '/':
baseurl += '/'
# A trailing space in url string may save users who are using
# RFC-1738 compliant MUA (Not Mozilla).
# Trailing space will definitely be a problem with format=flowed.
# Bracket the URL instead.
url = '<' + baseurl + '%s/%s%s%s>' % (dir, filebase, extra, ext)
return url
pass
# Queue the file for delivery by qrunner. Trying to deliver the
# message directly here can lead to a huge delay in web
# turnaround. Log the moderation and add a header.
msg['X-Mailman-Approved-At'] = email.Utils.formatdate(localtime=1)
syslog('vette', '%s: held message approved, message-id: %s',
self.internal_name(), msg.get('message-id', 'n/a'))
# Stick the message back in the incoming queue for further
# processing.
inq = get_switchboard(mm_cfg.INQUEUE_DIR)
inq.enqueue(msg, _metadata=msgdata)
elif value == mm_cfg.REJECT:
# Rejected
rejection = 'Refused'
lang = self.getMemberLanguage(sender)
subject = Utils.oneline(subject, Utils.GetCharSet(lang))
self.__refuse(_('Posting of your message titled "%(subject)s"'),
sender,
comment or _('[No reason given]'),
lang=lang)
else:
assert value == mm_cfg.DISCARD
# Discarded
rejection = 'Discarded'
# Forward the message
if forward and addr:
# If we've approved the message, we need to be sure to craft a
# completely unique second message for the forwarding operation,
# since we don't want to share any state or information with the
# normal delivery.
try:
def send_i18n_digests(mlist, mboxfp):
mbox = Mailbox(mboxfp)
# Prepare common information (first lang/charset)
lang = mlist.preferred_language
lcset = Utils.GetCharSet(lang)
lcset_out = Charset(lcset).output_charset or lcset
# Common Information (contd)
realname = mlist.real_name
volume = mlist.volume
issue = mlist.next_digest_number
digestid = _('%(realname)s Digest, Vol %(volume)d, Issue %(issue)d')
digestsubj = Header(digestid, lcset, header_name='Subject')
# Set things up for the MIME digest. Only headers not added by
# CookHeaders need be added here.
# Date/Message-ID should be added here also.
mimemsg = Message.Message()
mimemsg['Content-Type'] = 'multipart/mixed'
mimemsg['MIME-Version'] = '1.0'
mimemsg['From'] = mlist.GetRequestEmail()
mimemsg['Subject'] = digestsubj
mimemsg['To'] = mlist.GetListEmail()
mimemsg['Reply-To'] = mlist.GetListEmail()
mimemsg['Date'] = formatdate(localtime=1)
mimemsg['Message-ID'] = Utils.unique_message_id(mlist)
# Set things up for the rfc1153 digest
plainmsg = StringIO()
rfc1153msg = Message.Message()
rfc1153msg['From'] = mlist.GetRequestEmail()
rfc1153msg['Subject'] = digestsubj
rfc1153msg['To'] = mlist.GetListEmail()
rfc1153msg['Reply-To'] = mlist.GetListEmail()
rfc1153msg['Date'] = formatdate(localtime=1)
rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist)
separator70 = '-' * 70
separator30 = '-' * 30
# In the rfc1153 digest, the masthead contains the digest boilerplate plus
# any digest header. In the MIME digests, the masthead and digest header
# are separate MIME subobjects. In either case, it's the first thing in
# the digest, and we can calculate it now, so go ahead and add it now.
mastheadtxt = Utils.maketext(
'masthead.txt',
{'real_name' : mlist.real_name,
'got_list_email': mlist.GetListEmail(),
'got_listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
'got_request_email': mlist.GetRequestEmail(),
'got_owner_email': mlist.GetOwnerEmail(),
}, mlist=mlist)
# MIME
masthead = MIMEText(mastheadtxt, _charset=lcset)
masthead['Content-Description'] = digestid
mimemsg.attach(masthead)
# RFC 1153
print >> plainmsg, mastheadtxt
print >> plainmsg
# Now add the optional digest header but only if more than whitespace.
if re.sub('\s', '', mlist.digest_header):
headertxt = decorate(mlist, mlist.digest_header, _('digest header'))
# MIME
header = MIMEText(headertxt, _charset=lcset)
header['Content-Description'] = _('Digest Header')
mimemsg.attach(header)
# RFC 1153
print >> plainmsg, headertxt
print >> plainmsg
# Now we have to cruise through all the messages accumulated in the
# mailbox file. We can't add these messages to the plainmsg and mimemsg
# yet, because we first have to calculate the table of contents
# (i.e. grok out all the Subjects). Store the messages in a list until
# we're ready for them.
#
# Meanwhile prepare things for the table of contents
toc = StringIO()
print >> toc, _("Today's Topics:\n")
# Now cruise through all the messages in the mailbox of digest messages,
# building the MIME payload and core of the RFC 1153 digest. We'll also
# accumulate Subject: headers and authors for the table-of-contents.
messages = []
msgcount = 0
msg = mbox.next()
while msg is not None:
if msg == '':
# It was an unparseable message
msg = mbox.next()
continue
msgcount += 1
messages.append(msg)
# Get the Subject header
msgsubj = msg.get('subject', _('(no subject)'))
subject = Utils.oneline(msgsubj, lcset)
# Don't include the redundant subject prefix in the toc
mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix),
subject, re.IGNORECASE)
if mo:
subject = subject[:mo.start(2)] + subject[mo.end(2):]
username = ''
addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)])
# Take only the first author we find
if isinstance(addresses, ListType) and addresses:
username = addresses[0][0]
if not username:
username = addresses[0][1]
if username:
username = '******' % username
# Put count and Wrap the toc subject line
wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65)
slines = wrapped.split('\n')
# See if the user's name can fit on the last line
if len(slines[-1]) + len(username) > 70:
slines.append(username)
else:
slines[-1] += username
# Add this subject to the accumulating topics
first = True
for line in slines:
if first:
print >> toc, ' ', line
first = False
else:
print >> toc, ' ', line.lstrip()
# We do not want all the headers of the original message to leak
# through in the digest messages. For this phase, we'll leave the
# same set of headers in both digests, i.e. those required in RFC 1153
# plus a couple of other useful ones. We also need to reorder the
# headers according to RFC 1153. Later, we'll strip out headers for
# for the specific MIME or plain digests.
keeper = {}
all_keepers = {}
for header in (mm_cfg.MIME_DIGEST_KEEP_HEADERS +
mm_cfg.PLAIN_DIGEST_KEEP_HEADERS):
all_keepers[header] = True
all_keepers = all_keepers.keys()
for keep in all_keepers:
keeper[keep] = msg.get_all(keep, [])
# Now remove all unkempt headers :)
for header in msg.keys():
del msg[header]
# And add back the kept header in the RFC 1153 designated order
for keep in all_keepers:
for field in keeper[keep]:
msg[keep] = field
# And a bit of extra stuff
msg['Message'] = `msgcount`
# Get the next message in the digest mailbox
msg = mbox.next()
# Now we're finished with all the messages in the digest. First do some
# sanity checking and then on to adding the toc.
if msgcount == 0:
# Why did we even get here?
return
toctext = to_cset_out(toc.getvalue(), lcset)
# MIME
tocpart = MIMEText(toctext, _charset=lcset)
tocpart['Content-Description']= _("Today's Topics (%(msgcount)d messages)")
mimemsg.attach(tocpart)
# RFC 1153
print >> plainmsg, toctext
print >> plainmsg
# For RFC 1153 digests, we now need the standard separator
print >> plainmsg, separator70
print >> plainmsg
# Now go through and add each message
mimedigest = MIMEBase('multipart', 'digest')
mimemsg.attach(mimedigest)
first = True
for msg in messages:
# MIME. Make a copy of the message object since the rfc1153
# processing scrubs out attachments.
mimedigest.attach(MIMEMessage(copy.deepcopy(msg)))
# rfc1153
if first:
first = False
else:
print >> plainmsg, separator30
print >> plainmsg
# Use Mailman.Handlers.Scrubber.process() to get plain text
try:
msg = scrubber(mlist, msg)
except Errors.DiscardMessage:
print >> plainmsg, _('[Message discarded by content filter]')
continue
# Honor the default setting
for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS:
if msg[h]:
uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset)))
uh = '\n\t'.join(uh.split('\n'))
print >> plainmsg, uh
print >> plainmsg
# If decoded payload is empty, this may be multipart message.
# -- just stringfy it.
payload = msg.get_payload(decode=True) \
or msg.as_string().split('\n\n',1)[1]
mcset = msg.get_content_charset('')
if mcset and mcset <> lcset and mcset <> lcset_out:
try:
payload = unicode(payload, mcset, 'replace'
).encode(lcset, 'replace')
except (UnicodeError, LookupError):
# TK: Message has something unknown charset.
# _out means charset in 'outer world'.
payload = unicode(payload, lcset_out, 'replace'
).encode(lcset, 'replace')
print >> plainmsg, payload
if not payload.endswith('\n'):
print >> plainmsg
# Now add the footer but only if more than whitespace.
if re.sub('\s', '', mlist.digest_footer):
footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
# MIME
footer = MIMEText(footertxt, _charset=lcset)
footer['Content-Description'] = _('Digest Footer')
mimemsg.attach(footer)
# RFC 1153
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print >> plainmsg, separator30
print >> plainmsg
print >> plainmsg, 'Subject: ' + _('Digest Footer')
print >> plainmsg
print >> plainmsg, footertxt
print >> plainmsg
print >> plainmsg, separator30
print >> plainmsg
# Do the last bit of stuff for each digest type
signoff = _('End of ') + digestid
# MIME
# BAW: This stuff is outside the normal MIME goo, and it's what the old
# MIME digester did. No one seemed to complain, probably because you
# won't see it in an MUA that can't display the raw message. We've never
# got complaints before, but if we do, just wax this. It's primarily
# included for (marginally useful) backwards compatibility.
mimemsg.postamble = signoff
# rfc1153
print >> plainmsg, signoff
print >> plainmsg, '*' * len(signoff)
# Do our final bit of housekeeping, and then send each message to the
# outgoing queue for delivery.
mlist.next_digest_number += 1
virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
# Calculate the recipients lists
plainrecips = []
mimerecips = []
drecips = mlist.getDigestMemberKeys() + mlist.one_last_digest.keys()
for user in mlist.getMemberCPAddresses(drecips):
# user might be None if someone who toggled off digest delivery
# subsequently unsubscribed from the mailing list. Also, filter out
# folks who have disabled delivery.
if user is None or mlist.getDeliveryStatus(user) <> ENABLED:
continue
# Otherwise, decide whether they get MIME or RFC 1153 digests
if mlist.getMemberOption(user, mm_cfg.DisableMime):
plainrecips.append(user)
else:
mimerecips.append(user)
# Zap this since we're now delivering the last digest to these folks.
mlist.one_last_digest.clear()
# MIME
virginq.enqueue(mimemsg,
recips=mimerecips,
listname=mlist.internal_name(),
isdigest=True)
# RFC 1153
rfc1153msg.set_payload(to_cset_out(plainmsg.getvalue(), lcset), lcset)
virginq.enqueue(rfc1153msg,
recips=plainrecips,
listname=mlist.internal_name(),
isdigest=True)
def show_helds_overview(mlist, form):
# Sort the held messages by sender
bysender = helds_by_sender(mlist)
if not bysender:
return 0
form.AddItem('<hr>')
form.AddItem(Center(Header(2, _('Held Messages'))))
# Add the by-sender overview tables
admindburl = mlist.GetScriptURL('admindb', absolute=1)
table = Table(border=0)
form.AddItem(table)
senders = bysender.keys()
senders.sort()
for sender in senders:
qsender = quote_plus(sender)
esender = Utils.websafe(sender)
senderurl = admindburl + '?sender=' + qsender
# The encompassing sender table
stable = Table(border=1)
stable.AddRow([Center(Bold(_('From:')).Format() + esender)])
stable.AddCellInfo(stable.GetCurrentRowIndex(), 0, colspan=2)
left = Table(border=0)
left.AddRow([_('Action to take on all these held messages:')])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
btns = hacky_radio_buttons(
'senderaction-' + qsender,
(_('Defer'), _('Accept'), _('Reject'), _('Discard')),
(mm_cfg.DEFER, mm_cfg.APPROVE, mm_cfg.REJECT, mm_cfg.DISCARD),
(1, 0, 0, 0))
left.AddRow([btns])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
left.AddRow([
CheckBox('senderpreserve-' + qsender, 1).Format() + ' ' +
_('Preserve messages for the site administrator')
])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
left.AddRow([
CheckBox('senderforward-' + qsender, 1).Format() + ' ' +
_('Forward messages (individually) to:')
])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
left.AddRow([
TextBox('senderforwardto-' + qsender, value=mlist.GetOwnerEmail())
])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
# If the sender is a member and the message is being held due to a
# moderation bit, give the admin a chance to clear the member's mod
# bit. If this sender is not a member and is not already on one of
# the sender filters, then give the admin a chance to add this sender
# to one of the filters.
if mlist.isMember(sender):
if mlist.getMemberOption(sender, mm_cfg.Moderate):
left.AddRow([
CheckBox('senderclearmodp-' + qsender, 1).Format() +
' ' + _("Clear this member's <em>moderate</em> flag")
])
else:
left.AddRow(
[_('<em>The sender is now a member of this list</em>')])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
elif sender not in (mlist.accept_these_nonmembers +
mlist.hold_these_nonmembers +
mlist.reject_these_nonmembers +
mlist.discard_these_nonmembers):
left.AddRow([
CheckBox('senderfilterp-' + qsender, 1).Format() + ' ' +
_('Add <b>%(esender)s</b> to one of these sender filters:')
])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
btns = hacky_radio_buttons(
'senderfilter-' + qsender,
(_('Accepts'), _('Holds'), _('Rejects'), _('Discards')),
(mm_cfg.ACCEPT, mm_cfg.HOLD, mm_cfg.REJECT, mm_cfg.DISCARD),
(0, 0, 0, 1))
left.AddRow([btns])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
if sender not in mlist.ban_list:
left.AddRow([
CheckBox('senderbanp-' + qsender, 1).Format() + ' ' +
_("""Ban <b>%(esender)s</b> from ever subscribing to this
mailing list""")
])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
right = Table(border=0)
right.AddRow([
_("""Click on the message number to view the individual
message, or you can """) +
Link(senderurl, _('view all messages from %(esender)s')).Format()
])
right.AddCellInfo(right.GetCurrentRowIndex(), 0, colspan=2)
right.AddRow([' ', ' '])
counter = 1
for id in bysender[sender]:
info = mlist.GetRecord(id)
ptime, sender, subject, reason, filename, msgdata = info
# BAW: This is really the size of the message pickle, which should
# be close, but won't be exact. Sigh, good enough.
try:
size = os.path.getsize(os.path.join(mm_cfg.DATA_DIR, filename))
except OSError, e:
if e.errno <> errno.ENOENT: raise
# This message must have gotten lost, i.e. it's already been
# handled by the time we got here.
mlist.HandleRequest(id, mm_cfg.DISCARD)
continue
dispsubj = Utils.oneline(
subject, Utils.GetCharSet(mlist.preferred_language))
t = Table(border=0)
t.AddRow([
Link(admindburl + '?msgid=%d' % id, '[%d]' % counter),
Bold(_('Subject:')),
Utils.websafe(dispsubj)
])
t.AddRow([' ', Bold(_('Size:')), str(size) + _(' bytes')])
if reason:
reason = _(reason)
else:
reason = _('not available')
t.AddRow([' ', Bold(_('Reason:')), reason])
# Include the date we received the message, if available
when = msgdata.get('received_time')
if when:
t.AddRow([' ', Bold(_('Received:')), time.ctime(when)])
t.AddRow([InputObj(qsender, 'hidden', str(id), False).Format()])
counter += 1
right.AddRow([t])
stable.AddRow([left, right])
table.AddRow([stable])
def _decode(h):
if not h:
return h
return Utils.oneline(h, Utils.GetCharSet(mlist.preferred_language))
def save_attachment(mlist, msg, dir, filter_html=True):
fsdir = os.path.join(mlist.archive_dir(), dir)
makedirs(fsdir)
# Figure out the attachment type and get the decoded data
decodedpayload = msg.get_payload(decode=True)
# BAW: mimetypes ought to handle non-standard, but commonly found types,
# e.g. image/jpg (should be image/jpeg). For now we just store such
# things as application/octet-streams since that seems the safest.
ctype = msg.get_content_type()
# i18n file name is encoded
lcset = Utils.GetCharSet(mlist.preferred_language)
filename = Utils.oneline(msg.get_filename(''), lcset)
filename, fnext = os.path.splitext(filename)
# For safety, we should confirm this is valid ext for content-type
# but we can use fnext if we introduce fnext filtering
if mm_cfg.SCRUBBER_USE_ATTACHMENT_FILENAME_EXTENSION:
# HTML message doesn't have filename :-(
ext = fnext or guess_extension(ctype, fnext)
else:
ext = guess_extension(ctype, fnext)
if not ext:
# We don't know what it is, so assume it's just a shapeless
# application/octet-stream, unless the Content-Type: is
# message/rfc822, in which case we know we'll coerce the type to
# text/plain below.
if ctype == 'message/rfc822':
ext = '.txt'
else:
ext = '.bin'
# Allow only alphanumerics, dash, underscore, and dot
ext = sre.sub('', ext)
path = None
# We need a lock to calculate the next attachment number
lockfile = os.path.join(fsdir, 'attachments.lock')
lock = LockFile.LockFile(lockfile)
lock.lock()
try:
# Now base the filename on what's in the attachment, uniquifying it if
# necessary.
if not filename or mm_cfg.SCRUBBER_DONT_USE_ATTACHMENT_FILENAME:
filebase = 'attachment'
else:
# Sanitize the filename given in the message headers
parts = pre.split(filename)
filename = parts[-1]
# Strip off leading dots
filename = dre.sub('', filename)
# Allow only alphanumerics, dash, underscore, and dot
filename = sre.sub('', filename)
# If the filename's extension doesn't match the type we guessed,
# which one should we go with? For now, let's go with the one we
# guessed so attachments can't lie about their type. Also, if the
# filename /has/ no extension, then tack on the one we guessed.
# The extension was removed from the name above.
filebase = filename
# Now we're looking for a unique name for this file on the file
# system. If msgdir/filebase.ext isn't unique, we'll add a counter
# after filebase, e.g. msgdir/filebase-cnt.ext
counter = 0
extra = ''
while True:
path = os.path.join(fsdir, filebase + extra + ext)
# Generally it is not a good idea to test for file existance
# before just trying to create it, but the alternatives aren't
# wonderful (i.e. os.open(..., O_CREAT | O_EXCL) isn't
# NFS-safe). Besides, we have an exclusive lock now, so we're
# guaranteed that no other process will be racing with us.
if os.path.exists(path):
counter += 1
extra = '-%04d' % counter
else:
break
finally:
lock.unlock()
# `path' now contains the unique filename for the attachment. There's
# just one more step we need to do. If the part is text/html and
# ARCHIVE_HTML_SANITIZER is a string (which it must be or we wouldn't be
# here), then send the attachment through the filter program for
# sanitization
if filter_html and ctype == 'text/html':
base, ext = os.path.splitext(path)
tmppath = base + '-tmp' + ext
fp = open(tmppath, 'wb')
try:
fp.write(decodedpayload)
fp.close()
cmd = mm_cfg.ARCHIVE_HTML_SANITIZER % {'filename': tmppath}
progfp = os.popen(cmd, 'r')
decodedpayload = progfp.read()
status = progfp.close()
if status:
syslog('error',
'HTML sanitizer exited with non-zero status: %s',
status)
finally:
os.unlink(tmppath)
# BAW: Since we've now sanitized the document, it should be plain
# text. Blarg, we really want the sanitizer to tell us what the type
# if the return data is. :(
ext = '.txt'
path = base + '.txt'
# Is it a message/rfc822 attachment?
elif ctype == 'message/rfc822':
submsg = msg.get_payload()
# BAW: I'm sure we can eventually do better than this. :(
decodedpayload = Utils.websafe(str(submsg))
fp = open(path, 'wb')
fp.write(decodedpayload)
fp.close()
# Now calculate the url
baseurl = mlist.GetBaseArchiveURL()
# Private archives will likely have a trailing slash. Normalize.
if baseurl[-1] != '/':
baseurl += '/'
# A trailing space in url string may save users who are using
# RFC-1738 compliant MUA (Not Mozilla).
# Trailing space will definitely be a problem with format=flowed.
# Bracket the URL instead.
url = '<' + baseurl + '%s/%s%s%s>' % (dir, filebase, extra, ext)
return url
lcset = Utils.GetCharSet(mlist.preferred_language)
if mcset <> lcset:
try:
body = unicode(body, mcset).encode(lcset, 'replace')
except (LookupError, UnicodeError, ValueError):
pass
hdrtxt = NL.join(['%s: %s' % (k, v) for k, v in msg.items()])
hdrtxt = Utils.websafe(hdrtxt)
# Okay, we've reconstituted the message just fine. Now for the fun part!
t = Table(cellspacing=0, cellpadding=0, width='100%')
t.AddRow([Bold(_('From:')), sender])
row, col = t.GetCurrentRowIndex(), t.GetCurrentCellIndex()
t.AddCellInfo(row, col - 1, align='right')
t.AddRow(
[Bold(_('Subject:')),
Utils.websafe(Utils.oneline(subject, lcset))])
t.AddCellInfo(row + 1, col - 1, align='right')
t.AddRow([Bold(_('Reason:')), _(reason)])
t.AddCellInfo(row + 2, col - 1, align='right')
when = msgdata.get('received_time')
if when:
t.AddRow([Bold(_('Received:')), time.ctime(when)])
t.AddCellInfo(row + 2, col - 1, align='right')
# We can't use a RadioButtonArray here because horizontal placement can be
# confusing to the user and vertical placement takes up too much
# real-estate. This is a hack!
buttons = Table(cellspacing="5", cellpadding="0")
buttons.AddRow(
map(lambda x, s=' ' * 5: s + x + s,
(_('Defer'), _('Approve'), _('Reject'), _('Discard'))))
buttons.AddRow([
def get_attachment_fname(mlist, msg):
# i18n file name is encoded
lcset = Utils.GetCharSet(mlist.preferred_language)
filename = Utils.oneline(msg.get_filename(''), lcset)
# filename can be 'str' or unicode
return remove_accents(filename).encode('ascii')
def process(mlist, msg, msgdata=None):
sanitize = mm_cfg.ARCHIVE_HTML_SANITIZER
outer = True
in_pipeline = False
patches = None
sigs = None
if msgdata is None:
msgdata = {}
if msgdata:
# msgdata is available if it is in GLOBAL_PIPELINE
# ie. not in digest or archiver
# check if the list owner want to scrub regular delivery
# Disabled - function split, attachments saved in pipeline,
# if not mlist.scrub_nondigest:
# return
in_pipeline = True
patches = []
sigs = []
dir = calculate_attachments_dir(mlist, msg, msgdata)
charset = None
lcset = Utils.GetCharSet(mlist.preferred_language)
lcset_out = Charset(lcset).output_charset or lcset
# Now walk over all subparts of this message and scrub out various types
format = delsp = None
for part in msg.walk():
ctype = part.get_content_type()
# If the part is text/plain, we leave it alone
if ctype == "text/plain":
# We need to choose a charset for the scrubbed message, so we'll
# arbitrarily pick the charset of the first text/plain part in the
# message.
# MAS: Also get the RFC 3676 stuff from this part. This seems to
# work OK for scrub_nondigest. It will also work as far as
# scrubbing messages for the archive is concerned, but pipermail
# doesn't pay any attention to the RFC 3676 parameters. The plain
# format digest is going to be a disaster in any case as some of
# messages will be format="flowed" and some not. ToDigest creates
# its own Content-Type: header for the plain digest which won't
# have RFC 3676 parameters. If the message Content-Type: headers
# are retained for display in the digest, the parameters will be
# there for information, but not for the MUA. This is the best we
# can do without having get_payload() process the parameters.
if charset is None:
charset = part.get_content_charset(lcset)
format = part.get_param("format")
delsp = part.get_param("delsp")
# TK: if part is attached then check charset and scrub if none
if part.get("content-disposition") and not part.get_content_charset():
if in_pipeline:
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir, patches=patches, sigs=sigs)
part[mm_cfg.SCRUBBER_URL_HEADER] = url
finally:
os.umask(omask)
else:
url = "<" + part.get(mm_cfg.SCRUBBER_URL_HEADER, "N/A") + ">"
filename = part.get_filename(_("not available"))
filename = Utils.oneline(filename, lcset)
replace_payload_by_text(
part,
_(
"""\
An embedded and charset-unspecified text was scrubbed...
Name: %(filename)s
URL: %(url)s
"""
),
lcset,
)
elif mm_cfg.SCRUBBER_ARCHIVE_ALL_TEXT:
if in_pipeline:
# clearsigned or attached plaintext that will be shown, still archive the copy
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir, patches=patches, sigs=sigs)
part[mm_cfg.SCRUBBER_URL_HEADER] = url
finally:
os.umask(omask)
elif outer and not msg.is_multipart():
# whole email is only one plaintext. add URL here
payload = msg.get_payload(decode=True)
del msg["content-type"]
del msg["content-transfer-encoding"]
payload = "URL: <" + part.get(mm_cfg.SCRUBBER_URL_HEADER, "N/A") + ">\n" + payload
msg.set_payload(payload)
elif ctype == "text/html" and isinstance(sanitize, IntType):
if sanitize == 0:
if outer:
raise DiscardMessage
replace_payload_by_text(
part,
_("HTML attachment scrubbed and removed"),
# Adding charset arg and removing content-type
# sets content-type to text/plain
lcset,
)
elif sanitize == 2:
# By leaving it alone, Pipermail will automatically escape it
pass
elif sanitize == 3:
if in_pipeline:
# Pull it out as an attachment but leave it unescaped. This
# is dangerous, but perhaps useful for heavily moderated
# lists.
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir, filter_html=False, patches=patches, sigs=sigs)
part[mm_cfg.SCRUBBER_URL_HEADER] = url
finally:
os.umask(omask)
else:
url = "<" + part.get(mm_cfg.SCRUBBER_URL_HEADER, "N/A") + ">"
replace_payload_by_text(
part,
_(
"""\
An HTML attachment was scrubbed...
URL: %(url)s
"""
),
lcset,
)
else:
if in_pipeline:
# TODO if in_pipeline should preserve original attachment. but no biggie
# HTML-escape it and store it as an attachment, but make it
# look a /little/ bit prettier. :(
payload = Utils.websafe(part.get_payload(decode=True))
# For whitespace in the margin, change spaces into
# non-breaking spaces, and tabs into 8 of those. Then use a
# mono-space font. Still looks hideous to me, but then I'd
# just as soon discard them.
def doreplace(s):
return s.expandtabs(8).replace(" ", " ")
lines = [doreplace(s) for s in payload.split("\n")]
payload = "<tt>\n" + BR.join(lines) + "\n</tt>\n"
part.set_payload(payload)
# We're replacing the payload with the decoded payload so this
# will just get in the way.
del part["content-transfer-encoding"]
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir, filter_html=False, patches=patches, sigs=sigs)
part[mm_cfg.SCRUBBER_URL_HEADER] = url
finally:
os.umask(omask)
else:
url = "<" + part.get(mm_cfg.SCRUBBER_URL_HEADER, "N/A") + ">"
replace_payload_by_text(
part,
_(
"""\
An HTML attachment was scrubbed...
URL: %(url)s
"""
),
lcset,
)
elif ctype == "message/rfc822":
if in_pipeline:
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir)
part[mm_cfg.SCRUBBER_URL_HEADER] = url
finally:
os.umask(omask)
else:
# This part contains a submessage, so it too needs scrubbing
submsg = part.get_payload(0)
url = "<" + part.get(mm_cfg.SCRUBBER_URL_HEADER, "N/A") + ">"
subject = submsg.get("subject", _("no subject"))
subject = Utils.oneline(subject, lcset)
date = submsg.get("date", _("no date"))
who = submsg.get("from", _("unknown sender"))
size = len(str(submsg))
replace_payload_by_text(
part,
_(
"""\
An embedded message was scrubbed...
From: %(who)s
Subject: %(subject)s
Date: %(date)s
Size: %(size)s
URL: %(url)s
"""
),
lcset,
)
# If the message isn't a multipart, then we'll strip it out as an
# attachment that would have to be separately downloaded. Pipermail
# will transform the url into a hyperlink.
elif part.get_payload() and not part.is_multipart():
payload = part.get_payload(decode=True)
# XXX Under email 2.5, it is possible that payload will be None.
# This can happen when you have a Content-Type: multipart/* with
# only one part and that part has two blank lines between the
# first boundary and the end boundary. In email 3.0 you end up
# with a string in the payload. I think in this case it's safe to
# ignore the part.
if payload is None or payload.strip() == "":
continue
if in_pipeline:
omask = os.umask(002)
try:
url = save_attachment(mlist, part, dir, patches=patches, sigs=sigs)
part[mm_cfg.SCRUBBER_URL_HEADER] = url
finally:
os.umask(omask)
else:
ctype = part.get_content_type()
size = len(payload)
url = "<" + part.get(mm_cfg.SCRUBBER_URL_HEADER, "N/A") + ">"
desc = part.get("content-description", _("not available"))
desc = Utils.oneline(desc, lcset)
filename = part.get_filename(_("not available"))
filename = Utils.oneline(filename, lcset)
replace_payload_by_text(
part,
_(
"""\
A non-text attachment was scrubbed...
Name: %(filename)s
Type: %(ctype)s
Size: %(size)d bytes
Desc: %(desc)s
URL: %(url)s
"""
),
lcset,
)
outer = False
# We still have to sanitize multipart messages to flat text because
# Pipermail can't handle messages with list payloads. This is a kludge;
# def (n) clever hack ;).
if msg.is_multipart() and not in_pipeline:
# By default we take the charset of the first text/plain part in the
# message, but if there was none, we'll use the list's preferred
# language's charset.
if not charset or charset == "us-ascii":
charset = lcset_out
else:
# normalize to the output charset if input/output are different
charset = Charset(charset).output_charset or charset
# We now want to concatenate all the parts which have been scrubbed to
# text/plain, into a single text/plain payload. We need to make sure
# all the characters in the concatenated string are in the same
# encoding, so we'll use the 'replace' key in the coercion call.
# BAW: Martin's original patch suggested we might want to try
# generalizing to utf-8, and that's probably a good idea (eventually).
text = []
for part in msg.walk():
# TK: bug-id 1099138 and multipart
# MAS test payload - if part may fail if there are no headers.
if not part.get_payload() or part.is_multipart():
continue
# All parts should be scrubbed to text/plain by now, except
# if sanitize == 2, there could be text/html parts so keep them
# but skip any other parts.
partctype = part.get_content_type()
if partctype <> "text/plain" and (partctype <> "text/html" or sanitize <> 2):
text.append(_("Skipped content of type %(partctype)s\n"))
continue
try:
t = part.get_payload(decode=True) or ""
# MAS: TypeError exception can occur if payload is None. This
# was observed with a message that contained an attached
# message/delivery-status part. Because of the special parsing
# of this type, this resulted in a text/plain sub-part with a
# null body. See bug 1430236.
except (binascii.Error, TypeError):
t = part.get_payload() or ""
# TK: get_content_charset() returns 'iso-2022-jp' for internally
# crafted (scrubbed) 'euc-jp' text part. So, first try
# get_charset(), then get_content_charset() for the parts
# which are already embeded in the incoming message.
partcharset = part.get_charset()
if partcharset:
partcharset = str(partcharset)
else:
partcharset = part.get_content_charset()
if partcharset and partcharset <> charset:
try:
t = unicode(t, partcharset, "replace")
except (UnicodeError, LookupError, ValueError, AssertionError):
# We can get here if partcharset is bogus in come way.
# Replace funny characters. We use errors='replace'
t = unicode(t, "ascii", "replace")
try:
# Should use HTML-Escape, or try generalizing to UTF-8
t = t.encode(charset, "replace")
except (UnicodeError, LookupError, ValueError, AssertionError):
# if the message charset is bogus, use the list's.
t = t.encode(lcset, "replace")
# Separation is useful
if isinstance(t, StringType):
# omit empty parts
if t.strip(" \t\r\n") != "":
if not t.endswith("\n"):
t += "\n"
if mm_cfg.SCRUBBER_ARCHIVE_ALL_TEXT:
# Add link to archived part if it wasn't archived already
url = "URL: <" + part.get(mm_cfg.SCRUBBER_URL_HEADER, "N/A") + ">\n"
if not t.endswith(url):
t = url + t
filename = part.get_filename()
if filename:
filename = Utils.oneline(filename, lcset)
t = "Name: " + filename + "\n" + t
text.append(t)
# Now join the text and set the payload
sep = _("-------------- next part --------------\n")
# The i18n separator is in the list's charset. Coerce it to the
# message charset.
try:
s = unicode(sep, lcset, "replace")
sep = s.encode(charset, "replace")
except (UnicodeError, LookupError, ValueError, AssertionError):
pass
replace_payload_by_text(msg, sep.join(text), charset)
if format:
msg.set_param("Format", format)
if delsp:
msg.set_param("DelSp", delsp)
if in_pipeline:
(patches, sigs) = process_signatures(mlist, patches, sigs)
if patches:
# X-Patches-Received: PatchID1=filename; PatchID2=filename
processed = {}
for p in patches:
processed[p["id"]] = p["file"]
msg.add_header("X-Patches-Received", None, **processed)
if sigs:
# output header in RFC compliant way, if multiple sigs with multiple keys per one patch, then delimited by dot, like:
# X-Sigs-Received: PatchID1=KeyID1; PatchID2=KeyID2.KeyID3
processed = {}
for s in sigs:
processed[s["phash"]] = processed.get(s["phash"], []) + [s["key"]]
for k in processed.keys():
# dot does not need escaping
processed[k] = ".".join(processed[k])
msg.add_header("X-Sigs-Received", None, **processed)
return msg
def __handlepost(self, record, value, comment, preserve, forward, addr):
# For backwards compatibility with pre 2.0beta3
ptime, sender, subject, reason, filename, msgdata = record
path = os.path.join(mm_cfg.DATA_DIR, filename)
# Handle message preservation
if preserve:
parts = os.path.split(path)[1].split(DASH)
parts[0] = 'spam'
spamfile = DASH.join(parts)
# Preserve the message as plain text, not as a pickle
try:
fp = open(path)
except IOError as e:
if e.errno != errno.ENOENT: raise
return LOST
try:
if path.endswith('.pck'):
msg = pickle.load(fp)
else:
assert path.endswith('.txt'), '%s not .pck or .txt' % path
msg = fp.read()
finally:
fp.close()
# Save the plain text to a .msg file, not a .pck file
outpath = os.path.join(mm_cfg.SPAM_DIR, spamfile)
head, ext = os.path.splitext(outpath)
outpath = head + '.msg'
outfp = open(outpath, 'wb')
try:
if path.endswith('.pck'):
g = Generator(outfp)
g.flatten(msg, 1)
else:
outfp.write(msg)
finally:
outfp.close()
# Now handle updates to the database
rejection = None
fp = None
msg = None
status = REMOVE
if value == mm_cfg.DEFER:
# Defer
status = DEFER
elif value == mm_cfg.APPROVE:
# Approved.
try:
msg = readMessage(path)
except IOError as e:
if e.errno != errno.ENOENT: raise
return LOST
msg = readMessage(path)
msgdata['approved'] = 1
# adminapproved is used by the Emergency handler
msgdata['adminapproved'] = 1
# Calculate a new filebase for the approved message, otherwise
# delivery errors will cause duplicates.
try:
del msgdata['filebase']
except KeyError:
pass
# Queue the file for delivery by qrunner. Trying to deliver the
# message directly here can lead to a huge delay in web
# turnaround. Log the moderation and add a header.
msg['X-Mailman-Approved-At'] = email.utils.formatdate(localtime=1)
syslog('vette', '%s: held message approved, message-id: %s',
self.internal_name(), msg.get('message-id', 'n/a'))
# Stick the message back in the incoming queue for further
# processing.
inq = get_switchboard(mm_cfg.INQUEUE_DIR)
inq.enqueue(msg, _metadata=msgdata)
elif value == mm_cfg.REJECT:
# Rejected
rejection = 'Refused'
lang = self.getMemberLanguage(sender)
subject = Utils.oneline(subject, Utils.GetCharSet(lang))
self.__refuse(_('Posting of your message titled "%(subject)s"'),
sender,
comment or _('[No reason given]'),
lang=lang)
else:
assert value == mm_cfg.DISCARD
# Discarded
rejection = 'Discarded'
# Forward the message
if forward and addr:
# If we've approved the message, we need to be sure to craft a
# completely unique second message for the forwarding operation,
# since we don't want to share any state or information with the
# normal delivery.
try:
copy = readMessage(path)
except IOError as e:
if e.errno != errno.ENOENT: raise
raise Errors.LostHeldMessage(path)
# It's possible the addr is a comma separated list of addresses.
addrs = getaddresses([addr])
if len(addrs) == 1:
realname, addr = addrs[0]
# If the address getting the forwarded message is a member of
# the list, we want the headers of the outer message to be
# encoded in their language. Otherwise it'll be the preferred
# language of the mailing list.
lang = self.getMemberLanguage(addr)
else:
# Throw away the realnames
addr = [a for realname, a in addrs]
# Which member language do we attempt to use? We could use
# the first match or the first address, but in the face of
# ambiguity, let's just use the list's preferred language
lang = self.preferred_language
otrans = i18n.get_translation()
i18n.set_language(lang)
try:
fmsg = Message.UserNotification(
addr,
self.GetBouncesEmail(),
_('Forward of moderated message'),
lang=lang)
finally:
i18n.set_translation(otrans)
fmsg.set_type('message/rfc822')
fmsg.attach(copy)
fmsg.send(self)
# Log the rejection
if rejection:
note = '''%(listname)s: %(rejection)s posting:
tFrom: %(sender)s
tSubject: %(subject)s''' % {
'listname': self.internal_name(),
'rejection': rejection,
'sender': str(sender).replace('%', '%%'),
'subject': str(subject).replace('%', '%%'),
}
if comment:
note += '\n\tReason: ' + comment.replace('%', '%%')
syslog('vette', note)
# Always unlink the file containing the message text. It's not
# necessary anymore, regardless of the disposition of the message.
if status != DEFER:
try:
os.unlink(path)
except OSError as e:
if e.errno != errno.ENOENT: raise
# We lost the message text file. Clean up our housekeeping
# and inform of this status.
return LOST
return status
def show_helds_overview(mlist, form):
# Sort the held messages by sender
bysender = helds_by_sender(mlist)
if not bysender:
return 0
form.AddItem('<hr>')
form.AddItem(Center(Header(2, _('Held Messages'))))
# Add the by-sender overview tables
admindburl = mlist.GetScriptURL('admindb', absolute=1)
table = Table(border=0)
form.AddItem(table)
senders = bysender.keys()
senders.sort()
for sender in senders:
qsender = quote_plus(sender)
esender = Utils.websafe(sender)
senderurl = admindburl + '?sender=' + qsender
# The encompassing sender table
stable = Table(border=1)
stable.AddRow([Center(Bold(_('From:')).Format() + esender)])
stable.AddCellInfo(stable.GetCurrentRowIndex(), 0, colspan=2)
left = Table(border=0)
left.AddRow([_('Action to take on all these held messages:')])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
btns = hacky_radio_buttons(
'senderaction-' + qsender,
(_('Defer'), _('Accept'), _('Reject'), _('Discard')),
(mm_cfg.DEFER, mm_cfg.APPROVE, mm_cfg.REJECT, mm_cfg.DISCARD),
(1, 0, 0, 0))
left.AddRow([btns])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
left.AddRow([
CheckBox('senderpreserve-' + qsender, 1).Format() +
' ' +
_('Preserve messages for the site administrator')
])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
left.AddRow([
CheckBox('senderforward-' + qsender, 1).Format() +
' ' +
_('Forward messages (individually) to:')
])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
left.AddRow([
TextBox('senderforwardto-' + qsender,
value=mlist.GetOwnerEmail())
])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
# If the sender is a member and the message is being held due to a
# moderation bit, give the admin a chance to clear the member's mod
# bit. If this sender is not a member and is not already on one of
# the sender filters, then give the admin a chance to add this sender
# to one of the filters.
if mlist.isMember(sender):
if mlist.getMemberOption(sender, mm_cfg.Moderate):
left.AddRow([
CheckBox('senderclearmodp-' + qsender, 1).Format() +
' ' +
_("Clear this member's <em>moderate</em> flag")
])
else:
left.AddRow(
[_('<em>The sender is now a member of this list</em>')])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
elif sender not in (mlist.accept_these_nonmembers +
mlist.hold_these_nonmembers +
mlist.reject_these_nonmembers +
mlist.discard_these_nonmembers):
left.AddRow([
CheckBox('senderfilterp-' + qsender, 1).Format() +
' ' +
_('Add <b>%(esender)s</b> to one of these sender filters:')
])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
btns = hacky_radio_buttons(
'senderfilter-' + qsender,
(_('Accepts'), _('Holds'), _('Rejects'), _('Discards')),
(mm_cfg.ACCEPT, mm_cfg.HOLD, mm_cfg.REJECT, mm_cfg.DISCARD),
(0, 0, 0, 1))
left.AddRow([btns])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
if sender not in mlist.ban_list:
left.AddRow([
CheckBox('senderbanp-' + qsender, 1).Format() +
' ' +
_("""Ban <b>%(esender)s</b> from ever subscribing to this
mailing list""")])
left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
right = Table(border=0)
right.AddRow([
_("""Click on the message number to view the individual
message, or you can """) +
Link(senderurl, _('view all messages from %(esender)s')).Format()
])
right.AddCellInfo(right.GetCurrentRowIndex(), 0, colspan=2)
right.AddRow([' ', ' '])
counter = 1
for id in bysender[sender]:
info = mlist.GetRecord(id)
ptime, sender, subject, reason, filename, msgdata = info
# BAW: This is really the size of the message pickle, which should
# be close, but won't be exact. Sigh, good enough.
try:
size = os.path.getsize(os.path.join(mm_cfg.DATA_DIR, filename))
except OSError, e:
if e.errno <> errno.ENOENT: raise
# This message must have gotten lost, i.e. it's already been
# handled by the time we got here.
mlist.HandleRequest(id, mm_cfg.DISCARD)
continue
dispsubj = Utils.oneline(
subject, Utils.GetCharSet(mlist.preferred_language))
t = Table(border=0)
t.AddRow([Link(admindburl + '?msgid=%d' % id, '[%d]' % counter),
Bold(_('Subject:')),
Utils.websafe(dispsubj)
])
t.AddRow([' ', Bold(_('Size:')), str(size) + _(' bytes')])
if reason:
reason = _(reason)
else:
reason = _('not available')
t.AddRow([' ', Bold(_('Reason:')), reason])
# Include the date we received the message, if available
when = msgdata.get('received_time')
if when:
t.AddRow([' ', Bold(_('Received:')),
time.ctime(when)])
t.AddRow([InputObj(qsender, 'hidden', str(id), False).Format()])
counter += 1
right.AddRow([t])
stable.AddRow([left, right])
table.AddRow([stable])
def process(mlist, msg, msgdata=None):
sanitize = mm_cfg.ARCHIVE_HTML_SANITIZER
outer = True
if msgdata is None:
msgdata = {}
if msgdata:
# msgdata is available if it is in GLOBAL_PIPELINE
# ie. not in digest or archiver
# check if the list owner want to scrub regular delivery
if not mlist.scrub_nondigest:
return
dir = calculate_attachments_dir(mlist, msg, msgdata)
charset = None
lcset = Utils.GetCharSet(mlist.preferred_language)
lcset_out = Charset(lcset).output_charset or lcset
# Now walk over all subparts of this message and scrub out various types
format = delsp = None
for part in msg.walk():
ctype = part.get_content_type()
# If the part is text/plain, we leave it alone
if ctype == 'text/plain':
# We need to choose a charset for the scrubbed message, so we'll
# arbitrarily pick the charset of the first text/plain part in the
# message.
# MAS: Also get the RFC 3676 stuff from this part. This seems to
# work OK for scrub_nondigest. It will also work as far as
# scrubbing messages for the archive is concerned, but pipermail
# doesn't pay any attention to the RFC 3676 parameters. The plain
# format digest is going to be a disaster in any case as some of
# messages will be format="flowed" and some not. ToDigest creates
# its own Content-Type: header for the plain digest which won't
# have RFC 3676 parameters. If the message Content-Type: headers
# are retained for display in the digest, the parameters will be
# there for information, but not for the MUA. This is the best we
# can do without having get_payload() process the parameters.
if charset is None:
charset = part.get_content_charset(lcset)
format = part.get_param('format')
delsp = part.get_param('delsp')
# TK: if part is attached then check charset and scrub if none
if part.get('content-disposition') and \
not part.get_content_charset():
omask = os.umask(0o02)
try:
url = save_attachment(mlist, part, dir)
finally:
os.umask(omask)
filename = part.get_filename(_('not available'))
filename = Utils.oneline(filename, lcset)
replace_payload_by_text(
part,
_("""\
An embedded and charset-unspecified text was scrubbed...
Name: %(filename)s
URL: %(url)s
"""), lcset)
elif ctype == 'text/html' and isinstance(sanitize, int):
if sanitize == 0:
if outer:
raise DiscardMessage
replace_payload_by_text(
part,
_('HTML attachment scrubbed and removed'),
# Adding charset arg and removing content-type
# sets content-type to text/plain
lcset)
elif sanitize == 2:
# By leaving it alone, Pipermail will automatically escape it
pass
elif sanitize == 3:
# Pull it out as an attachment but leave it unescaped. This
# is dangerous, but perhaps useful for heavily moderated
# lists.
omask = os.umask(0o02)
try:
url = save_attachment(mlist, part, dir, filter_html=False)
finally:
os.umask(omask)
replace_payload_by_text(
part,
_("""\
An HTML attachment was scrubbed...
URL: %(url)s
"""), lcset)
else:
# HTML-escape it and store it as an attachment, but make it
# look a /little/ bit prettier. :(
payload = Utils.websafe(part.get_payload(decode=True))
# For whitespace in the margin, change spaces into
# non-breaking spaces, and tabs into 8 of those. Then use a
# mono-space font. Still looks hideous to me, but then I'd
# just as soon discard them.
def doreplace(s):
return s.expandtabs(8).replace(' ', ' ')
lines = [doreplace(s) for s in payload.split('\n')]
payload = '<tt>\n' + BR.join(lines) + '\n</tt>\n'
part.set_payload(payload)
# We're replacing the payload with the decoded payload so this
# will just get in the way.
del part['content-transfer-encoding']
omask = os.umask(0o02)
try:
url = save_attachment(mlist, part, dir, filter_html=False)
finally:
os.umask(omask)
replace_payload_by_text(
part,
_("""\
An HTML attachment was scrubbed...
URL: %(url)s
"""), lcset)
elif ctype == 'message/rfc822':
# This part contains a submessage, so it too needs scrubbing
submsg = part.get_payload(0)
omask = os.umask(0o02)
try:
url = save_attachment(mlist, part, dir)
finally:
os.umask(omask)
subject = submsg.get('subject', _('no subject'))
subject = Utils.oneline(subject, lcset)
date = submsg.get('date', _('no date'))
who = submsg.get('from', _('unknown sender'))
size = len(str(submsg))
replace_payload_by_text(
part,
_("""\
An embedded message was scrubbed...
From: %(who)s
Subject: %(subject)s
Date: %(date)s
Size: %(size)s
URL: %(url)s
"""), lcset)
# If the message isn't a multipart, then we'll strip it out as an
# attachment that would have to be separately downloaded. Pipermail
# will transform the url into a hyperlink.
elif part.get_payload() and not part.is_multipart():
payload = part.get_payload(decode=True)
ctype = part.get_content_type()
# XXX Under email 2.5, it is possible that payload will be None.
# This can happen when you have a Content-Type: multipart/* with
# only one part and that part has two blank lines between the
# first boundary and the end boundary. In email 3.0 you end up
# with a string in the payload. I think in this case it's safe to
# ignore the part.
if payload is None:
continue
size = len(payload)
omask = os.umask(0o02)
try:
url = save_attachment(mlist, part, dir)
finally:
os.umask(omask)
desc = part.get('content-description', _('not available'))
desc = Utils.oneline(desc, lcset)
filename = part.get_filename(_('not available'))
filename = Utils.oneline(filename, lcset)
replace_payload_by_text(
part,
_("""\
A non-text attachment was scrubbed...
Name: %(filename)s
Type: %(ctype)s
Size: %(size)d bytes
Desc: %(desc)s
URL: %(url)s
"""), lcset)
outer = False
# We still have to sanitize multipart messages to flat text because
# Pipermail can't handle messages with list payloads. This is a kludge;
# def (n) clever hack ;).
if msg.is_multipart():
# By default we take the charset of the first text/plain part in the
# message, but if there was none, we'll use the list's preferred
# language's charset.
if not charset or charset == 'us-ascii':
charset = lcset_out
else:
# normalize to the output charset if input/output are different
charset = Charset(charset).output_charset or charset
# We now want to concatenate all the parts which have been scrubbed to
# text/plain, into a single text/plain payload. We need to make sure
# all the characters in the concatenated string are in the same
# encoding, so we'll use the 'replace' key in the coercion call.
# BAW: Martin's original patch suggested we might want to try
# generalizing to utf-8, and that's probably a good idea (eventually).
text = []
for part in msg.walk():
# TK: bug-id 1099138 and multipart
# MAS test payload - if part may fail if there are no headers.
if not part.get_payload() or part.is_multipart():
continue
# All parts should be scrubbed to text/plain by now, except
# if sanitize == 2, there could be text/html parts so keep them
# but skip any other parts.
partctype = part.get_content_type()
if partctype != 'text/plain' and (partctype != 'text/html'
or sanitize != 2):
text.append(_('Skipped content of type %(partctype)s\n'))
continue
try:
t = part.get_payload(decode=True) or ''
# MAS: TypeError exception can occur if payload is None. This
# was observed with a message that contained an attached
# message/delivery-status part. Because of the special parsing
# of this type, this resulted in a text/plain sub-part with a
# null body. See bug 1430236.
except (binascii.Error, TypeError):
t = part.get_payload() or ''
# TK: get_content_charset() returns 'iso-2022-jp' for internally
# crafted (scrubbed) 'euc-jp' text part. So, first try
# get_charset(), then get_content_charset() for the parts
# which are already embeded in the incoming message.
partcharset = part.get_charset()
if partcharset:
partcharset = str(partcharset)
else:
partcharset = part.get_content_charset()
if partcharset and partcharset != charset:
try:
t = str(t, partcharset, 'replace')
except (UnicodeError, LookupError, ValueError, AssertionError):
# We can get here if partcharset is bogus in come way.
# Replace funny characters. We use errors='replace'
t = str(t, 'ascii', 'replace')
try:
# Should use HTML-Escape, or try generalizing to UTF-8
t = t.encode(charset, 'replace')
except (UnicodeError, LookupError, ValueError, AssertionError):
# if the message charset is bogus, use the list's.
t = t.encode(lcset, 'replace')
# Separation is useful
if isinstance(t, str):
if not t.endswith('\n'):
t += '\n'
text.append(t)
# Now join the text and set the payload
sep = _('-------------- next part --------------\n')
# The i18n separator is in the list's charset. Coerce it to the
# message charset.
try:
s = str(sep, lcset, 'replace')
sep = s.encode(charset, 'replace')
except (UnicodeError, LookupError, ValueError, AssertionError):
pass
replace_payload_by_text(msg, sep.join(text), charset)
if format:
msg.set_param('Format', format)
if delsp:
msg.set_param('DelSp', delsp)
return msg
pass
# Queue the file for delivery by qrunner. Trying to deliver the
# message directly here can lead to a huge delay in web
# turnaround. Log the moderation and add a header.
msg['X-Mailman-Approved-At'] = email.Utils.formatdate(localtime=1)
syslog('vette', 'held message approved, message-id: %s',
msg.get('message-id', 'n/a'))
# Stick the message back in the incoming queue for further
# processing.
inq = get_switchboard(mm_cfg.INQUEUE_DIR)
inq.enqueue(msg, _metadata=msgdata)
elif value == mm_cfg.REJECT:
# Rejected
rejection = 'Refused'
lang = self.getMemberLanguage(sender)
subject = Utils.oneline(subject, Utils.GetCharSet(lang))
self.__refuse(_('Posting of your message titled "%(subject)s"'),
sender, comment or _('[No reason given]'),
lang=lang)
else:
assert value == mm_cfg.DISCARD
# Discarded
rejection = 'Discarded'
# Forward the message
if forward and addr:
# If we've approved the message, we need to be sure to craft a
# completely unique second message for the forwarding operation,
# since we don't want to share any state or information with the
# normal delivery.
try:
copy = readMessage(path)
def hold_for_approval(mlist, msg, msgdata, exc):
# BAW: This should really be tied into the email confirmation system so
# that the message can be approved or denied via email as well as the
# web.
#
# XXX We use the weird type(type) construct below because in Python 2.1,
# type is a function not a type and so can't be used as the second
# argument in isinstance(). However, in Python 2.5, exceptions are
# new-style classes and so are not of ClassType.
# FIXME pzv
if isinstance(exc, type) or isinstance(exc, type(type)):
# Go ahead and instantiate it now.
exc = exc()
listname = mlist.real_name
sender = msgdata.get('sender', msg.get_sender())
usersubject = msg.get('subject')
charset = Utils.GetCharSet(mlist.preferred_language)
if usersubject:
usersubject = Utils.oneline(usersubject, charset)
else:
usersubject = _('(no subject)')
message_id = msg.get('message-id', 'n/a')
owneraddr = mlist.GetOwnerEmail()
adminaddr = mlist.GetBouncesEmail()
requestaddr = mlist.GetRequestEmail()
# We need to send both the reason and the rejection notice through the
# translator again, because of the games we play above
reason = Utils.wrap(exc.reason_notice())
if isinstance(exc, NonMemberPost) and mlist.nonmember_rejection_notice:
msgdata['rejection_notice'] = Utils.wrap(
mlist.nonmember_rejection_notice.replace('%(listowner)s',
owneraddr))
else:
msgdata['rejection_notice'] = Utils.wrap(exc.rejection_notice(mlist))
id = mlist.HoldMessage(msg, reason, msgdata)
# Now we need to craft and send a message to the list admin so they can
# deal with the held message.
d = {
'listname': listname,
'hostname': mlist.host_name,
'reason': _(reason),
'sender': sender,
'subject': usersubject,
'admindb_url': mlist.GetScriptURL('admindb', absolute=1),
}
# We may want to send a notification to the original sender too
fromusenet = msgdata.get('fromusenet')
# Since we're sending two messages, which may potentially be in different
# languages (the user's preferred and the list's preferred for the admin),
# we need to play some i18n games here. Since the current language
# context ought to be set up for the user, let's craft his message first.
cookie = mlist.pend_new(Pending.HELD_MESSAGE, id)
if not fromusenet and ackp(msg) and mlist.respond_to_post_requests and \
mlist.autorespondToSender(sender, mlist.getMemberLanguage(sender)):
# Get a confirmation cookie
d['confirmurl'] = '%s/%s' % (mlist.GetScriptURL('confirm',
absolute=1), cookie)
lang = msgdata.get('lang', mlist.getMemberLanguage(sender))
subject = _('Your message to %(listname)s awaits moderator approval')
text = Utils.maketext('postheld.txt', d, lang=lang, mlist=mlist)
nmsg = Message.UserNotification(sender, owneraddr, subject, text, lang)
nmsg.send(mlist)
# Now the message for the list owners. Be sure to include the list
# moderators in this message. This one should appear to come from
# <list>-owner since we really don't need to do bounce processing on it.
if mlist.admin_immed_notify:
# Now let's temporarily set the language context to that which the
# admin is expecting.
otranslation = i18n.get_translation()
i18n.set_language(mlist.preferred_language)
try:
lang = mlist.preferred_language
charset = Utils.GetCharSet(lang)
# We need to regenerate or re-translate a few values in d
d['reason'] = _(reason)
d['subject'] = usersubject
# craft the admin notification message and deliver it
subject = _('%(listname)s post from %(sender)s requires approval')
nmsg = Message.UserNotification(owneraddr,
owneraddr,
subject,
lang=lang)
nmsg.set_type('multipart/mixed')
text = MIMEText(Utils.maketext('postauth.txt',
d,
raw=1,
mlist=mlist),
_charset=charset)
dmsg = MIMEText(Utils.wrap(
_("""\
If you reply to this message, keeping the Subject: header intact, Mailman will
discard the held message. Do this if the message is spam. If you reply to
this message and include an Approved: header with the list password in it, the
message will be approved for posting to the list. The Approved: header can
also appear in the first line of the body of the reply.""")),
_charset=Utils.GetCharSet(lang))
dmsg['Subject'] = 'confirm ' + cookie
dmsg['Sender'] = requestaddr
dmsg['From'] = requestaddr
dmsg['Date'] = email.utils.formatdate(localtime=True)
dmsg['Message-ID'] = Utils.unique_message_id(mlist)
nmsg.attach(text)
nmsg.attach(message.MIMEMessage(msg))
nmsg.attach(message.MIMEMessage(dmsg))
nmsg.send(mlist, **{'tomoderators': 1})
finally:
i18n.set_translation(otranslation)
# Log the held message
syslog('vette', '%s post from %s held, message-id=%s: %s', listname,
sender, message_id, reason)
# raise the specific MessageHeld exception to exit out of the message
# delivery pipeline
raise exc
def save_attachment(mlist, msg, dir, filter_html=True, patches=None, sigs=None):
fsdir = os.path.join(mlist.archive_dir(), dir)
makedirs(fsdir)
# Figure out the attachment type and get the decoded data
decodedpayload = msg.get_payload(decode=True)
# BAW: mimetypes ought to handle non-standard, but commonly found types,
# e.g. image/jpg (should be image/jpeg). For now we just store such
# things as application/octet-streams since that seems the safest.
ctype = msg.get_content_type()
# i18n file name is encoded
lcset = Utils.GetCharSet(mlist.preferred_language)
filename = Utils.oneline(msg.get_filename(""), lcset)
# filename, fnext = os.path.splitext(filename) #won't work with double extensions like .patch.sig
fnext = None
try:
(filename, fnext) = filename.split(".", 1)
if filename == "":
filename = None
if fnext != "":
fnext = "." + fnext
except:
pass
# For safety, we should confirm this is valid ext for content-type
# but we can use fnext if we introduce fnext filtering
if mm_cfg.SCRUBBER_USE_ATTACHMENT_FILENAME_EXTENSION:
# HTML message doesn't have filename :-(
# if it's text/plain, use '.txt', otherwise we'd end with '.ksh' or so
ext = fnext or guess_extension(ctype, ".txt")
else:
ext = guess_extension(ctype, fnext)
if not ext:
# We don't know what it is, so assume it's just a shapeless
# application/octet-stream, unless the Content-Type: is
# message/rfc822, in which case we know we'll coerce the type to
# text/plain below.
if ctype == "message/rfc822":
ext = ".txt"
else:
ext = ".bin"
# Allow only alphanumerics, dash, underscore, and dot
ext = sre.sub("", ext)
path = None
extra = ""
sha = ""
msgfrom = ""
do_write_file = True
if mm_cfg.SCRUBBER_ADD_PAYLOAD_HASH_FILENAME or not filename:
sha = msg.get(mm_cfg.SCRUBBER_SHA1SUM_HEADER)
if sha:
# no need to clutter headers
del msg[mm_cfg.SCRUBBER_SHA1SUM_HEADER]
msgfrom = msg[mm_cfg.SCRUBBER_SIGNEDBY_HEADER]
else:
sha = sha_new(decodedpayload).hexdigest()
# We need a lock to calculate the next attachment number
lockfile = os.path.join(fsdir, "attachments.lock")
lock = LockFile.LockFile(lockfile)
lock.lock()
try:
# Now base the filename on what's in the attachment, uniquifying it if
# necessary.
if not filename or mm_cfg.SCRUBBER_DONT_USE_ATTACHMENT_FILENAME:
filebase = "attachment"
else:
# Sanitize the filename given in the message headers
parts = pre.split(filename)
filename = parts[-1]
# Strip off leading dots
filename = dre.sub("", filename)
# Allow only alphanumerics, dash, underscore, and dot
filename = sre.sub("", filename)
# If the filename's extension doesn't match the type we guessed,
# which one should we go with? For now, let's go with the one we
# guessed so attachments can't lie about their type. Also, if the
# filename /has/ no extension, then tack on the one we guessed.
# The extension was removed from the name above.
filebase = filename
# Now we're looking for a unique name for this file on the file
# system. If msgdir/filebase.ext isn't unique, we'll add a counter
# after filebase, e.g. msgdir/filebase-cnt.ext
counter = 0
while True:
path = os.path.join(fsdir, filebase + extra + ext)
# Generally it is not a good idea to test for file existance
# before just trying to create it, but the alternatives aren't
# wonderful (i.e. os.open(..., O_CREAT | O_EXCL) isn't
# NFS-safe). Besides, we have an exclusive lock now, so we're
# guaranteed that no other process will be racing with us.
if os.path.exists(path):
counter += 1
extra = "-%04d" % counter
else:
break
filename = filebase + extra + ext
if mm_cfg.SCRUBBER_ADD_PAYLOAD_HASH_FILENAME:
# Make content hash to attachment
linkdir = os.path.join(fsdir, "..", "links")
makedirs(linkdir)
if msgfrom:
dst = os.path.join(linkdir, msgfrom + "_" + sha)
else:
dst = os.path.join(linkdir, sha)
src = os.path.join(fsdir, filename)
try:
os.symlink(src, dst)
except:
syslog("gpg", "Duplicate attachment: %s/%s msgfrom: %s sha1: %s" % (fsdir, filename, msgfrom, sha))
# To deduplicate would need to parse, etc.
# filename = os.readlink(dst)
# do_write_file = False
finally:
lock.unlock()
if do_write_file:
# `path' now contains the unique filename for the attachment. There's
# just one more step we need to do. If the part is text/html and
# ARCHIVE_HTML_SANITIZER is a string (which it must be or we wouldn't be
# here), then send the attachment through the filter program for
# sanitization
if filter_html and ctype == "text/html":
base, ext = os.path.splitext(path)
tmppath = base + "-tmp" + ext
fp = open(tmppath, "w")
try:
fp.write(decodedpayload)
fp.close()
cmd = mm_cfg.ARCHIVE_HTML_SANITIZER % {"filename": tmppath}
progfp = os.popen(cmd, "r")
decodedpayload = progfp.read()
status = progfp.close()
if status:
syslog("error", "HTML sanitizer exited with non-zero status: %s", status)
finally:
os.unlink(tmppath)
# BAW: Since we've now sanitized the document, it should be plain
# text. Blarg, we really want the sanitizer to tell us what the type
# if the return data is. :(
ext = ".txt"
path = base + ".txt"
# Is it a message/rfc822 attachment?
elif ctype == "message/rfc822":
submsg = msg.get_payload()
# BAW: I'm sure we can eventually do better than this. :(
decodedpayload = Utils.websafe(str(submsg))
fp = open(path, "w")
fp.write(decodedpayload)
fp.close()
# Now calculate the url
baseurl = mlist.GetBaseArchiveURL()
# Private archives will likely have a trailing slash. Normalize.
if baseurl[-1] <> "/":
baseurl += "/"
url = baseurl + "%s/%s%s%s" % (dir, filebase, extra, ext)
if sha:
url += "?sha1=" + sha
if sigs is not None and (ext.endswith(".sig") or ctype == "application/pgp-signature"):
sigs.append({"id": sha, "name": filebase, "file": os.path.join(dir, filename), "url": url})
elif patches is not None and ctype != "text/html" and ctype != "message/rfc822":
patches.append({"id": sha, "name": filebase, "file": os.path.join(dir, filename), "url": url})
# A trailing space in url string may save users who are using
# RFC-1738 compliant MUA (Not Mozilla).
# Trailing space will definitely be a problem with format=flowed.
# Bracket the URL instead.
# '<' + url + '>' Done by caller instead.
return url
def show_post_requests(mlist, id, info, total, count, form):
# Mailman.ListAdmin.__handlepost no longer tests for pre 2.0beta3
ptime, sender, subject, reason, filename, msgdata = info
form.AddItem('<hr>')
# Header shown on each held posting (including count of total)
msg = _('Posting Held for Approval')
if total != 1:
msg += _(' (%(count)d of %(total)d)')
form.AddItem(Center(Header(2, msg)))
# We need to get the headers and part of the textual body of the message
# being held. The best way to do this is to use the email.parser to get
# an actual object, which will be easier to deal with. We probably could
# just do raw reads on the file.
try:
msg = readMessage(os.path.join(mm_cfg.DATA_DIR, filename))
except IOError as e:
if e.errno != errno.ENOENT:
raise
form.AddItem(_('<em>Message with id #%(id)d was lost.'))
form.AddItem('<p>')
# BAW: kludge to remove id from requests.db.
try:
mlist.HandleRequest(id, mm_cfg.DISCARD)
except Errors.LostHeldMessage:
pass
return
except email.errors.MessageParseError:
form.AddItem(_('<em>Message with id #%(id)d is corrupted.'))
# BAW: Should we really delete this, or shuttle it off for site admin
# to look more closely at?
form.AddItem('<p>')
# BAW: kludge to remove id from requests.db.
try:
mlist.HandleRequest(id, mm_cfg.DISCARD)
except Errors.LostHeldMessage:
pass
return
# Get the header text and the message body excerpt
lines = []
chars = 0
# A negative value means, include the entire message regardless of size
limit = mm_cfg.ADMINDB_PAGE_TEXT_LIMIT
for line in email.iterators.body_line_iterator(msg, decode=True):
lines.append(line)
chars += len(line)
if chars >= limit > 0:
break
# We may have gone over the limit on the last line, but keep the full line
# anyway to avoid losing part of a multibyte character.
body = EMPTYSTRING.join(lines)
# Get message charset and try encode in list charset
# We get it from the first text part.
# We need to replace invalid characters here or we can throw an uncaught
# exception in doc.Format().
for part in msg.walk():
if part.get_content_maintype() == 'text':
# Watchout for charset= with no value.
mcset = part.get_content_charset() or 'us-ascii'
break
else:
mcset = 'us-ascii'
lcset = Utils.GetCharSet(mlist.preferred_language)
if mcset != lcset:
try:
body = str(body, mcset, 'replace')
except (LookupError, UnicodeError, ValueError):
pass
hdrtxt = NL.join(['%s: %s' % (k, v) for k, v in list(msg.items())])
hdrtxt = Utils.websafe(hdrtxt)
# Okay, we've reconstituted the message just fine. Now for the fun part!
t = Table(cellspacing=0, cellpadding=0, width='100%')
t.AddRow([Bold(_('From:')), sender])
row, col = t.GetCurrentRowIndex(), t.GetCurrentCellIndex()
t.AddCellInfo(row, col-1, align='right')
t.AddRow([Bold(_('Subject:')),
Utils.websafe(Utils.oneline(subject, lcset))])
t.AddCellInfo(row+1, col-1, align='right')
t.AddRow([Bold(_('Reason:')), _(reason)])
t.AddCellInfo(row+2, col-1, align='right')
when = msgdata.get('received_time')
if when:
t.AddRow([Bold(_('Received:')), time.ctime(when)])
t.AddCellInfo(row+3, col-1, align='right')
buttons = hacky_radio_buttons(id,
(_('Defer'), _('Approve'), _('Reject'), _('Discard')),
(mm_cfg.DEFER, mm_cfg.APPROVE, mm_cfg.REJECT, mm_cfg.DISCARD),
(1, 0, 0, 0),
spacing=5)
t.AddRow([Bold(_('Action:')), buttons])
t.AddCellInfo(t.GetCurrentRowIndex(), col-1, align='right')
t.AddRow([' ',
'<label>' +
CheckBox('preserve-%d' % id, 'on', 0).Format() +
' ' + _('Preserve message for site administrator') +
'</label>'
])
t.AddRow([' ',
'<label>' +
CheckBox('forward-%d' % id, 'on', 0).Format() +
' ' + _('Additionally, forward this message to: ') +
'</label>' +
TextBox('forward-addr-%d' % id, size=47,
value=mlist.GetOwnerEmail()).Format()
])
notice = msgdata.get('rejection_notice', _('[No explanation given]'))
t.AddRow([
Bold(_('If you reject this post,<br>please explain (optional):')),
TextArea('comment-%d' % id, rows=4, cols=EXCERPT_WIDTH,
text = Utils.wrap(_(notice), column=80))
])
row, col = t.GetCurrentRowIndex(), t.GetCurrentCellIndex()
t.AddCellInfo(row, col-1, align='right')
t.AddRow([Bold(_('Message Headers:')),
TextArea('headers-%d' % id, hdrtxt,
rows=EXCERPT_HEIGHT, cols=EXCERPT_WIDTH, readonly=1)])
row, col = t.GetCurrentRowIndex(), t.GetCurrentCellIndex()
t.AddCellInfo(row, col-1, align='right')
t.AddRow([Bold(_('Message Excerpt:')),
TextArea('fulltext-%d' % id, Utils.websafe(body),
rows=EXCERPT_HEIGHT, cols=EXCERPT_WIDTH, readonly=1)])
t.AddCellInfo(row+1, col-1, align='right')
form.AddItem(t)
form.AddItem('<p>')