def setMemberOption(self, member, flag, value):
assert self.__mlist.Locked()
self.__assertIsMember(member)
memberkey = member.lower()
# There's one extra gotcha we have to deal with. If the user is
# toggling the Digests flag, then we need to move their entry from
# mlist.members to mlist.digest_members or vice versa. Blarg. Do
# this before the flag setting below in case it fails.
if flag == mm_cfg.Digests:
if value:
# Be sure the list supports digest delivery
if not self.__mlist.digestable:
raise Errors.CantDigestError
# The user is turning on digest mode
if memberkey in self.__mlist.digest_members:
raise Errors.AlreadyReceivingDigests(member)
cpuser = self.__mlist.members.get(memberkey)
if cpuser is None:
raise Errors.NotAMemberError(member)
del self.__mlist.members[memberkey]
self.__mlist.digest_members[memberkey] = cpuser
# If we recently turned off digest mode and are now
# turning it back on, the member may be in one_last_digest.
# If so, remove it so the member doesn't get a dup of the
# next digest.
if memberkey in self.__mlist.one_last_digest:
del self.__mlist.one_last_digest[memberkey]
else:
# Be sure the list supports regular delivery
if not self.__mlist.nondigestable:
raise Errors.MustDigestError
# The user is turning off digest mode
if memberkey in self.__mlist.members:
raise Errors.AlreadyReceivingRegularDeliveries(member)
cpuser = self.__mlist.digest_members.get(memberkey)
if cpuser is None:
raise Errors.NotAMemberError(member)
del self.__mlist.digest_members[memberkey]
self.__mlist.members[memberkey] = cpuser
# When toggling off digest delivery, we want to be sure to set
# things up so that the user receives one last digest,
# otherwise they may lose some email
self.__mlist.one_last_digest[memberkey] = cpuser
# We don't need to touch user_options because the digest state
# isn't kept as a bitfield flag.
return
# This is a bit kludgey because the semantics are that if the user has
# no options set (i.e. the value would be 0), then they have no entry
# in the user_options dict. We use setdefault() here, and then del
# the entry below just to make things (questionably) cleaner.
self.__mlist.user_options.setdefault(memberkey, 0)
if value:
self.__mlist.user_options[memberkey] |= flag
else:
self.__mlist.user_options[memberkey] &= ~flag
if not self.__mlist.user_options[memberkey]:
del self.__mlist.user_options[memberkey]
def do_reject(mlist):
listowner = mlist.GetOwnerEmail()
if mlist.nonmember_rejection_notice:
raise Errors.RejectMessage(Utils.wrap(_(mlist.nonmember_rejection_notice)))
else:
raise Errors.RejectMessage(Utils.wrap(_("""\
Your message has been rejected, probably because you are not subscribed to the
mailing list and the list's policy is to prohibit non-members from posting to
it. If you think that your messages are being rejected in error, contact the
mailing list owner at %(listowner)s.""")))
def process(mlist, msg, msgdata):
# Short circuit if we've already calculated the recipients list,
# regardless of whether the list is empty or not.
if 'recips' in msgdata:
return
# Should the original sender should be included in the recipients list?
include_sender = 1
sender = msg.get_sender()
try:
if mlist.getMemberOption(sender, mm_cfg.DontReceiveOwnPosts):
include_sender = 0
except Errors.NotAMemberError:
pass
# Support for urgent messages, which bypasses digests and disabled
# delivery and forces an immediate delivery to all members Right Now. We
# are specifically /not/ allowing the site admins password to work here
# because we want to discourage the practice of sending the site admin
# password through email in the clear. (see also Approve.py)
missing = []
password = msg.get('urgent', missing)
if password is not missing:
if mlist.Authenticate((mm_cfg.AuthListPoster, mm_cfg.AuthListModerator,
mm_cfg.AuthListAdmin), password):
recips = mlist.getMemberCPAddresses(mlist.getRegularMemberKeys() +
mlist.getDigestMemberKeys())
msgdata['recips'] = recips
return
else:
# Bad Urgent: password, so reject it instead of passing it on. I
# think it's better that the sender know they screwed up than to
# deliver it normally.
realname = mlist.real_name
text = _("""\
Your urgent message to the %(realname)s mailing list was not authorized for
delivery. The original message as received by Mailman is attached.
""")
raise Errors.RejectMessage(Utils.wrap(text))
# Calculate the regular recipients of the message
recips = [
mlist.getMemberCPAddress(m) for m in mlist.getRegularMemberKeys()
if mlist.getDeliveryStatus(m) == ENABLED
]
# Remove the sender if they don't want to receive their own posts
if not include_sender:
try:
recips.remove(mlist.getMemberCPAddress(sender))
except (Errors.NotAMemberError, ValueError):
# Sender does not want to get copies of their own messages (not
# metoo), but delivery to their address is disabled (nomail). Or
# the sender is not a member of the mailing list.
pass
# Handle topic classifications
do_topic_filters(mlist, msg, msgdata, recips)
# Regular delivery exclude/include (if in/not_in To: or Cc:) lists
recips = do_exclude(mlist, msg, msgdata, recips)
recips = do_include(mlist, msg, msgdata, recips)
# Bookkeeping
msgdata['recips'] = recips
def process(mlist, msg, msgdata):
if msgdata.get('approved'):
return
# First do site hard coded header spam checks
for header, regex in mm_cfg.KNOWN_SPAMMERS:
cre = re.compile(regex, re.IGNORECASE)
for value in msg.get_all(header, []):
mo = cre.search(value)
if mo:
# we've detected spam, so throw the message away
raise SpamDetected
# Now do header_filter_rules
# TK: Collect headers in sub-parts because attachment filename
# extension may be a clue to possible virus/spam.
headers = ''
# Get the character set of the lists preferred language for headers
lcset = GetCharSet(mlist.preferred_language)
for p in msg.walk():
headers += getDecodedHeaders(p, lcset)
for patterns, action, empty in mlist.header_filter_rules:
if action == mm_cfg.DEFER:
continue
for pattern in patterns.splitlines():
if pattern.startswith('#'):
continue
# ignore 'empty' patterns
if not pattern.strip():
continue
if re.search(pattern, headers, re.IGNORECASE | re.MULTILINE):
if action == mm_cfg.DISCARD:
raise Errors.DiscardMessage
if action == mm_cfg.REJECT:
if msgdata.get('toowner'):
# Don't send rejection notice if addressed to '-owner'
# because it may trigger a loop of notices if the
# sender address is forged. We just discard it here.
raise Errors.DiscardMessage
raise Errors.RejectMessage(
_('Message rejected by filter rule match'))
if action == mm_cfg.HOLD:
if msgdata.get('toowner'):
# Don't hold '-owner' addressed message. We just
# pass it here but list-owner can set this to be
# discarded on the GUI if he wants.
return
hold_for_approval(mlist, msg, msgdata, HeaderMatchHold)
if action == mm_cfg.ACCEPT:
return
def addNewMember(self, member, **kws):
assert self.__mlist.Locked()
# Make sure this address isn't already a member
if self.isMember(member):
raise Errors.MMAlreadyAMember(member)
# Parse the keywords
digest = 0
password = Utils.MakeRandomPassword()
language = self.__mlist.preferred_language
realname = None
if 'digest' in kws:
digest = kws['digest']
del kws['digest']
if 'password' in kws:
password = kws['password']
del kws['password']
if 'language' in kws:
language = kws['language']
del kws['language']
if 'realname' in kws:
realname = kws['realname']
del kws['realname']
# Assert that no other keywords are present
if kws:
raise ValueError(list(kws.keys()))
# If the localpart has uppercase letters in it, then the value in the
# members (or digest_members) dict is the case preserved address.
# Otherwise the value is 0. Note that the case of the domain part is
# of course ignored.
if Utils.LCDomain(member) == member.lower():
value = 0
else:
value = member
member = member.lower()
if digest:
self.__mlist.digest_members[member] = value
else:
self.__mlist.members[member] = value
self.setMemberPassword(member, password)
self.setMemberLanguage(member, language)
if realname:
self.setMemberName(member, realname)
# Set the member's default set of options
if self.__mlist.new_member_options:
self.__mlist.user_options[member] = self.__mlist.new_member_options
def dispose(mlist, msg, msgdata, why):
# filter_action == 0 just discards, see below
if mlist.filter_action == 1:
# Bounce the message to the original author
raise Errors.RejectMessage(why)
if mlist.filter_action == 2:
# Forward it on to the list owner
listname = mlist.internal_name()
mlist.ForwardMessage(
msg,
text=_("""\
The attached message matched the %(listname)s mailing list's content filtering
rules and was prevented from being forwarded on to the list membership. You
are receiving the only remaining copy of the discarded message.
"""),
subject=_('Content filtered message notification'))
if mlist.filter_action == 3 and \
mm_cfg.OWNERS_CAN_PRESERVE_FILTERED_MESSAGES:
badq = get_switchboard(mm_cfg.BADQUEUE_DIR)
badq.enqueue(msg, msgdata)
# Most cases also discard the message
raise Errors.DiscardMessage
def getMemberKey(self, member):
cpaddr, where = self.__get_cp_member(member)
if cpaddr is None:
raise Errors.NotAMemberError(member)
return member.lower()
def __assertIsMember(self, member):
if not self.isMember(member):
raise Errors.NotAMemberError(member)
def process(mlist, msg, msgdata):
if msgdata.get('approved'):
return
# Is the poster a member or not?
for sender in msg.get_senders():
if mlist.isMember(sender):
break
for sender in Utils.check_eq_domains(sender,
mlist.equivalent_domains):
if mlist.isMember(sender):
break
if mlist.isMember(sender):
break
else:
sender = None
if sender:
# If the member's moderation flag is on, then perform the moderation
# action.
if mlist.getMemberOption(sender, mm_cfg.Moderate):
# Note that for member_moderation_action, 0==Hold, 1=Reject,
# 2==Discard
if mlist.member_moderation_action == 0:
# Hold. BAW: WIBNI we could add the member_moderation_notice
# to the notice sent back to the sender?
msgdata['sender'] = sender
Hold.hold_for_approval(mlist, msg, msgdata,
ModeratedMemberPost)
elif mlist.member_moderation_action == 1:
# Reject
text = mlist.member_moderation_notice
if text:
text = Utils.wrap(text)
else:
# Use the default RejectMessage notice string
text = None
raise Errors.RejectMessage(text)
elif mlist.member_moderation_action == 2:
# Discard. BAW: Again, it would be nice if we could send a
# discard notice to the sender
raise Errors.DiscardMessage
else:
assert 0, 'bad member_moderation_action'
# Should we do anything explict to mark this message as getting past
# this point? No, because further pipeline handlers will need to do
# their own thing.
return
else:
sender = msg.get_sender()
# From here on out, we're dealing with non-members.
listname = mlist.internal_name()
if mlist.GetPattern(sender,
mlist.accept_these_nonmembers,
at_list='accept_these_nonmembers'
):
return
if mlist.GetPattern(sender,
mlist.hold_these_nonmembers,
at_list='hold_these_nonmembers'
):
Hold.hold_for_approval(mlist, msg, msgdata, Hold.NonMemberPost)
# No return
if mlist.GetPattern(sender,
mlist.reject_these_nonmembers,
at_list='reject_these_nonmembers'
):
do_reject(mlist)
# No return
if mlist.GetPattern(sender,
mlist.discard_these_nonmembers,
at_list='discard_these_nonmembers'
):
do_discard(mlist, msg)
# No return
# Okay, so the sender wasn't specified explicitly by any of the non-member
# moderation configuration variables. Handle by way of generic non-member
# action.
assert 0 <= mlist.generic_nonmember_action <= 4
if mlist.generic_nonmember_action == 0 or msgdata.get('fromusenet'):
# Accept
return
elif mlist.generic_nonmember_action == 1:
Hold.hold_for_approval(mlist, msg, msgdata, Hold.NonMemberPost)
elif mlist.generic_nonmember_action == 2:
do_reject(mlist)
elif mlist.generic_nonmember_action == 3:
do_discard(mlist, msg)
def _getValidValue(self, mlist, property, wtype, val):
# Coerce and validate the new value.
#
# Radio buttons and boolean toggles both have integral type
if wtype in (mm_cfg.Radio, mm_cfg.Toggle):
# Let ValueErrors propagate
return int(val)
# String and Text widgets both just return their values verbatim
if wtype in (mm_cfg.String, mm_cfg.Text):
return val
# This widget contains a single email address
if wtype == mm_cfg.Email:
# BAW: We must allow blank values otherwise reply_to_address can't
# be cleared. This is currently the only mm_cfg.Email type widget
# in the interface, so watch out if we ever add any new ones.
if val:
# Let MMBadEmailError and MMHostileAddress propagate
Utils.ValidateEmail(val)
return val
# These widget types contain lists of email addresses, one per line.
# The EmailListEx allows each line to contain either an email address
# or a regular expression
if wtype in (mm_cfg.EmailList, mm_cfg.EmailListEx):
# BAW: value might already be a list, if this is coming from
# config_list input. Sigh.
if isinstance(val, list):
return val
addrs = []
bad_addrs = []
for addr in [s.strip() for s in val.split(NL)]:
# Discard empty lines
if not addr:
continue
try:
# This throws an exception if the address is invalid
Utils.ValidateEmail(addr)
except Errors.EmailAddressError:
# See if this is a context that accepts regular
# expressions, and that the re is legal
if wtype == mm_cfg.EmailListEx and addr.startswith('^'):
try:
re.compile(addr)
except re.error:
bad_addrs.append(addr)
elif (wtype == mm_cfg.EmailListEx and addr.startswith('@')
and (property.endswith('_these_nonmembers')
or property == 'subscribe_auto_approval')):
# XXX Needs to be reviewed for list@domain names.
# don't reference your own list
if addr[1:] == mlist.internal_name():
bad_addrs.append(addr)
# check for existence of list? For now allow
# reference to list before creating it.
else:
bad_addrs.append(addr)
if property in ('regular_exclude_lists',
'regular_include_lists'):
if addr.lower() == mlist.GetListEmail().lower():
bad_addrs.append(addr)
addrs.append(addr)
if bad_addrs:
raise Errors.EmailAddressError(', '.join(bad_addrs))
return addrs
# This is a host name, i.e. verbatim
if wtype == mm_cfg.Host:
return val
# This is a number, either a float or an integer
if wtype == mm_cfg.Number:
# The int/float code below doesn't work if we are called from
# config_list with a value that is already a float. It will
# truncate the value to an int.
if isinstance(val, float):
return val
num = -1
try:
num = int(val)
except ValueError:
# Let ValueErrors percolate up
num = float(val)
if num < 0:
return getattr(mlist, property)
return num
# This widget is a select box, i.e. verbatim
if wtype == mm_cfg.Select:
return val
# Checkboxes return a list of the selected items, even if only one is
# selected.
if wtype == mm_cfg.Checkbox:
if isinstance(val, list):
return val
return [val]
if wtype == mm_cfg.FileUpload:
return val
if wtype == mm_cfg.Topics:
return val
if wtype == mm_cfg.HeaderFilter:
return val
# Should never get here
assert 0, 'Bad gui widget type: %s' % wtype
def process(mlist, msg, msgdata):
# Short circuits
# Do not short circuit. The problem is SpamDetect comes before Approve.
# Suppose a message with an Approved: header is held by SpamDetect (or
# any other handler that might come before Approve) and then approved
# by a moderator. When the approved message reaches Approve in the
# pipeline, we still need to remove the Approved: (pseudo-)header, so
# we can't short circuit.
#if msgdata.get('approved'):
# Digests, Usenet postings, and some other messages come pre-approved.
# TBD: we may want to further filter Usenet messages, so the test
# above may not be entirely correct.
#return
# See if the message has an Approved or Approve header with a valid
# list-moderator, list-admin. Also look at the first non-whitespace line
# in the file to see if it looks like an Approved header. We are
# specifically /not/ allowing the site admins password to work here
# because we want to discourage the practice of sending the site admin
# password through email in the clear.
missing = []
for hdr in ('approved', 'approve', 'x-approved', 'x-approve'):
passwd = msg.get(hdr, missing)
if passwd is not missing:
break
if passwd is missing:
# Find the first text/plain part in the message
part = None
stripped = False
for part in typed_subpart_iterator(msg, 'text', 'plain'):
break
# XXX I'm not entirely sure why, but it is possible for the payload of
# the part to be None, and you can't splitlines() on None.
if part is not None and part.get_payload() is not None:
lines = part.get_payload(decode=True).splitlines()
line = ''
for lineno, line in zip(list(range(len(lines))), lines):
if line.strip():
break
i = line.find(':')
if i >= 0:
name = line[:i]
value = line[i + 1:]
if name.lower() in (
'approve',
'approved',
'x-approve',
'x-approved',
):
passwd = value.lstrip()
# Now strip the first line from the payload so the
# password doesn't leak.
del lines[lineno]
reset_payload(part, NL.join(lines))
stripped = True
if stripped:
# MAS: Bug 1181161 - Now try all the text parts in case it's
# multipart/alternative with the approved line in HTML or other
# text part. We make a pattern from the Approved line and delete
# it from all text/* parts in which we find it. It would be
# better to just iterate forward, but email compatability for pre
# Python 2.2 returns a list, not a true iterator. Also, there
# are pathological MUAs that put the HTML part first.
#
# This will process all the multipart/alternative parts in the
# message as well as all other text parts. We shouldn't find the
# pattern outside the mp/a parts, but if we do, it is probably
# best to delete it anyway as it does contain the password.
#
# Make a pattern to delete. We can't just delete a line because
# line of HTML or other fancy text may include additional message
# text. This pattern works with HTML. It may not work with rtf
# or whatever else is possible.
#
# If we don't find the pattern in the decoded part, but we do
# find it after stripping HTML tags, we don't know how to remove
# it, so we just reject the post.
pattern = name + ':(\xA0|\s| )*' + re.escape(passwd)
for part in typed_subpart_iterator(msg, 'text'):
if part is not None and part.get_payload() is not None:
lines = part.get_payload(decode=True)
if re.search(pattern, lines):
reset_payload(part, re.sub(pattern, '', lines))
elif re.search(pattern, re.sub('(?s)<.*?>', '', lines)):
raise Errors.RejectMessage(REJECT)
if passwd is not missing and mlist.Authenticate(
(mm_cfg.AuthListPoster, mm_cfg.AuthListModerator,
mm_cfg.AuthListAdmin), passwd):
# BAW: should we definitely deny if the password exists but does not
# match? For now we'll let it percolate up for further determination.
msgdata['approved'] = 1
# Used by the Emergency module
msgdata['adminapproved'] = 1
# has this message already been posted to this list?
beentheres = [s.strip().lower() for s in msg.get_all('x-beenthere', [])]
if mlist.GetListEmail().lower() in beentheres:
raise Errors.LoopError
def __handlepost(self, record, value, comment, preserve, forward, addr):
# For backwards compatibility with pre 2.0beta3
ptime, sender, subject, reason, filename, msgdata = record
path = os.path.join(mm_cfg.DATA_DIR, filename)
# Handle message preservation
if preserve:
parts = os.path.split(path)[1].split(DASH)
parts[0] = 'spam'
spamfile = DASH.join(parts)
# Preserve the message as plain text, not as a pickle
try:
fp = open(path)
except IOError as e:
if e.errno != errno.ENOENT: raise
return LOST
try:
if path.endswith('.pck'):
msg = pickle.load(fp)
else:
assert path.endswith('.txt'), '%s not .pck or .txt' % path
msg = fp.read()
finally:
fp.close()
# Save the plain text to a .msg file, not a .pck file
outpath = os.path.join(mm_cfg.SPAM_DIR, spamfile)
head, ext = os.path.splitext(outpath)
outpath = head + '.msg'
outfp = open(outpath, 'wb')
try:
if path.endswith('.pck'):
g = Generator(outfp)
g.flatten(msg, 1)
else:
outfp.write(msg)
finally:
outfp.close()
# Now handle updates to the database
rejection = None
fp = None
msg = None
status = REMOVE
if value == mm_cfg.DEFER:
# Defer
status = DEFER
elif value == mm_cfg.APPROVE:
# Approved.
try:
msg = readMessage(path)
except IOError as e:
if e.errno != errno.ENOENT: raise
return LOST
msg = readMessage(path)
msgdata['approved'] = 1
# adminapproved is used by the Emergency handler
msgdata['adminapproved'] = 1
# Calculate a new filebase for the approved message, otherwise
# delivery errors will cause duplicates.
try:
del msgdata['filebase']
except KeyError:
pass
# Queue the file for delivery by qrunner. Trying to deliver the
# message directly here can lead to a huge delay in web
# turnaround. Log the moderation and add a header.
msg['X-Mailman-Approved-At'] = email.utils.formatdate(localtime=1)
syslog('vette', '%s: held message approved, message-id: %s',
self.internal_name(), msg.get('message-id', 'n/a'))
# Stick the message back in the incoming queue for further
# processing.
inq = get_switchboard(mm_cfg.INQUEUE_DIR)
inq.enqueue(msg, _metadata=msgdata)
elif value == mm_cfg.REJECT:
# Rejected
rejection = 'Refused'
lang = self.getMemberLanguage(sender)
subject = Utils.oneline(subject, Utils.GetCharSet(lang))
self.__refuse(_('Posting of your message titled "%(subject)s"'),
sender,
comment or _('[No reason given]'),
lang=lang)
else:
assert value == mm_cfg.DISCARD
# Discarded
rejection = 'Discarded'
# Forward the message
if forward and addr:
# If we've approved the message, we need to be sure to craft a
# completely unique second message for the forwarding operation,
# since we don't want to share any state or information with the
# normal delivery.
try:
copy = readMessage(path)
except IOError as e:
if e.errno != errno.ENOENT: raise
raise Errors.LostHeldMessage(path)
# It's possible the addr is a comma separated list of addresses.
addrs = getaddresses([addr])
if len(addrs) == 1:
realname, addr = addrs[0]
# If the address getting the forwarded message is a member of
# the list, we want the headers of the outer message to be
# encoded in their language. Otherwise it'll be the preferred
# language of the mailing list.
lang = self.getMemberLanguage(addr)
else:
# Throw away the realnames
addr = [a for realname, a in addrs]
# Which member language do we attempt to use? We could use
# the first match or the first address, but in the face of
# ambiguity, let's just use the list's preferred language
lang = self.preferred_language
otrans = i18n.get_translation()
i18n.set_language(lang)
try:
fmsg = Message.UserNotification(
addr,
self.GetBouncesEmail(),
_('Forward of moderated message'),
lang=lang)
finally:
i18n.set_translation(otrans)
fmsg.set_type('message/rfc822')
fmsg.attach(copy)
fmsg.send(self)
# Log the rejection
if rejection:
note = '''%(listname)s: %(rejection)s posting:
tFrom: %(sender)s
tSubject: %(subject)s''' % {
'listname': self.internal_name(),
'rejection': rejection,
'sender': str(sender).replace('%', '%%'),
'subject': str(subject).replace('%', '%%'),
}
if comment:
note += '\n\tReason: ' + comment.replace('%', '%%')
syslog('vette', note)
# Always unlink the file containing the message text. It's not
# necessary anymore, regardless of the disposition of the message.
if status != DEFER:
try:
os.unlink(path)
except OSError as e:
if e.errno != errno.ENOENT: raise
# We lost the message text file. Clean up our housekeeping
# and inform of this status.
return LOST
return status
lang=lang)
else:
assert value == mm_cfg.DISCARD
# Discarded
rejection = 'Discarded'
# Forward the message
if forward and addr:
# If we've approved the message, we need to be sure to craft a
# completely unique second message for the forwarding operation,
# since we don't want to share any state or information with the
# normal delivery.
try:
copy = readMessage(path)
except IOError, e:
if e.errno <> errno.ENOENT: raise
raise Errors.LostHeldMessage(path)
# It's possible the addr is a comma separated list of addresses.
addrs = getaddresses([addr])
if len(addrs) == 1:
realname, addr = addrs[0]
# If the address getting the forwarded message is a member of
# the list, we want the headers of the outer message to be
# encoded in their language. Otherwise it'll be the preferred
# language of the mailing list.
lang = self.getMemberLanguage(addr)
else:
# Throw away the realnames
addr = [a for realname, a in addrs]
# Which member language do we attempt to use? We could use
# the first match or the first address, but in the face of
# ambiguity, let's just use the list's preferred language
def getMemberCPAddress(self, member):
cpaddr, where = self.__get_cp_member(member)
if cpaddr is None:
raise Errors.NotAMemberError(member)
return cpaddr
def process(mlist, msg, msgdata):
recips = msgdata.get('recips')
if not recips:
# Nobody to deliver to!
return
# Calculate the non-VERP envelope sender.
envsender = msgdata.get('envsender')
if envsender is None:
if mlist:
envsender = mlist.GetBouncesEmail()
else:
envsender = Utils.get_site_email(extra='bounces')
# Time to split up the recipient list. If we're personalizing or VERPing
# then each chunk will have exactly one recipient. We'll then hand craft
# an envelope sender and stitch a message together in memory for each one
# separately. If we're not VERPing, then we'll chunkify based on
# SMTP_MAX_RCPTS. Note that most MTAs have a limit on the number of
# recipients they'll swallow in a single transaction.
deliveryfunc = None
if ('personalize' not in msgdata or msgdata['personalize']) and (
msgdata.get('verp') or mlist.personalize):
chunks = [[recip] for recip in recips]
msgdata['personalize'] = 1
deliveryfunc = verpdeliver
elif mm_cfg.SMTP_MAX_RCPTS <= 0:
chunks = [recips]
else:
chunks = chunkify(recips, mm_cfg.SMTP_MAX_RCPTS)
# See if this is an unshunted message for which some were undelivered
if 'undelivered' in msgdata:
chunks = msgdata['undelivered']
# If we're doing bulk delivery, then we can stitch up the message now.
if deliveryfunc is None:
# Be sure never to decorate the message more than once!
if not msgdata.get('decorated'):
Decorate.process(mlist, msg, msgdata)
msgdata['decorated'] = True
deliveryfunc = bulkdeliver
refused = {}
t0 = time.time()
# Open the initial connection
origrecips = msgdata['recips']
# MAS: get the message sender now for logging. If we're using 'sender'
# and not 'from', bulkdeliver changes it for bounce processing. If we're
# VERPing, it doesn't matter because bulkdeliver is working on a copy, but
# otherwise msg gets changed. If the list is anonymous, the original
# sender is long gone, but Cleanse.py has logged it.
origsender = msgdata.get('original_sender', msg.get_sender())
# `undelivered' is a copy of chunks that we pop from to do deliveries.
# This seems like a good tradeoff between robustness and resource
# utilization. If delivery really fails (i.e. qfiles/shunt type
# failures), then we'll pick up where we left off with `undelivered'.
# This means at worst, the last chunk for which delivery was attempted
# could get duplicates but not every one, and no recips should miss the
# message.
conn = Connection()
try:
msgdata['undelivered'] = chunks
while chunks:
chunk = chunks.pop()
msgdata['recips'] = chunk
try:
deliveryfunc(mlist, msg, msgdata, envsender, refused, conn)
except Exception:
# If /anything/ goes wrong, push the last chunk back on the
# undelivered list and re-raise the exception. We don't know
# how many of the last chunk might receive the message, so at
# worst, everyone in this chunk will get a duplicate. Sigh.
chunks.append(chunk)
raise
del msgdata['undelivered']
finally:
conn.quit()
msgdata['recips'] = origrecips
# Log the successful post
t1 = time.time()
d = MsgSafeDict(msg, {'time' : t1-t0,
# BAW: Urg. This seems inefficient.
'size' : len(msg.as_string()),
'#recips' : len(recips),
'#refused': len(refused),
'listname': mlist.internal_name(),
'sender' : origsender,
})
# We have to use the copy() method because extended call syntax requires a
# concrete dictionary object; it does not allow a generic mapping. It's
# still worthwhile doing the interpolation in syslog() because it'll catch
# any catastrophic exceptions due to bogus format strings.
if mm_cfg.SMTP_LOG_EVERY_MESSAGE:
syslog.write_ex(mm_cfg.SMTP_LOG_EVERY_MESSAGE[0],
mm_cfg.SMTP_LOG_EVERY_MESSAGE[1], kws=d)
if refused:
if mm_cfg.SMTP_LOG_REFUSED:
syslog.write_ex(mm_cfg.SMTP_LOG_REFUSED[0],
mm_cfg.SMTP_LOG_REFUSED[1], kws=d)
elif msgdata.get('tolist'):
# Log the successful post, but only if it really was a post to the
# mailing list. Don't log sends to the -owner, or -admin addrs.
# -request addrs should never get here. BAW: it may be useful to log
# the other messages, but in that case, we should probably have a
# separate configuration variable to control that.
if mm_cfg.SMTP_LOG_SUCCESS:
syslog.write_ex(mm_cfg.SMTP_LOG_SUCCESS[0],
mm_cfg.SMTP_LOG_SUCCESS[1], kws=d)
# Process any failed deliveries.
tempfailures = []
permfailures = []
for recip, (code, smtpmsg) in list(refused.items()):
# DRUMS is an internet draft, but it says:
#
# [RFC-821] incorrectly listed the error where an SMTP server
# exhausts its implementation limit on the number of RCPT commands
# ("too many recipients") as having reply code 552. The correct
# reply code for this condition is 452. Clients SHOULD treat a 552
# code in this case as a temporary, rather than permanent failure
# so the logic below works.
#
if code >= 500 and code != 552:
# A permanent failure
permfailures.append(recip)
else:
# Deal with persistent transient failures by queuing them up for
# future delivery. TBD: this could generate lots of log entries!
tempfailures.append(recip)
if mm_cfg.SMTP_LOG_EACH_FAILURE:
d.update({'recipient': recip,
'failcode' : code,
'failmsg' : smtpmsg})
syslog.write_ex(mm_cfg.SMTP_LOG_EACH_FAILURE[0],
mm_cfg.SMTP_LOG_EACH_FAILURE[1], kws=d)
# Return the results
if tempfailures or permfailures:
raise Errors.SomeRecipientsFailed(tempfailures, permfailures)
def getMemberPassword(self, member):
secret = self.__mlist.passwords.get(member.lower())
if secret is None:
raise Errors.NotAMemberError(member)
return secret
def process(mlist, msg, msgdata):
# Before anything else, check DMARC if necessary. We do this as early
# as possible so reject/discard actions trump other holds/approvals and
# wrap/munge actions get flagged even for approved messages.
# But not for owner mail which should not be subject to DMARC reject or
# discard actions.
if not msgdata.get('toowner'):
msgdata['from_is_list'] = 0
dn, addr = parseaddr(msg.get('from'))
if addr and mlist.dmarc_moderation_action > 0:
if Utils.IsDMARCProhibited(mlist, addr):
# Note that for dmarc_moderation_action, 0 = Accept,
# 1 = Munge, 2 = Wrap, 3 = Reject, 4 = Discard
if mlist.dmarc_moderation_action == 1:
msgdata['from_is_list'] = 1
elif mlist.dmarc_moderation_action == 2:
msgdata['from_is_list'] = 2
elif mlist.dmarc_moderation_action == 3:
# Reject
text = mlist.dmarc_moderation_notice
if text:
text = Utils.wrap(text)
else:
listowner = mlist.GetOwnerEmail()
text = Utils.wrap(
_("""You are not allowed to post to this mailing list From: a domain which
publishes a DMARC policy of reject or quarantine, and your message has been
automatically rejected. If you think that your messages are being rejected in
error, contact the mailing list owner at %(listowner)s."""))
raise Errors.RejectMessage, text
elif mlist.dmarc_moderation_action == 4:
raise Errors.DiscardMessage
# Get member address if any.
for sender in msg.get_senders():
if mlist.isMember(sender):
break
else:
sender = msg.get_sender()
if (mlist.member_verbosity_threshold > 0
and Utils.IsVerboseMember(mlist, sender)):
mlist.setMemberOption(sender, mm_cfg.Moderate, 1)
syslog('vette',
'%s: Automatically Moderated %s for verbose postings.',
mlist.real_name, sender)
if msgdata.get('approved'):
return
# First do site hard coded header spam checks
for header, regex in mm_cfg.KNOWN_SPAMMERS:
cre = re.compile(regex, re.IGNORECASE)
for value in msg.get_all(header, []):
mo = cre.search(value)
if mo:
# we've detected spam, so throw the message away
raise SpamDetected
# Now do header_filter_rules
# TK: Collect headers in sub-parts because attachment filename
# extension may be a clue to possible virus/spam.
headers = u''
# Get the character set of the lists preferred language for headers
lcset = Utils.GetCharSet(mlist.preferred_language)
for p in msg.walk():
headers += getDecodedHeaders(p, lcset)
for patterns, action, empty in mlist.header_filter_rules:
if action == mm_cfg.DEFER:
continue
for pattern in patterns.splitlines():
if pattern.startswith('#'):
continue
# ignore 'empty' patterns
if not pattern.strip():
continue
pattern = Utils.xml_to_unicode(pattern, lcset)
pattern = normalize(mm_cfg.NORMALIZE_FORM, pattern)
try:
mo = re.search(pattern, headers,
re.IGNORECASE | re.MULTILINE | re.UNICODE)
except (re.error, TypeError):
syslog('error',
'ignoring header_filter_rules invalid pattern: %s',
pattern)
if mo:
if action == mm_cfg.DISCARD:
raise Errors.DiscardMessage
if action == mm_cfg.REJECT:
if msgdata.get('toowner'):
# Don't send rejection notice if addressed to '-owner'
# because it may trigger a loop of notices if the
# sender address is forged. We just discard it here.
raise Errors.DiscardMessage
raise Errors.RejectMessage(
_('Message rejected by filter rule match'))
if action == mm_cfg.HOLD:
if msgdata.get('toowner'):
# Don't hold '-owner' addressed message. We just
# pass it here but list-owner can set this to be
# discarded on the GUI if he wants.
return
hold_for_approval(mlist, msg, msgdata,
HeaderMatchHold(pattern))
if action == mm_cfg.ACCEPT:
return