def moderate(self, request):
try:
validator = Validator(action=enum_validator(Action))
arguments = validator(request)
except ValueError as error:
return http.bad_request([], str(error))
requests = IListRequests(self._mlist)
try:
request_id = int(self._request_id)
except ValueError:
return http.bad_request()
results = requests.get_request(request_id)
if results is None:
return http.not_found()
key, data = results
try:
request_type = RequestType(data['_request_type'])
except ValueError:
return http.bad_request()
if request_type is RequestType.subscription:
handle_subscription(self._mlist, request_id, **arguments)
elif request_type is RequestType.unsubscription:
handle_unsubscription(self._mlist, request_id, **arguments)
else:
return http.bad_request()
return no_content()
def _make_resource(self, request_id):
requests = IListRequests(self._mlist)
results = requests.get_request(request_id)
if results is None:
return None
key, data = results
resource = dict(key=key, request_id=request_id)
# Flatten the IRequest payload into the JSON representation.
if data is not None:
resource.update(data)
# Check for a matching request type, and insert the type name into the
# resource.
try:
request_type = RequestType[resource.pop('_request_type', None)]
except KeyError:
request_type = None
if request_type is not RequestType.held_message:
return None
resource['type'] = RequestType.held_message.name
# This key isn't what you think it is. Usually, it's the Pendable
# record's row id, which isn't helpful at all. If it's not there,
# that's fine too.
resource.pop('id', None)
# Add a self_link.
resource['self_link'] = self.api.path_to(
'lists/{}/held/{}'.format(self._mlist.list_id, request_id))
return resource
def hold_subscription(mlist, address, display_name, password, mode, language):
data = dict(when=now().isoformat(),
address=address,
display_name=display_name,
password=password,
delivery_mode=mode.name,
language=language)
# Now hold this request. We'll use the address as the key.
requestsdb = IListRequests(mlist)
request_id = requestsdb.hold_request(
RequestType.subscription, address, data)
vlog.info('%s: held subscription request from %s',
mlist.fqdn_listname, address)
# Possibly notify the administrator in default list language
if mlist.admin_immed_notify:
subject = _(
'New subscription request to $mlist.display_name from $address')
text = make('subauth.txt',
mailing_list=mlist,
username=address,
listname=mlist.fqdn_listname,
admindb_url=mlist.script_url('admindb'),
)
# This message should appear to come from the <list>-owner so as
# to avoid any useless bounce processing.
msg = UserNotification(
mlist.owner_address, mlist.owner_address,
subject, text, mlist.preferred_language)
msg.send(mlist, tomoderators=True)
return request_id
def on_post(self, request, response):
try:
validator = Validator(action=enum_validator(Action))
arguments = validator(request)
except ValueError as error:
bad_request(response, str(error))
return
requests = IListRequests(self._mlist)
try:
request_id = int(self._request_id)
except ValueError:
bad_request(response)
return
results = requests.get_request(request_id)
if results is None:
not_found(response)
return
key, data = results
try:
request_type = RequestType[data['_request_type']]
except ValueError:
bad_request(response)
return
if request_type is RequestType.subscription:
handle_subscription(self._mlist, request_id, **arguments)
elif request_type is RequestType.unsubscription:
handle_unsubscription(self._mlist, request_id, **arguments)
else:
bad_request(response)
return
no_content(response)
def test_request_is_not_held_message(self):
requests = IListRequests(self._mlist)
with transaction():
request_id = requests.hold_request(RequestType.subscription, 'foo')
with self.assertRaises(HTTPError) as cm:
call_api('http://localhost:9001/3.0/lists/ant.example.com'
'/held/{}'.format(request_id))
self.assertEqual(cm.exception.code, 404)
def _get_collection(self, request):
requests = IListRequests(self._mlist)
self._requests = requests
items = []
for request_type in MEMBERSHIP_CHANGE_REQUESTS:
for request in requests.of_type(request_type):
items.append(request)
return items
def handle_ListDeletingEvent(event):
if not isinstance(event, ListDeletingEvent):
return
# Get the held requests database for the mailing list. Since the mailing
# list is about to get deleted, we can delete all associated requests.
requestsdb = IListRequests(event.mailing_list)
for request in requestsdb.held_requests:
requestsdb.delete_request(request.id)
def test_lp_1031391(self):
# LP: #1031391 msgdata['received_time'] gets added by the LMTP server.
# The value is a datetime. If this message gets held, it will break
# pending requests since they require string keys and values.
received_time = now()
msgdata = dict(received_time=received_time)
request_id = hold_message(self._mlist, self._msg, msgdata)
requests_db = IListRequests(self._mlist)
key, data = requests_db.get_request(request_id)
self.assertEqual(data['received_time'], received_time)
def test_hold_action_alias_for_defer(self):
# In handle_message(), the 'hold' action is the same as 'defer' for
# purposes of this API.
request_id = hold_message(self._mlist, self._msg)
handle_message(self._mlist, request_id, Action.defer)
# The message is still in the pending requests.
requests_db = IListRequests(self._mlist)
key, data = requests_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
handle_message(self._mlist, request_id, Action.hold)
key, data = requests_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
def auto_discard(mlist):
# Discard old held messages
discard_count = 0
expire = config.days(mlist.max_days_to_hold)
requestsdb = IListRequests(mlist)
heldmsgs = list(requestsdb.of_type(RequestType.held_message))
if expire and heldmsgs:
for request in heldmsgs:
key, data = requestsdb.get_request(request.id)
if now - data['date'] > expire:
handle_request(mlist, request.id, config.DISCARD)
discard_count += 1
mlist.Save()
return discard_count
def moderate(self, request):
try:
validator = Validator(action=enum_validator(Action))
arguments = validator(request)
except ValueError as error:
return http.bad_request([], str(error))
requests = IListRequests(self._mlist)
try:
request_id = int(self._request_id)
except ValueError:
return http.bad_request()
results = requests.get_request(request_id, RequestType.held_message)
if results is None:
return http.not_found()
handle_message(self._mlist, request_id, **arguments)
return no_content()
def test_get_request_with_type(self):
# get_request() takes an optional request type.
request_id = hold_message(self._mlist, self._msg)
requests_db = IListRequests(self._mlist)
# Submit a request with a non-matching type. This should return None
# as if there were no matches.
response = requests_db.get_request(
request_id, RequestType.subscription)
self.assertEqual(response, None)
# Submit the same request with a matching type.
key, data = requests_db.get_request(
request_id, RequestType.held_message)
self.assertEqual(key, '<alpha>')
# It should also succeed with no optional request type given.
key, data = requests_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
def details(self, request):
requests = IListRequests(self._mlist)
try:
request_id = int(self._request_id)
except ValueError:
return http.bad_request()
results = requests.get_request(request_id, RequestType.held_message)
if results is None:
return http.not_found()
key, data = results
msg = getUtility(IMessageStore).get_message_by_id(key)
resource = dict(
key=key,
data=data,
msg=msg.as_string(),
id=request_id,
)
return http.ok([], etag(resource))
def on_post(self, request, response):
try:
validator = Validator(action=enum_validator(Action))
arguments = validator(request)
except ValueError as error:
bad_request(response, str(error))
return
requests = IListRequests(self._mlist)
try:
request_id = int(self._request_id)
except ValueError:
bad_request(response)
return
results = requests.get_request(request_id, RequestType.held_message)
if results is None:
not_found(response)
else:
handle_message(self._mlist, request_id, **arguments)
no_content(response)
def hold_message(mlist, msg, msgdata=None, reason=None):
"""Hold a message for moderator approval.
The message is added to the mailing list's request database.
:param mlist: The mailing list to hold the message on.
:param msg: The message to hold.
:param msgdata: Optional message metadata to hold. If not given, a new
metadata dictionary is created and held with the message.
:param reason: Optional string reason why the message is being held. If
not given, the empty string is used.
:return: An id used to handle the held message later.
"""
if msgdata is None:
msgdata = {}
else:
# Make a copy of msgdata so that subsequent changes won't corrupt the
# request database. TBD: remove the `filebase' key since this will
# not be relevant when the message is resurrected.
msgdata = msgdata.copy()
if reason is None:
reason = ''
# Add the message to the message store. It is required to have a
# Message-ID header.
message_id = msg.get('message-id')
if message_id is None:
msg['Message-ID'] = message_id = unicode(make_msgid())
assert isinstance(message_id, unicode), (
'Message-ID is not a unicode: %s' % message_id)
getUtility(IMessageStore).add(msg)
# Prepare the message metadata with some extra information needed only by
# the moderation interface.
msgdata['_mod_message_id'] = message_id
msgdata['_mod_fqdn_listname'] = mlist.fqdn_listname
msgdata['_mod_sender'] = msg.sender
msgdata['_mod_subject'] = msg.get('subject', _('(no subject)'))
msgdata['_mod_reason'] = reason
msgdata['_mod_hold_date'] = now().isoformat()
# Now hold this request. We'll use the message_id as the key.
requestsdb = IListRequests(mlist)
request_id = requestsdb.hold_request(
RequestType.held_message, message_id, msgdata)
return request_id
def handle_subscription(mlist, id, action, comment=None):
requestdb = IListRequests(mlist)
if action is Action.defer:
# Nothing to do.
return
elif action is Action.discard:
# Nothing to do except delete the request from the database.
pass
elif action is Action.reject:
key, data = requestdb.get_request(id)
_refuse(mlist, _('Subscription request'),
data['address'],
comment or _('[No reason given]'),
lang=getUtility(ILanguageManager)[data['language']])
elif action is Action.accept:
key, data = requestdb.get_request(id)
enum_value = data['delivery_mode'].split('.')[-1]
delivery_mode = DeliveryMode(enum_value)
address = data['address']
display_name = data['display_name']
language = getUtility(ILanguageManager)[data['language']]
password = data['password']
try:
add_member(mlist, address, display_name, password,
delivery_mode, language)
except AlreadySubscribedError:
# The address got subscribed in some other way after the original
# request was made and accepted.
pass
else:
if mlist.send_welcome_message:
send_welcome_message(mlist, address, language, delivery_mode)
if mlist.admin_notify_mchanges:
send_admin_subscription_notice(
mlist, address, display_name, language)
slog.info('%s: new %s, %s %s', mlist.fqdn_listname,
delivery_mode, formataddr((display_name, address)),
'via admin approval')
else:
raise AssertionError('Unexpected action: {0}'.format(action))
# Delete the request from the database.
requestdb.delete_request(id)
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._requests_db = IListRequests(self._mlist)
self._msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: Something
Message-ID: <alpha>
Something else.
""")
def _make_resource(self, request_id, expected_request_types):
requests = IListRequests(self._mlist)
results = requests.get_request(request_id)
if results is None:
return None
key, data = results
resource = dict(key=key, request_id=request_id)
# Flatten the IRequest payload into the JSON representation.
resource.update(data)
# Check for a matching request type, and insert the type name into the
# resource.
request_type = RequestType(resource.pop('_request_type'))
if request_type not in expected_request_types:
return None
resource['type'] = request_type.name
# This key isn't what you think it is. Usually, it's the Pendable
# record's row id, which isn't helpful at all. If it's not there,
# that's fine too.
resource.pop('id', None)
return resource
def test_requests_are_deleted_when_mailing_list_is_deleted(self):
# When a mailing list is deleted, its requests database is deleted
# too, e.g. all its message hold requests (but not the messages
# themselves).
msg = specialized_message_from_string("""\
From: [email protected]
To: [email protected]
Subject: Hold me
Message-ID: <argon>
""")
request_id = hold_message(self._ant, msg)
getUtility(IListManager).delete(self._ant)
# This is a hack. ListRequests don't access self._mailinglist in
# their get_request() method.
requestsdb = IListRequests(self._bee)
request = requestsdb.get_request(request_id)
self.assertEqual(request, None)
saved_message = getUtility(IMessageStore).get_message_by_id('<argon>')
self.assertEqual(saved_message.as_string(), msg.as_string())
def handle_unsubscription(mlist, id, action, comment=None):
requestdb = IListRequests(mlist)
key, data = requestdb.get_request(id)
address = data['address']
if action is Action.defer:
# Nothing to do.
return
elif action is Action.discard:
# Nothing to do except delete the request from the database.
pass
elif action is Action.reject:
key, data = requestdb.get_request(id)
_refuse(mlist, _('Unsubscription request'), address,
comment or _('[No reason given]'))
elif action is Action.accept:
key, data = requestdb.get_request(id)
try:
delete_member(mlist, address)
except NotAMemberError:
# User has already been unsubscribed.
pass
slog.info('%s: deleted %s', mlist.fqdn_listname, address)
else:
raise AssertionError('Unexpected action: {0}'.format(action))
# Delete the request from the database.
requestdb.delete_request(id)
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._request_db = IListRequests(self._mlist)
self._msg = specialized_message_from_string("""\
From: [email protected]
To: [email protected]
Subject: hold me
Message-ID: <alpha>
""")
self._in = make_testable_runner(IncomingRunner, 'in')
self._pipeline = make_testable_runner(PipelineRunner, 'pipeline')
self._out = make_testable_runner(OutgoingRunner, 'out')
def handle_subscription(mlist, id, action, comment=None):
requestdb = IListRequests(mlist)
if action is Action.defer:
# Nothing to do.
return
elif action is Action.discard:
# Nothing to do except delete the request from the database.
pass
elif action is Action.reject:
key, data = requestdb.get_request(id)
send_rejection(
mlist, _('Subscription request'),
data['email'],
comment or _('[No reason given]'),
lang=getUtility(ILanguageManager)[data['language']])
elif action is Action.accept:
key, data = requestdb.get_request(id)
delivery_mode = DeliveryMode[data['delivery_mode']]
email = data['email']
display_name = data['display_name']
language = getUtility(ILanguageManager)[data['language']]
try:
add_member(
mlist,
RequestRecord(email, display_name, delivery_mode, language))
except AlreadySubscribedError:
# The address got subscribed in some other way after the original
# request was made and accepted.
pass
slog.info('%s: new %s, %s %s', mlist.fqdn_listname,
delivery_mode, formataddr((display_name, email)),
'via admin approval')
else:
raise AssertionError('Unexpected action: {0}'.format(action))
# Delete the request from the database.
requestdb.delete_request(id)
def hold_unsubscription(mlist, address):
data = dict(address=address)
requestsdb = IListRequests(mlist)
request_id = requestsdb.hold_request(
RequestType.unsubscription, address, data)
vlog.info('%s: held unsubscription request from %s',
mlist.fqdn_listname, address)
# Possibly notify the administrator of the hold
if mlist.admin_immed_notify:
subject = _(
'New unsubscription request from $mlist.display_name by $address')
text = make('unsubauth.txt',
mailing_list=mlist,
address=address,
listname=mlist.fqdn_listname,
admindb_url=mlist.script_url('admindb'),
)
# This message should appear to come from the <list>-owner so as
# to avoid any useless bounce processing.
msg = UserNotification(
mlist.owner_address, mlist.owner_address,
subject, text, mlist.preferred_language)
msg.send(mlist, tomoderators=True)
return request_id
class TestRequests(unittest.TestCase):
layer = ConfigLayer
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._requests_db = IListRequests(self._mlist)
self._msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: Something
Message-ID: <alpha>
Something else.
""")
def test_get_request_with_type(self):
# get_request() takes an optional request type.
request_id = hold_message(self._mlist, self._msg)
# Submit a request with a non-matching type. This should return None
# as if there were no matches.
response = self._requests_db.get_request(
request_id, RequestType.subscription)
self.assertEqual(response, None)
# Submit the same request with a matching type.
key, data = self._requests_db.get_request(
request_id, RequestType.held_message)
self.assertEqual(key, '<alpha>')
# It should also succeed with no optional request type given.
key, data = self._requests_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
def test_hold_with_bogus_type(self):
# Calling hold_request() with a bogus request type is an error.
with self.assertRaises(TypeError) as cm:
self._requests_db.hold_request(5, 'foo')
self.assertEqual(cm.exception.args[0], 5)
def test_delete_missing_request(self):
# Trying to delete a missing request is an error.
with self.assertRaises(KeyError) as cm:
self._requests_db.delete_request(801)
self.assertEqual(cm.exception.args[0], 801)
def test_only_return_this_lists_requests(self):
# Issue #161: get_requests() returns requests that are not specific to
# the mailing list in question.
request_id = hold_message(self._mlist, self._msg)
bee = create_list('*****@*****.**')
self.assertIsNone(IListRequests(bee).get_request(request_id))
def on_get(self, request, response):
requests = IListRequests(self._mlist)
count = requests.count_of(RequestType.held_message)
okay(response, etag(dict(count=count)))
def handle_message(mlist, id, action,
comment=None, preserve=False, forward=None):
message_store = getUtility(IMessageStore)
requestdb = IListRequests(mlist)
key, msgdata = requestdb.get_request(id)
# Handle the action.
rejection = None
message_id = msgdata['_mod_message_id']
sender = msgdata['_mod_sender']
subject = msgdata['_mod_subject']
if action in (Action.defer, Action.hold):
# Nothing to do, but preserve the message for later.
preserve = True
elif action is Action.discard:
rejection = 'Discarded'
elif action is Action.reject:
rejection = 'Refused'
member = mlist.members.get_member(sender)
if member:
language = member.preferred_language
else:
language = None
_refuse(mlist, _('Posting of your message titled "$subject"'),
sender, comment or _('[No reason given]'), language)
elif action is Action.accept:
# Start by getting the message from the message store.
msg = message_store.get_message_by_id(message_id)
# Delete moderation-specific entries from the message metadata.
for key in msgdata.keys():
if key.startswith('_mod_'):
del msgdata[key]
# Add some metadata to indicate this message has now been approved.
msgdata['approved'] = True
msgdata['moderator_approved'] = True
# Calculate a new filebase for the approved message, otherwise
# delivery errors will cause duplicates.
if 'filebase' in msgdata:
del msgdata['filebase']
# Queue the file for delivery. Trying to deliver the message directly
# here can lead to a huge delay in web turnaround. Log the moderation
# and add a header.
msg['X-Mailman-Approved-At'] = formatdate(
time.mktime(now().timetuple()), localtime=True)
vlog.info('held message approved, message-id: %s',
msg.get('message-id', 'n/a'))
# Stick the message back in the incoming queue for further
# processing.
config.switchboards['pipeline'].enqueue(msg, _metadata=msgdata)
else:
raise AssertionError('Unexpected action: {0}'.format(action))
# Forward the message.
if forward:
# Get a copy of the original message from the message store.
msg = message_store.get_message_by_id(message_id)
# It's possible the forwarding address list is a comma separated list
# of display_name/address pairs.
addresses = [addr[1] for addr in getaddresses(forward)]
language = mlist.preferred_language
if len(addresses) == 1:
# If the address getting the forwarded message is a member of
# the list, we want the headers of the outer message to be
# encoded in their language. Otherwise it'll be the preferred
# language of the mailing list. This is better than sending a
# separate message per recipient.
member = mlist.members.get_member(addresses[0])
if member:
language = member.preferred_language
with _.using(language.code):
fmsg = UserNotification(
addresses, mlist.bounces_address,
_('Forward of moderated message'),
lang=language)
fmsg.set_type('message/rfc822')
fmsg.attach(msg)
fmsg.send(mlist)
# Delete the message from the message store if it is not being preserved.
if not preserve:
message_store.delete_message(message_id)
requestdb.delete_request(id)
# Log the rejection
if rejection:
note = """%s: %s posting:
\tFrom: %s
\tSubject: %s"""
if comment:
note += '\n\tReason: ' + comment
vlog.info(note, mlist.fqdn_listname, rejection, sender, subject)
class TestRequests(unittest.TestCase):
layer = ConfigLayer
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._requests_db = IListRequests(self._mlist)
self._msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: Something
Message-ID: <alpha>
Something else.
""")
def test_get_request_with_type(self):
# get_request() takes an optional request type.
request_id = hold_message(self._mlist, self._msg)
# Submit a request with a non-matching type. This should return None
# as if there were no matches.
response = self._requests_db.get_request(
request_id, RequestType.subscription)
self.assertEqual(response, None)
# Submit the same request with a matching type.
key, data = self._requests_db.get_request(
request_id, RequestType.held_message)
self.assertEqual(key, '<alpha>')
# It should also succeed with no optional request type given.
key, data = self._requests_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
def test_hold_with_bogus_type(self):
# Calling hold_request() with a bogus request type is an error.
with self.assertRaises(TypeError) as cm:
self._requests_db.hold_request(5, 'foo')
self.assertEqual(cm.exception.args[0], 5)
def test_delete_missing_request(self):
# Trying to delete a missing request is an error.
with self.assertRaises(KeyError) as cm:
self._requests_db.delete_request(801)
self.assertEqual(cm.exception.args[0], 801)
def test_only_return_this_lists_requests(self):
# Issue #161: get_requests() returns requests that are not specific to
# the mailing list in question.
request_id = hold_message(self._mlist, self._msg)
bee = create_list('*****@*****.**')
self.assertIsNone(IListRequests(bee).get_request(request_id))
def test_request_order(self):
# Requests must be sorted in creation order.
#
# This test only "works" for PostgreSQL, in the sense that if you
# remove the fix in ../requests.py, it will still pass in SQLite.
# Apparently SQLite auto-sorts results by ID but PostgreSQL autosorts
# by insertion time. It's still worth keeping the test to prevent
# regressions.
#
# We modify the auto-incremented ids by listening to SQLAlchemy's
# flush event, and hacking all the _Request object id's to the next
# value in a descending counter.
request_ids = []
counter = count(200, -1)
def id_hacker(session, flush_context, instances): # noqa
for instance in session.new:
if isinstance(instance, _Request):
instance.id = next(counter)
with before_flush(id_hacker):
for index in range(10):
msg = mfs(self._msg.as_string())
msg.replace_header('Message-ID', '<alpha{}>'.format(index))
request_ids.append(hold_message(self._mlist, msg))
config.db.store.flush()
# Make sure that our ID are not already sorted.
self.assertNotEqual(request_ids, sorted(request_ids))
# Get requests and check their order.
requests = self._requests_db.of_type(RequestType.held_message)
self.assertEqual([r.id for r in requests], sorted(request_ids))
def pending_requests(mlist):
# Must return a byte string
lcset = mlist.preferred_language.charset
pending = []
first = True
requestsdb = IListRequests(mlist)
for request in requestsdb.of_type(RequestType.subscription):
if first:
pending.append(_('Pending subscriptions:'))
first = False
key, data = requestsdb.get_request(request.id)
when = data['when']
addr = data['addr']
fullname = data['fullname']
passwd = data['passwd']
digest = data['digest']
lang = data['lang']
if fullname:
if isinstance(fullname, unicode):
fullname = fullname.encode(lcset, 'replace')
fullname = ' (%s)' % fullname
pending.append(' %s%s %s' % (addr, fullname, time.ctime(when)))
first = True
for request in requestsdb.of_type(RequestType.held_message):
if first:
pending.append(_('\nPending posts:'))
first = False
key, data = requestsdb.get_request(request.id)
when = data['when']
sender = data['sender']
subject = data['subject']
reason = data['reason']
text = data['text']
msgdata = data['msgdata']
subject = Utils.oneline(subject, lcset)
date = time.ctime(when)
reason = _(reason)
pending.append(_("""\
From: $sender on $date
Subject: $subject
Cause: $reason"""))
pending.append('')
# Coerce all items in pending to a Unicode so we can join them
upending = []
charset = mlist.preferred_language.charset
for s in pending:
if isinstance(s, unicode):
upending.append(s)
else:
upending.append(unicode(s, charset, 'replace'))
# Make sure that the text we return from here can be encoded to a byte
# string in the charset of the list's language. This could fail if for
# example, the request was pended while the list's language was French,
# but then it was changed to English before checkdbs ran.
text = NL.join(upending)
charset = Charset(mlist.preferred_language.charset)
incodec = charset.input_codec or 'ascii'
outcodec = charset.output_codec or 'ascii'
if isinstance(text, unicode):
return text.encode(outcodec, 'replace')
# Be sure this is a byte string encodeable in the list's charset
utext = unicode(text, incodec, 'replace')
return utext.encode(outcodec, 'replace')
def pending_requests(mlist):
# Must return a byte string
lcset = mlist.preferred_language.charset
pending = []
first = True
requestsdb = IListRequests(mlist)
for request in requestsdb.of_type(RequestType.subscription):
if first:
pending.append(_('Pending subscriptions:'))
first = False
key, data = requestsdb.get_request(request.id)
when = data['when']
addr = data['addr']
fullname = data['fullname']
passwd = data['passwd']
digest = data['digest']
lang = data['lang']
if fullname:
if isinstance(fullname, unicode):
fullname = fullname.encode(lcset, 'replace')
fullname = ' (%s)' % fullname
pending.append(' %s%s %s' % (addr, fullname, time.ctime(when)))
first = True
for request in requestsdb.of_type(RequestType.held_message):
if first:
pending.append(_('\nPending posts:'))
first = False
key, data = requestsdb.get_request(request.id)
when = data['when']
sender = data['sender']
subject = data['subject']
reason = data['reason']
text = data['text']
msgdata = data['msgdata']
subject = Utils.oneline(subject, lcset)
date = time.ctime(when)
reason = _(reason)
pending.append(
_("""\
From: $sender on $date
Subject: $subject
Cause: $reason"""))
pending.append('')
# Coerce all items in pending to a Unicode so we can join them
upending = []
charset = mlist.preferred_language.charset
for s in pending:
if isinstance(s, unicode):
upending.append(s)
else:
upending.append(unicode(s, charset, 'replace'))
# Make sure that the text we return from here can be encoded to a byte
# string in the charset of the list's language. This could fail if for
# example, the request was pended while the list's language was French,
# but then it was changed to English before checkdbs ran.
text = NL.join(upending)
charset = Charset(mlist.preferred_language.charset)
incodec = charset.input_codec or 'ascii'
outcodec = charset.output_codec or 'ascii'
if isinstance(text, unicode):
return text.encode(outcodec, 'replace')
# Be sure this is a byte string encodeable in the list's charset
utext = unicode(text, incodec, 'replace')
return utext.encode(outcodec, 'replace')
def _get_collection(self, request):
requests = IListRequests(self._mlist)
self._requests = requests
return list(requests.of_type(RequestType.held_message))
class TestModeration(unittest.TestCase):
"""Test moderation functionality."""
layer = SMTPLayer
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._request_db = IListRequests(self._mlist)
self._msg = specialized_message_from_string("""\
From: [email protected]
To: [email protected]
Subject: hold me
Message-ID: <alpha>
""")
self._in = make_testable_runner(IncomingRunner, 'in')
self._pipeline = make_testable_runner(PipelineRunner, 'pipeline')
self._out = make_testable_runner(OutgoingRunner, 'out')
def test_accepted_message_gets_posted(self):
# A message that is accepted by the moderator should get posted to the
# mailing list. LP: #827697
msgdata = dict(listname='*****@*****.**',
recipients=['*****@*****.**'])
request_id = hold_message(self._mlist, self._msg, msgdata)
handle_message(self._mlist, request_id, Action.accept)
self._in.run()
self._pipeline.run()
self._out.run()
messages = list(SMTPLayer.smtpd.messages)
self.assertEqual(len(messages), 1)
message = messages[0]
# We don't need to test the entire posted message, just the bits that
# prove it got sent out.
self.assertTrue('x-mailman-version' in message)
self.assertTrue('x-peer' in message)
# The X-Mailman-Approved-At header has local timezone information in
# it, so test that separately.
self.assertEqual(message['x-mailman-approved-at'][:-5],
'Mon, 01 Aug 2005 07:49:23 ')
del message['x-mailman-approved-at']
# The Message-ID matches the original.
self.assertEqual(message['message-id'], '<alpha>')
# Anne sent the message and the mailing list received it.
self.assertEqual(message['from'], '*****@*****.**')
self.assertEqual(message['to'], '*****@*****.**')
# The Subject header has the list's prefix.
self.assertEqual(message['subject'], '[Test] hold me')
# The list's -bounce address is the actual sender, and Bart is the
# only actual recipient. These headers are added by the testing
# framework and don't show up in production. They match the RFC 5321
# envelope.
self.assertEqual(message['x-mailfrom'], '*****@*****.**')
self.assertEqual(message['x-rcptto'], '*****@*****.**')
def test_hold_action_alias_for_defer(self):
# In handle_message(), the 'hold' action is the same as 'defer' for
# purposes of this API.
request_id = hold_message(self._mlist, self._msg)
handle_message(self._mlist, request_id, Action.defer)
# The message is still in the pending requests.
key, data = self._request_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
handle_message(self._mlist, request_id, Action.hold)
key, data = self._request_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
def test_lp_1031391(self):
# LP: #1031391 msgdata['received_time'] gets added by the LMTP server.
# The value is a datetime. If this message gets held, it will break
# pending requests since they require string keys and values.
received_time = now()
msgdata = dict(received_time=received_time)
request_id = hold_message(self._mlist, self._msg, msgdata)
key, data = self._request_db.get_request(request_id)
self.assertEqual(data['received_time'], received_time)
def test_non_preserving_disposition(self):
# By default, disposed messages are not preserved.
request_id = hold_message(self._mlist, self._msg)
handle_message(self._mlist, request_id, Action.discard)
message_store = getUtility(IMessageStore)
self.assertIsNone(message_store.get_message_by_id('<alpha>'))
def test_preserving_disposition(self):
# Preserving a message keeps it in the store.
request_id = hold_message(self._mlist, self._msg)
handle_message(self._mlist, request_id, Action.discard, preserve=True)
message_store = getUtility(IMessageStore)
preserved_message = message_store.get_message_by_id('<alpha>')
self.assertEqual(preserved_message['message-id'], '<alpha>')
def test_preserve_and_forward(self):
# We can both preserve and forward the message.
request_id = hold_message(self._mlist, self._msg)
handle_message(self._mlist,
request_id,
Action.discard,
preserve=True,
forward=['*****@*****.**'])
# The message is preserved in the store.
message_store = getUtility(IMessageStore)
preserved_message = message_store.get_message_by_id('<alpha>')
self.assertEqual(preserved_message['message-id'], '<alpha>')
# And the forwarded message lives in the virgin queue.
messages = get_queue_messages('virgin')
self.assertEqual(len(messages), 1)
self.assertEqual(str(messages[0].msg['subject']),
'Forward of moderated message')
self.assertEqual(messages[0].msgdata['recipients'],
['*****@*****.**'])
class TestModeration(unittest.TestCase):
"""Test moderation functionality."""
layer = SMTPLayer
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._request_db = IListRequests(self._mlist)
self._msg = specialized_message_from_string("""\
From: [email protected]
To: [email protected]
Subject: hold me
Message-ID: <alpha>
""")
self._in = make_testable_runner(IncomingRunner, 'in')
self._pipeline = make_testable_runner(PipelineRunner, 'pipeline')
self._out = make_testable_runner(OutgoingRunner, 'out')
def test_accepted_message_gets_posted(self):
# A message that is accepted by the moderator should get posted to the
# mailing list. LP: #827697
msgdata = dict(listname='*****@*****.**',
recipients=['*****@*****.**'])
request_id = hold_message(self._mlist, self._msg, msgdata)
handle_message(self._mlist, request_id, Action.accept)
self._in.run()
self._pipeline.run()
self._out.run()
messages = list(SMTPLayer.smtpd.messages)
self.assertEqual(len(messages), 1)
message = messages[0]
# We don't need to test the entire posted message, just the bits that
# prove it got sent out.
self.assertTrue('x-mailman-version' in message)
self.assertTrue('x-peer' in message)
# The X-Mailman-Approved-At header has local timezone information in
# it, so test that separately.
self.assertEqual(message['x-mailman-approved-at'][:-5],
'Mon, 01 Aug 2005 07:49:23 ')
del message['x-mailman-approved-at']
# The Message-ID matches the original.
self.assertEqual(message['message-id'], '<alpha>')
# Anne sent the message and the mailing list received it.
self.assertEqual(message['from'], '*****@*****.**')
self.assertEqual(message['to'], '*****@*****.**')
# The Subject header has the list's prefix.
self.assertEqual(message['subject'], '[Test] hold me')
# The list's -bounce address is the actual sender, and Bart is the
# only actual recipient. These headers are added by the testing
# framework and don't show up in production. They match the RFC 5321
# envelope.
self.assertEqual(message['x-mailfrom'], '*****@*****.**')
self.assertEqual(message['x-rcptto'], '*****@*****.**')
def test_hold_action_alias_for_defer(self):
# In handle_message(), the 'hold' action is the same as 'defer' for
# purposes of this API.
request_id = hold_message(self._mlist, self._msg)
handle_message(self._mlist, request_id, Action.defer)
# The message is still in the pending requests.
key, data = self._request_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
handle_message(self._mlist, request_id, Action.hold)
key, data = self._request_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
def test_lp_1031391(self):
# LP: #1031391 msgdata['received_time'] gets added by the LMTP server.
# The value is a datetime. If this message gets held, it will break
# pending requests since they require string keys and values.
received_time = now()
msgdata = dict(received_time=received_time)
request_id = hold_message(self._mlist, self._msg, msgdata)
key, data = self._request_db.get_request(request_id)
self.assertEqual(data['received_time'], received_time)
def test_non_preserving_disposition(self):
# By default, disposed messages are not preserved.
request_id = hold_message(self._mlist, self._msg)
handle_message(self._mlist, request_id, Action.discard)
message_store = getUtility(IMessageStore)
self.assertIsNone(message_store.get_message_by_id('<alpha>'))
def test_preserving_disposition(self):
# Preserving a message keeps it in the store.
request_id = hold_message(self._mlist, self._msg)
handle_message(self._mlist, request_id, Action.discard, preserve=True)
message_store = getUtility(IMessageStore)
preserved_message = message_store.get_message_by_id('<alpha>')
self.assertEqual(preserved_message['message-id'], '<alpha>')
def test_preserve_and_forward(self):
# We can both preserve and forward the message.
request_id = hold_message(self._mlist, self._msg)
handle_message(self._mlist, request_id, Action.discard,
preserve=True, forward=['*****@*****.**'])
# The message is preserved in the store.
message_store = getUtility(IMessageStore)
preserved_message = message_store.get_message_by_id('<alpha>')
self.assertEqual(preserved_message['message-id'], '<alpha>')
# And the forwarded message lives in the virgin queue.
messages = get_queue_messages('virgin')
self.assertEqual(len(messages), 1)
self.assertEqual(str(messages[0].msg['subject']),
'Forward of moderated message')
self.assertEqual(messages[0].msgdata['recipients'],
['*****@*****.**'])
def _get_collection(self, request):
requests = IListRequests(self._mlist)
return requests.of_type(RequestType.held_message)
class TestRequests(unittest.TestCase):
layer = ConfigLayer
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._requests_db = IListRequests(self._mlist)
self._msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: Something
Message-ID: <alpha>
Something else.
""")
def test_get_request_with_type(self):
# get_request() takes an optional request type.
request_id = hold_message(self._mlist, self._msg)
# Submit a request with a non-matching type. This should return None
# as if there were no matches.
response = self._requests_db.get_request(request_id,
RequestType.subscription)
self.assertEqual(response, None)
# Submit the same request with a matching type.
key, data = self._requests_db.get_request(request_id,
RequestType.held_message)
self.assertEqual(key, '<alpha>')
# It should also succeed with no optional request type given.
key, data = self._requests_db.get_request(request_id)
self.assertEqual(key, '<alpha>')
def test_hold_with_bogus_type(self):
# Calling hold_request() with a bogus request type is an error.
with self.assertRaises(TypeError) as cm:
self._requests_db.hold_request(5, 'foo')
self.assertEqual(cm.exception.args[0], 5)
def test_delete_missing_request(self):
# Trying to delete a missing request is an error.
with self.assertRaises(KeyError) as cm:
self._requests_db.delete_request(801)
self.assertEqual(cm.exception.args[0], 801)
def test_only_return_this_lists_requests(self):
# Issue #161: get_requests() returns requests that are not specific to
# the mailing list in question.
request_id = hold_message(self._mlist, self._msg)
bee = create_list('*****@*****.**')
self.assertIsNone(IListRequests(bee).get_request(request_id))
def test_request_order(self):
# Requests must be sorted in creation order.
#
# This test only "works" for PostgreSQL, in the sense that if you
# remove the fix in ../requests.py, it will still pass in SQLite.
# Apparently SQLite auto-sorts results by ID but PostgreSQL autosorts
# by insertion time. It's still worth keeping the test to prevent
# regressions.
#
# We modify the auto-incremented ids by listening to SQLAlchemy's
# flush event, and hacking all the _Request object id's to the next
# value in a descending counter.
request_ids = []
counter = count(200, -1)
def id_hacker(session, flush_context, instances): # noqa: E301
for instance in session.new:
if isinstance(instance, _Request):
instance.id = next(counter)
with before_flush(id_hacker):
for index in range(10):
msg = mfs(self._msg.as_string())
msg.replace_header('Message-ID', '<alpha{}>'.format(index))
request_ids.append(hold_message(self._mlist, msg))
config.db.store.flush()
# Make sure that our ID are not already sorted.
self.assertNotEqual(request_ids, sorted(request_ids))
# Get requests and check their order.
requests = self._requests_db.of_type(RequestType.held_message)
self.assertEqual([r.id for r in requests], sorted(request_ids))
def test_only_return_this_lists_requests(self):
# Issue #161: get_requests() returns requests that are not specific to
# the mailing list in question.
request_id = hold_message(self._mlist, self._msg)
bee = create_list('*****@*****.**')
self.assertIsNone(IListRequests(bee).get_request(request_id))
def handle_message(mlist,
id,
action,
comment=None,
preserve=False,
forward=None):
message_store = getUtility(IMessageStore)
requestdb = IListRequests(mlist)
key, msgdata = requestdb.get_request(id)
# Handle the action.
rejection = None
message_id = msgdata['_mod_message_id']
sender = msgdata['_mod_sender']
subject = msgdata['_mod_subject']
if action in (Action.defer, Action.hold):
# Nothing to do, but preserve the message for later.
preserve = True
elif action is Action.discard:
rejection = 'Discarded'
elif action is Action.reject:
rejection = 'Refused'
member = mlist.members.get_member(sender)
if member:
language = member.preferred_language
else:
language = None
send_rejection(mlist, _('Posting of your message titled "$subject"'),
sender, comment or _('[No reason given]'), language)
elif action is Action.accept:
# Start by getting the message from the message store.
msg = message_store.get_message_by_id(message_id)
# Delete moderation-specific entries from the message metadata.
for key in list(msgdata):
if key.startswith('_mod_'):
del msgdata[key]
# Add some metadata to indicate this message has now been approved.
msgdata['approved'] = True
msgdata['moderator_approved'] = True
# Calculate a new filebase for the approved message, otherwise
# delivery errors will cause duplicates.
if 'filebase' in msgdata:
del msgdata['filebase']
# Queue the file for delivery. Trying to deliver the message directly
# here can lead to a huge delay in web turnaround. Log the moderation
# and add a header.
msg['X-Mailman-Approved-At'] = formatdate(time.mktime(
now().timetuple()),
localtime=True)
vlog.info('held message approved, message-id: %s',
msg.get('message-id', 'n/a'))
# Stick the message back in the incoming queue for further
# processing.
config.switchboards['pipeline'].enqueue(msg, _metadata=msgdata)
else:
raise AssertionError('Unexpected action: {0}'.format(action))
# Forward the message.
if forward:
# Get a copy of the original message from the message store.
msg = message_store.get_message_by_id(message_id)
# It's possible the forwarding address list is a comma separated list
# of display_name/address pairs.
addresses = [addr[1] for addr in getaddresses(forward)]
language = mlist.preferred_language
if len(addresses) == 1:
# If the address getting the forwarded message is a member of
# the list, we want the headers of the outer message to be
# encoded in their language. Otherwise it'll be the preferred
# language of the mailing list. This is better than sending a
# separate message per recipient.
member = mlist.members.get_member(addresses[0])
if member:
language = member.preferred_language
with _.using(language.code):
fmsg = UserNotification(addresses,
mlist.bounces_address,
_('Forward of moderated message'),
lang=language)
fmsg.set_type('message/rfc822')
fmsg.attach(msg)
fmsg.send(mlist)
# Delete the message from the message store if it is not being preserved.
if not preserve:
message_store.delete_message(message_id)
requestdb.delete_request(id)
# Log the rejection
if rejection:
note = """%s: %s posting:
\tFrom: %s
\tSubject: %s"""
if comment:
note += '\n\tReason: ' + comment
vlog.info(note, mlist.fqdn_listname, rejection, sender, subject)