def on_post(self, request, response):
try:
validator = Validator(action=enum_validator(Action))
arguments = validator(request)
except ValueError as error:
bad_request(response, str(error))
return
requests = IListRequests(self._mlist)
try:
request_id = int(self._request_id)
except ValueError:
bad_request(response)
return
results = requests.get_request(request_id)
if results is None:
not_found(response)
return
key, data = results
try:
request_type = RequestType[data['_request_type']]
except ValueError:
bad_request(response)
return
if request_type is RequestType.subscription:
handle_subscription(self._mlist, request_id, **arguments)
elif request_type is RequestType.unsubscription:
handle_unsubscription(self._mlist, request_id, **arguments)
else:
bad_request(response)
return
no_content(response)
def notify(ctx, list_ids, dry_run, verbose):
list_manager = getUtility(IListManager)
if list_ids:
lists = []
for spec in list_ids:
# We'll accept list-ids or fqdn list names.
if '@' in spec:
mlist = list_manager.get(spec)
else:
mlist = list_manager.get_by_list_id(spec)
if mlist is None:
print(_('No such list found: $spec'), file=sys.stderr)
else:
lists.append(mlist)
else:
lists = list_manager.mailing_lists
for mlist in lists:
requestdb = IListRequests(mlist)
count = requestdb.count
subs, unsubs = _get_subs(mlist)
count += len(subs) + len(unsubs)
if verbose:
print(_('The {} list has {} moderation requests waiting.').format(
mlist.fqdn_listname, count))
if count > 0 and not dry_run:
detail = _build_detail(requestdb, subs, unsubs)
_send_notice(mlist, count, detail)
def _make_resource(self, request_id):
requests = IListRequests(self._mlist)
results = requests.get_request(request_id)
if results is None:
return None
key, data = results
resource = dict(key=key, request_id=request_id)
# Flatten the IRequest payload into the JSON representation.
if data is not None:
resource.update(data)
# Check for a matching request type, and insert the type name into the
# resource.
try:
request_type = RequestType[resource.pop('_request_type', None)]
except KeyError:
request_type = None
if request_type is not RequestType.held_message:
return None
resource['type'] = RequestType.held_message.name
# This key isn't what you think it is. Usually, it's the Pendable
# record's row id, which isn't helpful at all. If it's not there,
# that's fine too.
resource.pop('id', None)
# Add a self_link.
resource['self_link'] = self.api.path_to(
'lists/{}/held/{}'.format(self._mlist.list_id, request_id))
return resource
def handle_unsubscription(mlist, id, action, comment=None):
requestdb = IListRequests(mlist)
key, data = requestdb.get_request(id)
email = data['email']
if action is Action.defer:
# Nothing to do.
return
elif action is Action.discard:
# Nothing to do except delete the request from the database.
pass
elif action is Action.reject:
key, data = requestdb.get_request(id)
send_rejection(
mlist, _('Unsubscription request'), email,
comment or _('[No reason given]'))
elif action is Action.accept:
key, data = requestdb.get_request(id)
try:
delete_member(mlist, email)
except NotAMemberError:
# User has already been unsubscribed.
pass
slog.info('%s: deleted %s', mlist.fqdn_listname, email)
else:
raise AssertionError('Unexpected action: {0}'.format(action))
# Delete the request from the database.
requestdb.delete_request(id)
def hold_unsubscription(mlist, email):
data = dict(email=email)
requestsdb = IListRequests(mlist)
request_id = requestsdb.hold_request(RequestType.unsubscription, email,
data)
vlog.info('%s: held unsubscription request from %s', mlist.fqdn_listname,
email)
# Possibly notify the administrator of the hold
if mlist.admin_immed_notify:
subject = _(
'New unsubscription request from $mlist.display_name by $email')
template = getUtility(ITemplateLoader).get(
'list:admin:action:unsubscribe', mlist)
text = wrap(
expand(
template,
mlist,
dict(
# For backward compatibility.
mailing_list=mlist,
member=email,
email=email,
)))
# This message should appear to come from the <list>-owner so as
# to avoid any useless bounce processing.
msg = UserNotification(mlist.owner_address, mlist.owner_address,
subject, text, mlist.preferred_language)
msg.send(mlist)
return request_id
def hold_subscription(mlist, address, display_name, password, mode, language):
data = dict(when=now().isoformat(),
address=address,
display_name=display_name,
password=password,
delivery_mode=mode.name,
language=language)
# Now hold this request. We'll use the address as the key.
requestsdb = IListRequests(mlist)
request_id = requestsdb.hold_request(RequestType.subscription, address,
data)
vlog.info('%s: held subscription request from %s', mlist.fqdn_listname,
address)
# Possibly notify the administrator in default list language
if mlist.admin_immed_notify:
subject = _(
'New subscription request to $mlist.display_name from $address')
text = make(
'subauth.txt',
mailing_list=mlist,
username=address,
listname=mlist.fqdn_listname,
admindb_url=mlist.script_url('admindb'),
)
# This message should appear to come from the <list>-owner so as
# to avoid any useless bounce processing.
msg = UserNotification(mlist.owner_address, mlist.owner_address,
subject, text, mlist.preferred_language)
msg.send(mlist, tomoderators=True)
return request_id
def test_request_is_not_held_message(self):
requests = IListRequests(self._mlist)
with transaction():
request_id = requests.hold_request(RequestType.subscription, 'foo')
with self.assertRaises(HTTPError) as cm:
call_api('http://localhost:9001/3.0/lists/ant.example.com'
'/held/{}'.format(request_id))
self.assertEqual(cm.exception.code, 404)
def _get_collection(self, request):
requests = IListRequests(self._mlist)
self._requests = requests
items = []
for request_type in MEMBERSHIP_CHANGE_REQUESTS:
for request in requests.of_type(request_type):
items.append(request)
return items
def handle_ListDeletingEvent(event):
if not isinstance(event, ListDeletingEvent):
return
# Get the held requests database for the mailing list. Since the mailing
# list is about to get deleted, we can delete all associated requests.
requestsdb = IListRequests(event.mailing_list)
for request in requestsdb.held_requests:
requestsdb.delete_request(request.id)
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._requests_db = IListRequests(self._mlist)
self._msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: Something
Message-ID: <alpha>
Something else.
""")
def main():
opts, args, parser = parseargs()
initialize(opts.config)
for name in config.list_manager.names:
# The list must be locked in order to open the requests database
mlist = MailList.MailList(name)
try:
count = IListRequests(mlist).count
# While we're at it, let's evict yesterday's autoresponse data
midnight_today = midnight()
evictions = []
for sender in mlist.hold_and_cmd_autoresponses.keys():
date, respcount = mlist.hold_and_cmd_autoresponses[sender]
if midnight(date) < midnight_today:
evictions.append(sender)
if evictions:
for sender in evictions:
del mlist.hold_and_cmd_autoresponses[sender]
# This is the only place we've changed the list's database
mlist.Save()
if count:
# Set the default language the the list's preferred language.
_.default = mlist.preferred_language
realname = mlist.real_name
discarded = auto_discard(mlist)
if discarded:
count = count - discarded
text = _('Notice: $discarded old request(s) '
'automatically expired.\n\n')
else:
text = ''
if count:
text += Utils.maketext(
'checkdbs.txt', {
'count': count,
'mail_host': mlist.mail_host,
'adminDB': mlist.GetScriptURL('admindb',
absolute=1),
'real_name': realname,
},
mlist=mlist)
text += '\n' + pending_requests(mlist)
subject = _('$count $realname moderator '
'request(s) waiting')
else:
subject = _('$realname moderator request check result')
msg = UserNotification(mlist.GetOwnerEmail(),
mlist.GetBouncesEmail(), subject, text,
mlist.preferred_language)
msg.send(mlist, **{'tomoderators': True})
finally:
mlist.Unlock()
def delete(self, store, mlist):
"""See `IListManager`."""
fqdn_listname = mlist.fqdn_listname
notify(ListDeletingEvent(mlist))
# First delete information associated with the mailing list.
IAcceptableAliasSet(mlist).clear()
IListRequests(mlist).clear()
store.query(AutoResponseRecord).filter_by(mailing_list=mlist).delete()
store.query(ContentFilter).filter_by(mailing_list=mlist).delete()
store.query(ListArchiver).filter_by(mailing_list=mlist).delete()
store.query(Ban).filter_by(list_id=mlist.list_id).delete()
store.delete(mlist)
notify(ListDeletedEvent(fqdn_listname))
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._request_db = IListRequests(self._mlist)
self._msg = specialized_message_from_string("""\
From: [email protected]
To: [email protected]
Subject: hold me
Message-ID: <alpha>
""")
self._in = make_testable_runner(IncomingRunner, 'in')
self._pipeline = make_testable_runner(PipelineRunner, 'pipeline')
self._out = make_testable_runner(OutgoingRunner, 'out')
def auto_discard(mlist):
# Discard old held messages
discard_count = 0
expire = config.days(mlist.max_days_to_hold)
requestsdb = IListRequests(mlist)
heldmsgs = list(requestsdb.of_type(RequestType.held_message))
if expire and heldmsgs:
for request in heldmsgs:
key, data = requestsdb.get_request(request.id)
if now - data['date'] > expire:
handle_request(mlist, request.id, config.DISCARD)
discard_count += 1
mlist.Save()
return discard_count
def test_hold_chain_no_reasons_given(self):
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: A message
Message-ID: <ant>
MIME-Version: 1.0
A message body.
""")
process_chain(self._mlist, msg, {}, start_chain='hold')
# No reason was given, so a default is used.
requests = IListRequests(self._mlist)
self.assertEqual(requests.count_of(RequestType.held_message), 1)
request = requests.of_type(RequestType.held_message)[0]
key, data = requests.get_request(request.id)
self.assertEqual(data.get('_mod_reason'), 'n/a')
def test_hold_chain(self):
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: A message
Message-ID: <ant>
MIME-Version: 1.0
A message body.
""")
msgdata = dict(moderation_reasons=[
'TEST-REASON-1',
'TEST-REASON-2',
('TEST-{}-REASON-{}', 'FORMAT', 3),
])
logfile = LogFileMark('mailman.vette')
process_chain(self._mlist, msg, msgdata, start_chain='hold')
messages = get_queue_messages('virgin', expected_count=2)
payloads = {}
for item in messages:
if item.msg['to'] == '*****@*****.**':
part = item.msg.get_payload(0)
payloads['owner'] = part.get_payload().splitlines()
elif item.msg['To'] == '*****@*****.**':
payloads['sender'] = item.msg.get_payload().splitlines()
else:
self.fail('Unexpected message: %s' % item.msg)
self.assertIn(' TEST-REASON-1', payloads['owner'])
self.assertIn(' TEST-REASON-2', payloads['owner'])
self.assertIn(' TEST-FORMAT-REASON-3', payloads['owner'])
self.assertIn(' TEST-REASON-1', payloads['sender'])
self.assertIn(' TEST-REASON-2', payloads['sender'])
self.assertIn(' TEST-FORMAT-REASON-3', payloads['sender'])
logged = logfile.read()
self.assertIn('TEST-REASON-1', logged)
self.assertIn('TEST-REASON-2', logged)
self.assertIn('TEST-FORMAT-REASON-3', logged)
# Check the reason passed to hold_message().
requests = IListRequests(self._mlist)
self.assertEqual(requests.count_of(RequestType.held_message), 1)
request = requests.of_type(RequestType.held_message)[0]
key, data = requests.get_request(request.id)
self.assertEqual(
data.get('_mod_reason'),
'TEST-REASON-1; TEST-REASON-2; TEST-FORMAT-REASON-3')
def hold_message(mlist, msg, msgdata=None, reason=None):
"""Hold a message for moderator approval.
The message is added to the mailing list's request database.
:param mlist: The mailing list to hold the message on.
:param msg: The message to hold.
:param msgdata: Optional message metadata to hold. If not given, a new
metadata dictionary is created and held with the message.
:param reason: Optional string reason why the message is being held. If
not given, the empty string is used.
:return: An id used to handle the held message later.
"""
if msgdata is None:
msgdata = {}
else:
# Make a copy of msgdata so that subsequent changes won't corrupt the
# request database. TBD: remove the `filebase' key since this will
# not be relevant when the message is resurrected.
msgdata = msgdata.copy()
if reason is None:
reason = ''
# Add the message to the message store. It is required to have a
# Message-ID header.
message_id = msg.get('message-id')
if message_id is None:
msg['Message-ID'] = message_id = make_msgid()
elif isinstance(message_id, bytes):
message_id = message_id.decode('ascii')
getUtility(IMessageStore).add(msg)
# Prepare the message metadata with some extra information needed only by
# the moderation interface.
msgdata['_mod_message_id'] = message_id
msgdata['_mod_listid'] = mlist.list_id
msgdata['_mod_sender'] = msg.sender
# The subject can sometimes be a Header instance. Stringify it.
msgdata['_mod_subject'] = str(msg.get('subject', _('(no subject)')))
msgdata['_mod_reason'] = reason
msgdata['_mod_hold_date'] = now().isoformat()
# Now hold this request. We'll use the message_id as the key.
requestsdb = IListRequests(mlist)
request_id = requestsdb.hold_request(RequestType.held_message, message_id,
msgdata)
return request_id
def on_post(self, request, response):
try:
validator = Validator(action=enum_validator(Action))
arguments = validator(request)
except ValueError as error:
bad_request(response, str(error))
return
requests = IListRequests(self._mlist)
try:
request_id = int(self._request_id)
except ValueError:
not_found(response)
return
results = requests.get_request(request_id, RequestType.held_message)
if results is None:
not_found(response)
else:
handle_message(self._mlist, request_id, **arguments)
no_content(response)
def test_requests_are_deleted_when_mailing_list_is_deleted(self):
# When a mailing list is deleted, its requests database is deleted
# too, e.g. all its message hold requests (but not the messages
# themselves).
msg = specialized_message_from_string("""\
From: [email protected]
To: [email protected]
Subject: Hold me
Message-ID: <argon>
""")
request_id = hold_message(self._ant, msg)
getUtility(IListManager).delete(self._ant)
# This is a hack. ListRequests don't access self._mailinglist in
# their get_request() method.
requestsdb = IListRequests(self._bee)
request = requestsdb.get_request(request_id)
self.assertEqual(request, None)
saved_message = getUtility(IMessageStore).get_message_by_id('<argon>')
self.assertEqual(saved_message.as_string(), msg.as_string())
def _make_resource(self, request_id):
requests = IListRequests(self._mlist)
results = requests.get_request(request_id)
if results is None:
return None
key, data = results
resource = dict(key=key, request_id=request_id)
# Flatten the IRequest payload into the JSON representation.
resource.update(data)
# Check for a matching request type, and insert the type name into the
# resource.
request_type = RequestType[resource.pop('_request_type')]
if request_type is not RequestType.held_message:
return None
resource['type'] = RequestType.held_message.name
# This key isn't what you think it is. Usually, it's the Pendable
# record's row id, which isn't helpful at all. If it's not there,
# that's fine too.
resource.pop('id', None)
return resource
def handle_subscription(mlist, id, action, comment=None):
requestdb = IListRequests(mlist)
if action is Action.defer:
# Nothing to do.
return
elif action is Action.discard:
# Nothing to do except delete the request from the database.
pass
elif action is Action.reject:
key, data = requestdb.get_request(id)
_refuse(mlist,
_('Subscription request'),
data['address'],
comment or _('[No reason given]'),
lang=getUtility(ILanguageManager)[data['language']])
elif action is Action.accept:
key, data = requestdb.get_request(id)
delivery_mode = DeliveryMode[data['delivery_mode']]
address = data['address']
display_name = data['display_name']
language = getUtility(ILanguageManager)[data['language']]
password = data['password']
try:
add_member(mlist, address, display_name, password, delivery_mode,
language)
except AlreadySubscribedError:
# The address got subscribed in some other way after the original
# request was made and accepted.
pass
else:
if mlist.admin_notify_mchanges:
send_admin_subscription_notice(mlist, address, display_name,
language)
slog.info('%s: new %s, %s %s', mlist.fqdn_listname, delivery_mode,
formataddr((display_name, address)), 'via admin approval')
else:
raise AssertionError('Unexpected action: {0}'.format(action))
# Delete the request from the database.
requestdb.delete_request(id)
def hold_unsubscription(mlist, email):
data = dict(email=email)
requestsdb = IListRequests(mlist)
request_id = requestsdb.hold_request(
RequestType.unsubscription, email, data)
vlog.info('%s: held unsubscription request from %s',
mlist.fqdn_listname, email)
# Possibly notify the administrator of the hold
if mlist.admin_immed_notify:
subject = _(
'New unsubscription request from $mlist.display_name by $email')
text = make('unsubauth.txt',
mailing_list=mlist,
email=email,
listname=mlist.fqdn_listname,
admindb_url=mlist.script_url('admindb'),
)
# This message should appear to come from the <list>-owner so as
# to avoid any useless bounce processing.
msg = UserNotification(
mlist.owner_address, mlist.owner_address,
subject, text, mlist.preferred_language)
msg.send(mlist, tomoderators=True)
return request_id
def pending_requests(mlist):
# Must return a byte string
lcset = mlist.preferred_language.charset
pending = []
first = True
requestsdb = IListRequests(mlist)
for request in requestsdb.of_type(RequestType.subscription):
if first:
pending.append(_('Pending subscriptions:'))
first = False
key, data = requestsdb.get_request(request.id)
when = data['when']
addr = data['addr']
fullname = data['fullname']
passwd = data['passwd']
digest = data['digest']
lang = data['lang']
if fullname:
if isinstance(fullname, unicode):
fullname = fullname.encode(lcset, 'replace')
fullname = ' (%s)' % fullname
pending.append(' %s%s %s' % (addr, fullname, time.ctime(when)))
first = True
for request in requestsdb.of_type(RequestType.held_message):
if first:
pending.append(_('\nPending posts:'))
first = False
key, data = requestsdb.get_request(request.id)
when = data['when']
sender = data['sender']
subject = data['subject']
reason = data['reason']
text = data['text']
msgdata = data['msgdata']
subject = Utils.oneline(subject, lcset)
date = time.ctime(when)
reason = _(reason)
pending.append(
_("""\
From: $sender on $date
Subject: $subject
Cause: $reason"""))
pending.append('')
# Coerce all items in pending to a Unicode so we can join them
upending = []
charset = mlist.preferred_language.charset
for s in pending:
if isinstance(s, unicode):
upending.append(s)
else:
upending.append(unicode(s, charset, 'replace'))
# Make sure that the text we return from here can be encoded to a byte
# string in the charset of the list's language. This could fail if for
# example, the request was pended while the list's language was French,
# but then it was changed to English before checkdbs ran.
text = NL.join(upending)
charset = Charset(mlist.preferred_language.charset)
incodec = charset.input_codec or 'ascii'
outcodec = charset.output_codec or 'ascii'
if isinstance(text, unicode):
return text.encode(outcodec, 'replace')
# Be sure this is a byte string encodeable in the list's charset
utext = unicode(text, incodec, 'replace')
return utext.encode(outcodec, 'replace')
def handle_message(mlist, id, action, comment=None, forward=None):
message_store = getUtility(IMessageStore)
requestdb = IListRequests(mlist)
key, msgdata = requestdb.get_request(id)
# Handle the action.
rejection = None
message_id = msgdata['_mod_message_id']
sender = msgdata['_mod_sender']
subject = msgdata['_mod_subject']
keep = False
if action in (Action.defer, Action.hold):
# Nothing to do, but preserve the message for later.
keep = True
elif action is Action.discard:
rejection = 'Discarded'
elif action is Action.reject:
rejection = 'Refused'
member = mlist.members.get_member(sender)
if member:
language = member.preferred_language
else:
language = None
send_rejection(
mlist, _('Posting of your message titled "$subject"'),
sender, comment or _('[No reason given]'), language)
elif action is Action.accept:
# Start by getting the message from the message store.
msg = message_store.get_message_by_id(message_id)
# Delete moderation-specific entries from the message metadata.
for key in list(msgdata):
if key.startswith('_mod_'):
del msgdata[key]
# Add some metadata to indicate this message has now been approved.
msgdata['approved'] = True
msgdata['moderator_approved'] = True
# Calculate a new filebase for the approved message, otherwise
# delivery errors will cause duplicates.
if 'filebase' in msgdata:
del msgdata['filebase']
# Queue the file for delivery. Trying to deliver the message directly
# here can lead to a huge delay in web turnaround. Log the moderation
# and add a header.
msg['X-Mailman-Approved-At'] = formatdate(
time.mktime(now().timetuple()), localtime=True)
vlog.info('held message approved, message-id: %s',
msg.get('message-id', 'n/a'))
# Stick the message back in the incoming queue for further
# processing.
config.switchboards['pipeline'].enqueue(msg, _metadata=msgdata)
else:
raise AssertionError('Unexpected action: {0}'.format(action))
# Forward the message.
if forward:
# Get a copy of the original message from the message store.
msg = message_store.get_message_by_id(message_id)
# It's possible the forwarding address list is a comma separated list
# of display_name/address pairs.
addresses = [addr[1] for addr in getaddresses(forward)]
language = mlist.preferred_language
if len(addresses) == 1:
# If the address getting the forwarded message is a member of
# the list, we want the headers of the outer message to be
# encoded in their language. Otherwise it'll be the preferred
# language of the mailing list. This is better than sending a
# separate message per recipient.
member = mlist.members.get_member(addresses[0])
if member:
language = member.preferred_language
with _.using(language.code):
fmsg = UserNotification(
addresses, mlist.bounces_address,
_('Forward of moderated message'),
lang=language)
fmsg.set_type('message/rfc822')
fmsg.attach(msg)
fmsg.send(mlist)
# Delete the request if it's not being kept.
if not keep:
requestdb.delete_request(id)
# Log the rejection
if rejection:
note = """%s: %s posting:
\tFrom: %s
\tSubject: %s"""
if comment:
note += '\n\tReason: ' + comment
vlog.info(note, mlist.fqdn_listname, rejection, sender, subject)
def on_get(self, request, response):
requests = IListRequests(self._mlist)
count = requests.count_of(RequestType.held_message)
okay(response, etag(dict(count=count)))
def test_only_return_this_lists_requests(self):
# Issue #161: get_requests() returns requests that are not specific to
# the mailing list in question.
request_id = hold_message(self._mlist, self._msg)
bee = create_list('*****@*****.**')
self.assertIsNone(IListRequests(bee).get_request(request_id))
def _get_collection(self, request):
requests = IListRequests(self._mlist)
return requests.of_type(RequestType.held_message)
