def test_delete_list_with_acceptable_aliases(self):
# LP: #1432239 - deleting a mailing list with acceptable aliases
# causes a SQLAlchemy error. The aliases must be deleted first.
with transaction():
alias_set = IAcceptableAliasSet(self._mlist)
alias_set.add('*****@*****.**')
self.assertEqual(['*****@*****.**'], list(alias_set.aliases))
getUtility(IListManager).delete(self._mlist)
self.assertEqual(len(list(alias_set.aliases)), 0)
def test_delete_list_with_acceptable_aliases(self):
# LP: #1432239 - deleting a mailing list with acceptable aliases
# causes a SQLAlchemy error. The aliases must be deleted first.
with transaction():
alias_set = IAcceptableAliasSet(self._mlist)
alias_set.add('*****@*****.**')
self.assertEqual(['*****@*****.**'], list(alias_set.aliases))
getUtility(IListManager).delete(self._mlist)
self.assertEqual(len(list(alias_set.aliases)), 0)
def test_delete_list_with_acceptable_aliases(self):
# LP: #1432239 - deleting a mailing list with acceptable aliases
# causes a SQLAlchemy error. The aliases must be deleted first.
with transaction():
alias_set = IAcceptableAliasSet(self._mlist)
alias_set.add('*****@*****.**')
call_api('http://*****:*****@example.com'))
self.assertEqual(config.db.store.query(AcceptableAlias).count(), 0)
def test_delete_list_with_acceptable_aliases(self):
# LP: #1432239 - deleting a mailing list with acceptable aliases
# causes a SQLAlchemy error. The aliases must be deleted first.
with transaction():
alias_set = IAcceptableAliasSet(self._mlist)
alias_set.add('*****@*****.**')
call_api('http://*****:*****@example.com'))
self.assertEqual(config.db.store.query(AcceptableAlias).count(), 0)
def put(self, mlist, attribute, value):
"""Change the acceptable aliases.
Because this is a PUT operation, all previous aliases are cleared
first. Thus, this is an overwrite. The keys in the request are
ignored.
"""
assert attribute == "acceptable_aliases", "Unexpected attribute: {0}".format(attribute)
alias_set = IAcceptableAliasSet(mlist)
alias_set.clear()
for alias in value:
alias_set.add(unicode(alias))
def test_acceptable_aliases_invalid(self):
# Values without an '@' sign used to be matched against the local
# part, now we need to add the '^' sign to indicate it's a regexp.
aliases = ['invalid-value']
self._pckdict['acceptable_aliases'] = list_to_string(aliases)
self._import()
alias_set = IAcceptableAliasSet(self._mlist)
self.assertEqual(sorted(alias_set.aliases),
[('^' + alias) for alias in aliases])
def test_acceptable_aliases_as_list(self):
# In some versions of the pickle, this can be a list, not a string
# (seen in the wild).
aliases = [b'*****@*****.**', b'*****@*****.**']
self._pckdict['acceptable_aliases'] = aliases
self._import()
alias_set = IAcceptableAliasSet(self._mlist)
self.assertEqual(sorted(alias_set.aliases),
sorted(a.decode('utf-8') for a in aliases))
def check(self, mlist, msg, msgdata):
"""See `IRule`."""
# Implicit destination checking must be enabled in the mailing list.
if not mlist.require_explicit_destination:
return False
# Messages gated from NNTP will always have an implicit destination so
# are never checked.
if msgdata.get('fromusenet'):
return False
# Calculate the list of acceptable aliases. If the alias starts with
# a caret (i.e. ^), then it's a regular expression to match against.
aliases = set()
alias_patterns = set()
# Adapt the mailing list to the appropriate interface.
alias_set = IAcceptableAliasSet(mlist)
for alias in alias_set.aliases:
if alias.startswith('^'):
alias_patterns.add(alias)
else:
aliases.add(alias)
# Add the list's posting address, i.e. the explicit address, to the
# set of acceptable aliases.
aliases.add(mlist.posting_address)
# Look at all the recipients. If the recipient is any acceptable
# alias (or the explicit posting address), then this rule does not
# match. If not, then add it to the set of recipients we'll check
# against the alias patterns later.
recipients = set()
for header in ('to', 'cc', 'resent-to', 'resent-cc'):
for fullname, address in getaddresses(msg.get_all(header, [])):
if isinstance(address, bytes):
address = address.decode('ascii')
address = address.lower()
if address in aliases:
return False
recipients.add(address)
# Now for all alias patterns, see if any of the recipients matches a
# pattern. If so, then this rule does not match.
for pattern in alias_patterns:
escaped = re.escape(pattern)
for recipient in recipients:
try:
if re.match(pattern, recipient, re.IGNORECASE):
return False
except re.error:
# The pattern is a malformed regular expression. Try
# matching again with the pattern escaped.
with suppress(re.error):
if re.match(escaped, recipient, re.IGNORECASE):
return False
# Nothing matched.
msgdata['moderation_sender'] = msg.sender
with _.defer_translation():
# This will be translated at the point of use.
msgdata.setdefault('moderation_reasons', []).append(
_('Message has implicit destination'))
return True
def test_put_configuration(self):
# When using PUT, all writable attributes must be included.
resource, response = call_api(
'http://localhost:9001/3.0/lists/ant.example.com/config', RESOURCE,
'PUT')
self.assertEqual(response.status, 204)
self.assertEqual(self._mlist.display_name, 'Fnords')
# All three acceptable aliases were set.
self.assertEqual(set(IAcceptableAliasSet(self._mlist).aliases),
set(RESOURCE['acceptable_aliases']))
def test_acceptable_aliases(self):
# This used to be a plain-text field (values are newline-separated).
aliases = ['*****@*****.**',
'*****@*****.**',
'non-ascii-\[email protected]',
]
self._pckdict['acceptable_aliases'] = list_to_string(aliases)
self._import()
alias_set = IAcceptableAliasSet(self._mlist)
self.assertEqual(sorted(alias_set.aliases), aliases)
def test_put_configuration(self):
aliases = [
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
]
# When using PUT, all writable attributes must be included.
resource, response = call_api(
'http://*****:*****@example.com/config',
dict(
acceptable_aliases=aliases,
admin_immed_notify=False,
admin_notify_mchanges=True,
administrivia=False,
advertised=False,
anonymous_list=True,
archive_policy='never',
autorespond_owner='respond_and_discard',
autorespond_postings='respond_and_continue',
autorespond_requests='respond_and_discard',
autoresponse_grace_period='45d',
autoresponse_owner_text='the owner',
autoresponse_postings_text='the mailing list',
autoresponse_request_text='the robot',
display_name='Fnords',
description='This is my mailing list',
include_rfc2369_headers=False,
allow_list_posts=False,
digest_size_threshold=10.5,
posting_pipeline='virgin',
filter_content=True,
first_strip_reply_to=True,
convert_html_to_plaintext=True,
collapse_alternatives=False,
reply_goes_to_list='point_to_list',
reply_to_address='*****@*****.**',
send_welcome_message=False,
subject_prefix='[ant]',
welcome_message_uri='mailman:///welcome.txt',
default_member_action='hold',
default_nonmember_action='discard',
),
'PUT')
self.assertEqual(response.status, 204)
self.assertEqual(self._mlist.display_name, 'Fnords')
# All three acceptable aliases were set.
self.assertEqual(set(IAcceptableAliasSet(self._mlist).aliases),
set(aliases))
def put(self, mlist, attribute, value):
"""Change the acceptable aliases.
Because this is a PUT operation, all previous aliases are cleared
first. Thus, this is an overwrite. The keys in the request are
ignored.
"""
assert attribute == 'acceptable_aliases', (
'Unexpected attribute: {}'.format(attribute)) # pragma: nocover
alias_set = IAcceptableAliasSet(mlist)
alias_set.clear()
for alias in value:
alias_set.add(alias)
def on_delete(self, request, response):
if self._attribute is None:
bad_request(response,
'Cannot delete the list configuration itself')
return
if self._attribute not in VALIDATORS:
bad_request(response,
'Read-only attribute: {}'.format(self._attribute))
return
# This kind of sucks because it doesn't scale if the list of attributes
# which can be deleted grows. So if we get too many we'll have to use
# a lookup table. For now, this is good enough.
if self._attribute != 'acceptable_aliases':
bad_request(
response,
'Attribute cannot be DELETEd: {}'.format(self._attribute))
return
IAcceptableAliasSet(self._mlist).clear()
no_content(response)
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
global key
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == 'preferred_language' or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
if column is not None and isinstance(column.type, SAUnicode):
converter = maybe_truncate_mysql
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value),
file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get('default_member_moderation', 0)):
mlist.default_member_action = member_moderation_action_mapping(
config_dict.get('member_moderation_action'))
else:
mlist.default_member_action = Action.defer
# Handle DMARC mitigations.
# This would be straightforward except for from_is_list. The issue
# is in MM 2.1 the from_is_list action applies if dmarc_moderation_action
# doesn't apply and they can be different.
# We will map as follows:
# from_is_list > dmarc_moderation_action
# dmarc_mitigate_action = from_is_list action
# dmarc_mitigate_unconditionally = True
# from_is_list <= dmarc_moderation_action
# dmarc_mitigate_action = dmarc_moderation_action
# dmarc_mitigate_unconditionally = False
# The text attributes are handled above.
if (config_dict.get('from_is_list', 0) > config_dict.get(
'dmarc_moderation_action', 0)):
mlist.dmarc_mitigate_action = dmarc_action_mapping(
config_dict.get('from_is_list', 0))
mlist.dmarc_mitigate_unconditionally = True
else:
mlist.dmarc_mitigate_action = dmarc_action_mapping(
config_dict.get('dmarc_moderation_action', 0))
mlist.dmarc_mitigate_unconditionally = False
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode('utf-8')
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
# All 2.1 acceptable aliases are regexps whether or not they start
# with '^' or contain '@'.
if not address.startswith('^'):
address = '^' + address
# This used to be in a try which would catch ValueError and add a '^',
# but .add() would not raise ValueError if address contained '@' and
# that needs the '^' too as it could be a regexp with an '@' in it.
alias_set.add(address)
# Handle roster visibility.
mapping = member_roster_visibility_mapping(
config_dict.get('private_roster', None))
if mapping is not None:
mlist.member_roster_visibility = mapping
# Handle header_filter_rules conversion to header_matches.
header_matches = IHeaderMatchList(mlist)
header_filter_rules = config_dict.get('header_filter_rules', [])
for line_patterns, action, _unused in header_filter_rules:
try:
chain = action_to_chain(action)
except KeyError:
log.warning('Unsupported header_filter_rules action: %r', action)
continue
# Now split the line into a header and a pattern.
for line_pattern in line_patterns.splitlines():
if len(line_pattern.strip()) == 0:
continue
for sep in (': ', ':.*', ':.', ':'):
header, sep, pattern = line_pattern.partition(sep)
if sep:
# We found it.
break
else:
# Matches any header, which is not supported. XXX
log.warning('Unsupported header_filter_rules pattern: %r',
line_pattern)
continue
header = header.strip().lstrip('^').lower()
header = header.replace('\\', '')
if not header:
log.warning(
'Cannot parse the header in header_filter_rule: %r',
line_pattern)
continue
if len(pattern) == 0:
# The line matched only the header, therefore the header can
# be anything.
pattern = '.*'
try:
re.compile(pattern)
except re.error:
log.warning(
'Skipping header_filter rule because of an '
'invalid regular expression: %r', line_pattern)
continue
try:
header_matches.append(header, pattern, chain)
except ValueError:
log.warning('Skipping duplicate header_filter rule: %r',
line_pattern)
continue
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# strings. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path, then add the template to the template manager.
# We are intentionally omitting the 2.1 welcome_msg here because the
# string is actually interpolated into a larger template and there's
# no good way to figure where in the default template to insert it.
convert_to_uri = {
'goodbye_msg': 'list:user:notice:goodbye',
'msg_header': 'list:member:regular:header',
'msg_footer': 'list:member:regular:footer',
'digest_header': 'list:member:digest:header',
'digest_footer': 'list:member:digest:footer',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
# First convert \r\n that may have been set by a browser to \n.
('\r\n', '\n'),
('%(real_name)s@%(host_name)s',
'To unsubscribe send an email to ${short_listname}-leave@${domain}'),
('%(real_name)s mailing list',
'$display_name mailing list -- $listname'),
# The generic footers no longer have URLs in them.
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s\n', ''),
]
# Collect defaults.
manager = getUtility(ITemplateManager)
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getUtility(ITemplateLoader).get(newvar, mlist)
if not default_value:
continue
# Get the decorated default text
try:
default_text = decorate_template(mlist, default_value)
except (URLError, KeyError): # pragma: nocover
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = default_text
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_text = defaults.get(newvar, None)
if not text and not default_text:
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError: # pragma: nocover
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file and add it to the template
# manager for real.
base_uri = 'mailman:///$listname/$language/'
filename = '{}.txt'.format(newvar)
manager.set(newvar, mlist.list_id, base_uri + filename)
with ExitStack() as resources:
filepath = list(search(resources, filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get('members', {}))
digesters_set = set(config_dict.get('digest_members', {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages or notify admins when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
admin_notify_mchanges = mlist.admin_notify_mchanges
mlist.admin_notify_mchanges = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ('accept', 'hold', 'reject', 'discard'):
prop_name = '{}_these_nonmembers'.format(action_name)
emails = [
addr for addr in config_dict.get(prop_name, [])
if not addr.startswith('^')
]
# MM 2.1 accept maps to MM 3 defer
if action_name == 'accept':
action_name = 'defer'
import_roster(mlist, config_dict, emails, MemberRole.nonmember,
Action[action_name])
# Now add the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for addr in config_dict.get(prop_name, []):
if addr.startswith('^'):
list_prop.append(addr)
finally:
mlist.send_welcome_message = send_welcome_message
mlist.admin_notify_mchanges = admin_notify_mchanges
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == 'preferred_language' or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value),
file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get('default_member_moderation', 0)):
mlist.default_member_action = member_moderation_action_mapping(
config_dict.get('member_moderation_action'))
else:
mlist.default_member_action = Action.defer
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode('utf-8')
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
'welcome_msg': 'welcome_message_uri',
'goodbye_msg': 'goodbye_message_uri',
'msg_header': 'header_uri',
'msg_footer': 'footer_uri',
'digest_header': 'digest_header_uri',
'digest_footer': 'digest_footer_uri',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
('%(real_name)s@%(host_name)s', '$fqdn_listname'),
('%(real_name)s', '$display_name'),
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
'$listinfo_uri'),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = 'mailman:///$listname/$language/'
if default_value:
filename = default_value.rpartition('/')[2]
else:
filename = '{}.txt'.format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get('members', {}))
digesters_set = set(config_dict.get('digest_members', {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ('accept', 'hold', 'reject', 'discard'):
prop_name = '{}_these_nonmembers'.format(action_name)
emails = [
addr for addr in config_dict.get(prop_name, [])
if not addr.startswith('^')
]
import_roster(mlist, config_dict, emails, MemberRole.nonmember,
Action[action_name])
# Only keep the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for email in emails:
list_prop.remove(email)
finally:
mlist.send_welcome_message = send_welcome_message
def get(self, mlist, attribute):
"""Return the mailing list's acceptable aliases."""
assert attribute == 'acceptable_aliases', (
'Unexpected attribute: {}'.format(attribute)) # pragma: nocover
aliases = IAcceptableAliasSet(mlist)
return sorted(aliases.aliases)
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == "preferred_language" or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(key, value), file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == "last_post_time":
setattr(mlist, "last_post_at", value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get("default_member_moderation", 0)):
mlist.default_member_action = member_moderation_action_mapping(config_dict.get("member_moderation_action"))
else:
mlist.default_member_action = Action.defer
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get("archive"):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get("archive_private", True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get("ban_list", []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get("acceptable_aliases", "")
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode("utf-8")
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add("^" + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
"welcome_msg": "welcome_message_uri",
"goodbye_msg": "goodbye_message_uri",
"msg_header": "header_uri",
"msg_footer": "footer_uri",
"digest_header": "digest_header_uri",
"digest_footer": "digest_footer_uri",
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
("%(real_name)s@%(host_name)s", "$fqdn_listname"),
("%(real_name)s", "$display_name"),
("%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s", "$listinfo_uri"),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print(
"Unable to convert mailing list attribute:",
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr,
)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode("utf-8", "replace")
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print("Unable to convert mailing list attribute:", oldvar, 'with value "{}"'.format(text), file=sys.stderr)
continue
if expanded_text and default_text and expanded_text.strip() == default_text.strip():
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = "mailman:///$listname/$language/"
if default_value:
filename = default_value.rpartition("/")[2]
else:
filename = "{}.txt".format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, "w", encoding="utf-8") as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get("members", {}))
digesters_set = set(config_dict.get("digest_members", {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get("owner", []), MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get("moderator", []), MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ("accept", "hold", "reject", "discard"):
prop_name = "{}_these_nonmembers".format(action_name)
emails = [addr for addr in config_dict.get(prop_name, []) if not addr.startswith("^")]
import_roster(mlist, config_dict, emails, MemberRole.nonmember, Action[action_name])
# Only keep the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for email in emails:
list_prop.remove(email)
finally:
mlist.send_welcome_message = send_welcome_message
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if hasattr(mlist, key) or key == 'preferred_language':
if isinstance(value, str):
value = str_to_unicode(value)
# Some types require conversion.
converter = TYPES.get(key)
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value), file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(str_to_unicode(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, basestring):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = str_to_unicode(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
'welcome_msg': 'welcome_message_uri',
'goodbye_msg': 'goodbye_message_uri',
'msg_header': 'header_uri',
'msg_footer': 'footer_uri',
'digest_header': 'digest_header_uri',
'digest_footer': 'digest_footer_uri',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
('%(real_name)s@%(host_name)s', '$fqdn_listname'),
('%(real_name)s', '$display_name'),
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
'$listinfo_uri'),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = 'mailman:///$listname/$language/'
if default_value:
filename = default_value.rpartition('/')[2]
else:
filename = '{}.txt'.format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
members = set(config_dict.get('members', {}).keys()
+ config_dict.get('digest_members', {}).keys())
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
