def _stripped_search_order(self,
template_file,
mailing_list=None,
language=None):
# Return the search path order for a given template, possibly using
# the mailing list and the language as context. Note that this only
# returns the search path, and does not check for whether the paths
# exist or not.
#
# Replace the tempdir prefix with a placeholder for more readable and
# reproducible tests. Essentially the paths below are rooted at
# $var_dir, except those files that live within Mailman's source
# tree. The former will use /v/ as the root and the latter will use
# /m/ as the root.
with ExitStack() as resources:
in_tree = str(
resources.enter_context(resource_path('mailman',
'templates')).parent)
raw_search_order = search(resources, template_file, mailing_list,
language)
for path in raw_search_order:
if path.startswith(self.var_dir):
path = '/v' + path[len(self.var_dir):]
elif path.startswith(in_tree):
path = '/m' + path[len(in_tree):]
else:
# This will cause tests to fail, so keep the full bogus
# pathname for better debugging.
pass
yield path
def _stripped_search_order(self, template_file,
mailing_list=None, language=None):
# Return the search path order for a given template, possibly using
# the mailing list and the language as context. Note that this only
# returns the search path, and does not check for whether the paths
# exist or not.
#
# Replace the tempdir prefix with a placeholder for more readable and
# reproducible tests. Essentially the paths below are rooted at
# $var_dir, except those files that live within Mailman's source
# tree. The former will use /v/ as the root and the latter will use
# /m/ as the root.
in_tree = os.path.dirname(resource_filename('mailman', 'templates'))
raw_search_order = search(template_file, mailing_list, language)
for path in raw_search_order:
if path.startswith(self.var_dir):
path = '/v' + path[len(self.var_dir):]
elif path.startswith(in_tree):
path = '/m' + path[len(in_tree):]
else:
# This will cause tests to fail, so keep the full bogus
# pathname for better debugging.
pass
yield path
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
global key
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == 'preferred_language' or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
if column is not None and isinstance(column.type, SAUnicode):
converter = maybe_truncate_mysql
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value),
file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get('default_member_moderation', 0)):
mlist.default_member_action = member_moderation_action_mapping(
config_dict.get('member_moderation_action'))
else:
mlist.default_member_action = Action.defer
# Handle DMARC mitigations.
# This would be straightforward except for from_is_list. The issue
# is in MM 2.1 the from_is_list action applies if dmarc_moderation_action
# doesn't apply and they can be different.
# We will map as follows:
# from_is_list > dmarc_moderation_action
# dmarc_mitigate_action = from_is_list action
# dmarc_mitigate_unconditionally = True
# from_is_list <= dmarc_moderation_action
# dmarc_mitigate_action = dmarc_moderation_action
# dmarc_mitigate_unconditionally = False
# The text attributes are handled above.
if (config_dict.get('from_is_list', 0) > config_dict.get(
'dmarc_moderation_action', 0)):
mlist.dmarc_mitigate_action = dmarc_action_mapping(
config_dict.get('from_is_list', 0))
mlist.dmarc_mitigate_unconditionally = True
else:
mlist.dmarc_mitigate_action = dmarc_action_mapping(
config_dict.get('dmarc_moderation_action', 0))
mlist.dmarc_mitigate_unconditionally = False
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode('utf-8')
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
# All 2.1 acceptable aliases are regexps whether or not they start
# with '^' or contain '@'.
if not address.startswith('^'):
address = '^' + address
# This used to be in a try which would catch ValueError and add a '^',
# but .add() would not raise ValueError if address contained '@' and
# that needs the '^' too as it could be a regexp with an '@' in it.
alias_set.add(address)
# Handle roster visibility.
mapping = member_roster_visibility_mapping(
config_dict.get('private_roster', None))
if mapping is not None:
mlist.member_roster_visibility = mapping
# Handle header_filter_rules conversion to header_matches.
header_matches = IHeaderMatchList(mlist)
header_filter_rules = config_dict.get('header_filter_rules', [])
for line_patterns, action, _unused in header_filter_rules:
try:
chain = action_to_chain(action)
except KeyError:
log.warning('Unsupported header_filter_rules action: %r', action)
continue
# Now split the line into a header and a pattern.
for line_pattern in line_patterns.splitlines():
if len(line_pattern.strip()) == 0:
continue
for sep in (': ', ':.*', ':.', ':'):
header, sep, pattern = line_pattern.partition(sep)
if sep:
# We found it.
break
else:
# Matches any header, which is not supported. XXX
log.warning('Unsupported header_filter_rules pattern: %r',
line_pattern)
continue
header = header.strip().lstrip('^').lower()
header = header.replace('\\', '')
if not header:
log.warning(
'Cannot parse the header in header_filter_rule: %r',
line_pattern)
continue
if len(pattern) == 0:
# The line matched only the header, therefore the header can
# be anything.
pattern = '.*'
try:
re.compile(pattern)
except re.error:
log.warning(
'Skipping header_filter rule because of an '
'invalid regular expression: %r', line_pattern)
continue
try:
header_matches.append(header, pattern, chain)
except ValueError:
log.warning('Skipping duplicate header_filter rule: %r',
line_pattern)
continue
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# strings. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path, then add the template to the template manager.
# We are intentionally omitting the 2.1 welcome_msg here because the
# string is actually interpolated into a larger template and there's
# no good way to figure where in the default template to insert it.
convert_to_uri = {
'goodbye_msg': 'list:user:notice:goodbye',
'msg_header': 'list:member:regular:header',
'msg_footer': 'list:member:regular:footer',
'digest_header': 'list:member:digest:header',
'digest_footer': 'list:member:digest:footer',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
# First convert \r\n that may have been set by a browser to \n.
('\r\n', '\n'),
('%(real_name)s@%(host_name)s',
'To unsubscribe send an email to ${short_listname}-leave@${domain}'),
('%(real_name)s mailing list',
'$display_name mailing list -- $listname'),
# The generic footers no longer have URLs in them.
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s\n', ''),
]
# Collect defaults.
manager = getUtility(ITemplateManager)
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getUtility(ITemplateLoader).get(newvar, mlist)
if not default_value:
continue
# Get the decorated default text
try:
default_text = decorate_template(mlist, default_value)
except (URLError, KeyError): # pragma: nocover
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = default_text
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_text = defaults.get(newvar, None)
if not text and not default_text:
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError: # pragma: nocover
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file and add it to the template
# manager for real.
base_uri = 'mailman:///$listname/$language/'
filename = '{}.txt'.format(newvar)
manager.set(newvar, mlist.list_id, base_uri + filename)
with ExitStack() as resources:
filepath = list(search(resources, filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get('members', {}))
digesters_set = set(config_dict.get('digest_members', {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages or notify admins when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
admin_notify_mchanges = mlist.admin_notify_mchanges
mlist.admin_notify_mchanges = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ('accept', 'hold', 'reject', 'discard'):
prop_name = '{}_these_nonmembers'.format(action_name)
emails = [
addr for addr in config_dict.get(prop_name, [])
if not addr.startswith('^')
]
# MM 2.1 accept maps to MM 3 defer
if action_name == 'accept':
action_name = 'defer'
import_roster(mlist, config_dict, emails, MemberRole.nonmember,
Action[action_name])
# Now add the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for addr in config_dict.get(prop_name, []):
if addr.startswith('^'):
list_prop.append(addr)
finally:
mlist.send_welcome_message = send_welcome_message
mlist.admin_notify_mchanges = admin_notify_mchanges
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if hasattr(mlist, key) or key == 'preferred_language':
if isinstance(value, str):
value = str_to_unicode(value)
# Some types require conversion.
converter = TYPES.get(key)
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value), file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(str_to_unicode(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, basestring):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = str_to_unicode(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
'welcome_msg': 'welcome_message_uri',
'goodbye_msg': 'goodbye_message_uri',
'msg_header': 'header_uri',
'msg_footer': 'footer_uri',
'digest_header': 'digest_header_uri',
'digest_footer': 'digest_footer_uri',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
('%(real_name)s@%(host_name)s', '$fqdn_listname'),
('%(real_name)s', '$display_name'),
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
'$listinfo_uri'),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = 'mailman:///$listname/$language/'
if default_value:
filename = default_value.rpartition('/')[2]
else:
filename = '{}.txt'.format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
members = set(config_dict.get('members', {}).keys()
+ config_dict.get('digest_members', {}).keys())
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == 'preferred_language' or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value),
file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get('default_member_moderation', 0)):
mlist.default_member_action = member_moderation_action_mapping(
config_dict.get('member_moderation_action'))
else:
mlist.default_member_action = Action.defer
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode('utf-8')
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
'welcome_msg': 'welcome_message_uri',
'goodbye_msg': 'goodbye_message_uri',
'msg_header': 'header_uri',
'msg_footer': 'footer_uri',
'digest_header': 'digest_header_uri',
'digest_footer': 'digest_footer_uri',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
('%(real_name)s@%(host_name)s', '$fqdn_listname'),
('%(real_name)s', '$display_name'),
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
'$listinfo_uri'),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = 'mailman:///$listname/$language/'
if default_value:
filename = default_value.rpartition('/')[2]
else:
filename = '{}.txt'.format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get('members', {}))
digesters_set = set(config_dict.get('digest_members', {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ('accept', 'hold', 'reject', 'discard'):
prop_name = '{}_these_nonmembers'.format(action_name)
emails = [
addr for addr in config_dict.get(prop_name, [])
if not addr.startswith('^')
]
import_roster(mlist, config_dict, emails, MemberRole.nonmember,
Action[action_name])
# Only keep the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for email in emails:
list_prop.remove(email)
finally:
mlist.send_welcome_message = send_welcome_message
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == "preferred_language" or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(key, value), file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == "last_post_time":
setattr(mlist, "last_post_at", value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get("default_member_moderation", 0)):
mlist.default_member_action = member_moderation_action_mapping(config_dict.get("member_moderation_action"))
else:
mlist.default_member_action = Action.defer
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get("archive"):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get("archive_private", True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get("ban_list", []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get("acceptable_aliases", "")
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode("utf-8")
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add("^" + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
"welcome_msg": "welcome_message_uri",
"goodbye_msg": "goodbye_message_uri",
"msg_header": "header_uri",
"msg_footer": "footer_uri",
"digest_header": "digest_header_uri",
"digest_footer": "digest_footer_uri",
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
("%(real_name)s@%(host_name)s", "$fqdn_listname"),
("%(real_name)s", "$display_name"),
("%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s", "$listinfo_uri"),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print(
"Unable to convert mailing list attribute:",
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr,
)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode("utf-8", "replace")
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print("Unable to convert mailing list attribute:", oldvar, 'with value "{}"'.format(text), file=sys.stderr)
continue
if expanded_text and default_text and expanded_text.strip() == default_text.strip():
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = "mailman:///$listname/$language/"
if default_value:
filename = default_value.rpartition("/")[2]
else:
filename = "{}.txt".format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, "w", encoding="utf-8") as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get("members", {}))
digesters_set = set(config_dict.get("digest_members", {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get("owner", []), MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get("moderator", []), MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ("accept", "hold", "reject", "discard"):
prop_name = "{}_these_nonmembers".format(action_name)
emails = [addr for addr in config_dict.get(prop_name, []) if not addr.startswith("^")]
import_roster(mlist, config_dict, emails, MemberRole.nonmember, Action[action_name])
# Only keep the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for email in emails:
list_prop.remove(email)
finally:
mlist.send_welcome_message = send_welcome_message