def __init__(self, name, queue_directory,
slice=None, numslices=1, recover=False):
"""Create a switchboard object.
:param name: The queue name.
:type name: str
:param queue_directory: The queue directory.
:type queue_directory: str
:param slice: The slice number for this switchboard, or None. If not
None, it must be [0..`numslices`).
:type slice: int or None
:param numslices: The total number of slices to split this queue
directory into. It must be a power of 2.
:type numslices: int
:param recover: True if backup files should be recovered.
:type recover: bool
"""
assert (numslices & (numslices - 1)) == 0, (
'Not a power of 2: {0}'.format(numslices))
self.name = name
self.queue_directory = queue_directory
# If configured to, create the directory if it doesn't yet exist.
if config.create_paths:
makedirs(self.queue_directory, 0770)
# Fast track for no slices
self._lower = None
self._upper = None
# BAW: test performance and end-cases of this algorithm
if numslices <> 1:
self._lower = ((shamax + 1) * slice) / numslices
self._upper = (((shamax + 1) * (slice + 1)) / numslices) - 1
if recover:
self.recover_backup_files()
def test_makedirs_race_condition(self):
"""Mocks os.makedirs behaviour to create willingly a race condition in
filesystem.makedirs and test it."""
first_inexistent_directory(self.baz)
makedirs(self.bar)
with unittest.mock.patch('os.makedirs', new=fake_makedirs):
with self.assertRaises(FileExistsError):
makedirs(self.baz)
def test_unicode_in_default(self):
# What if the default template is already in UTF-8? For example, if
# you import it twice.
footer = b'\xe4\xb8\xad $listinfo_uri'
footer_path = os.path.join(config.VAR_DIR, 'templates', 'lists',
'*****@*****.**', 'en', 'footer.txt')
makedirs(os.path.dirname(footer_path))
with open(footer_path, 'wb') as fp:
fp.write(footer)
self._pckdict['msg_footer'] = b'NEW-VALUE'
import_config_pck(self._mlist, self._pckdict)
text = decorate('list:member:regular:footer', self._mlist)
self.assertEqual(text, 'NEW-VALUE')
def add(self, store, message):
# Ensure that the message has the requisite headers.
message_ids = message.get_all('message-id', [])
if len(message_ids) != 1:
raise ValueError('Exactly one Message-ID header required')
# Calculate and insert the X-Message-ID-Hash.
message_id = message_ids[0]
if isinstance(message_id, bytes):
message_id = message_id.decode('ascii')
# Complain if the Message-ID already exists in the storage.
existing = store.query(Message).filter(
Message.message_id == message_id).first()
if existing is not None:
raise ValueError(
'Message ID already exists in message store: {0}'.format(
message_id))
shaobj = hashlib.sha1(message_id.encode('utf-8'))
hash32 = base64.b32encode(shaobj.digest()).decode('utf-8')
del message['X-Message-ID-Hash']
message['X-Message-ID-Hash'] = hash32
# Calculate the path on disk where we're going to store this message
# object, in pickled format.
parts = []
split = list(hash32)
while split and len(parts) < MAX_SPLITS:
parts.append(split.pop(0) + split.pop(0))
parts.append(hash32)
relpath = os.path.join(*parts)
# Store the message in the database. This relies on the database
# providing a unique serial number, but to get this information, we
# have to use a straight insert instead of relying on Elixir to create
# the object.
Message(message_id=message_id,
message_id_hash=hash32,
path=relpath)
# Now calculate the full file system path.
path = os.path.join(config.MESSAGES_DIR, relpath)
# Write the file to the path, but catch the appropriate exception in
# case the parent directories don't yet exist. In that case, create
# them and try again.
while True:
try:
with open(path, 'wb') as fp:
# -1 says to use the highest protocol available.
pickle.dump(message, fp, -1)
break
except IOError as error:
if error.errno != errno.ENOENT:
raise
makedirs(os.path.dirname(path))
return hash32
def test_unicode_in_default(self):
# What if the default template is already in UTF-8? For example, if
# you import it twice.
footer = b'\xe4\xb8\xad $listinfo_uri'
footer_path = os.path.join(
config.VAR_DIR, 'templates', 'lists',
'*****@*****.**', 'en', 'footer-generic.txt')
makedirs(os.path.dirname(footer_path))
with open(footer_path, 'wb') as fp:
fp.write(footer)
self._pckdict[b'msg_footer'] = b'NEW-VALUE'
import_config_pck(self._mlist, self._pckdict)
text = decorate(self._mlist, self._mlist.footer_uri)
self.assertEqual(text, 'NEW-VALUE')
def add(self, store, message):
# Ensure that the message has the requisite headers.
message_ids = message.get_all('message-id', [])
if len(message_ids) != 1:
raise ValueError('Exactly one Message-ID header required')
# Calculate and insert the X-Message-ID-Hash.
message_id = message_ids[0]
if isinstance(message_id, bytes):
message_id = message_id.decode('ascii')
# Complain if the Message-ID already exists in the storage.
existing = store.query(Message).filter(
Message.message_id == message_id).first()
if existing is not None:
raise ValueError(
'Message ID already exists in message store: {0}'.format(
message_id))
shaobj = hashlib.sha1(message_id.encode('utf-8'))
hash32 = base64.b32encode(shaobj.digest()).decode('utf-8')
del message['X-Message-ID-Hash']
message['X-Message-ID-Hash'] = hash32
# Calculate the path on disk where we're going to store this message
# object, in pickled format.
parts = []
split = list(hash32)
while split and len(parts) < MAX_SPLITS:
parts.append(split.pop(0) + split.pop(0))
parts.append(hash32)
relpath = os.path.join(*parts)
# Store the message in the database. This relies on the database
# providing a unique serial number, but to get this information, we
# have to use a straight insert instead of relying on Elixir to create
# the object.
Message(message_id=message_id, message_id_hash=hash32, path=relpath)
# Now calculate the full file system path.
path = os.path.join(config.MESSAGES_DIR, relpath)
# Write the file to the path, but catch the appropriate exception in
# case the parent directories don't yet exist. In that case, create
# them and try again.
while True:
try:
with open(path, 'wb') as fp:
# -1 says to use the highest protocol available.
pickle.dump(message, fp, -1)
break
except IOError as error:
if error.errno != errno.ENOENT:
raise
makedirs(os.path.dirname(path))
return hash32
def __init__(self, fqdn_listname):
super(MailingList, self).__init__()
listname, at, hostname = fqdn_listname.partition('@')
assert hostname, 'Bad list name: {0}'.format(fqdn_listname)
self.list_name = listname
self.mail_host = hostname
self._list_id = '{0}.{1}'.format(listname, hostname)
# For the pending database
self.next_request_id = 1
# We need to set up the rosters. Normally, this method will get called
# when the MailingList object is loaded from the database, but when the
# constructor is called, SQLAlchemy's `load` event isn't triggered.
# Thus we need to set up the rosters explicitly.
self._post_load()
makedirs(self.data_path)
def __init__(self, fqdn_listname):
super().__init__()
listname, at, hostname = fqdn_listname.partition('@')
assert hostname, 'Bad list name: {0}'.format(fqdn_listname)
self.list_name = listname
self.mail_host = hostname
self._list_id = '{0}.{1}'.format(listname, hostname)
# For the pending database
self.next_request_id = 1
# We need to set up the rosters. Normally, this method will get called
# when the MailingList object is loaded from the database, but when the
# constructor is called, SQLAlchemy's `load` event isn't triggered.
# Thus we need to set up the rosters explicitly.
self._post_load()
makedirs(self.data_path)
def __init__(self, fqdn_listname):
super(MailingList, self).__init__()
listname, at, hostname = fqdn_listname.partition('@')
assert hostname, 'Bad list name: {0}'.format(fqdn_listname)
self.list_name = listname
self.mail_host = hostname
self._list_id = '{0}.{1}'.format(listname, hostname)
# For the pending database
self.next_request_id = 1
# We need to set up the rosters. Normally, this method will get
# called when the MailingList object is loaded from the database, but
# that's not the case when the constructor is called. So, set up the
# rosters explicitly.
self.__storm_loaded__()
makedirs(self.data_path)
def __init__(self, fqdn_listname):
super(MailingList, self).__init__()
listname, at, hostname = fqdn_listname.partition('@')
assert hostname, 'Bad list name: {0}'.format(fqdn_listname)
self.list_name = listname
self.mail_host = hostname
# For the pending database
self.next_request_id = 1
# We need to set up the rosters. Normally, this method will get
# called when the MailingList object is loaded from the database, but
# that's not the case when the constructor is called. So, set up the
# rosters explicitly.
self.__storm_loaded__()
self.personalize = Personalization.none
self.display_name = string.capwords(
SPACE.join(listname.split(UNDERSCORE)))
makedirs(self.data_path)
def ensure_directories_exist(self):
"""Create all path directories if they do not exist."""
if self.create_paths:
for variable, directory in self.paths.items():
makedirs(directory)
# Avoid circular imports.
from mailman.utilities.datetime import now
# Create a mailman.cfg template file if it doesn't already exist.
# LBYL: <boo hiss>, but it's probably okay because the directories
# likely didn't exist before the above loop, and we'll create a
# temporary lock.
lock_file = os.path.join(self.LOCK_DIR, 'mailman-cfg.lck')
mailman_cfg = os.path.join(self.ETC_DIR, 'mailman.cfg')
with Lock(lock_file):
if not os.path.exists(mailman_cfg):
with open(mailman_cfg, 'w') as fp:
print(MAILMAN_CFG_TEMPLATE.format(
now().replace(microsecond=0)), file=fp)
def ensure_directories_exist(self):
"""Create all path directories if they do not exist."""
if self.create_paths:
for variable, directory in self.paths.items():
makedirs(directory)
# Avoid circular imports.
from mailman.utilities.datetime import now
# Create a mailman.cfg template file if it doesn't already exist.
# LBYL: <boo hiss>, but it's probably okay because the directories
# likely didn't exist before the above loop, and we'll create a
# temporary lock.
lock_file = os.path.join(self.LOCK_DIR, 'mailman-cfg.lck')
mailman_cfg = os.path.join(self.ETC_DIR, 'mailman.cfg')
with Lock(lock_file):
if not os.path.exists(mailman_cfg):
with open(mailman_cfg, 'w') as fp:
print(MAILMAN_CFG_TEMPLATE.format(
now().replace(microsecond=0)), file=fp)
def __init__(self,
name,
queue_directory,
slice=None,
numslices=1,
recover=False):
"""Create a switchboard object.
:param name: The queue name.
:type name: str
:param queue_directory: The queue directory.
:type queue_directory: str
:param slice: The slice number for this switchboard, or None. If not
None, it must be [0..`numslices`).
:type slice: int or None
:param numslices: The total number of slices to split this queue
directory into. It must be a power of 2.
:type numslices: int
:param recover: True if backup files should be recovered.
:type recover: bool
"""
assert (numslices & (numslices - 1)) == 0, (
'Not a power of 2: {}'.format(numslices))
self.name = name
self.queue_directory = queue_directory
# If configured to, create the directory if it doesn't yet exist.
if config.create_paths:
makedirs(self.queue_directory, 0o770)
# Fast track for no slices
self._lower = None
self._upper = None
# BAW: test performance and end-cases of this algorithm
if numslices != 1:
self._lower = ((shamax + 1) * slice) / numslices
self._upper = (((shamax + 1) * (slice + 1)) / numslices) - 1
if recover:
self.recover_backup_files()
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == 'preferred_language' or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value),
file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get('default_member_moderation', 0)):
mlist.default_member_action = member_moderation_action_mapping(
config_dict.get('member_moderation_action'))
else:
mlist.default_member_action = Action.defer
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode('utf-8')
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
'welcome_msg': 'welcome_message_uri',
'goodbye_msg': 'goodbye_message_uri',
'msg_header': 'header_uri',
'msg_footer': 'footer_uri',
'digest_header': 'digest_header_uri',
'digest_footer': 'digest_footer_uri',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
('%(real_name)s@%(host_name)s', '$fqdn_listname'),
('%(real_name)s', '$display_name'),
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
'$listinfo_uri'),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = 'mailman:///$listname/$language/'
if default_value:
filename = default_value.rpartition('/')[2]
else:
filename = '{}.txt'.format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get('members', {}))
digesters_set = set(config_dict.get('digest_members', {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ('accept', 'hold', 'reject', 'discard'):
prop_name = '{}_these_nonmembers'.format(action_name)
emails = [
addr for addr in config_dict.get(prop_name, [])
if not addr.startswith('^')
]
import_roster(mlist, config_dict, emails, MemberRole.nonmember,
Action[action_name])
# Only keep the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for email in emails:
list_prop.remove(email)
finally:
mlist.send_welcome_message = send_welcome_message
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == "preferred_language" or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(key, value), file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == "last_post_time":
setattr(mlist, "last_post_at", value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get("default_member_moderation", 0)):
mlist.default_member_action = member_moderation_action_mapping(config_dict.get("member_moderation_action"))
else:
mlist.default_member_action = Action.defer
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get("archive"):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get("archive_private", True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get("ban_list", []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get("acceptable_aliases", "")
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode("utf-8")
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add("^" + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
"welcome_msg": "welcome_message_uri",
"goodbye_msg": "goodbye_message_uri",
"msg_header": "header_uri",
"msg_footer": "footer_uri",
"digest_header": "digest_header_uri",
"digest_footer": "digest_footer_uri",
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
("%(real_name)s@%(host_name)s", "$fqdn_listname"),
("%(real_name)s", "$display_name"),
("%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s", "$listinfo_uri"),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print(
"Unable to convert mailing list attribute:",
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr,
)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode("utf-8", "replace")
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print("Unable to convert mailing list attribute:", oldvar, 'with value "{}"'.format(text), file=sys.stderr)
continue
if expanded_text and default_text and expanded_text.strip() == default_text.strip():
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = "mailman:///$listname/$language/"
if default_value:
filename = default_value.rpartition("/")[2]
else:
filename = "{}.txt".format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, "w", encoding="utf-8") as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get("members", {}))
digesters_set = set(config_dict.get("digest_members", {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get("owner", []), MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get("moderator", []), MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ("accept", "hold", "reject", "discard"):
prop_name = "{}_these_nonmembers".format(action_name)
emails = [addr for addr in config_dict.get(prop_name, []) if not addr.startswith("^")]
import_roster(mlist, config_dict, emails, MemberRole.nonmember, Action[action_name])
# Only keep the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for email in emails:
list_prop.remove(email)
finally:
mlist.send_welcome_message = send_welcome_message
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if hasattr(mlist, key) or key == 'preferred_language':
if isinstance(value, str):
value = str_to_unicode(value)
# Some types require conversion.
converter = TYPES.get(key)
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value), file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(str_to_unicode(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, basestring):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = str_to_unicode(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
'welcome_msg': 'welcome_message_uri',
'goodbye_msg': 'goodbye_message_uri',
'msg_header': 'header_uri',
'msg_footer': 'footer_uri',
'digest_header': 'digest_header_uri',
'digest_footer': 'digest_footer_uri',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
('%(real_name)s@%(host_name)s', '$fqdn_listname'),
('%(real_name)s', '$display_name'),
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
'$listinfo_uri'),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = 'mailman:///$listname/$language/'
if default_value:
filename = default_value.rpartition('/')[2]
else:
filename = '{}.txt'.format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
members = set(config_dict.get('members', {}).keys()
+ config_dict.get('digest_members', {}).keys())
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
global key
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == 'preferred_language' or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
if column is not None and isinstance(column.type, SAUnicode):
converter = maybe_truncate_mysql
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value),
file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get('default_member_moderation', 0)):
mlist.default_member_action = member_moderation_action_mapping(
config_dict.get('member_moderation_action'))
else:
mlist.default_member_action = Action.defer
# Handle DMARC mitigations.
# This would be straightforward except for from_is_list. The issue
# is in MM 2.1 the from_is_list action applies if dmarc_moderation_action
# doesn't apply and they can be different.
# We will map as follows:
# from_is_list > dmarc_moderation_action
# dmarc_mitigate_action = from_is_list action
# dmarc_mitigate_unconditionally = True
# from_is_list <= dmarc_moderation_action
# dmarc_mitigate_action = dmarc_moderation_action
# dmarc_mitigate_unconditionally = False
# The text attributes are handled above.
if (config_dict.get('from_is_list', 0) > config_dict.get(
'dmarc_moderation_action', 0)):
mlist.dmarc_mitigate_action = dmarc_action_mapping(
config_dict.get('from_is_list', 0))
mlist.dmarc_mitigate_unconditionally = True
else:
mlist.dmarc_mitigate_action = dmarc_action_mapping(
config_dict.get('dmarc_moderation_action', 0))
mlist.dmarc_mitigate_unconditionally = False
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode('utf-8')
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
# All 2.1 acceptable aliases are regexps whether or not they start
# with '^' or contain '@'.
if not address.startswith('^'):
address = '^' + address
# This used to be in a try which would catch ValueError and add a '^',
# but .add() would not raise ValueError if address contained '@' and
# that needs the '^' too as it could be a regexp with an '@' in it.
alias_set.add(address)
# Handle roster visibility.
mapping = member_roster_visibility_mapping(
config_dict.get('private_roster', None))
if mapping is not None:
mlist.member_roster_visibility = mapping
# Handle header_filter_rules conversion to header_matches.
header_matches = IHeaderMatchList(mlist)
header_filter_rules = config_dict.get('header_filter_rules', [])
for line_patterns, action, _unused in header_filter_rules:
try:
chain = action_to_chain(action)
except KeyError:
log.warning('Unsupported header_filter_rules action: %r', action)
continue
# Now split the line into a header and a pattern.
for line_pattern in line_patterns.splitlines():
if len(line_pattern.strip()) == 0:
continue
for sep in (': ', ':.*', ':.', ':'):
header, sep, pattern = line_pattern.partition(sep)
if sep:
# We found it.
break
else:
# Matches any header, which is not supported. XXX
log.warning('Unsupported header_filter_rules pattern: %r',
line_pattern)
continue
header = header.strip().lstrip('^').lower()
header = header.replace('\\', '')
if not header:
log.warning(
'Cannot parse the header in header_filter_rule: %r',
line_pattern)
continue
if len(pattern) == 0:
# The line matched only the header, therefore the header can
# be anything.
pattern = '.*'
try:
re.compile(pattern)
except re.error:
log.warning(
'Skipping header_filter rule because of an '
'invalid regular expression: %r', line_pattern)
continue
try:
header_matches.append(header, pattern, chain)
except ValueError:
log.warning('Skipping duplicate header_filter rule: %r',
line_pattern)
continue
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# strings. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path, then add the template to the template manager.
# We are intentionally omitting the 2.1 welcome_msg here because the
# string is actually interpolated into a larger template and there's
# no good way to figure where in the default template to insert it.
convert_to_uri = {
'goodbye_msg': 'list:user:notice:goodbye',
'msg_header': 'list:member:regular:header',
'msg_footer': 'list:member:regular:footer',
'digest_header': 'list:member:digest:header',
'digest_footer': 'list:member:digest:footer',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
# First convert \r\n that may have been set by a browser to \n.
('\r\n', '\n'),
('%(real_name)s@%(host_name)s',
'To unsubscribe send an email to ${short_listname}-leave@${domain}'),
('%(real_name)s mailing list',
'$display_name mailing list -- $listname'),
# The generic footers no longer have URLs in them.
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s\n', ''),
]
# Collect defaults.
manager = getUtility(ITemplateManager)
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getUtility(ITemplateLoader).get(newvar, mlist)
if not default_value:
continue
# Get the decorated default text
try:
default_text = decorate_template(mlist, default_value)
except (URLError, KeyError): # pragma: nocover
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = default_text
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_text = defaults.get(newvar, None)
if not text and not default_text:
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError: # pragma: nocover
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file and add it to the template
# manager for real.
base_uri = 'mailman:///$listname/$language/'
filename = '{}.txt'.format(newvar)
manager.set(newvar, mlist.list_id, base_uri + filename)
with ExitStack() as resources:
filepath = list(search(resources, filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get('members', {}))
digesters_set = set(config_dict.get('digest_members', {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages or notify admins when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
admin_notify_mchanges = mlist.admin_notify_mchanges
mlist.admin_notify_mchanges = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ('accept', 'hold', 'reject', 'discard'):
prop_name = '{}_these_nonmembers'.format(action_name)
emails = [
addr for addr in config_dict.get(prop_name, [])
if not addr.startswith('^')
]
# MM 2.1 accept maps to MM 3 defer
if action_name == 'accept':
action_name = 'defer'
import_roster(mlist, config_dict, emails, MemberRole.nonmember,
Action[action_name])
# Now add the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for addr in config_dict.get(prop_name, []):
if addr.startswith('^'):
list_prop.append(addr)
finally:
mlist.send_welcome_message = send_welcome_message
mlist.admin_notify_mchanges = admin_notify_mchanges
def ensure_directories_exist(self):
"""Create all path directories if they do not exist."""
if self.create_paths:
for variable, directory in self.paths.items():
makedirs(directory)