def __init__(self, mlist, volume, digest_number):
self._mlist = mlist
self._charset = mlist.preferred_language.charset
# This will be used in the Subject, so use $-strings.
self._digest_id = _(
'$mlist.display_name Digest, Vol $volume, Issue $digest_number')
self._subject = Header(self._digest_id,
self._charset,
header_name='Subject')
self._message = self._make_message()
self._digest_part = self._make_digest_part()
self._message['From'] = mlist.request_address
self._message['Subject'] = self._subject
self._message['To'] = mlist.posting_address
self._message['Reply-To'] = mlist.posting_address
self._message['Date'] = formatdate(localtime=True)
self._message['Message-ID'] = make_msgid()
# In the rfc1153 digest, the masthead contains the digest boilerplate
# plus any digest header. In the MIME digests, the masthead and
# digest header are separate MIME subobjects. In either case, it's
# the first thing in the digest, and we can calculate it now, so go
# ahead and add it now.
template = getUtility(ITemplateLoader).get(
'list:member:digest:masthead', mlist)
self._masthead = wrap(expand(template, mlist, dict(
# For backward compatibility.
got_list_email=mlist.posting_address,
got_request_email=mlist.request_address,
got_owner_email=mlist.owner_address,
)))
# Set things up for the table of contents.
self._header = decorate('list:member:digest:header', mlist)
self._toc = StringIO()
print(_("Today's Topics:\n"), file=self._toc)
def finish(self):
"""Finish up the digest, producing the email-ready copy."""
if self._mlist.digest_footer_uri is not None:
try:
footer_text = decorate(self._mlist, self._mlist.digest_footer_uri)
except URLError:
log.exception(
"Digest footer decorator URI not found ({0}): {1}".format(
self._mlist.fqdn_listname, self._mlist.digest_footer_uri
)
)
footer_text = ""
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print >>self._text, self._separator30
print >>self._text
print >>self._text, "Subject: " + _("Digest Footer")
print >>self._text
print >>self._text, footer_text
print >>self._text
print >>self._text, self._separator30
print >>self._text
# Add the sign-off.
sign_off = _("End of ") + self._digest_id
print >>self._text, sign_off
print >>self._text, "*" * len(sign_off)
# If the digest message can't be encoded by the list character set,
# fall back to utf-8.
text = self._text.getvalue()
try:
self._message.set_payload(text.encode(self._charset), charset=self._charset)
except UnicodeError:
self._message.set_payload(text.encode("utf-8"), charset="utf-8")
return self._message
def finish(self):
"""Finish up the digest, producing the email-ready copy."""
footer_text = decorate('list:member:digest:footer', self._mlist)
if len(footer_text) > 0:
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print(self._separator30, file=self._text)
print(file=self._text)
print('Subject: ' + _('Digest Footer'), file=self._text)
print(file=self._text)
print(footer_text, file=self._text)
print(file=self._text)
print(self._separator30, file=self._text)
print(file=self._text)
# Add the sign-off.
sign_off = _('End of ') + self._digest_id
print(sign_off, file=self._text)
print('*' * len(sign_off), file=self._text)
# If the digest message can't be encoded by the list character set,
# fall back to utf-8.
text = self._text.getvalue()
try:
self._message.set_payload(text.encode(self._charset),
charset=self._charset)
except UnicodeError:
self._message.set_payload(text.encode('utf-8'), charset='utf-8')
return self._message
def finish(self):
"""Finish up the digest, producing the email-ready copy."""
if self._mlist.digest_footer_uri is not None:
try:
footer_text = decorate(
self._mlist, self._mlist.digest_footer_uri)
except URLError:
log.exception(
'Digest footer decorator URI not found ({0}): {1}'.format(
self._mlist.fqdn_listname,
self._mlist.digest_footer_uri))
footer_text = ''
# This is not strictly conformant RFC 1153. The trailer is only
# supposed to contain two lines, i.e. the "End of ... Digest" line
# and the row of asterisks. If this screws up MUAs, the solution
# is to add the footer as the last message in the RFC 1153 digest.
# I just hate the way that VM does that and I think it's confusing
# to users, so don't do it unless there's a clamor.
print >> self._text, self._separator30
print >> self._text
print >> self._text, footer_text
print >> self._text
# Add the sign-off.
sign_off = _('End of ') + self._digest_id
print >> self._text, sign_off
print >> self._text, '*' * len(sign_off)
# If the digest message can't be encoded by the list character set,
# fall back to utf-8.
text = self._text.getvalue()
try:
self._message.set_payload(text.encode(self._charset),
charset=self._charset)
except UnicodeError:
self._message.set_payload(text.encode('utf-8'), charset='utf-8')
return self._message
def test_text_to_uri(self):
for oldvar, newvar in self._conf_mapping.items():
self._pckdict[str(oldvar)] = b'TEST VALUE'
import_config_pck(self._mlist, self._pckdict)
text = decorate(newvar, self._mlist)
self.assertEqual(
text, 'TEST VALUE',
'Old variable %s was not properly imported to %s' %
(oldvar, newvar))
def test_text_to_uri(self):
for oldvar, newvar in self._conf_mapping.items():
self._pckdict[str(oldvar)] = b'TEST VALUE'
import_config_pck(self._mlist, self._pckdict)
newattr = getattr(self._mlist, newvar)
text = decorate(self._mlist, newattr)
self.assertEqual(text, 'TEST VALUE',
'Old variable %s was not properly imported to %s'
% (oldvar, newvar))
def test_unicode(self):
# non-ascii templates
for oldvar in self._conf_mapping:
self._pckdict[str(oldvar)] = b'Ol\xe1!'
import_config_pck(self._mlist, self._pckdict)
for oldvar, newvar in self._conf_mapping.items():
newattr = getattr(self._mlist, newvar)
text = decorate(self._mlist, newattr)
expected = u'Ol\ufffd!'
self.assertEqual(text, expected)
def test_unicode(self):
# non-ascii templates
for oldvar in self._conf_mapping:
self._pckdict[str(oldvar)] = b'Ol\xe1!'
import_config_pck(self._mlist, self._pckdict)
for oldvar, newvar in self._conf_mapping.iteritems():
newattr = getattr(self._mlist, newvar)
text = decorate(self._mlist, newattr)
expected = u'Ol\ufffd!'
self.assertEqual(text, expected)
def test_unicode(self):
# non-ascii templates
for oldvar in self._conf_mapping:
self._pckdict[str(oldvar)] = b'Ol\xe1!'
import_config_pck(self._mlist, self._pckdict)
for oldvar, newvar in self._conf_mapping.items():
text = decorate(newvar, self._mlist)
expected = u'Ol\ufffd!'
self.assertEqual(
text, expected,
'{} -> {} did not get converted'.format(oldvar, newvar))
def test_unicode_in_default(self):
# What if the default template is already in UTF-8? For example, if
# you import it twice.
footer = b'\xe4\xb8\xad $listinfo_uri'
footer_path = os.path.join(config.VAR_DIR, 'templates', 'lists',
'*****@*****.**', 'en', 'footer.txt')
makedirs(os.path.dirname(footer_path))
with open(footer_path, 'wb') as fp:
fp.write(footer)
self._pckdict['msg_footer'] = b'NEW-VALUE'
import_config_pck(self._mlist, self._pckdict)
text = decorate('list:member:regular:footer', self._mlist)
self.assertEqual(text, 'NEW-VALUE')
def test_unicode_in_default(self):
# What if the default template is already in UTF-8? For example, if
# you import it twice.
footer = b'\xe4\xb8\xad $listinfo_uri'
footer_path = os.path.join(
config.VAR_DIR, 'templates', 'lists',
'*****@*****.**', 'en', 'footer-generic.txt')
makedirs(os.path.dirname(footer_path))
with open(footer_path, 'wb') as fp:
fp.write(footer)
self._pckdict[b'msg_footer'] = b'NEW-VALUE'
import_config_pck(self._mlist, self._pckdict)
text = decorate(self._mlist, self._mlist.footer_uri)
self.assertEqual(text, 'NEW-VALUE')
def finish(self):
"""Finish up the digest, producing the email-ready copy."""
self._message.attach(self._digest_part)
footer_text = decorate('list:member:digest:footer', self._mlist)
if len(footer_text) > 0:
footer = MIMEText(footer_text.encode(self._charset),
_charset=self._charset)
footer['Content-Description'] = _('Digest Footer')
self._message.attach(footer)
# This stuff is outside the normal MIME goo, and it's what the old
# MIME digester did. No one seemed to complain, probably because you
# won't see it in an MUA that can't display the raw message. We've
# never got complaints before, but if we do, just wax this. It's
# primarily included for (marginally useful) backwards compatibility.
self._message.postamble = _('End of ') + self._digest_id
return self._message
def __init__(self, mlist, volume, digest_number):
self._mlist = mlist
self._charset = mlist.preferred_language.charset
# This will be used in the Subject, so use $-strings.
self._digest_id = _(
'$mlist.display_name Digest, Vol $volume, Issue $digest_number')
self._subject = Header(self._digest_id,
self._charset,
header_name='Subject')
self._message = self._make_message()
self._digest_part = self._make_digest_part()
self._message['From'] = mlist.request_address
self._message['Subject'] = self._subject
self._message['To'] = mlist.posting_address
self._message['Reply-To'] = mlist.posting_address
self._message['Date'] = formatdate(localtime=True)
self._message['Message-ID'] = make_msgid()
# In the rfc1153 digest, the masthead contains the digest boilerplate
# plus any digest header. In the MIME digests, the masthead and
# digest header are separate MIME subobjects. In either case, it's
# the first thing in the digest, and we can calculate it now, so go
# ahead and add it now.
self._masthead = make('masthead.txt',
mailing_list=mlist,
display_name=mlist.display_name,
got_list_email=mlist.posting_address,
got_listinfo_url=mlist.script_url('listinfo'),
got_request_email=mlist.request_address,
got_owner_email=mlist.owner_address,
)
# Set things up for the table of contents.
if mlist.digest_header_uri is not None:
try:
self._header = decorate(mlist, mlist.digest_header_uri)
except URLError:
log.exception(
'Digest header decorator URI not found ({}): {}'.format(
mlist.fqdn_listname, mlist.digest_header_uri))
self._header = ''
self._toc = StringIO()
print(_("Today's Topics:\n"), file=self._toc)
def __init__(self, mlist, volume, digest_number):
self._mlist = mlist
self._charset = mlist.preferred_language.charset
# This will be used in the Subject, so use $-strings.
self._digest_id = _(
'$mlist.display_name Digest, Vol $volume, Issue $digest_number')
self._subject = Header(self._digest_id,
self._charset,
header_name='Subject')
self._message = self._make_message()
self._digest_part = self._make_digest_part()
self._message['From'] = mlist.request_address
self._message['Subject'] = self._subject
self._message['To'] = mlist.posting_address
self._message['Reply-To'] = mlist.posting_address
self._message['Date'] = formatdate(localtime=True)
self._message['Message-ID'] = make_msgid()
# In the rfc1153 digest, the masthead contains the digest boilerplate
# plus any digest header. In the MIME digests, the masthead and
# digest header are separate MIME subobjects. In either case, it's
# the first thing in the digest, and we can calculate it now, so go
# ahead and add it now.
self._masthead = make('masthead.txt',
mailing_list=mlist,
display_name=mlist.display_name,
got_list_email=mlist.posting_address,
got_listinfo_url=mlist.script_url('listinfo'),
got_request_email=mlist.request_address,
got_owner_email=mlist.owner_address,
)
# Set things up for the table of contents.
if mlist.digest_header_uri is not None:
try:
self._header = decorate(mlist, mlist.digest_header_uri)
except URLError:
log.exception(
'Digest header decorator URI not found ({}): {}'.format(
mlist.fqdn_listname, mlist.digest_header_uri))
self._header = ''
self._toc = StringIO()
print(_("Today's Topics:\n"), file=self._toc)
def finish(self):
"""Finish up the digest, producing the email-ready copy."""
if self._mlist.digest_footer_uri is not None:
try:
footer_text = decorate(self._mlist, self._mlist.digest_footer_uri)
except URLError:
log.exception(
"Digest footer decorator URI not found ({0}): {1}".format(
self._mlist.fqdn_listname, self._mlist.digest_footer_uri
)
)
footer_text = ""
footer = MIMEText(footer_text.encode(self._charset), _charset=self._charset)
footer["Content-Description"] = _("Digest Footer")
self._message.attach(footer)
# This stuff is outside the normal MIME goo, and it's what the old
# MIME digester did. No one seemed to complain, probably because you
# won't see it in an MUA that can't display the raw message. We've
# never got complaints before, but if we do, just wax this. It's
# primarily included for (marginally useful) backwards compatibility.
self._message.postamble = _("End of ") + self._digest_id
return self._message
def finish(self):
"""Finish up the digest, producing the email-ready copy."""
if self._mlist.digest_footer_uri is not None:
try:
footer_text = decorate(self._mlist,
self._mlist.digest_footer_uri)
except URLError:
log.exception(
'Digest footer decorator URI not found ({0}): {1}'.format(
self._mlist.fqdn_listname,
self._mlist.digest_footer_uri))
footer_text = ''
footer = MIMEText(footer_text.encode(self._charset),
_charset=self._charset)
footer['Content-Description'] = _('Digest Footer')
self._message.attach(footer)
# This stuff is outside the normal MIME goo, and it's what the old
# MIME digester did. No one seemed to complain, probably because you
# won't see it in an MUA that can't display the raw message. We've
# never got complaints before, but if we do, just wax this. It's
# primarily included for (marginally useful) backwards compatibility.
self._message.postamble = _('End of ') + self._digest_id
return self._message
def finish(self):
"""Finish up the digest, producing the email-ready copy."""
if self._mlist.digest_footer_uri is not None:
try:
footer_text = decorate(
self._mlist, self._mlist.digest_footer_uri)
except URLError:
log.exception(
'Digest footer decorator URI not found ({}): {}'.format(
self._mlist.fqdn_listname,
self._mlist.digest_footer_uri))
footer_text = ''
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print(self._separator30, file=self._text)
print(file=self._text)
print('Subject: ' + _('Digest Footer'), file=self._text)
print(file=self._text)
print(footer_text, file=self._text)
print(file=self._text)
print(self._separator30, file=self._text)
print(file=self._text)
# Add the sign-off.
sign_off = _('End of ') + self._digest_id
print(sign_off, file=self._text)
print('*' * len(sign_off), file=self._text)
# If the digest message can't be encoded by the list character set,
# fall back to utf-8.
text = self._text.getvalue()
try:
self._message.set_payload(text.encode(self._charset),
charset=self._charset)
except UnicodeError:
self._message.set_payload(text.encode('utf-8'), charset='utf-8')
return self._message
def finish(self):
"""Finish up the digest, producing the email-ready copy."""
if self._mlist.digest_footer_uri is not None:
try:
footer_text = decorate(
self._mlist, self._mlist.digest_footer_uri)
except URLError:
log.exception(
'Digest footer decorator URI not found ({}): {}'.format(
self._mlist.fqdn_listname,
self._mlist.digest_footer_uri))
footer_text = ''
# MAS: There is no real place for the digest_footer in an RFC 1153
# compliant digest, so add it as an additional message with
# Subject: Digest Footer
print(self._separator30, file=self._text)
print(file=self._text)
print('Subject: ' + _('Digest Footer'), file=self._text)
print(file=self._text)
print(footer_text, file=self._text)
print(file=self._text)
print(self._separator30, file=self._text)
print(file=self._text)
# Add the sign-off.
sign_off = _('End of ') + self._digest_id
print(sign_off, file=self._text)
print('*' * len(sign_off), file=self._text)
# If the digest message can't be encoded by the list character set,
# fall back to utf-8.
text = self._text.getvalue()
try:
self._message.set_payload(text.encode(self._charset),
charset=self._charset)
except UnicodeError:
self._message.set_payload(text.encode('utf-8'), charset='utf-8')
return self._message
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if hasattr(mlist, key) or key == 'preferred_language':
if isinstance(value, str):
value = str_to_unicode(value)
# Some types require conversion.
converter = TYPES.get(key)
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value), file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(str_to_unicode(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, basestring):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = str_to_unicode(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
'welcome_msg': 'welcome_message_uri',
'goodbye_msg': 'goodbye_message_uri',
'msg_header': 'header_uri',
'msg_footer': 'footer_uri',
'digest_header': 'digest_header_uri',
'digest_footer': 'digest_footer_uri',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
('%(real_name)s@%(host_name)s', '$fqdn_listname'),
('%(real_name)s', '$display_name'),
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
'$listinfo_uri'),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = 'mailman:///$listname/$language/'
if default_value:
filename = default_value.rpartition('/')[2]
else:
filename = '{}.txt'.format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
members = set(config_dict.get('members', {}).keys()
+ config_dict.get('digest_members', {}).keys())
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == "preferred_language" or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(key, value), file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == "last_post_time":
setattr(mlist, "last_post_at", value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get("default_member_moderation", 0)):
mlist.default_member_action = member_moderation_action_mapping(config_dict.get("member_moderation_action"))
else:
mlist.default_member_action = Action.defer
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get("archive"):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get("archive_private", True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get("ban_list", []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get("acceptable_aliases", "")
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode("utf-8")
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add("^" + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
"welcome_msg": "welcome_message_uri",
"goodbye_msg": "goodbye_message_uri",
"msg_header": "header_uri",
"msg_footer": "footer_uri",
"digest_header": "digest_header_uri",
"digest_footer": "digest_footer_uri",
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
("%(real_name)s@%(host_name)s", "$fqdn_listname"),
("%(real_name)s", "$display_name"),
("%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s", "$listinfo_uri"),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print(
"Unable to convert mailing list attribute:",
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr,
)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode("utf-8", "replace")
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print("Unable to convert mailing list attribute:", oldvar, 'with value "{}"'.format(text), file=sys.stderr)
continue
if expanded_text and default_text and expanded_text.strip() == default_text.strip():
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = "mailman:///$listname/$language/"
if default_value:
filename = default_value.rpartition("/")[2]
else:
filename = "{}.txt".format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, "w", encoding="utf-8") as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get("members", {}))
digesters_set = set(config_dict.get("digest_members", {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get("owner", []), MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get("moderator", []), MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ("accept", "hold", "reject", "discard"):
prop_name = "{}_these_nonmembers".format(action_name)
emails = [addr for addr in config_dict.get(prop_name, []) if not addr.startswith("^")]
import_roster(mlist, config_dict, emails, MemberRole.nonmember, Action[action_name])
# Only keep the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for email in emails:
list_prop.remove(email)
finally:
mlist.send_welcome_message = send_welcome_message
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == 'preferred_language' or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value),
file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get('default_member_moderation', 0)):
mlist.default_member_action = member_moderation_action_mapping(
config_dict.get('member_moderation_action'))
else:
mlist.default_member_action = Action.defer
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode('utf-8')
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
'welcome_msg': 'welcome_message_uri',
'goodbye_msg': 'goodbye_message_uri',
'msg_header': 'header_uri',
'msg_footer': 'footer_uri',
'digest_header': 'digest_header_uri',
'digest_footer': 'digest_footer_uri',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
('%(real_name)s@%(host_name)s', '$fqdn_listname'),
('%(real_name)s', '$display_name'),
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
'$listinfo_uri'),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = 'mailman:///$listname/$language/'
if default_value:
filename = default_value.rpartition('/')[2]
else:
filename = '{}.txt'.format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get('members', {}))
digesters_set = set(config_dict.get('digest_members', {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ('accept', 'hold', 'reject', 'discard'):
prop_name = '{}_these_nonmembers'.format(action_name)
emails = [
addr for addr in config_dict.get(prop_name, [])
if not addr.startswith('^')
]
import_roster(mlist, config_dict, emails, MemberRole.nonmember,
Action[action_name])
# Only keep the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for email in emails:
list_prop.remove(email)
finally:
mlist.send_welcome_message = send_welcome_message
