Beispiel #1
0
 def test_delete_list_with_acceptable_aliases(self):
     # LP: #1432239 - deleting a mailing list with acceptable aliases
     # causes a SQLAlchemy error.  The aliases must be deleted first.
     with transaction():
         alias_set = IAcceptableAliasSet(self._mlist)
         alias_set.add('*****@*****.**')
     self.assertEqual(['*****@*****.**'], list(alias_set.aliases))
     getUtility(IListManager).delete(self._mlist)
     self.assertEqual(len(list(alias_set.aliases)), 0)
Beispiel #2
0
 def test_delete_list_with_acceptable_aliases(self):
     # LP: #1432239 - deleting a mailing list with acceptable aliases
     # causes a SQLAlchemy error.  The aliases must be deleted first.
     with transaction():
         alias_set = IAcceptableAliasSet(self._mlist)
         alias_set.add('*****@*****.**')
     self.assertEqual(['*****@*****.**'], list(alias_set.aliases))
     getUtility(IListManager).delete(self._mlist)
     self.assertEqual(len(list(alias_set.aliases)), 0)
Beispiel #3
0
 def test_delete_list_with_acceptable_aliases(self):
     # LP: #1432239 - deleting a mailing list with acceptable aliases
     # causes a SQLAlchemy error.  The aliases must be deleted first.
     with transaction():
         alias_set = IAcceptableAliasSet(self._mlist)
         alias_set.add('*****@*****.**')
     call_api('http://*****:*****@example.com'))
     self.assertEqual(config.db.store.query(AcceptableAlias).count(), 0)
Beispiel #4
0
 def test_delete_list_with_acceptable_aliases(self):
     # LP: #1432239 - deleting a mailing list with acceptable aliases
     # causes a SQLAlchemy error.  The aliases must be deleted first.
     with transaction():
         alias_set = IAcceptableAliasSet(self._mlist)
         alias_set.add('*****@*****.**')
     call_api('http://*****:*****@example.com'))
     self.assertEqual(config.db.store.query(AcceptableAlias).count(), 0)
Beispiel #5
0
    def put(self, mlist, attribute, value):
        """Change the acceptable aliases.

        Because this is a PUT operation, all previous aliases are cleared
        first.  Thus, this is an overwrite.  The keys in the request are
        ignored.
        """
        assert attribute == "acceptable_aliases", "Unexpected attribute: {0}".format(attribute)
        alias_set = IAcceptableAliasSet(mlist)
        alias_set.clear()
        for alias in value:
            alias_set.add(unicode(alias))
Beispiel #6
0
 def test_acceptable_aliases_invalid(self):
     # Values without an '@' sign used to be matched against the local
     # part, now we need to add the '^' sign to indicate it's a regexp.
     aliases = ['invalid-value']
     self._pckdict['acceptable_aliases'] = list_to_string(aliases)
     self._import()
     alias_set = IAcceptableAliasSet(self._mlist)
     self.assertEqual(sorted(alias_set.aliases),
                      [('^' + alias) for alias in aliases])
Beispiel #7
0
 def test_acceptable_aliases_as_list(self):
     # In some versions of the pickle, this can be a list, not a string
     # (seen in the wild).
     aliases = [b'*****@*****.**', b'*****@*****.**']
     self._pckdict['acceptable_aliases'] = aliases
     self._import()
     alias_set = IAcceptableAliasSet(self._mlist)
     self.assertEqual(sorted(alias_set.aliases),
                      sorted(a.decode('utf-8') for a in aliases))
 def check(self, mlist, msg, msgdata):
     """See `IRule`."""
     # Implicit destination checking must be enabled in the mailing list.
     if not mlist.require_explicit_destination:
         return False
     # Messages gated from NNTP will always have an implicit destination so
     # are never checked.
     if msgdata.get('fromusenet'):
         return False
     # Calculate the list of acceptable aliases.  If the alias starts with
     # a caret (i.e. ^), then it's a regular expression to match against.
     aliases = set()
     alias_patterns = set()
     # Adapt the mailing list to the appropriate interface.
     alias_set = IAcceptableAliasSet(mlist)
     for alias in alias_set.aliases:
         if alias.startswith('^'):
             alias_patterns.add(alias)
         else:
             aliases.add(alias)
     # Add the list's posting address, i.e. the explicit address, to the
     # set of acceptable aliases.
     aliases.add(mlist.posting_address)
     # Look at all the recipients.  If the recipient is any acceptable
     # alias (or the explicit posting address), then this rule does not
     # match.  If not, then add it to the set of recipients we'll check
     # against the alias patterns later.
     recipients = set()
     for header in ('to', 'cc', 'resent-to', 'resent-cc'):
         for fullname, address in getaddresses(msg.get_all(header, [])):
             if isinstance(address, bytes):
                 address = address.decode('ascii')
             address = address.lower()
             if address in aliases:
                 return False
             recipients.add(address)
     # Now for all alias patterns, see if any of the recipients matches a
     # pattern.  If so, then this rule does not match.
     for pattern in alias_patterns:
         escaped = re.escape(pattern)
         for recipient in recipients:
             try:
                 if re.match(pattern, recipient, re.IGNORECASE):
                     return False
             except re.error:
                 # The pattern is a malformed regular expression.  Try
                 # matching again with the pattern escaped.
                 with suppress(re.error):
                     if re.match(escaped, recipient, re.IGNORECASE):
                         return False
     # Nothing matched.
     msgdata['moderation_sender'] = msg.sender
     with _.defer_translation():
         # This will be translated at the point of use.
         msgdata.setdefault('moderation_reasons', []).append(
             _('Message has implicit destination'))
     return True
Beispiel #9
0
 def test_put_configuration(self):
     # When using PUT, all writable attributes must be included.
     resource, response = call_api(
         'http://localhost:9001/3.0/lists/ant.example.com/config', RESOURCE,
         'PUT')
     self.assertEqual(response.status, 204)
     self.assertEqual(self._mlist.display_name, 'Fnords')
     # All three acceptable aliases were set.
     self.assertEqual(set(IAcceptableAliasSet(self._mlist).aliases),
                      set(RESOURCE['acceptable_aliases']))
Beispiel #10
0
 def test_acceptable_aliases(self):
     # This used to be a plain-text field (values are newline-separated).
     aliases = ['*****@*****.**',
                '*****@*****.**',
                'non-ascii-\[email protected]',
                ]
     self._pckdict['acceptable_aliases'] = list_to_string(aliases)
     self._import()
     alias_set = IAcceptableAliasSet(self._mlist)
     self.assertEqual(sorted(alias_set.aliases), aliases)
Beispiel #11
0
 def test_put_configuration(self):
     aliases = [
         '*****@*****.**',
         '*****@*****.**',
         '*****@*****.**',
         ]
     # When using PUT, all writable attributes must be included.
     resource, response = call_api(
         'http://*****:*****@example.com/config',
         dict(
             acceptable_aliases=aliases,
             admin_immed_notify=False,
             admin_notify_mchanges=True,
             administrivia=False,
             advertised=False,
             anonymous_list=True,
             archive_policy='never',
             autorespond_owner='respond_and_discard',
             autorespond_postings='respond_and_continue',
             autorespond_requests='respond_and_discard',
             autoresponse_grace_period='45d',
             autoresponse_owner_text='the owner',
             autoresponse_postings_text='the mailing list',
             autoresponse_request_text='the robot',
             display_name='Fnords',
             description='This is my mailing list',
             include_rfc2369_headers=False,
             allow_list_posts=False,
             digest_size_threshold=10.5,
             posting_pipeline='virgin',
             filter_content=True,
             first_strip_reply_to=True,
             convert_html_to_plaintext=True,
             collapse_alternatives=False,
             reply_goes_to_list='point_to_list',
             reply_to_address='*****@*****.**',
             send_welcome_message=False,
             subject_prefix='[ant]',
             welcome_message_uri='mailman:///welcome.txt',
             default_member_action='hold',
             default_nonmember_action='discard',
             ),
         'PUT')
     self.assertEqual(response.status, 204)
     self.assertEqual(self._mlist.display_name, 'Fnords')
     # All three acceptable aliases were set.
     self.assertEqual(set(IAcceptableAliasSet(self._mlist).aliases),
                      set(aliases))
Beispiel #12
0
    def put(self, mlist, attribute, value):
        """Change the acceptable aliases.

        Because this is a PUT operation, all previous aliases are cleared
        first.  Thus, this is an overwrite.  The keys in the request are
        ignored.
        """
        assert attribute == 'acceptable_aliases', (
            'Unexpected attribute: {}'.format(attribute))  # pragma: nocover
        alias_set = IAcceptableAliasSet(mlist)
        alias_set.clear()
        for alias in value:
            alias_set.add(alias)
Beispiel #13
0
 def on_delete(self, request, response):
     if self._attribute is None:
         bad_request(response,
                     'Cannot delete the list configuration itself')
         return
     if self._attribute not in VALIDATORS:
         bad_request(response,
                     'Read-only attribute: {}'.format(self._attribute))
         return
     # This kind of sucks because it doesn't scale if the list of attributes
     # which can be deleted grows.  So if we get too many we'll have to use
     # a lookup table.  For now, this is good enough.
     if self._attribute != 'acceptable_aliases':
         bad_request(
             response,
             'Attribute cannot be DELETEd: {}'.format(self._attribute))
         return
     IAcceptableAliasSet(self._mlist).clear()
     no_content(response)
def import_config_pck(mlist, config_dict):
    """Apply a config.pck configuration dictionary to a mailing list.

    :param mlist: The mailing list.
    :type mlist: IMailingList
    :param config_dict: The Mailman 2.1 configuration dictionary.
    :type config_dict: dict
    """
    global key
    for key, value in config_dict.items():
        # Some attributes must not be directly imported.
        if key in EXCLUDES:
            continue
        # These objects need explicit type conversions.
        if key in DATETIME_COLUMNS:
            continue
        # Some attributes from Mailman 2 were renamed in Mailman 3.
        key = NAME_MAPPINGS.get(key, key)
        # Handle the simple case where the key is an attribute of the
        # IMailingList and the types are the same (modulo 8-bit/unicode
        # strings).
        #
        # If the mailing list has a preferred language that isn't registered
        # in the configuration file, hasattr() will swallow the KeyError this
        # raises and return False.  Treat that attribute specially.
        if key == 'preferred_language' or hasattr(mlist, key):
            if isinstance(value, bytes):
                value = bytes_to_str(value)
            # Some types require conversion.
            converter = TYPES.get(key)
            if converter is None:
                column = getattr(mlist.__class__, key, None)
                if column is not None and isinstance(column.type, Boolean):
                    converter = bool
                if column is not None and isinstance(column.type, SAUnicode):
                    converter = maybe_truncate_mysql
            try:
                if converter is not None:
                    value = converter(value)
                setattr(mlist, key, value)
            except (TypeError, KeyError):
                print('Type conversion error for key "{}": {}'.format(
                    key, value),
                      file=sys.stderr)
    for key in DATETIME_COLUMNS:
        try:
            value = datetime.datetime.utcfromtimestamp(config_dict[key])
        except KeyError:
            continue
        if key == 'last_post_time':
            setattr(mlist, 'last_post_at', value)
            continue
        setattr(mlist, key, value)
    # Handle the moderation policy.
    #
    # The mlist.default_member_action and mlist.default_nonmember_action enum
    # values are different in Mailman 2.1, because they have been merged into a
    # single enum in Mailman 3.
    #
    # Unmoderated lists used to have default_member_moderation set to a false
    # value; this translates to the Defer default action.  Moderated lists with
    # the default_member_moderation set to a true value used to store the
    # action in the member_moderation_action flag, the values were: 0==Hold,
    # 1=Reject, 2==Discard
    if bool(config_dict.get('default_member_moderation', 0)):
        mlist.default_member_action = member_moderation_action_mapping(
            config_dict.get('member_moderation_action'))
    else:
        mlist.default_member_action = Action.defer
    # Handle DMARC mitigations.
    # This would be straightforward except for from_is_list.  The issue
    # is in MM 2.1 the from_is_list action applies if dmarc_moderation_action
    # doesn't apply and they can be different.
    # We will map as follows:
    # from_is_list > dmarc_moderation_action
    #    dmarc_mitigate_action = from_is_list action
    #    dmarc_mitigate_unconditionally = True
    # from_is_list <= dmarc_moderation_action
    #    dmarc_mitigate_action = dmarc_moderation_action
    #    dmarc_mitigate_unconditionally = False
    # The text attributes are handled above.
    if (config_dict.get('from_is_list', 0) > config_dict.get(
            'dmarc_moderation_action', 0)):
        mlist.dmarc_mitigate_action = dmarc_action_mapping(
            config_dict.get('from_is_list', 0))
        mlist.dmarc_mitigate_unconditionally = True
    else:
        mlist.dmarc_mitigate_action = dmarc_action_mapping(
            config_dict.get('dmarc_moderation_action', 0))
        mlist.dmarc_mitigate_unconditionally = False
    # Handle the archiving policy.  In MM2.1 there were two boolean options
    # but only three of the four possible states were valid.  Now there's just
    # an enum.
    if config_dict.get('archive'):
        # For maximum safety, if for some strange reason there's no
        # archive_private key, treat the list as having private archives.
        if config_dict.get('archive_private', True):
            mlist.archive_policy = ArchivePolicy.private
        else:
            mlist.archive_policy = ArchivePolicy.public
    else:
        mlist.archive_policy = ArchivePolicy.never
    # Handle ban list.
    ban_manager = IBanManager(mlist)
    for address in config_dict.get('ban_list', []):
        ban_manager.ban(bytes_to_str(address))
    # Handle acceptable aliases.
    acceptable_aliases = config_dict.get('acceptable_aliases', '')
    if isinstance(acceptable_aliases, bytes):
        acceptable_aliases = acceptable_aliases.decode('utf-8')
    if isinstance(acceptable_aliases, str):
        acceptable_aliases = acceptable_aliases.splitlines()
    alias_set = IAcceptableAliasSet(mlist)
    for address in acceptable_aliases:
        address = address.strip()
        if len(address) == 0:
            continue
        address = bytes_to_str(address)
        # All 2.1 acceptable aliases are regexps whether or not they start
        # with '^' or contain '@'.
        if not address.startswith('^'):
            address = '^' + address
        # This used to be in a try which would catch ValueError and add a '^',
        # but .add() would not raise ValueError if address contained '@' and
        # that needs the '^' too as it could be a regexp with an '@' in it.
        alias_set.add(address)
    # Handle roster visibility.
    mapping = member_roster_visibility_mapping(
        config_dict.get('private_roster', None))
    if mapping is not None:
        mlist.member_roster_visibility = mapping
    # Handle header_filter_rules conversion to header_matches.
    header_matches = IHeaderMatchList(mlist)
    header_filter_rules = config_dict.get('header_filter_rules', [])
    for line_patterns, action, _unused in header_filter_rules:
        try:
            chain = action_to_chain(action)
        except KeyError:
            log.warning('Unsupported header_filter_rules action: %r', action)
            continue
        # Now split the line into a header and a pattern.
        for line_pattern in line_patterns.splitlines():
            if len(line_pattern.strip()) == 0:
                continue
            for sep in (': ', ':.*', ':.', ':'):
                header, sep, pattern = line_pattern.partition(sep)
                if sep:
                    # We found it.
                    break
            else:
                # Matches any header, which is not supported.  XXX
                log.warning('Unsupported header_filter_rules pattern: %r',
                            line_pattern)
                continue
            header = header.strip().lstrip('^').lower()
            header = header.replace('\\', '')
            if not header:
                log.warning(
                    'Cannot parse the header in header_filter_rule: %r',
                    line_pattern)
                continue
            if len(pattern) == 0:
                # The line matched only the header, therefore the header can
                # be anything.
                pattern = '.*'
            try:
                re.compile(pattern)
            except re.error:
                log.warning(
                    'Skipping header_filter rule because of an '
                    'invalid regular expression: %r', line_pattern)
                continue
            try:
                header_matches.append(header, pattern, chain)
            except ValueError:
                log.warning('Skipping duplicate header_filter rule: %r',
                            line_pattern)
                continue
    # Handle conversion to URIs.  In MM2.1, the decorations are strings
    # containing placeholders, and there's no provision for language-specific
    # strings.  In MM3, template locations are specified by URLs with the
    # special `mailman:` scheme indicating a file system path.  What we do
    # here is look to see if the list's decoration is different than the
    # default, and if so, we'll write the new decoration template to a
    # `mailman:` scheme path, then add the template to the template manager.
    # We are intentionally omitting the 2.1 welcome_msg here because the
    # string is actually interpolated into a larger template and there's
    # no good way to figure where in the default template to insert it.
    convert_to_uri = {
        'goodbye_msg': 'list:user:notice:goodbye',
        'msg_header': 'list:member:regular:header',
        'msg_footer': 'list:member:regular:footer',
        'digest_header': 'list:member:digest:header',
        'digest_footer': 'list:member:digest:footer',
    }
    # The best we can do is convert only the most common ones.  These are
    # order dependent; the longer substitution with the common prefix must
    # show up earlier.
    convert_placeholders = [
        # First convert \r\n that may have been set by a browser to \n.
        ('\r\n', '\n'),
        ('%(real_name)s@%(host_name)s',
         'To unsubscribe send an email to ${short_listname}-leave@${domain}'),
        ('%(real_name)s mailing list',
         '$display_name mailing list -- $listname'),
        # The generic footers no longer have URLs in them.
        ('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s\n', ''),
    ]
    # Collect defaults.
    manager = getUtility(ITemplateManager)
    defaults = {}
    for oldvar, newvar in convert_to_uri.items():
        default_value = getUtility(ITemplateLoader).get(newvar, mlist)
        if not default_value:
            continue
        # Get the decorated default text
        try:
            default_text = decorate_template(mlist, default_value)
        except (URLError, KeyError):  # pragma: nocover
            # Use case: importing the old [email protected] into [email protected].  We can't
            # check if it changed from the default so don't import, we may do
            # more harm than good and it's easy to change if needed.
            # TESTME
            print('Unable to convert mailing list attribute:',
                  oldvar,
                  'with old value "{}"'.format(default_value),
                  file=sys.stderr)
            continue
        defaults[newvar] = default_text
    for oldvar, newvar in convert_to_uri.items():
        if oldvar not in config_dict:
            continue
        text = config_dict[oldvar]
        if isinstance(text, bytes):
            text = text.decode('utf-8', 'replace')
        for oldph, newph in convert_placeholders:
            text = text.replace(oldph, newph)
        default_text = defaults.get(newvar, None)
        if not text and not default_text:
            # Both are empty, leave it.
            continue
        # Check if the value changed from the default
        try:
            expanded_text = decorate_template(mlist, text)
        except KeyError:  # pragma: nocover
            # Use case: importing the old [email protected] into [email protected]
            # We can't check if it changed from the default
            # -> don't import, we may do more harm than good and it's easy to
            # change if needed
            # TESTME
            print('Unable to convert mailing list attribute:',
                  oldvar,
                  'with value "{}"'.format(text),
                  file=sys.stderr)
            continue
        if (expanded_text and default_text
                and expanded_text.strip() == default_text.strip()):
            # Keep the default.
            continue
        # Write the custom value to the right file and add it to the template
        # manager for real.
        base_uri = 'mailman:///$listname/$language/'
        filename = '{}.txt'.format(newvar)
        manager.set(newvar, mlist.list_id, base_uri + filename)
        with ExitStack() as resources:
            filepath = list(search(resources, filename, mlist))[0]
        makedirs(os.path.dirname(filepath))
        with open(filepath, 'w', encoding='utf-8') as fp:
            fp.write(text)
    # Import rosters.
    regulars_set = set(config_dict.get('members', {}))
    digesters_set = set(config_dict.get('digest_members', {}))
    members = regulars_set.union(digesters_set)
    # Don't send welcome messages or notify admins when we import the rosters.
    send_welcome_message = mlist.send_welcome_message
    mlist.send_welcome_message = False
    admin_notify_mchanges = mlist.admin_notify_mchanges
    mlist.admin_notify_mchanges = False
    try:
        import_roster(mlist, config_dict, members, MemberRole.member)
        import_roster(mlist, config_dict, config_dict.get('owner', []),
                      MemberRole.owner)
        import_roster(mlist, config_dict, config_dict.get('moderator', []),
                      MemberRole.moderator)
        # Now import the '*_these_nonmembers' properties, filtering out the
        # regexps which will remain in the property.
        for action_name in ('accept', 'hold', 'reject', 'discard'):
            prop_name = '{}_these_nonmembers'.format(action_name)
            emails = [
                addr for addr in config_dict.get(prop_name, [])
                if not addr.startswith('^')
            ]
            # MM 2.1 accept maps to MM 3 defer
            if action_name == 'accept':
                action_name = 'defer'
            import_roster(mlist, config_dict, emails, MemberRole.nonmember,
                          Action[action_name])
            # Now add the regexes in the legacy list property.
            list_prop = getattr(mlist, prop_name)
            for addr in config_dict.get(prop_name, []):
                if addr.startswith('^'):
                    list_prop.append(addr)
    finally:
        mlist.send_welcome_message = send_welcome_message
        mlist.admin_notify_mchanges = admin_notify_mchanges
Beispiel #15
0
def import_config_pck(mlist, config_dict):
    """Apply a config.pck configuration dictionary to a mailing list.

    :param mlist: The mailing list.
    :type mlist: IMailingList
    :param config_dict: The Mailman 2.1 configuration dictionary.
    :type config_dict: dict
    """
    for key, value in config_dict.items():
        # Some attributes must not be directly imported.
        if key in EXCLUDES:
            continue
        # These objects need explicit type conversions.
        if key in DATETIME_COLUMNS:
            continue
        # Some attributes from Mailman 2 were renamed in Mailman 3.
        key = NAME_MAPPINGS.get(key, key)
        # Handle the simple case where the key is an attribute of the
        # IMailingList and the types are the same (modulo 8-bit/unicode
        # strings).
        #
        # If the mailing list has a preferred language that isn't registered
        # in the configuration file, hasattr() will swallow the KeyError this
        # raises and return False.  Treat that attribute specially.
        if key == 'preferred_language' or hasattr(mlist, key):
            if isinstance(value, bytes):
                value = bytes_to_str(value)
            # Some types require conversion.
            converter = TYPES.get(key)
            if converter is None:
                column = getattr(mlist.__class__, key, None)
                if column is not None and isinstance(column.type, Boolean):
                    converter = bool
            try:
                if converter is not None:
                    value = converter(value)
                setattr(mlist, key, value)
            except (TypeError, KeyError):
                print('Type conversion error for key "{}": {}'.format(
                    key, value),
                      file=sys.stderr)
    for key in DATETIME_COLUMNS:
        try:
            value = datetime.datetime.utcfromtimestamp(config_dict[key])
        except KeyError:
            continue
        if key == 'last_post_time':
            setattr(mlist, 'last_post_at', value)
            continue
        setattr(mlist, key, value)
    # Handle the moderation policy.
    #
    # The mlist.default_member_action and mlist.default_nonmember_action enum
    # values are different in Mailman 2.1, because they have been merged into a
    # single enum in Mailman 3.
    #
    # Unmoderated lists used to have default_member_moderation set to a false
    # value; this translates to the Defer default action.  Moderated lists with
    # the default_member_moderation set to a true value used to store the
    # action in the member_moderation_action flag, the values were: 0==Hold,
    # 1=Reject, 2==Discard
    if bool(config_dict.get('default_member_moderation', 0)):
        mlist.default_member_action = member_moderation_action_mapping(
            config_dict.get('member_moderation_action'))
    else:
        mlist.default_member_action = Action.defer
    # Handle the archiving policy.  In MM2.1 there were two boolean options
    # but only three of the four possible states were valid.  Now there's just
    # an enum.
    if config_dict.get('archive'):
        # For maximum safety, if for some strange reason there's no
        # archive_private key, treat the list as having private archives.
        if config_dict.get('archive_private', True):
            mlist.archive_policy = ArchivePolicy.private
        else:
            mlist.archive_policy = ArchivePolicy.public
    else:
        mlist.archive_policy = ArchivePolicy.never
    # Handle ban list.
    ban_manager = IBanManager(mlist)
    for address in config_dict.get('ban_list', []):
        ban_manager.ban(bytes_to_str(address))
    # Handle acceptable aliases.
    acceptable_aliases = config_dict.get('acceptable_aliases', '')
    if isinstance(acceptable_aliases, bytes):
        acceptable_aliases = acceptable_aliases.decode('utf-8')
    if isinstance(acceptable_aliases, str):
        acceptable_aliases = acceptable_aliases.splitlines()
    alias_set = IAcceptableAliasSet(mlist)
    for address in acceptable_aliases:
        address = address.strip()
        if len(address) == 0:
            continue
        address = bytes_to_str(address)
        try:
            alias_set.add(address)
        except ValueError:
            # When .add() rejects this, the line probably contains a regular
            # expression.  Make that explicit for MM3.
            alias_set.add('^' + address)
    # Handle conversion to URIs.  In MM2.1, the decorations are strings
    # containing placeholders, and there's no provision for language-specific
    # templates.  In MM3, template locations are specified by URLs with the
    # special `mailman:` scheme indicating a file system path.  What we do
    # here is look to see if the list's decoration is different than the
    # default, and if so, we'll write the new decoration template to a
    # `mailman:` scheme path.
    convert_to_uri = {
        'welcome_msg': 'welcome_message_uri',
        'goodbye_msg': 'goodbye_message_uri',
        'msg_header': 'header_uri',
        'msg_footer': 'footer_uri',
        'digest_header': 'digest_header_uri',
        'digest_footer': 'digest_footer_uri',
    }
    # The best we can do is convert only the most common ones.  These are
    # order dependent; the longer substitution with the common prefix must
    # show up earlier.
    convert_placeholders = [
        ('%(real_name)s@%(host_name)s', '$fqdn_listname'),
        ('%(real_name)s', '$display_name'),
        ('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
         '$listinfo_uri'),
    ]
    # Collect defaults.
    defaults = {}
    for oldvar, newvar in convert_to_uri.items():
        default_value = getattr(mlist, newvar, None)
        if not default_value:
            continue
        # Check if the value changed from the default.
        try:
            default_text = decorate(mlist, default_value)
        except (URLError, KeyError):
            # Use case: importing the old [email protected] into [email protected].  We can't
            # check if it changed from the default so don't import, we may do
            # more harm than good and it's easy to change if needed.
            # TESTME
            print('Unable to convert mailing list attribute:',
                  oldvar,
                  'with old value "{}"'.format(default_value),
                  file=sys.stderr)
            continue
        defaults[newvar] = (default_value, default_text)
    for oldvar, newvar in convert_to_uri.items():
        if oldvar not in config_dict:
            continue
        text = config_dict[oldvar]
        if isinstance(text, bytes):
            text = text.decode('utf-8', 'replace')
        for oldph, newph in convert_placeholders:
            text = text.replace(oldph, newph)
        default_value, default_text = defaults.get(newvar, (None, None))
        if not text and not (default_value or default_text):
            # Both are empty, leave it.
            continue
        # Check if the value changed from the default
        try:
            expanded_text = decorate_template(mlist, text)
        except KeyError:
            # Use case: importing the old [email protected] into [email protected]
            # We can't check if it changed from the default
            # -> don't import, we may do more harm than good and it's easy to
            # change if needed
            # TESTME
            print('Unable to convert mailing list attribute:',
                  oldvar,
                  'with value "{}"'.format(text),
                  file=sys.stderr)
            continue
        if (expanded_text and default_text
                and expanded_text.strip() == default_text.strip()):
            # Keep the default.
            continue
        # Write the custom value to the right file.
        base_uri = 'mailman:///$listname/$language/'
        if default_value:
            filename = default_value.rpartition('/')[2]
        else:
            filename = '{}.txt'.format(newvar[:-4])
        if not default_value or not default_value.startswith(base_uri):
            setattr(mlist, newvar, base_uri + filename)
        filepath = list(search(filename, mlist))[0]
        makedirs(os.path.dirname(filepath))
        with codecs.open(filepath, 'w', encoding='utf-8') as fp:
            fp.write(text)
    # Import rosters.
    regulars_set = set(config_dict.get('members', {}))
    digesters_set = set(config_dict.get('digest_members', {}))
    members = regulars_set.union(digesters_set)
    # Don't send welcome messages when we import the rosters.
    send_welcome_message = mlist.send_welcome_message
    mlist.send_welcome_message = False
    try:
        import_roster(mlist, config_dict, members, MemberRole.member)
        import_roster(mlist, config_dict, config_dict.get('owner', []),
                      MemberRole.owner)
        import_roster(mlist, config_dict, config_dict.get('moderator', []),
                      MemberRole.moderator)
        # Now import the '*_these_nonmembers' properties, filtering out the
        # regexps which will remain in the property.
        for action_name in ('accept', 'hold', 'reject', 'discard'):
            prop_name = '{}_these_nonmembers'.format(action_name)
            emails = [
                addr for addr in config_dict.get(prop_name, [])
                if not addr.startswith('^')
            ]
            import_roster(mlist, config_dict, emails, MemberRole.nonmember,
                          Action[action_name])
            # Only keep the regexes in the legacy list property.
            list_prop = getattr(mlist, prop_name)
            for email in emails:
                list_prop.remove(email)
    finally:
        mlist.send_welcome_message = send_welcome_message
Beispiel #16
0
 def get(self, mlist, attribute):
     """Return the mailing list's acceptable aliases."""
     assert attribute == 'acceptable_aliases', (
         'Unexpected attribute: {}'.format(attribute))  # pragma: nocover
     aliases = IAcceptableAliasSet(mlist)
     return sorted(aliases.aliases)
Beispiel #17
0
def import_config_pck(mlist, config_dict):
    """Apply a config.pck configuration dictionary to a mailing list.

    :param mlist: The mailing list.
    :type mlist: IMailingList
    :param config_dict: The Mailman 2.1 configuration dictionary.
    :type config_dict: dict
    """
    for key, value in config_dict.items():
        # Some attributes must not be directly imported.
        if key in EXCLUDES:
            continue
        # These objects need explicit type conversions.
        if key in DATETIME_COLUMNS:
            continue
        # Some attributes from Mailman 2 were renamed in Mailman 3.
        key = NAME_MAPPINGS.get(key, key)
        # Handle the simple case where the key is an attribute of the
        # IMailingList and the types are the same (modulo 8-bit/unicode
        # strings).
        #
        # If the mailing list has a preferred language that isn't registered
        # in the configuration file, hasattr() will swallow the KeyError this
        # raises and return False.  Treat that attribute specially.
        if key == "preferred_language" or hasattr(mlist, key):
            if isinstance(value, bytes):
                value = bytes_to_str(value)
            # Some types require conversion.
            converter = TYPES.get(key)
            if converter is None:
                column = getattr(mlist.__class__, key, None)
                if column is not None and isinstance(column.type, Boolean):
                    converter = bool
            try:
                if converter is not None:
                    value = converter(value)
                setattr(mlist, key, value)
            except (TypeError, KeyError):
                print('Type conversion error for key "{}": {}'.format(key, value), file=sys.stderr)
    for key in DATETIME_COLUMNS:
        try:
            value = datetime.datetime.utcfromtimestamp(config_dict[key])
        except KeyError:
            continue
        if key == "last_post_time":
            setattr(mlist, "last_post_at", value)
            continue
        setattr(mlist, key, value)
    # Handle the moderation policy.
    #
    # The mlist.default_member_action and mlist.default_nonmember_action enum
    # values are different in Mailman 2.1, because they have been merged into a
    # single enum in Mailman 3.
    #
    # Unmoderated lists used to have default_member_moderation set to a false
    # value; this translates to the Defer default action.  Moderated lists with
    # the default_member_moderation set to a true value used to store the
    # action in the member_moderation_action flag, the values were: 0==Hold,
    # 1=Reject, 2==Discard
    if bool(config_dict.get("default_member_moderation", 0)):
        mlist.default_member_action = member_moderation_action_mapping(config_dict.get("member_moderation_action"))
    else:
        mlist.default_member_action = Action.defer
    # Handle the archiving policy.  In MM2.1 there were two boolean options
    # but only three of the four possible states were valid.  Now there's just
    # an enum.
    if config_dict.get("archive"):
        # For maximum safety, if for some strange reason there's no
        # archive_private key, treat the list as having private archives.
        if config_dict.get("archive_private", True):
            mlist.archive_policy = ArchivePolicy.private
        else:
            mlist.archive_policy = ArchivePolicy.public
    else:
        mlist.archive_policy = ArchivePolicy.never
    # Handle ban list.
    ban_manager = IBanManager(mlist)
    for address in config_dict.get("ban_list", []):
        ban_manager.ban(bytes_to_str(address))
    # Handle acceptable aliases.
    acceptable_aliases = config_dict.get("acceptable_aliases", "")
    if isinstance(acceptable_aliases, bytes):
        acceptable_aliases = acceptable_aliases.decode("utf-8")
    if isinstance(acceptable_aliases, str):
        acceptable_aliases = acceptable_aliases.splitlines()
    alias_set = IAcceptableAliasSet(mlist)
    for address in acceptable_aliases:
        address = address.strip()
        if len(address) == 0:
            continue
        address = bytes_to_str(address)
        try:
            alias_set.add(address)
        except ValueError:
            # When .add() rejects this, the line probably contains a regular
            # expression.  Make that explicit for MM3.
            alias_set.add("^" + address)
    # Handle conversion to URIs.  In MM2.1, the decorations are strings
    # containing placeholders, and there's no provision for language-specific
    # templates.  In MM3, template locations are specified by URLs with the
    # special `mailman:` scheme indicating a file system path.  What we do
    # here is look to see if the list's decoration is different than the
    # default, and if so, we'll write the new decoration template to a
    # `mailman:` scheme path.
    convert_to_uri = {
        "welcome_msg": "welcome_message_uri",
        "goodbye_msg": "goodbye_message_uri",
        "msg_header": "header_uri",
        "msg_footer": "footer_uri",
        "digest_header": "digest_header_uri",
        "digest_footer": "digest_footer_uri",
    }
    # The best we can do is convert only the most common ones.  These are
    # order dependent; the longer substitution with the common prefix must
    # show up earlier.
    convert_placeholders = [
        ("%(real_name)s@%(host_name)s", "$fqdn_listname"),
        ("%(real_name)s", "$display_name"),
        ("%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s", "$listinfo_uri"),
    ]
    # Collect defaults.
    defaults = {}
    for oldvar, newvar in convert_to_uri.items():
        default_value = getattr(mlist, newvar, None)
        if not default_value:
            continue
        # Check if the value changed from the default.
        try:
            default_text = decorate(mlist, default_value)
        except (URLError, KeyError):
            # Use case: importing the old [email protected] into [email protected].  We can't
            # check if it changed from the default so don't import, we may do
            # more harm than good and it's easy to change if needed.
            # TESTME
            print(
                "Unable to convert mailing list attribute:",
                oldvar,
                'with old value "{}"'.format(default_value),
                file=sys.stderr,
            )
            continue
        defaults[newvar] = (default_value, default_text)
    for oldvar, newvar in convert_to_uri.items():
        if oldvar not in config_dict:
            continue
        text = config_dict[oldvar]
        if isinstance(text, bytes):
            text = text.decode("utf-8", "replace")
        for oldph, newph in convert_placeholders:
            text = text.replace(oldph, newph)
        default_value, default_text = defaults.get(newvar, (None, None))
        if not text and not (default_value or default_text):
            # Both are empty, leave it.
            continue
        # Check if the value changed from the default
        try:
            expanded_text = decorate_template(mlist, text)
        except KeyError:
            # Use case: importing the old [email protected] into [email protected]
            # We can't check if it changed from the default
            # -> don't import, we may do more harm than good and it's easy to
            # change if needed
            # TESTME
            print("Unable to convert mailing list attribute:", oldvar, 'with value "{}"'.format(text), file=sys.stderr)
            continue
        if expanded_text and default_text and expanded_text.strip() == default_text.strip():
            # Keep the default.
            continue
        # Write the custom value to the right file.
        base_uri = "mailman:///$listname/$language/"
        if default_value:
            filename = default_value.rpartition("/")[2]
        else:
            filename = "{}.txt".format(newvar[:-4])
        if not default_value or not default_value.startswith(base_uri):
            setattr(mlist, newvar, base_uri + filename)
        filepath = list(search(filename, mlist))[0]
        makedirs(os.path.dirname(filepath))
        with codecs.open(filepath, "w", encoding="utf-8") as fp:
            fp.write(text)
    # Import rosters.
    regulars_set = set(config_dict.get("members", {}))
    digesters_set = set(config_dict.get("digest_members", {}))
    members = regulars_set.union(digesters_set)
    # Don't send welcome messages when we import the rosters.
    send_welcome_message = mlist.send_welcome_message
    mlist.send_welcome_message = False
    try:
        import_roster(mlist, config_dict, members, MemberRole.member)
        import_roster(mlist, config_dict, config_dict.get("owner", []), MemberRole.owner)
        import_roster(mlist, config_dict, config_dict.get("moderator", []), MemberRole.moderator)
        # Now import the '*_these_nonmembers' properties, filtering out the
        # regexps which will remain in the property.
        for action_name in ("accept", "hold", "reject", "discard"):
            prop_name = "{}_these_nonmembers".format(action_name)
            emails = [addr for addr in config_dict.get(prop_name, []) if not addr.startswith("^")]
            import_roster(mlist, config_dict, emails, MemberRole.nonmember, Action[action_name])
            # Only keep the regexes in the legacy list property.
            list_prop = getattr(mlist, prop_name)
            for email in emails:
                list_prop.remove(email)
    finally:
        mlist.send_welcome_message = send_welcome_message
Beispiel #18
0
def import_config_pck(mlist, config_dict):
    """Apply a config.pck configuration dictionary to a mailing list.

    :param mlist: The mailing list.
    :type mlist: IMailingList
    :param config_dict: The Mailman 2.1 configuration dictionary.
    :type config_dict: dict
    """
    for key, value in config_dict.items():
        # Some attributes must not be directly imported.
        if key in EXCLUDES:
            continue
        # These objects need explicit type conversions.
        if key in DATETIME_COLUMNS:
            continue
        # Some attributes from Mailman 2 were renamed in Mailman 3.
        key = NAME_MAPPINGS.get(key, key)
        # Handle the simple case where the key is an attribute of the
        # IMailingList and the types are the same (modulo 8-bit/unicode
        # strings).
        #
        # If the mailing list has a preferred language that isn't registered
        # in the configuration file, hasattr() will swallow the KeyError this
        # raises and return False.  Treat that attribute specially.
        if hasattr(mlist, key) or key == 'preferred_language':
            if isinstance(value, str):
                value = str_to_unicode(value)
            # Some types require conversion.
            converter = TYPES.get(key)
            try:
                if converter is not None:
                    value = converter(value)
                setattr(mlist, key, value)
            except (TypeError, KeyError):
                print('Type conversion error for key "{}": {}'.format(
                    key, value), file=sys.stderr)
    for key in DATETIME_COLUMNS:
        try:
            value = datetime.datetime.utcfromtimestamp(config_dict[key])
        except KeyError:
            continue
        if key == 'last_post_time':
            setattr(mlist, 'last_post_at', value)
            continue
        setattr(mlist, key, value)
    # Handle the archiving policy.  In MM2.1 there were two boolean options
    # but only three of the four possible states were valid.  Now there's just
    # an enum.
    if config_dict.get('archive'):
        # For maximum safety, if for some strange reason there's no
        # archive_private key, treat the list as having private archives.
        if config_dict.get('archive_private', True):
            mlist.archive_policy = ArchivePolicy.private
        else:
            mlist.archive_policy = ArchivePolicy.public
    else:
        mlist.archive_policy = ArchivePolicy.never
    # Handle ban list.
    ban_manager = IBanManager(mlist)
    for address in config_dict.get('ban_list', []):
        ban_manager.ban(str_to_unicode(address))
    # Handle acceptable aliases.
    acceptable_aliases = config_dict.get('acceptable_aliases', '')
    if isinstance(acceptable_aliases, basestring):
        acceptable_aliases = acceptable_aliases.splitlines()
    alias_set = IAcceptableAliasSet(mlist)
    for address in acceptable_aliases:
        address = address.strip()
        if len(address) == 0:
            continue
        address = str_to_unicode(address)
        try:
            alias_set.add(address)
        except ValueError:
            # When .add() rejects this, the line probably contains a regular
            # expression.  Make that explicit for MM3.
            alias_set.add('^' + address)
    # Handle conversion to URIs.  In MM2.1, the decorations are strings
    # containing placeholders, and there's no provision for language-specific
    # templates.  In MM3, template locations are specified by URLs with the
    # special `mailman:` scheme indicating a file system path.  What we do
    # here is look to see if the list's decoration is different than the
    # default, and if so, we'll write the new decoration template to a
    # `mailman:` scheme path.
    convert_to_uri = {
        'welcome_msg': 'welcome_message_uri',
        'goodbye_msg': 'goodbye_message_uri',
        'msg_header': 'header_uri',
        'msg_footer': 'footer_uri',
        'digest_header': 'digest_header_uri',
        'digest_footer': 'digest_footer_uri',
        }
    # The best we can do is convert only the most common ones.  These are
    # order dependent; the longer substitution with the common prefix must
    # show up earlier.
    convert_placeholders = [
        ('%(real_name)s@%(host_name)s', '$fqdn_listname'),
        ('%(real_name)s', '$display_name'),
        ('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
         '$listinfo_uri'),
        ]
    # Collect defaults.
    defaults = {}
    for oldvar, newvar in convert_to_uri.items():
        default_value = getattr(mlist, newvar, None)
        if not default_value:
            continue
        # Check if the value changed from the default.
        try:
            default_text = decorate(mlist, default_value)
        except (URLError, KeyError):
            # Use case: importing the old [email protected] into [email protected].  We can't
            # check if it changed from the default so don't import, we may do
            # more harm than good and it's easy to change if needed.
            # TESTME
            print('Unable to convert mailing list attribute:', oldvar,
                  'with old value "{}"'.format(default_value),
                  file=sys.stderr)
            continue
        defaults[newvar] = (default_value, default_text)
    for oldvar, newvar in convert_to_uri.items():
        if oldvar not in config_dict:
            continue
        text = config_dict[oldvar]
        text = text.decode('utf-8', 'replace')
        for oldph, newph in convert_placeholders:
            text = text.replace(oldph, newph)
        default_value, default_text  = defaults.get(newvar, (None, None))
        if not text and not (default_value or default_text):
            # Both are empty, leave it.
            continue
        # Check if the value changed from the default
        try:
            expanded_text = decorate_template(mlist, text)
        except KeyError:
            # Use case: importing the old [email protected] into [email protected]
            # We can't check if it changed from the default
            # -> don't import, we may do more harm than good and it's easy to
            # change if needed
            # TESTME
            print('Unable to convert mailing list attribute:', oldvar,
                  'with value "{}"'.format(text),
                  file=sys.stderr)
            continue
        if (expanded_text and default_text
                and expanded_text.strip() == default_text.strip()):
            # Keep the default.
            continue
        # Write the custom value to the right file.
        base_uri = 'mailman:///$listname/$language/'
        if default_value:
            filename = default_value.rpartition('/')[2]
        else:
            filename = '{}.txt'.format(newvar[:-4])
        if not default_value or not default_value.startswith(base_uri):
            setattr(mlist, newvar, base_uri + filename)
        filepath = list(search(filename, mlist))[0]
        makedirs(os.path.dirname(filepath))
        with codecs.open(filepath, 'w', encoding='utf-8') as fp:
            fp.write(text)
    # Import rosters.
    members = set(config_dict.get('members', {}).keys()
                + config_dict.get('digest_members', {}).keys())
    import_roster(mlist, config_dict, members, MemberRole.member)
    import_roster(mlist, config_dict, config_dict.get('owner', []),
                  MemberRole.owner)
    import_roster(mlist, config_dict, config_dict.get('moderator', []),
                  MemberRole.moderator)