def test_acceptable_aliases_as_list(self): # In some versions of the pickle, this can be a list, not a string # (seen in the wild). aliases = [b'*****@*****.**', b'*****@*****.**'] self._pckdict['acceptable_aliases'] = aliases self._import() alias_set = IAcceptableAliasSet(self._mlist) self.assertEqual(sorted(alias_set.aliases), sorted(a.decode('utf-8') for a in aliases))
def check(self, mlist, msg, msgdata): """See `IRule`.""" # Implicit destination checking must be enabled in the mailing list. if not mlist.require_explicit_destination: return False # Messages gated from NNTP will always have an implicit destination so # are never checked. if msgdata.get('fromusenet'): return False # Calculate the list of acceptable aliases. If the alias starts with # a caret (i.e. ^), then it's a regular expression to match against. aliases = set() alias_patterns = set() # Adapt the mailing list to the appropriate interface. alias_set = IAcceptableAliasSet(mlist) for alias in alias_set.aliases: if alias.startswith('^'): alias_patterns.add(alias) else: aliases.add(alias) # Add the list's posting address, i.e. the explicit address, to the # set of acceptable aliases. aliases.add(mlist.posting_address) # Look at all the recipients. If the recipient is any acceptable # alias (or the explicit posting address), then this rule does not # match. If not, then add it to the set of recipients we'll check # against the alias patterns later. recipients = set() for header in ('to', 'cc', 'resent-to', 'resent-cc'): for fullname, address in getaddresses(msg.get_all(header, [])): if isinstance(address, bytes): address = address.decode('ascii') address = address.lower() if address in aliases: return False recipients.add(address) # Now for all alias patterns, see if any of the recipients matches a # pattern. If so, then this rule does not match. for pattern in alias_patterns: escaped = re.escape(pattern) for recipient in recipients: try: if re.match(pattern, recipient, re.IGNORECASE): return False except re.error: # The pattern is a malformed regular expression. Try # matching again with the pattern escaped. with suppress(re.error): if re.match(escaped, recipient, re.IGNORECASE): return False # Nothing matched. msgdata['moderation_sender'] = msg.sender with _.defer_translation(): # This will be translated at the point of use. msgdata.setdefault('moderation_reasons', []).append( _('Message has implicit destination')) return True
def test_acceptable_aliases_invalid(self): # Values without an '@' sign used to be matched against the local # part, now we need to add the '^' sign to indicate it's a regexp. aliases = ['invalid-value'] self._pckdict['acceptable_aliases'] = list_to_string(aliases) self._import() alias_set = IAcceptableAliasSet(self._mlist) self.assertEqual(sorted(alias_set.aliases), [('^' + alias) for alias in aliases])
def test_delete_list_with_acceptable_aliases(self): # LP: #1432239 - deleting a mailing list with acceptable aliases # causes a SQLAlchemy error. The aliases must be deleted first. with transaction(): alias_set = IAcceptableAliasSet(self._mlist) alias_set.add('*****@*****.**') self.assertEqual(['*****@*****.**'], list(alias_set.aliases)) getUtility(IListManager).delete(self._mlist) self.assertEqual(len(list(alias_set.aliases)), 0)
def test_acceptable_aliases(self): # This used to be a plain-text field (values are newline-separated). aliases = ['*****@*****.**', '*****@*****.**', 'non-ascii-\[email protected]', ] self._pckdict['acceptable_aliases'] = list_to_string(aliases) self._import() alias_set = IAcceptableAliasSet(self._mlist) self.assertEqual(sorted(alias_set.aliases), aliases)
def test_put_configuration(self): # When using PUT, all writable attributes must be included. resource, response = call_api( 'http://localhost:9001/3.0/lists/ant.example.com/config', RESOURCE, 'PUT') self.assertEqual(response.status, 204) self.assertEqual(self._mlist.display_name, 'Fnords') # All three acceptable aliases were set. self.assertEqual(set(IAcceptableAliasSet(self._mlist).aliases), set(RESOURCE['acceptable_aliases']))
def test_delete_list_with_acceptable_aliases(self): # LP: #1432239 - deleting a mailing list with acceptable aliases # causes a SQLAlchemy error. The aliases must be deleted first. with transaction(): alias_set = IAcceptableAliasSet(self._mlist) alias_set.add('*****@*****.**') call_api('http://*****:*****@example.com')) self.assertEqual(config.db.store.query(AcceptableAlias).count(), 0)
def put(self, mlist, attribute, value): """Change the acceptable aliases. Because this is a PUT operation, all previous aliases are cleared first. Thus, this is an overwrite. The keys in the request are ignored. """ assert attribute == 'acceptable_aliases', ( 'Unexpected attribute: {}'.format(attribute)) # pragma: nocover alias_set = IAcceptableAliasSet(mlist) alias_set.clear() for alias in value: alias_set.add(alias)
def test_put_configuration(self): aliases = [ '*****@*****.**', '*****@*****.**', '*****@*****.**', ] # When using PUT, all writable attributes must be included. resource, response = call_api( 'http://*****:*****@example.com/config', dict( acceptable_aliases=aliases, admin_immed_notify=False, admin_notify_mchanges=True, administrivia=False, advertised=False, anonymous_list=True, archive_policy='never', autorespond_owner='respond_and_discard', autorespond_postings='respond_and_continue', autorespond_requests='respond_and_discard', autoresponse_grace_period='45d', autoresponse_owner_text='the owner', autoresponse_postings_text='the mailing list', autoresponse_request_text='the robot', display_name='Fnords', description='This is my mailing list', include_rfc2369_headers=False, allow_list_posts=False, digest_size_threshold=10.5, posting_pipeline='virgin', filter_content=True, first_strip_reply_to=True, convert_html_to_plaintext=True, collapse_alternatives=False, reply_goes_to_list='point_to_list', reply_to_address='*****@*****.**', send_welcome_message=False, subject_prefix='[ant]', welcome_message_uri='mailman:///welcome.txt', default_member_action='hold', default_nonmember_action='discard', ), 'PUT') self.assertEqual(response.status, 204) self.assertEqual(self._mlist.display_name, 'Fnords') # All three acceptable aliases were set. self.assertEqual(set(IAcceptableAliasSet(self._mlist).aliases), set(aliases))
def on_delete(self, request, response): if self._attribute is None: bad_request(response, 'Cannot delete the list configuration itself') return if self._attribute not in VALIDATORS: bad_request(response, 'Read-only attribute: {}'.format(self._attribute)) return # This kind of sucks because it doesn't scale if the list of attributes # which can be deleted grows. So if we get too many we'll have to use # a lookup table. For now, this is good enough. if self._attribute != 'acceptable_aliases': bad_request( response, 'Attribute cannot be DELETEd: {}'.format(self._attribute)) return IAcceptableAliasSet(self._mlist).clear() no_content(response)
def import_config_pck(mlist, config_dict): """Apply a config.pck configuration dictionary to a mailing list. :param mlist: The mailing list. :type mlist: IMailingList :param config_dict: The Mailman 2.1 configuration dictionary. :type config_dict: dict """ global key for key, value in config_dict.items(): # Some attributes must not be directly imported. if key in EXCLUDES: continue # These objects need explicit type conversions. if key in DATETIME_COLUMNS: continue # Some attributes from Mailman 2 were renamed in Mailman 3. key = NAME_MAPPINGS.get(key, key) # Handle the simple case where the key is an attribute of the # IMailingList and the types are the same (modulo 8-bit/unicode # strings). # # If the mailing list has a preferred language that isn't registered # in the configuration file, hasattr() will swallow the KeyError this # raises and return False. Treat that attribute specially. if key == 'preferred_language' or hasattr(mlist, key): if isinstance(value, bytes): value = bytes_to_str(value) # Some types require conversion. converter = TYPES.get(key) if converter is None: column = getattr(mlist.__class__, key, None) if column is not None and isinstance(column.type, Boolean): converter = bool if column is not None and isinstance(column.type, SAUnicode): converter = maybe_truncate_mysql try: if converter is not None: value = converter(value) setattr(mlist, key, value) except (TypeError, KeyError): print('Type conversion error for key "{}": {}'.format( key, value), file=sys.stderr) for key in DATETIME_COLUMNS: try: value = datetime.datetime.utcfromtimestamp(config_dict[key]) except KeyError: continue if key == 'last_post_time': setattr(mlist, 'last_post_at', value) continue setattr(mlist, key, value) # Handle the moderation policy. # # The mlist.default_member_action and mlist.default_nonmember_action enum # values are different in Mailman 2.1, because they have been merged into a # single enum in Mailman 3. # # Unmoderated lists used to have default_member_moderation set to a false # value; this translates to the Defer default action. Moderated lists with # the default_member_moderation set to a true value used to store the # action in the member_moderation_action flag, the values were: 0==Hold, # 1=Reject, 2==Discard if bool(config_dict.get('default_member_moderation', 0)): mlist.default_member_action = member_moderation_action_mapping( config_dict.get('member_moderation_action')) else: mlist.default_member_action = Action.defer # Handle DMARC mitigations. # This would be straightforward except for from_is_list. The issue # is in MM 2.1 the from_is_list action applies if dmarc_moderation_action # doesn't apply and they can be different. # We will map as follows: # from_is_list > dmarc_moderation_action # dmarc_mitigate_action = from_is_list action # dmarc_mitigate_unconditionally = True # from_is_list <= dmarc_moderation_action # dmarc_mitigate_action = dmarc_moderation_action # dmarc_mitigate_unconditionally = False # The text attributes are handled above. if (config_dict.get('from_is_list', 0) > config_dict.get( 'dmarc_moderation_action', 0)): mlist.dmarc_mitigate_action = dmarc_action_mapping( config_dict.get('from_is_list', 0)) mlist.dmarc_mitigate_unconditionally = True else: mlist.dmarc_mitigate_action = dmarc_action_mapping( config_dict.get('dmarc_moderation_action', 0)) mlist.dmarc_mitigate_unconditionally = False # Handle the archiving policy. In MM2.1 there were two boolean options # but only three of the four possible states were valid. Now there's just # an enum. if config_dict.get('archive'): # For maximum safety, if for some strange reason there's no # archive_private key, treat the list as having private archives. if config_dict.get('archive_private', True): mlist.archive_policy = ArchivePolicy.private else: mlist.archive_policy = ArchivePolicy.public else: mlist.archive_policy = ArchivePolicy.never # Handle ban list. ban_manager = IBanManager(mlist) for address in config_dict.get('ban_list', []): ban_manager.ban(bytes_to_str(address)) # Handle acceptable aliases. acceptable_aliases = config_dict.get('acceptable_aliases', '') if isinstance(acceptable_aliases, bytes): acceptable_aliases = acceptable_aliases.decode('utf-8') if isinstance(acceptable_aliases, str): acceptable_aliases = acceptable_aliases.splitlines() alias_set = IAcceptableAliasSet(mlist) for address in acceptable_aliases: address = address.strip() if len(address) == 0: continue address = bytes_to_str(address) # All 2.1 acceptable aliases are regexps whether or not they start # with '^' or contain '@'. if not address.startswith('^'): address = '^' + address # This used to be in a try which would catch ValueError and add a '^', # but .add() would not raise ValueError if address contained '@' and # that needs the '^' too as it could be a regexp with an '@' in it. alias_set.add(address) # Handle roster visibility. mapping = member_roster_visibility_mapping( config_dict.get('private_roster', None)) if mapping is not None: mlist.member_roster_visibility = mapping # Handle header_filter_rules conversion to header_matches. header_matches = IHeaderMatchList(mlist) header_filter_rules = config_dict.get('header_filter_rules', []) for line_patterns, action, _unused in header_filter_rules: try: chain = action_to_chain(action) except KeyError: log.warning('Unsupported header_filter_rules action: %r', action) continue # Now split the line into a header and a pattern. for line_pattern in line_patterns.splitlines(): if len(line_pattern.strip()) == 0: continue for sep in (': ', ':.*', ':.', ':'): header, sep, pattern = line_pattern.partition(sep) if sep: # We found it. break else: # Matches any header, which is not supported. XXX log.warning('Unsupported header_filter_rules pattern: %r', line_pattern) continue header = header.strip().lstrip('^').lower() header = header.replace('\\', '') if not header: log.warning( 'Cannot parse the header in header_filter_rule: %r', line_pattern) continue if len(pattern) == 0: # The line matched only the header, therefore the header can # be anything. pattern = '.*' try: re.compile(pattern) except re.error: log.warning( 'Skipping header_filter rule because of an ' 'invalid regular expression: %r', line_pattern) continue try: header_matches.append(header, pattern, chain) except ValueError: log.warning('Skipping duplicate header_filter rule: %r', line_pattern) continue # Handle conversion to URIs. In MM2.1, the decorations are strings # containing placeholders, and there's no provision for language-specific # strings. In MM3, template locations are specified by URLs with the # special `mailman:` scheme indicating a file system path. What we do # here is look to see if the list's decoration is different than the # default, and if so, we'll write the new decoration template to a # `mailman:` scheme path, then add the template to the template manager. # We are intentionally omitting the 2.1 welcome_msg here because the # string is actually interpolated into a larger template and there's # no good way to figure where in the default template to insert it. convert_to_uri = { 'goodbye_msg': 'list:user:notice:goodbye', 'msg_header': 'list:member:regular:header', 'msg_footer': 'list:member:regular:footer', 'digest_header': 'list:member:digest:header', 'digest_footer': 'list:member:digest:footer', } # The best we can do is convert only the most common ones. These are # order dependent; the longer substitution with the common prefix must # show up earlier. convert_placeholders = [ # First convert \r\n that may have been set by a browser to \n. ('\r\n', '\n'), ('%(real_name)s@%(host_name)s', 'To unsubscribe send an email to ${short_listname}-leave@${domain}'), ('%(real_name)s mailing list', '$display_name mailing list -- $listname'), # The generic footers no longer have URLs in them. ('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s\n', ''), ] # Collect defaults. manager = getUtility(ITemplateManager) defaults = {} for oldvar, newvar in convert_to_uri.items(): default_value = getUtility(ITemplateLoader).get(newvar, mlist) if not default_value: continue # Get the decorated default text try: default_text = decorate_template(mlist, default_value) except (URLError, KeyError): # pragma: nocover # Use case: importing the old [email protected] into [email protected]. We can't # check if it changed from the default so don't import, we may do # more harm than good and it's easy to change if needed. # TESTME print('Unable to convert mailing list attribute:', oldvar, 'with old value "{}"'.format(default_value), file=sys.stderr) continue defaults[newvar] = default_text for oldvar, newvar in convert_to_uri.items(): if oldvar not in config_dict: continue text = config_dict[oldvar] if isinstance(text, bytes): text = text.decode('utf-8', 'replace') for oldph, newph in convert_placeholders: text = text.replace(oldph, newph) default_text = defaults.get(newvar, None) if not text and not default_text: # Both are empty, leave it. continue # Check if the value changed from the default try: expanded_text = decorate_template(mlist, text) except KeyError: # pragma: nocover # Use case: importing the old [email protected] into [email protected] # We can't check if it changed from the default # -> don't import, we may do more harm than good and it's easy to # change if needed # TESTME print('Unable to convert mailing list attribute:', oldvar, 'with value "{}"'.format(text), file=sys.stderr) continue if (expanded_text and default_text and expanded_text.strip() == default_text.strip()): # Keep the default. continue # Write the custom value to the right file and add it to the template # manager for real. base_uri = 'mailman:///$listname/$language/' filename = '{}.txt'.format(newvar) manager.set(newvar, mlist.list_id, base_uri + filename) with ExitStack() as resources: filepath = list(search(resources, filename, mlist))[0] makedirs(os.path.dirname(filepath)) with open(filepath, 'w', encoding='utf-8') as fp: fp.write(text) # Import rosters. regulars_set = set(config_dict.get('members', {})) digesters_set = set(config_dict.get('digest_members', {})) members = regulars_set.union(digesters_set) # Don't send welcome messages or notify admins when we import the rosters. send_welcome_message = mlist.send_welcome_message mlist.send_welcome_message = False admin_notify_mchanges = mlist.admin_notify_mchanges mlist.admin_notify_mchanges = False try: import_roster(mlist, config_dict, members, MemberRole.member) import_roster(mlist, config_dict, config_dict.get('owner', []), MemberRole.owner) import_roster(mlist, config_dict, config_dict.get('moderator', []), MemberRole.moderator) # Now import the '*_these_nonmembers' properties, filtering out the # regexps which will remain in the property. for action_name in ('accept', 'hold', 'reject', 'discard'): prop_name = '{}_these_nonmembers'.format(action_name) emails = [ addr for addr in config_dict.get(prop_name, []) if not addr.startswith('^') ] # MM 2.1 accept maps to MM 3 defer if action_name == 'accept': action_name = 'defer' import_roster(mlist, config_dict, emails, MemberRole.nonmember, Action[action_name]) # Now add the regexes in the legacy list property. list_prop = getattr(mlist, prop_name) for addr in config_dict.get(prop_name, []): if addr.startswith('^'): list_prop.append(addr) finally: mlist.send_welcome_message = send_welcome_message mlist.admin_notify_mchanges = admin_notify_mchanges
def get(self, mlist, attribute): """Return the mailing list's acceptable aliases.""" assert attribute == 'acceptable_aliases', ( 'Unexpected attribute: {}'.format(attribute)) # pragma: nocover aliases = IAcceptableAliasSet(mlist) return sorted(aliases.aliases)
def import_config_pck(mlist, config_dict): """Apply a config.pck configuration dictionary to a mailing list. :param mlist: The mailing list. :type mlist: IMailingList :param config_dict: The Mailman 2.1 configuration dictionary. :type config_dict: dict """ for key, value in config_dict.items(): # Some attributes must not be directly imported. if key in EXCLUDES: continue # These objects need explicit type conversions. if key in DATETIME_COLUMNS: continue # Some attributes from Mailman 2 were renamed in Mailman 3. key = NAME_MAPPINGS.get(key, key) # Handle the simple case where the key is an attribute of the # IMailingList and the types are the same (modulo 8-bit/unicode # strings). # # If the mailing list has a preferred language that isn't registered # in the configuration file, hasattr() will swallow the KeyError this # raises and return False. Treat that attribute specially. if key == 'preferred_language' or hasattr(mlist, key): if isinstance(value, bytes): value = bytes_to_str(value) # Some types require conversion. converter = TYPES.get(key) if converter is None: column = getattr(mlist.__class__, key, None) if column is not None and isinstance(column.type, Boolean): converter = bool try: if converter is not None: value = converter(value) setattr(mlist, key, value) except (TypeError, KeyError): print('Type conversion error for key "{}": {}'.format( key, value), file=sys.stderr) for key in DATETIME_COLUMNS: try: value = datetime.datetime.utcfromtimestamp(config_dict[key]) except KeyError: continue if key == 'last_post_time': setattr(mlist, 'last_post_at', value) continue setattr(mlist, key, value) # Handle the moderation policy. # # The mlist.default_member_action and mlist.default_nonmember_action enum # values are different in Mailman 2.1, because they have been merged into a # single enum in Mailman 3. # # Unmoderated lists used to have default_member_moderation set to a false # value; this translates to the Defer default action. Moderated lists with # the default_member_moderation set to a true value used to store the # action in the member_moderation_action flag, the values were: 0==Hold, # 1=Reject, 2==Discard if bool(config_dict.get('default_member_moderation', 0)): mlist.default_member_action = member_moderation_action_mapping( config_dict.get('member_moderation_action')) else: mlist.default_member_action = Action.defer # Handle the archiving policy. In MM2.1 there were two boolean options # but only three of the four possible states were valid. Now there's just # an enum. if config_dict.get('archive'): # For maximum safety, if for some strange reason there's no # archive_private key, treat the list as having private archives. if config_dict.get('archive_private', True): mlist.archive_policy = ArchivePolicy.private else: mlist.archive_policy = ArchivePolicy.public else: mlist.archive_policy = ArchivePolicy.never # Handle ban list. ban_manager = IBanManager(mlist) for address in config_dict.get('ban_list', []): ban_manager.ban(bytes_to_str(address)) # Handle acceptable aliases. acceptable_aliases = config_dict.get('acceptable_aliases', '') if isinstance(acceptable_aliases, bytes): acceptable_aliases = acceptable_aliases.decode('utf-8') if isinstance(acceptable_aliases, str): acceptable_aliases = acceptable_aliases.splitlines() alias_set = IAcceptableAliasSet(mlist) for address in acceptable_aliases: address = address.strip() if len(address) == 0: continue address = bytes_to_str(address) try: alias_set.add(address) except ValueError: # When .add() rejects this, the line probably contains a regular # expression. Make that explicit for MM3. alias_set.add('^' + address) # Handle conversion to URIs. In MM2.1, the decorations are strings # containing placeholders, and there's no provision for language-specific # templates. In MM3, template locations are specified by URLs with the # special `mailman:` scheme indicating a file system path. What we do # here is look to see if the list's decoration is different than the # default, and if so, we'll write the new decoration template to a # `mailman:` scheme path. convert_to_uri = { 'welcome_msg': 'welcome_message_uri', 'goodbye_msg': 'goodbye_message_uri', 'msg_header': 'header_uri', 'msg_footer': 'footer_uri', 'digest_header': 'digest_header_uri', 'digest_footer': 'digest_footer_uri', } # The best we can do is convert only the most common ones. These are # order dependent; the longer substitution with the common prefix must # show up earlier. convert_placeholders = [ ('%(real_name)s@%(host_name)s', '$fqdn_listname'), ('%(real_name)s', '$display_name'), ('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s', '$listinfo_uri'), ] # Collect defaults. defaults = {} for oldvar, newvar in convert_to_uri.items(): default_value = getattr(mlist, newvar, None) if not default_value: continue # Check if the value changed from the default. try: default_text = decorate(mlist, default_value) except (URLError, KeyError): # Use case: importing the old [email protected] into [email protected]. We can't # check if it changed from the default so don't import, we may do # more harm than good and it's easy to change if needed. # TESTME print('Unable to convert mailing list attribute:', oldvar, 'with old value "{}"'.format(default_value), file=sys.stderr) continue defaults[newvar] = (default_value, default_text) for oldvar, newvar in convert_to_uri.items(): if oldvar not in config_dict: continue text = config_dict[oldvar] if isinstance(text, bytes): text = text.decode('utf-8', 'replace') for oldph, newph in convert_placeholders: text = text.replace(oldph, newph) default_value, default_text = defaults.get(newvar, (None, None)) if not text and not (default_value or default_text): # Both are empty, leave it. continue # Check if the value changed from the default try: expanded_text = decorate_template(mlist, text) except KeyError: # Use case: importing the old [email protected] into [email protected] # We can't check if it changed from the default # -> don't import, we may do more harm than good and it's easy to # change if needed # TESTME print('Unable to convert mailing list attribute:', oldvar, 'with value "{}"'.format(text), file=sys.stderr) continue if (expanded_text and default_text and expanded_text.strip() == default_text.strip()): # Keep the default. continue # Write the custom value to the right file. base_uri = 'mailman:///$listname/$language/' if default_value: filename = default_value.rpartition('/')[2] else: filename = '{}.txt'.format(newvar[:-4]) if not default_value or not default_value.startswith(base_uri): setattr(mlist, newvar, base_uri + filename) filepath = list(search(filename, mlist))[0] makedirs(os.path.dirname(filepath)) with codecs.open(filepath, 'w', encoding='utf-8') as fp: fp.write(text) # Import rosters. regulars_set = set(config_dict.get('members', {})) digesters_set = set(config_dict.get('digest_members', {})) members = regulars_set.union(digesters_set) # Don't send welcome messages when we import the rosters. send_welcome_message = mlist.send_welcome_message mlist.send_welcome_message = False try: import_roster(mlist, config_dict, members, MemberRole.member) import_roster(mlist, config_dict, config_dict.get('owner', []), MemberRole.owner) import_roster(mlist, config_dict, config_dict.get('moderator', []), MemberRole.moderator) # Now import the '*_these_nonmembers' properties, filtering out the # regexps which will remain in the property. for action_name in ('accept', 'hold', 'reject', 'discard'): prop_name = '{}_these_nonmembers'.format(action_name) emails = [ addr for addr in config_dict.get(prop_name, []) if not addr.startswith('^') ] import_roster(mlist, config_dict, emails, MemberRole.nonmember, Action[action_name]) # Only keep the regexes in the legacy list property. list_prop = getattr(mlist, prop_name) for email in emails: list_prop.remove(email) finally: mlist.send_welcome_message = send_welcome_message