def test_priority_site_over_list(self):
# Test that the site-wide checks take precedence over the list-specific
# checks.
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: A message
Message-ID: <ant>
Foo: foo
MIME-Version: 1.0
A message body.
""")
msgdata = {}
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Foo', 'foo', 'accept')
# This event subscriber records the event that occurs when the message
# is processed by the owner chain.
events = []
with event_subscribers(events.append):
process(self._mlist, msg, msgdata, start_chain='header-match')
self.assertEqual(len(events), 1)
event = events[0]
# Site-wide wants to hold the message, the list wants to accept it.
self.assertIsInstance(event, HoldEvent)
self.assertEqual(event.chain, config.chains['hold'])
def test_clear(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
with transaction():
header_matches.clear()
self.assertEqual(len(self._mlist.header_matches), 0)
def test_get_by_negative_index(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-1', 'pattern-1')
header_matches.append('header-2', 'pattern-2')
header_matches.append('header-3', 'pattern-3')
match = header_matches[-1]
self.assertEqual(match.header, 'header-3')
self.assertEqual(match.pattern, 'pattern-3')
def test_add_remove(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header1', 'pattern')
header_matches.append('header2', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 2)
self.assertEqual(len(header_matches), 2)
header_matches.remove('header1', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
self.assertEqual(len(header_matches), 1)
del header_matches[0]
self.assertEqual(len(self._mlist.header_matches), 0)
self.assertEqual(len(header_matches), 0)
def test_no_action_defaults_to_site_wide_action(self):
# If the list-specific header check matches, but there is no defined
# action, the site-wide antispam action is used.
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: A message
Message-ID: <ant>
Foo: foo
MIME-Version: 1.0
A message body.
""")
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Foo', 'foo')
# This event subscriber records the event that occurs when the message
# is processed by the owner chain, which holds its for approval.
events = []
def record_holds(event): # noqa
if not isinstance(event, HoldEvent):
return
events.append(event)
with event_subscribers(record_holds):
# Set the site-wide antispam action to hold the message.
with configuration('antispam', header_checks="""
Spam: [*]{3,}
""", jump_chain='hold'): # noqa
process(self._mlist, msg, {}, start_chain='header-match')
self.assertEqual(len(events), 1)
event = events[0]
self.assertIsInstance(event, HoldEvent)
self.assertEqual(event.chain, config.chains['hold'])
self.assertEqual(event.mlist, self._mlist)
self.assertEqual(event.msg, msg)
events = []
def record_discards(event): # noqa
if not isinstance(event, DiscardEvent):
return
events.append(event)
with event_subscribers(record_discards):
# Set the site-wide default to discard the message.
msg.replace_header('Message-Id', '<bee>')
with configuration('antispam', header_checks="""
Spam: [*]{3,}
""", jump_chain='discard'): # noqa
process(self._mlist, msg, {}, start_chain='header-match')
self.assertEqual(len(events), 1)
event = events[0]
self.assertIsInstance(event, DiscardEvent)
self.assertEqual(event.chain, config.chains['discard'])
self.assertEqual(event.mlist, self._mlist)
self.assertEqual(event.msg, msg)
def test_add_duplicate(self):
header_matches = IHeaderMatchList(self._mlist)
with transaction():
header_matches.append('header', 'pattern')
with self.assertRaises(HTTPError) as cm:
call_api('http://localhost:9001/3.0/lists/ant.example.com'
'/header-matches', {
'header': 'header',
'pattern': 'pattern',
})
self.assertEqual(cm.exception.code, 400)
self.assertEqual(cm.exception.reason,
b'This header match already exists')
def test_list_rule(self):
# Test that the header-match chain has the header checks from the
# mailing-list configuration.
chain = config.chains['header-match']
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Foo', 'a+')
links = [link for link in chain.get_links(self._mlist, Message(), {})
if link.rule.name != 'any']
self.assertEqual(len(links), 1)
self.assertEqual(links[0].action, LinkAction.jump)
self.assertEqual(links[0].chain.name, config.antispam.jump_chain)
self.assertEqual(links[0].rule.header, 'foo')
self.assertEqual(links[0].rule.pattern, 'a+')
self.assertTrue(links[0].rule.name.startswith(
'header-match-test.example.com-'))
def test_insert(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-0', 'pattern')
header_matches.append('header-1', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-1', 1),
])
header_matches.insert(1, 'header-2', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-2', 1),
('header-1', 2),
])
def test_list_rule(self):
# Test that the header-match chain has the header checks from the
# mailing-list configuration.
chain = config.chains['header-match']
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Foo', 'a+')
links = [
link for link in chain.get_links(self._mlist, Message(), {})
if link.rule.name != 'any'
]
self.assertEqual(len(links), 1)
self.assertEqual(links[0].action, LinkAction.jump)
self.assertEqual(links[0].chain.name, config.antispam.jump_chain)
self.assertEqual(links[0].rule.header, 'foo')
self.assertEqual(links[0].rule.pattern, 'a+')
self.assertTrue(
links[0].rule.name.startswith('header-match-test.example.com-'))
def test_iterator(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header', 'pattern')
header_matches.append('Subject', 'patt.*')
header_matches.append('From', '.*@example.com', 'discard')
header_matches.append('From', '.*@example.org', 'accept')
matches = [(match.header, match.pattern, match.chain)
for match in IHeaderMatchList(self._mlist)]
self.assertEqual(matches, [
('header', 'pattern', None),
('subject', 'patt.*', None),
('from', '.*@example.com', 'discard'),
('from', '.*@example.org', 'accept'),
])
def test_patch_bad_regexp(self):
header_matches = IHeaderMatchList(self._mlist)
with transaction():
header_matches.append('header', 'pattern')
with self.assertRaises(HTTPError) as cm:
call_api(
'http://localhost:9001/3.0/lists/ant.example.com'
'/header-matches/0', {
'header': 'header',
'pattern': '+invalid',
},
method='PATCH')
self.assertEqual(cm.exception.code, 400)
self.assertEqual(
cm.exception.reason, 'Invalid Parameter "pattern":'
' Expected a valid regexp, got +invalid.')
self.assertEqual(
cm.exception.reason, 'Invalid Parameter "pattern": '
'Expected a valid regexp, got +invalid.')
def test_move_identical(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-0', 'pattern')
header_matches.append('header-1', 'pattern')
header_matches.append('header-2', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-1', 1),
('header-2', 2),
])
header_match_1 = self._mlist.header_matches[1]
self.assertEqual(header_match_1.position, 1)
header_match_1.position = 1
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-1', 1),
('header-2', 2),
])
def test_clear(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
with transaction():
header_matches.clear()
self.assertEqual(len(self._mlist.header_matches), 0)
def test_header_in_subpart(self):
# Test that headers in sub-parts are also matched.
msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: A message
Message-ID: <ant>
Foo: foo
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="================12345=="
--================12345==
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
A message body.
--================12345==
Content-Type: application/junk
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
This is junk
--================12345==--
""")
msgdata = {}
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Content-Type', 'application/junk', 'hold')
# This event subscriber records the event that occurs when the message
# is processed by the owner chain.
events = []
with event_subscribers(events.append):
process(self._mlist, msg, msgdata, start_chain='header-match')
self.assertEqual(len(events), 1)
event = events[0]
self.assertIsInstance(event, HoldEvent)
self.assertEqual(event.chain, config.chains['hold'])
def test_get_by_negative_index(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-1', 'pattern-1')
header_matches.append('header-2', 'pattern-2')
header_matches.append('header-3', 'pattern-3')
match = header_matches[-1]
self.assertEqual(match.header, 'header-3')
self.assertEqual(match.pattern, 'pattern-3')
def test_move_up(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-0', 'pattern')
header_matches.append('header-1', 'pattern')
header_matches.append('header-2', 'pattern')
header_matches.append('header-3', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-1', 1),
('header-2', 2),
('header-3', 3),
])
header_match_2 = self._mlist.header_matches[2]
self.assertEqual(header_match_2.position, 2)
header_match_2.position = 1
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-2', 1),
('header-1', 2),
('header-3', 3),
])
def test_reuse_rules(self):
# Test that existing header-match rules are used instead of creating
# new ones.
chain = config.chains['header-match']
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header2', 'b+')
header_matches.append('Header3', 'c+')
def get_links(): # noqa
return [
link for link in chain.get_links(self._mlist, Message(), {})
if link.rule.name != 'any'
]
links_1 = get_links()
self.assertEqual(len(links_1), 3)
links_2 = get_links()
# The link rules both have the same name...
self.assertEqual(
[l.rule.name for l in links_1],
[l.rule.name for l in links_2],
)
# ...and are actually the identical objects.
for link1, link2 in zip(links_1, links_2):
self.assertIs(link1.rule, link2.rule)
def test_get_all_returns_non_string(self):
# Test case where msg.get_all() returns header instance.
msg = message_from_bytes(b"""\
From: [email protected]
To: [email protected]
Subject: Bad \x96 subject
Message-ID: <ant>
body
""", Message)
msgdata = {}
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Subject', 'Bad', 'hold')
# This event subscriber records the event that occurs when the message
# is processed by the owner chain.
events = []
with event_subscribers(events.append):
process(self._mlist, msg, msgdata, start_chain='header-match')
self.assertEqual(len(events), 1)
event = events[0]
self.assertIsInstance(event, HoldEvent)
self.assertEqual(event.chain, config.chains['hold'])
def test_reuse_rules(self):
# Test that existing header-match rules are used instead of creating
# new ones.
chain = config.chains['header-match']
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header2', 'b+')
header_matches.append('Header3', 'c+')
def get_links(): # noqa: E306
return [
link for link in chain.get_links(self._mlist, Message(), {})
if link.rule.name != 'any'
]
links_1 = get_links()
self.assertEqual(len(links_1), 3)
links_2 = get_links()
# The link rules both have the same name...
self.assertEqual(
[l.rule.name for l in links_1],
[l.rule.name for l in links_2],
)
# ...and are actually the identical objects.
for link1, link2 in zip(links_1, links_2):
self.assertIs(link1.rule, link2.rule)
def test_list_complex_rule_reorder(self):
# Test that the mailing-list header-match complex rules are read
# properly after reordering.
chain = config.chains['header-match']
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Foo', 'a+', 'reject')
header_matches.append('Bar', 'b+', 'discard')
header_matches.append('Baz', 'z+', 'accept')
links = [
link for link in chain.get_links(self._mlist, Message(), {})
if link.rule.name != 'any'
]
self.assertEqual(len(links), 3)
self.assertEqual([
(link.rule.header, link.rule.pattern, link.action, link.chain.name)
for link in links
], [
('foo', 'a+', LinkAction.jump, 'reject'),
('bar', 'b+', LinkAction.jump, 'discard'),
('baz', 'z+', LinkAction.jump, 'accept'),
]) # noqa: E124
del header_matches[0]
header_matches.append('Foo', 'a+', 'reject')
links = [
link for link in chain.get_links(self._mlist, Message(), {})
if link.rule.name != 'any'
]
self.assertEqual(len(links), 3)
self.assertEqual([
(link.rule.header, link.rule.pattern, link.action, link.chain.name)
for link in links
], [
('bar', 'b+', LinkAction.jump, 'discard'),
('baz', 'z+', LinkAction.jump, 'accept'),
('foo', 'a+', LinkAction.jump, 'reject'),
]) # noqa: E124
def test_add_header_match_with_no_action(self):
_, resp = call_api(
'http://localhost:9001/3.0/lists/ant.example.com'
'/header-matches', {
'header': 'header-1',
'pattern': '^Yes',
'action': '',
'tag': 'tag1',
},
method='POST')
self.assertEqual(resp.status_code, 201)
header_matches = IHeaderMatchList(self._mlist)
self.assertEqual([(match.header, match.pattern, match.chain, match.tag)
for match in header_matches],
[('header-1', '^Yes', None, 'tag1')])
def test_rfc2047_encodedheader(self):
# Test case where msg.get_all() returns raw rfc2047 encoded string.
msg = message_from_bytes(
b"""\
From: [email protected]
To: [email protected]
Subject: =?utf-8?b?SSBsaWtlIElrZQo=?=
Message-ID: <ant>
body
""", Message)
msgdata = {}
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Subject', 'I Like Ike', 'hold')
# This event subscriber records the event that occurs when the message
# is processed by the owner chain.
events = []
with event_subscribers(events.append):
process(self._mlist, msg, msgdata, start_chain='header-match')
self.assertEqual(len(events), 1)
event = events[0]
self.assertIsInstance(event, HoldEvent)
self.assertEqual(event.chain, config.chains['hold'])
def test_iterator(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header', 'pattern')
header_matches.append('Subject', 'patt.*')
header_matches.append('From', '.*@example.com', 'discard')
header_matches.append('From', '.*@example.org', 'accept')
matches = [(match.header, match.pattern, match.chain)
for match in IHeaderMatchList(self._mlist)]
self.assertEqual(
matches, [
('header', 'pattern', None),
('subject', 'patt.*', None),
('from', '.*@example.com', 'discard'),
('from', '.*@example.org', 'accept'),
])
def test_add_remove(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header1', 'pattern')
header_matches.append('header2', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 2)
self.assertEqual(len(header_matches), 2)
header_matches.remove('header1', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
self.assertEqual(len(header_matches), 1)
del header_matches[0]
self.assertEqual(len(self._mlist.header_matches), 0)
self.assertEqual(len(header_matches), 0)
def test_get_header_match_by_tag(self):
header_matches = IHeaderMatchList(self._mlist)
with transaction():
header_matches.append('header-1', 'pattern-1')
header_matches.append('header-2',
'pattern-2',
chain='hold',
tag='tag')
header_matches.append('header-3', 'pattern-3', chain='accept')
content, resp = call_api(
'http://localhost:9001/3.0/lists/ant.example.com'
'/header-matches/find', {'tag': 'tag'})
self.assertEqual(resp.status_code, 200)
self.assertIsNotNone(content)
self.assertEqual(len(content['entries']), 1)
self.assertEqual(content['entries'][0]['header'], 'header-2')
self.assertEqual(content['entries'][0]['pattern'], 'pattern-2')
self.assertEqual(content['entries'][0]['action'], 'hold')
def test_rebuild_sequence_after_remove(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-0', 'pattern')
header_matches.append('header-1', 'pattern')
header_matches.append('header-2', 'pattern')
self.assertEqual([(match.header, match.position)
for match in header_matches], [
('header-0', 0),
('header-1', 1),
('header-2', 2),
])
del header_matches[0]
self.assertEqual([(match.header, match.position)
for match in header_matches], [
('header-1', 0),
('header-2', 1),
])
header_matches.remove('header-1', 'pattern')
self.assertEqual([(match.header, match.position)
for match in header_matches], [('header-2', 0)])
def test_list_complex_rule(self):
# Test that the mailing-list header-match complex rules are read
# properly.
chain = config.chains['header-match']
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Foo', 'a+', 'reject')
header_matches.append('Bar', 'b+', 'discard')
header_matches.append('Baz', 'z+', 'accept')
links = [link for link in chain.get_links(self._mlist, Message(), {})
if link.rule.name != 'any']
self.assertEqual(len(links), 3)
self.assertEqual([
(link.rule.header, link.rule.pattern, link.action, link.chain.name)
for link in links
],
[('foo', 'a+', LinkAction.jump, 'reject'),
('bar', 'b+', LinkAction.jump, 'discard'),
('baz', 'z+', LinkAction.jump, 'accept'),
]) # noqa
def test_insert(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-0', 'pattern')
header_matches.append('header-1', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-1', 1),
])
header_matches.insert(1, 'header-2', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-2', 1),
('header-1', 2),
])
def test_rebuild_sequence_after_remove(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-0', 'pattern')
header_matches.append('header-1', 'pattern')
header_matches.append('header-2', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-1', 1),
('header-2', 2),
])
del header_matches[0]
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-1', 0),
('header-2', 1),
])
header_matches.remove('header-1', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches],
[('header-2', 0)])
def test_move_identical(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-0', 'pattern')
header_matches.append('header-1', 'pattern')
header_matches.append('header-2', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-1', 1),
('header-2', 2),
])
header_match_1 = self._mlist.header_matches[1]
self.assertEqual(header_match_1.position, 1)
header_match_1.position = 1
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-1', 1),
('header-2', 2),
])
def test_move_up(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-0', 'pattern')
header_matches.append('header-1', 'pattern')
header_matches.append('header-2', 'pattern')
header_matches.append('header-3', 'pattern')
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-1', 1),
('header-2', 2),
('header-3', 3),
])
header_match_2 = self._mlist.header_matches[2]
self.assertEqual(header_match_2.position, 2)
header_match_2.position = 1
self.assertEqual(
[(match.header, match.position) for match in header_matches], [
('header-0', 0),
('header-2', 1),
('header-1', 2),
('header-3', 3),
])
def test_move_invalid(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header', 'pattern')
header_match = self._mlist.header_matches[0]
with self.assertRaises(ValueError):
header_match.position = 2
def test_chain_defaults_to_none(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
self.assertIsNone(self._mlist.header_matches[0].chain)
def test_lowercase_header(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
self.assertEqual(self._mlist.header_matches[0].header, 'header')
def __init__(self, mlist):
self._mlist = mlist
self.header_matches = IHeaderMatchList(self._mlist)
def test_move_invalid(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header', 'pattern')
header_match = self._mlist.header_matches[0]
with self.assertRaises(ValueError):
header_match.position = 2
def test_filter(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header-1', 'pattern-1', tag='tag1')
header_matches.append('header-1', 'pattern-3', tag='tag1')
header_matches.append('header-2', 'pattern-')
header_matches.append('header-3', 'pattern-2', tag='tag1')
header_matches.append('header-3', 'pattern-3', chain='hold')
match_tag = header_matches.filter(header='header-1', tag='tag1')
self.assertEqual(len(list(match_tag)), 2)
match_tag = header_matches.filter(tag='tag1')
self.assertEqual(len(list(match_tag)), 3)
match_tag = header_matches.filter(chain='hold')
self.assertEqual(len(list(match_tag)), 1)
def test_duplicate(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header', 'pattern')
self.assertRaises(
ValueError, header_matches.append, 'Header', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
def test_lowercase_header(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
self.assertEqual(self._mlist.header_matches[0].header, 'header')
def test_remove_non_existent_by_index(self):
header_matches = IHeaderMatchList(self._mlist)
with self.assertRaises(IndexError):
del header_matches[0]
def test_duplicate(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('Header', 'pattern')
self.assertRaises(
ValueError, header_matches.append, 'Header', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
def test_chain_defaults_to_none(self):
header_matches = IHeaderMatchList(self._mlist)
header_matches.append('header', 'pattern')
self.assertEqual(len(self._mlist.header_matches), 1)
self.assertIsNone(self._mlist.header_matches[0].chain)
def test_remove_non_existent(self):
header_matches = IHeaderMatchList(self._mlist)
self.assertRaises(
ValueError, header_matches.remove, 'header', 'pattern')
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
global key
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == 'preferred_language' or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
if column is not None and isinstance(column.type, SAUnicode):
converter = maybe_truncate_mysql
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value),
file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get('default_member_moderation', 0)):
mlist.default_member_action = member_moderation_action_mapping(
config_dict.get('member_moderation_action'))
else:
mlist.default_member_action = Action.defer
# Handle DMARC mitigations.
# This would be straightforward except for from_is_list. The issue
# is in MM 2.1 the from_is_list action applies if dmarc_moderation_action
# doesn't apply and they can be different.
# We will map as follows:
# from_is_list > dmarc_moderation_action
# dmarc_mitigate_action = from_is_list action
# dmarc_mitigate_unconditionally = True
# from_is_list <= dmarc_moderation_action
# dmarc_mitigate_action = dmarc_moderation_action
# dmarc_mitigate_unconditionally = False
# The text attributes are handled above.
if (config_dict.get('from_is_list', 0) > config_dict.get(
'dmarc_moderation_action', 0)):
mlist.dmarc_mitigate_action = dmarc_action_mapping(
config_dict.get('from_is_list', 0))
mlist.dmarc_mitigate_unconditionally = True
else:
mlist.dmarc_mitigate_action = dmarc_action_mapping(
config_dict.get('dmarc_moderation_action', 0))
mlist.dmarc_mitigate_unconditionally = False
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode('utf-8')
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
# All 2.1 acceptable aliases are regexps whether or not they start
# with '^' or contain '@'.
if not address.startswith('^'):
address = '^' + address
# This used to be in a try which would catch ValueError and add a '^',
# but .add() would not raise ValueError if address contained '@' and
# that needs the '^' too as it could be a regexp with an '@' in it.
alias_set.add(address)
# Handle roster visibility.
mapping = member_roster_visibility_mapping(
config_dict.get('private_roster', None))
if mapping is not None:
mlist.member_roster_visibility = mapping
# Handle header_filter_rules conversion to header_matches.
header_matches = IHeaderMatchList(mlist)
header_filter_rules = config_dict.get('header_filter_rules', [])
for line_patterns, action, _unused in header_filter_rules:
try:
chain = action_to_chain(action)
except KeyError:
log.warning('Unsupported header_filter_rules action: %r', action)
continue
# Now split the line into a header and a pattern.
for line_pattern in line_patterns.splitlines():
if len(line_pattern.strip()) == 0:
continue
for sep in (': ', ':.*', ':.', ':'):
header, sep, pattern = line_pattern.partition(sep)
if sep:
# We found it.
break
else:
# Matches any header, which is not supported. XXX
log.warning('Unsupported header_filter_rules pattern: %r',
line_pattern)
continue
header = header.strip().lstrip('^').lower()
header = header.replace('\\', '')
if not header:
log.warning(
'Cannot parse the header in header_filter_rule: %r',
line_pattern)
continue
if len(pattern) == 0:
# The line matched only the header, therefore the header can
# be anything.
pattern = '.*'
try:
re.compile(pattern)
except re.error:
log.warning(
'Skipping header_filter rule because of an '
'invalid regular expression: %r', line_pattern)
continue
try:
header_matches.append(header, pattern, chain)
except ValueError:
log.warning('Skipping duplicate header_filter rule: %r',
line_pattern)
continue
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# strings. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path, then add the template to the template manager.
# We are intentionally omitting the 2.1 welcome_msg here because the
# string is actually interpolated into a larger template and there's
# no good way to figure where in the default template to insert it.
convert_to_uri = {
'goodbye_msg': 'list:user:notice:goodbye',
'msg_header': 'list:member:regular:header',
'msg_footer': 'list:member:regular:footer',
'digest_header': 'list:member:digest:header',
'digest_footer': 'list:member:digest:footer',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
# First convert \r\n that may have been set by a browser to \n.
('\r\n', '\n'),
('%(real_name)s@%(host_name)s',
'To unsubscribe send an email to ${short_listname}-leave@${domain}'),
('%(real_name)s mailing list',
'$display_name mailing list -- $listname'),
# The generic footers no longer have URLs in them.
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s\n', ''),
]
# Collect defaults.
manager = getUtility(ITemplateManager)
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getUtility(ITemplateLoader).get(newvar, mlist)
if not default_value:
continue
# Get the decorated default text
try:
default_text = decorate_template(mlist, default_value)
except (URLError, KeyError): # pragma: nocover
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = default_text
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_text = defaults.get(newvar, None)
if not text and not default_text:
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError: # pragma: nocover
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:',
oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text
and expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file and add it to the template
# manager for real.
base_uri = 'mailman:///$listname/$language/'
filename = '{}.txt'.format(newvar)
manager.set(newvar, mlist.list_id, base_uri + filename)
with ExitStack() as resources:
filepath = list(search(resources, filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get('members', {}))
digesters_set = set(config_dict.get('digest_members', {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages or notify admins when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
admin_notify_mchanges = mlist.admin_notify_mchanges
mlist.admin_notify_mchanges = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ('accept', 'hold', 'reject', 'discard'):
prop_name = '{}_these_nonmembers'.format(action_name)
emails = [
addr for addr in config_dict.get(prop_name, [])
if not addr.startswith('^')
]
# MM 2.1 accept maps to MM 3 defer
if action_name == 'accept':
action_name = 'defer'
import_roster(mlist, config_dict, emails, MemberRole.nonmember,
Action[action_name])
# Now add the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for addr in config_dict.get(prop_name, []):
if addr.startswith('^'):
list_prop.append(addr)
finally:
mlist.send_welcome_message = send_welcome_message
mlist.admin_notify_mchanges = admin_notify_mchanges
def import_config_pck(mlist, config_dict):
"""Apply a config.pck configuration dictionary to a mailing list.
:param mlist: The mailing list.
:type mlist: IMailingList
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
continue
# These objects need explicit type conversions.
if key in DATETIME_COLUMNS:
continue
# Some attributes from Mailman 2 were renamed in Mailman 3.
key = NAME_MAPPINGS.get(key, key)
# Handle the simple case where the key is an attribute of the
# IMailingList and the types are the same (modulo 8-bit/unicode
# strings).
#
# If the mailing list has a preferred language that isn't registered
# in the configuration file, hasattr() will swallow the KeyError this
# raises and return False. Treat that attribute specially.
if key == 'preferred_language' or hasattr(mlist, key):
if isinstance(value, bytes):
value = bytes_to_str(value)
# Some types require conversion.
converter = TYPES.get(key)
if converter is None:
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
try:
if converter is not None:
value = converter(value)
setattr(mlist, key, value)
except (TypeError, KeyError):
print('Type conversion error for key "{}": {}'.format(
key, value), file=sys.stderr)
for key in DATETIME_COLUMNS:
try:
value = datetime.datetime.utcfromtimestamp(config_dict[key])
except KeyError:
continue
if key == 'last_post_time':
setattr(mlist, 'last_post_at', value)
continue
setattr(mlist, key, value)
# Handle the moderation policy.
#
# The mlist.default_member_action and mlist.default_nonmember_action enum
# values are different in Mailman 2.1, because they have been merged into a
# single enum in Mailman 3.
#
# Unmoderated lists used to have default_member_moderation set to a false
# value; this translates to the Defer default action. Moderated lists with
# the default_member_moderation set to a true value used to store the
# action in the member_moderation_action flag, the values were: 0==Hold,
# 1=Reject, 2==Discard
if bool(config_dict.get('default_member_moderation', 0)):
mlist.default_member_action = member_moderation_action_mapping(
config_dict.get('member_moderation_action'))
else:
mlist.default_member_action = Action.defer
# Handle the archiving policy. In MM2.1 there were two boolean options
# but only three of the four possible states were valid. Now there's just
# an enum.
if config_dict.get('archive'):
# For maximum safety, if for some strange reason there's no
# archive_private key, treat the list as having private archives.
if config_dict.get('archive_private', True):
mlist.archive_policy = ArchivePolicy.private
else:
mlist.archive_policy = ArchivePolicy.public
else:
mlist.archive_policy = ArchivePolicy.never
# Handle ban list.
ban_manager = IBanManager(mlist)
for address in config_dict.get('ban_list', []):
ban_manager.ban(bytes_to_str(address))
# Handle acceptable aliases.
acceptable_aliases = config_dict.get('acceptable_aliases', '')
if isinstance(acceptable_aliases, bytes):
acceptable_aliases = acceptable_aliases.decode('utf-8')
if isinstance(acceptable_aliases, str):
acceptable_aliases = acceptable_aliases.splitlines()
alias_set = IAcceptableAliasSet(mlist)
for address in acceptable_aliases:
address = address.strip()
if len(address) == 0:
continue
address = bytes_to_str(address)
try:
alias_set.add(address)
except ValueError:
# When .add() rejects this, the line probably contains a regular
# expression. Make that explicit for MM3.
alias_set.add('^' + address)
# Handle header_filter_rules conversion to header_matches.
header_matches = IHeaderMatchList(mlist)
header_filter_rules = config_dict.get('header_filter_rules', [])
for line_patterns, action, _unused in header_filter_rules:
try:
chain = action_to_chain(action)
except KeyError:
log.warning('Unsupported header_filter_rules action: %r',
action)
continue
# Now split the line into a header and a pattern.
for line_pattern in line_patterns.splitlines():
if len(line_pattern.strip()) == 0:
continue
for sep in (': ', ':.', ':'):
header, sep, pattern = line_pattern.partition(sep)
if sep:
# We found it.
break
else:
# Matches any header, which is not supported. XXX
log.warning('Unsupported header_filter_rules pattern: %r',
line_pattern)
continue
header = header.strip().lstrip('^').lower()
header = header.replace('\\', '')
if not header:
log.warning(
'Cannot parse the header in header_filter_rule: %r',
line_pattern)
continue
if len(pattern) == 0:
# The line matched only the header, therefore the header can
# be anything.
pattern = '.*'
try:
re.compile(pattern)
except re.error:
log.warning('Skipping header_filter rule because of an '
'invalid regular expression: %r', line_pattern)
continue
try:
header_matches.append(header, pattern, chain)
except ValueError:
log.warning('Skipping duplicate header_filter rule: %r',
line_pattern)
continue
# Handle conversion to URIs. In MM2.1, the decorations are strings
# containing placeholders, and there's no provision for language-specific
# templates. In MM3, template locations are specified by URLs with the
# special `mailman:` scheme indicating a file system path. What we do
# here is look to see if the list's decoration is different than the
# default, and if so, we'll write the new decoration template to a
# `mailman:` scheme path.
convert_to_uri = {
'welcome_msg': 'welcome_message_uri',
'goodbye_msg': 'goodbye_message_uri',
'msg_header': 'header_uri',
'msg_footer': 'footer_uri',
'digest_header': 'digest_header_uri',
'digest_footer': 'digest_footer_uri',
}
# The best we can do is convert only the most common ones. These are
# order dependent; the longer substitution with the common prefix must
# show up earlier.
convert_placeholders = [
('%(real_name)s@%(host_name)s', '$fqdn_listname'),
('%(real_name)s', '$display_name'),
('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
'$listinfo_uri'),
]
# Collect defaults.
defaults = {}
for oldvar, newvar in convert_to_uri.items():
default_value = getattr(mlist, newvar, None)
if not default_value:
continue
# Check if the value changed from the default.
try:
default_text = decorate(mlist, default_value)
except (URLError, KeyError):
# Use case: importing the old [email protected] into [email protected]. We can't
# check if it changed from the default so don't import, we may do
# more harm than good and it's easy to change if needed.
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with old value "{}"'.format(default_value),
file=sys.stderr)
continue
defaults[newvar] = (default_value, default_text)
for oldvar, newvar in convert_to_uri.items():
if oldvar not in config_dict:
continue
text = config_dict[oldvar]
if isinstance(text, bytes):
text = text.decode('utf-8', 'replace')
for oldph, newph in convert_placeholders:
text = text.replace(oldph, newph)
default_value, default_text = defaults.get(newvar, (None, None))
if not text and not (default_value or default_text):
# Both are empty, leave it.
continue
# Check if the value changed from the default
try:
expanded_text = decorate_template(mlist, text)
except KeyError:
# Use case: importing the old [email protected] into [email protected]
# We can't check if it changed from the default
# -> don't import, we may do more harm than good and it's easy to
# change if needed
# TESTME
print('Unable to convert mailing list attribute:', oldvar,
'with value "{}"'.format(text),
file=sys.stderr)
continue
if (expanded_text and default_text and
expanded_text.strip() == default_text.strip()):
# Keep the default.
continue
# Write the custom value to the right file.
base_uri = 'mailman:///$listname/$language/'
if default_value:
filename = default_value.rpartition('/')[2]
else:
filename = '{}.txt'.format(newvar[:-4])
if not default_value or not default_value.startswith(base_uri):
setattr(mlist, newvar, base_uri + filename)
filepath = list(search(filename, mlist))[0]
makedirs(os.path.dirname(filepath))
with codecs.open(filepath, 'w', encoding='utf-8') as fp:
fp.write(text)
# Import rosters.
regulars_set = set(config_dict.get('members', {}))
digesters_set = set(config_dict.get('digest_members', {}))
members = regulars_set.union(digesters_set)
# Don't send welcome messages when we import the rosters.
send_welcome_message = mlist.send_welcome_message
mlist.send_welcome_message = False
try:
import_roster(mlist, config_dict, members, MemberRole.member)
import_roster(mlist, config_dict, config_dict.get('owner', []),
MemberRole.owner)
import_roster(mlist, config_dict, config_dict.get('moderator', []),
MemberRole.moderator)
# Now import the '*_these_nonmembers' properties, filtering out the
# regexps which will remain in the property.
for action_name in ('accept', 'hold', 'reject', 'discard'):
prop_name = '{}_these_nonmembers'.format(action_name)
emails = [addr
for addr in config_dict.get(prop_name, [])
if not addr.startswith('^')]
import_roster(mlist, config_dict, emails, MemberRole.nonmember,
Action[action_name])
# Only keep the regexes in the legacy list property.
list_prop = getattr(mlist, prop_name)
for email in emails:
list_prop.remove(email)
finally:
mlist.send_welcome_message = send_welcome_message
