def setUp(self):
# Set up a temporary directory for the archiver so that it's
# easier to clean up.
self._tempdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self._tempdir)
config.push(
'hyperkitty', """
[paths.testing]
archive_dir: {tmpdir}/archive
[archiver.hyperkitty]
class: mailman_hyperkitty.Archiver
enable: yes
configuration: {tmpdir}/mailman-hyperkitty.cfg
""".format(tmpdir=self._tempdir))
self.addCleanup(config.pop, 'hyperkitty')
with open(os.path.join(self._tempdir, "mailman-hyperkitty.cfg"),
"w") as conf_h:
conf_h.write(
dedent("""
[general]
base_url: http://localhost
api_key: DummyKey
"""))
# Create the archiver
self.archiver = Archiver()
self.mlist = FakeList("*****@*****.**")
# Patch requests
self.requests_patcher = patch("mailman_hyperkitty.requests")
self.requests = self.requests_patcher.start()
self.fake_response = None
self.requests.get.side_effect = \
lambda url, *a, **kw: self.fake_response
self.requests.post.side_effect = \
lambda url, *a, **kw: self.fake_response
def setUp(self):
self.archiver = Archiver()
self.archiver._base_url = "http://lists.example.com"
self.archiver._api_key = "DummyKey"
self.mlist = FakeList("*****@*****.**")
# Patch requests
self.requests_patcher = patch("mailman_hyperkitty.requests")
self.requests = self.requests_patcher.start()
self.fake_response = None
self.requests.get.side_effect = lambda url, *a, **kw: self.fake_response
self.requests.post.side_effect = lambda url, *a, **kw: self.fake_response
def setUp(self):
self.archiver = Archiver()
self.archiver._base_url = "http://lists.example.com"
self.archiver._api_key = "DummyKey"
self.mlist = FakeList("*****@*****.**")
# Patch requests
self.requests_patcher = patch("mailman_hyperkitty.requests")
self.requests = self.requests_patcher.start()
self.fake_response = None
self.requests.get.side_effect = lambda url, *a, **kw: self.fake_response
self.requests.post.side_effect = lambda url, *a, **kw: self.fake_response
class ArchiverTestCase(TestCase):
layer = ConfigLayer
def setUp(self):
# Set up a temporary directory for the archiver so that it's
# easier to clean up.
self._tempdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self._tempdir)
config.push(
'hyperkitty', """
[paths.testing]
archive_dir: {tmpdir}/archive
[archiver.hyperkitty]
class: mailman_hyperkitty.Archiver
enable: yes
configuration: {tmpdir}/mailman-hyperkitty.cfg
""".format(tmpdir=self._tempdir))
self.addCleanup(config.pop, 'hyperkitty')
with open(os.path.join(self._tempdir, "mailman-hyperkitty.cfg"),
"w") as conf_h:
conf_h.write(
dedent("""
[general]
base_url: http://localhost
api_key: DummyKey
"""))
# Create the archiver
self.archiver = Archiver()
self.mlist = FakeList("*****@*****.**")
# Patch requests
self.requests_patcher = patch("mailman_hyperkitty.requests")
self.requests = self.requests_patcher.start()
self.fake_response = None
self.requests.get.side_effect = \
lambda url, *a, **kw: self.fake_response
self.requests.post.side_effect = \
lambda url, *a, **kw: self.fake_response
def tearDown(self):
self.requests_patcher.stop()
def _get_msg(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["Message-ID-Hash"] = "QKODQBCADMDSP5YPOPKECXQWEQAMXZL3"
msg.set_payload("Dummy message")
return msg
def test_list_url(self):
self.fake_response = FakeResponse(
200, {"url": "http://example.com/list/[email protected]/"})
self.assertEqual(self.archiver.list_url(self.mlist),
"http://example.com/list/[email protected]/")
self.requests.get.assert_called_with(
"http://localhost/api/mailman/urls",
params={
'key': 'DummyKey',
'mlist': '*****@*****.**'
})
def test_permalink(self):
msg = self._get_msg()
url = ("http://example.com/list/[email protected]/"
"message/{}/".format(msg["Message-ID-Hash"]))
self.fake_response = FakeResponse(200, {"url": url})
self.assertEqual(self.archiver.permalink(self.mlist, msg), url)
self.requests.get.assert_called_with(
"http://localhost/api/mailman/urls",
params={
'key': 'DummyKey',
'msgid': 'dummy',
'mlist': '*****@*****.**'
})
def test_archive_message(self):
msg = self._get_msg()
url = ("http://example.com/list/[email protected]/"
"message/{}/".format(msg["Message-ID-Hash"]))
self.fake_response = FakeResponse(200, {"url": url})
with patch("mailman_hyperkitty.logger") as logger:
archive_url = self.archiver.archive_message(self.mlist, msg)
self.assertTrue(logger.info.called)
self.assertEqual(archive_url, url)
self.requests.post.assert_called_with(
"http://localhost/api/mailman/archive",
params={'key': 'DummyKey'},
data={'mlist': '*****@*****.**'},
files={'message': ('message.txt', msg.as_string())},
)
# Check that the archive directory was created.
self.assertTrue(
os.path.exists(self.archiver._switchboard.queue_directory))
# Make sure it is empty, since the message has been successfuly
# archived.
self.assertEqual(
len(os.listdir(self.archiver._switchboard.queue_directory)), 0)
self.assertEqual(len(self.archiver._switchboard.files), 0)
def test_list_url_permalink_error(self):
# Don't raise exceptions for list_url and permalink
self.fake_response = FakeResponse(500, "Fake error")
with patch("mailman_hyperkitty.logger") as logger:
self.assertEqual(self.archiver.list_url(self.mlist), "")
self.assertEqual(
self.archiver.permalink(self.mlist, self._get_msg()), "")
# Check error log
self.assertEqual(logger.error.call_count, 2)
for call in logger.error.call_args_list:
self.assertEqual(call[0][3], 500)
def test_list_url_permalink_invalid(self):
self.fake_response = Mock()
self.fake_response.status_code = 200
self.fake_response.json.side_effect = ValueError
with patch("mailman_hyperkitty.logger") as logger:
self.assertEqual(self.archiver.list_url(self.mlist), "")
self.assertEqual(
self.archiver.permalink(self.mlist, self._get_msg()), "")
# Check error log
self.assertEqual(logger.exception.call_count, 2)
for call in logger.error.call_args_list:
self.assertTrue(isinstance(call[0][2], ValueError))
def test_archive_message_error(self):
self.fake_response = FakeResponse(500, "Fake error")
with patch("mailman_hyperkitty.logger") as logger:
self.archiver.archive_message(self.mlist, self._get_msg())
# Check error log
self.assertEqual(logger.error.call_count, 3)
self.assertTrue(
isinstance(logger.error.call_args_list[1][0][1], ValueError))
# Check that the message is stored in the spool.
self.assertEqual(
len(os.listdir(self.archiver._switchboard.queue_directory)), 1)
self.assertEqual(len(self.archiver._switchboard.files), 1)
def test_archive_message_unavailable(self):
self.requests.exceptions.RequestException = \
requests.exceptions.RequestException
self.requests.post.side_effect = requests.exceptions.RequestException
with patch("mailman_hyperkitty.logger") as logger:
self.archiver.archive_message(self.mlist, self._get_msg())
# Check error log
self.assertTrue(logger.error.called)
self.assertTrue(
isinstance(logger.error.call_args_list[0][0][1],
requests.exceptions.RequestException))
# Check that the message is stored in the spool.
self.assertEqual(
len(os.listdir(self.archiver._switchboard.queue_directory)), 1)
self.assertEqual(len(self.archiver._switchboard.files), 1)
def test_archive_message_invalid(self):
self.fake_response = Mock()
self.fake_response.status_code = 200
self.fake_response.json.side_effect = ValueError
with patch("mailman_hyperkitty.logger") as logger:
self.archiver.archive_message(self.mlist, self._get_msg())
# Check error log
self.assertEqual(logger.exception.call_count, 1)
self.assertTrue(
isinstance(logger.exception.call_args_list[0][0][2], ValueError))
# Check that the message is stored in the spool.
self.assertEqual(
len(os.listdir(self.archiver._switchboard.queue_directory)), 1)
self.assertEqual(len(self.archiver._switchboard.files), 1)
def test_archive_message_replay(self):
# If there are messages in the spool directory, they must be processed
# before any other message.
# Create a previously failed message in the spool queue.
msg_1 = self._get_msg()
msg_1["Message-ID"] = "<dummy-1>"
del msg_1["Message-ID-Hash"]
add_message_hash(msg_1)
self.archiver._switchboard.enqueue(msg_1, mlist=self.mlist)
# Now send another message
msg_2 = self._get_msg()
msg_2["Message-ID"] = "<dummy-2>"
del msg_2["Message-ID-Hash"]
add_message_hash(msg_2)
self.fake_response = FakeResponse(200, {"url": "dummy"})
with patch("mailman_hyperkitty.logger") as logger:
self.archiver.archive_message(self.mlist, msg_2)
# Two messages must have been archived
self.assertEqual(logger.info.call_count, 2)
self.assertEqual(self.requests.post.call_args_list, [
(("http://localhost/api/mailman/archive", ),
dict(
params={'key': 'DummyKey'},
data={'mlist': '*****@*****.**'},
files={'message': ('message.txt', msg_1.as_string())},
)),
(("http://localhost/api/mailman/archive", ),
dict(
params={'key': 'DummyKey'},
data={'mlist': '*****@*****.**'},
files={'message': ('message.txt', msg_2.as_string())},
)),
])
# Make sure the spool directory is empty now.
self.assertEqual(
len(os.listdir(self.archiver._switchboard.queue_directory)), 0)
self.assertEqual(len(self.archiver._switchboard.files), 0)
def test_queued_archive_message_error(self):
# If a queue message is being retried and the archiving fails again, it
# stays in the queue.
self.fake_response = FakeResponse(500, "Fake error")
self.archiver._switchboard.enqueue(self._get_msg(), mlist=self.mlist)
with patch("mailman_hyperkitty.logger") as logger:
self.archiver.process_queue()
# Check error log
self.assertEqual(logger.error.call_count, 3)
self.assertEqual(logger.error.call_args_list[0][0][3], 500)
self.assertTrue(
isinstance(logger.error.call_args_list[1][0][1], ValueError))
# Check that the message is still stored in the spool.
self.assertEqual(
len(os.listdir(self.archiver._switchboard.queue_directory)), 1)
self.assertEqual(len(self.archiver._switchboard.files), 1)
def test_queued_message_unparseable(self):
self.fake_response = FakeResponse(200, {"url": "dummy"})
with open(
os.path.join(self.archiver._switchboard.queue_directory,
"123456789+dummy.pck"), "w") as fh:
fh.write("invalid pickle data")
self.assertEqual(len(self.archiver._switchboard.files), 1)
# Now process the queue.
with patch("mailman_hyperkitty.logger") as logger:
self.archiver.process_queue()
# Check error log
self.assertEqual(logger.error.call_count, 3)
# Check that the message has been preserved for analysis
self.assertEqual(
len(
glob.glob(
os.path.join(config.switchboards["bad"].queue_directory,
"*.psv"))), 1)
def test_queued_message_enqueue_exception(self):
self.fake_response = FakeResponse(500, "Fake error")
self.archiver._switchboard.enqueue(self._get_msg(), mlist=self.mlist)
# Now, cause .enqueue() to throw an exception and process the queue.
self.archiver._switchboard.enqueue = Mock()
self.archiver._switchboard.enqueue.side_effect = OSError('Oops!')
with patch("mailman_hyperkitty.logger") as logger:
self.archiver.process_queue()
self.assertEqual(logger.error.call_count, 6)
# Check error log
self.assertEqual(logger.error.call_args_list[0][0][3], 500)
self.assertTrue(
isinstance(logger.error.call_args_list[1][0][1], ValueError))
self.assertTrue(
isinstance(logger.error.call_args_list[3][0][1], OSError))
# Check that the message has been preserved for analysis
self.assertEqual(
len(
glob.glob(
os.path.join(config.switchboards["bad"].queue_directory,
"*.psv"))), 1)
def test_queued_message_finish_exception(self):
self.fake_response = FakeResponse(200, {"url": "dummy"})
self.archiver._switchboard.enqueue(self._get_msg(), mlist=self.mlist)
# Now, cause .finish() to throw an exception and process the queue.
with patch('mailman.core.switchboard.os.rename',
side_effect=OSError('Oops!')), \
patch("mailman_hyperkitty.logger") as logger:
self.archiver.process_queue()
# Check error log
self.assertEqual(logger.error.call_count, 3)
self.assertTrue(
isinstance(logger.error.call_args_list[0][0][1], OSError))
# Check that the message is still stored in the spool.
self.assertEqual(
len(os.listdir(self.archiver._switchboard.queue_directory)), 1)
self.assertEqual(len(self.archiver._switchboard.files), 1)
def test_archive_message_unserializable(self):
msg = self._get_msg()
msg["content-type"] = 'text/plain; charset="UTF-8"'
msg.set_payload(b"this contains encoded unicode \xc3\xa9 \xc3\xa0")
# If you try to serialize this message to text, it will cause a:
# KeyError: 'content-transfer-encoding'
with patch("mailman_hyperkitty.logger") as logger:
self.archiver.archive_message(self.mlist, msg)
# Check error log
self.assertEqual(logger.error.call_count, 1)
self.assertTrue(
isinstance(logger.error.call_args_list[0][0][2], KeyError))
# Check that the message is not stored in the spool.
self.assertEqual(
len(os.listdir(self.archiver._switchboard.queue_directory)), 0)
self.assertEqual(len(self.archiver._switchboard.files), 0)
class ArchiverTestCase(TestCase):
def setUp(self):
self.archiver = Archiver()
self.archiver._base_url = "http://lists.example.com"
self.archiver._api_key = "DummyKey"
self.mlist = FakeList("*****@*****.**")
# Patch requests
self.requests_patcher = patch("mailman_hyperkitty.requests")
self.requests = self.requests_patcher.start()
self.fake_response = None
self.requests.get.side_effect = lambda url, *a, **kw: self.fake_response
self.requests.post.side_effect = lambda url, *a, **kw: self.fake_response
def tearDown(self):
self.requests_patcher.stop()
def _get_msg(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["Message-ID-Hash"] = "QKODQBCADMDSP5YPOPKECXQWEQAMXZL3"
msg.set_payload("Dummy message")
return msg
def test_list_url(self):
self.fake_response = FakeResponse(
200, {"url": "/list/[email protected]/"})
self.assertEqual(
self.archiver.list_url(self.mlist),
"http://lists.example.com/list/[email protected]/")
#print(self.requests.get.call_args_list)
self.requests.get.assert_called_with(
"http://lists.example.com/api/mailman/urls",
params={
'key': 'DummyKey',
'mlist': '*****@*****.**'
})
def test_permalink(self):
msg = self._get_msg()
relative_url = "/list/[email protected]/message/{}/".format(
msg["Message-ID-Hash"])
self.fake_response = FakeResponse(200, {"url": relative_url})
self.assertEqual(self.archiver.permalink(self.mlist, msg),
"http://lists.example.com" + relative_url)
#print(self.requests.get.call_args_list)
self.requests.get.assert_called_with(
"http://lists.example.com/api/mailman/urls",
params={
'key': 'DummyKey',
'msgid': 'dummy',
'mlist': '*****@*****.**'
})
def test_archive_message(self):
msg = self._get_msg()
relative_url = "/list/[email protected]/message/{}/".format(
msg["Message-ID-Hash"])
self.fake_response = FakeResponse(200, {"url": relative_url})
with patch("mailman_hyperkitty.logger") as logger:
url = self.archiver.archive_message(self.mlist, msg)
self.assertTrue(logger.info.called)
self.assertEqual(url, "http://lists.example.com" + relative_url)
#print(self.requests.post.call_args_list)
self.requests.post.assert_called_with( # pylint: disable=no-member
"http://lists.example.com/api/mailman/archive",
params={'key': 'DummyKey'},
data={'mlist': '*****@*****.**'},
files={'message': ('message.txt', msg.as_string())},
)
def test_list_url_permalink_error(self):
# Don't raise exceptions for list_url and permalink
self.fake_response = FakeResponse(500, "Fake error")
with patch("mailman_hyperkitty.logger") as logger:
self.assertEqual(self.archiver.list_url(self.mlist), "")
self.assertEqual(
self.archiver.permalink(self.mlist, self._get_msg()), "")
self.assertEqual(logger.error.call_count, 2)
def test_archive_message_error(self):
self.fake_response = FakeResponse(500, "Fake error")
with patch("mailman_hyperkitty.logger") as logger:
self.assertRaises(ValueError, self.archiver.archive_message,
self.mlist, self._get_msg())
self.assertTrue(logger.error.called)
class ArchiverTestCase(TestCase):
def setUp(self):
self.archiver = Archiver()
self.archiver._base_url = "http://lists.example.com"
self.archiver._api_key = "DummyKey"
self.mlist = FakeList("*****@*****.**")
# Patch requests
self.requests_patcher = patch("mailman_hyperkitty.requests")
self.requests = self.requests_patcher.start()
self.fake_response = None
self.requests.get.side_effect = lambda url, *a, **kw: self.fake_response
self.requests.post.side_effect = lambda url, *a, **kw: self.fake_response
def tearDown(self):
self.requests_patcher.stop()
def _get_msg(self):
msg = Message()
msg["From"] = "*****@*****.**"
msg["Message-ID"] = "<dummy>"
msg["Message-ID-Hash"] = "QKODQBCADMDSP5YPOPKECXQWEQAMXZL3"
msg.set_payload("Dummy message")
return msg
def test_list_url(self):
self.fake_response = FakeResponse(
200, {"url": "/list/[email protected]/"})
self.assertEqual(self.archiver.list_url(self.mlist),
"http://lists.example.com/list/[email protected]/"
)
#print(self.requests.get.call_args_list)
self.requests.get.assert_called_with(
"http://lists.example.com/api/mailman/urls",
params={'key': 'DummyKey', 'mlist': '*****@*****.**'}
)
def test_permalink(self):
msg = self._get_msg()
relative_url = "/list/[email protected]/message/{}/".format(
msg["Message-ID-Hash"])
self.fake_response = FakeResponse(200, {"url": relative_url})
self.assertEqual(self.archiver.permalink(self.mlist, msg),
"http://lists.example.com" + relative_url)
#print(self.requests.get.call_args_list)
self.requests.get.assert_called_with(
"http://lists.example.com/api/mailman/urls",
params={'key': 'DummyKey', 'msgid': 'dummy',
'mlist': '*****@*****.**'}
)
def test_archive_message(self):
msg = self._get_msg()
relative_url = "/list/[email protected]/message/{}/".format(
msg["Message-ID-Hash"])
self.fake_response = FakeResponse(200, {"url": relative_url})
with patch("mailman_hyperkitty.logger") as logger:
url = self.archiver.archive_message(self.mlist, msg)
self.assertTrue(logger.info.called)
self.assertEqual(url, "http://lists.example.com" + relative_url)
#print(self.requests.post.call_args_list)
self.requests.post.assert_called_with( # pylint: disable=no-member
"http://lists.example.com/api/mailman/archive",
params={'key': 'DummyKey'},
data={'mlist': '*****@*****.**'},
files={'message': ('message.txt', msg.as_string())},
)
def test_list_url_permalink_error(self):
# Don't raise exceptions for list_url and permalink
self.fake_response = FakeResponse(500, "Fake error")
with patch("mailman_hyperkitty.logger") as logger:
self.assertEqual(self.archiver.list_url(self.mlist), "")
self.assertEqual(
self.archiver.permalink(self.mlist, self._get_msg()), "")
self.assertEqual(logger.error.call_count, 2)
def test_archive_message_error(self):
self.fake_response = FakeResponse(500, "Fake error")
with patch("mailman_hyperkitty.logger") as logger:
self.assertRaises(ValueError, self.archiver.archive_message,
self.mlist, self._get_msg())
self.assertTrue(logger.error.called)