def test_search(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
received = [(Mock(), Document(sn='1')), (Mock(), Document(sn='2')),
(Mock(), Document(sn='3'))]
# test
url = TEST_URL
node = Node('')
sn = received[1][1].sn
reader = Reader(node, url)
reader.next = Mock(side_effect=received)
document = reader.search(sn, timeout=10)
# validation
next_calls = reader.next.call_args_list
self.assertEqual(len(next_calls), 2)
self.assertEqual(document, received[1][1])
for call in next_calls:
self.assertEqual(call[0][0], 10)
self.assertTrue(received[0][0].ack.called)
self.assertTrue(received[1][0].ack.called)
self.assertFalse(received[2][0].ack.called)
def test_search_timeout(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
received = [
(Mock(), Document(sn='1')),
(Mock(), Document(sn='2')),
(None, None)
]
# test
url = TEST_URL
node = Node('')
reader = Reader(node, url)
reader.next = Mock(side_effect=received)
document = reader.search('', timeout=10)
# validation
next_calls = reader.next.call_args_list
self.assertEqual(len(next_calls), len(received))
self.assertEqual(document, None)
for call in next_calls:
self.assertEqual(call[0][0], 10)
self.assertTrue(received[0][0].ack.called)
self.assertTrue(received[1][0].ack.called)
def test_repair(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
url = TEST_URL
node = Node('test')
reader = Reader(node, url)
reader.repair()
_impl.repair.assert_called_with()
def test_open(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
url = TEST_URL
node = Node('')
reader = Reader(node, url)
reader.open()
_impl.open.assert_called_once_with()
def test_repair(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
url = TEST_URL
node = Node('test')
reader = Reader(node, url)
reader.repair()
_impl.repair.assert_called_with()
def test_open(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
url = TEST_URL
node = Node('')
reader = Reader(node, url)
reader.open()
_impl.open.assert_called_once_with()
def test_reject(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
message = Mock()
url = TEST_URL
node = Node('')
reader = Reader(node, url)
reader.reject(message, 29)
message.reject.assert_called_with(29)
def test_reject(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
message = Mock()
url = TEST_URL
node = Node('')
reader = Reader(node, url)
reader.reject(message, 29)
message.reject.assert_called_with(29)
def test_ack(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
message = Mock()
url = TEST_URL
node = Node('')
reader = Reader(node, url)
reader.ack(message)
message.ack.assert_called_once_with()
def test_close(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
url = TEST_URL
node = Node('')
reader = Reader(node, url)
# soft
reader.close()
_impl.close.assert_called_with()
def test_ack(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
message = Mock()
url = TEST_URL
node = Node('')
reader = Reader(node, url)
reader.ack(message)
message.ack.assert_called_once_with()
def test_close(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
url = TEST_URL
node = Node('')
reader = Reader(node, url)
# soft
reader.close()
_impl.close.assert_called_with()
def run(self):
"""
Main consumer loop.
"""
self.reader = Reader(self.node, self.url)
self.reader.authenticator = self.authenticator
self.open()
try:
while not Thread.aborted():
self.read()
finally:
self.close()
def test_next_not_found(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
# test
reader = Reader(Node(''))
reader.get = Mock(return_value=None)
reader.authenticator = Mock()
_message, _document = reader.next(10)
# validation
reader.get.assert_called_once_with(10)
self.assertEqual(_message, None)
self.assertEqual(_document, None)
def test_next_not_found(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
# test
reader = Reader(Node(''))
reader.get = Mock(return_value=None)
reader.authenticator = Mock()
_message, _document = reader.next(10)
# validation
reader.get.assert_called_once_with(10)
self.assertEqual(_message, None)
self.assertEqual(_document, None)
def test_get(self, _find):
message = Mock()
_impl = Mock()
_impl.get.return_value = message
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
node = Node('')
url = TEST_URL
# test
reader = Reader(node, url)
# validation
m = reader.get(10)
_impl.get.assert_called_with(10)
self.assertEqual(m, message)
def test_get(self, _find):
message = Mock()
_impl = Mock()
_impl.get.return_value = message
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
node = Node('')
url = TEST_URL
# test
reader = Reader(node, url)
# validation
m = reader.get(10)
_impl.get.assert_called_with(10)
self.assertEqual(m, message)
def test_next_auth_rejected(self, _find, auth, validate):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
message = Mock(body='test-content')
auth.validate.side_effect = ModelError
# test
reader = Reader(Node(''))
reader.get = Mock(return_value=message)
reader.authenticator = Mock()
self.assertRaises(ModelError, reader.next, 10)
# validation
reader.get.assert_called_once_with(10)
auth.validate.assert_called_once_with(reader.authenticator, message.body)
message.ack.assert_called_once_with()
self.assertFalse(validate.called)
def test_next(self, _find, auth, validate):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
message = Mock(body='test-content')
document = Mock()
auth.validate.return_value = document
# test
reader = Reader(Node(''))
reader.get = Mock(return_value=message)
reader.authenticator = Mock()
_message, _document = reader.next(10)
# validation
reader.get.assert_called_once_with(10)
auth.validate.assert_called_once_with(reader.authenticator, message.body)
validate.assert_called_once_with(document)
self.assertEqual(_message, reader.get.return_value)
self.assertEqual(_document, document)
def run(self):
"""
Main consumer loop.
"""
self.reader = Reader(self.node, self.url)
self.reader.authenticator = self.authenticator
self.open()
try:
while not Thread.aborted():
self.read()
finally:
self.close()
def test_init(self, _find):
_impl = Mock()
plugin = Mock()
plugin.Reader.return_value = _impl
_find.return_value = plugin
node = Node('test')
url = TEST_URL
# test
reader = Reader(node, url)
# validation
_find.assert_called_with(url)
plugin.Reader.assert_called_with(node, url)
self.assertEqual(reader.authenticator, None)
self.assertTrue(isinstance(reader, BaseReader))
class ConsumerThread(Thread):
"""
An AMQP (abstract) consumer.
"""
def __init__(self, node, url):
"""
:param node: An AMQP queue.
:type node: gofer.messaging.adapter.model.Node
:param url: The broker URL.
:type url: str
"""
Thread.__init__(self, name=node.name)
self.url = url
self.node = node
self.authenticator = None
self._reader = None
self.setDaemon(True)
def shutdown(self):
"""
Shutdown the consumer.
"""
self.abort()
@released
def run(self):
"""
Main consumer loop.
"""
self._reader = Reader(self.node, self.url)
self._reader.authenticator = self.authenticator
self._open()
try:
while not Thread.aborted():
self._read()
finally:
self._close()
def _open(self):
"""
Open the reader.
"""
while not Thread.aborted():
try:
self._reader.open()
break
except Exception:
log.exception(self.getName())
sleep(60)
def _close(self):
"""
Close the reader.
"""
try:
self._reader.close()
except Exception:
log.exception(self.getName())
def _read(self):
"""
Read and process incoming documents.
"""
try:
message, document = self._reader.next(10)
if message is None:
return
log.debug('{%s} read: %s', self.getName(), document)
self.dispatch(document)
message.ack()
except InvalidDocument, invalid:
self._rejected(invalid.code, invalid.description, invalid.document,
invalid.details)
except Exception:
log.exception(self.getName())
sleep(60)
self._close()
self._open()
class ConsumerThread(Thread):
"""
An AMQP (abstract) consumer.
"""
def __init__(self, node, url, wait=3):
"""
:param node: An AMQP queue.
:type node: gofer.messaging.adapter.model.Node
:param url: The broker URL.
:type url: str
:param wait: Number of seconds to wait for a message.
:type wait: int
"""
Thread.__init__(self, name=node.name)
self.url = url
self.node = node
self.wait = wait
self.authenticator = None
self.reader = None
self.setDaemon(True)
def shutdown(self):
"""
Shutdown the consumer.
"""
self.abort()
@released
def run(self):
"""
Main consumer loop.
"""
self.reader = Reader(self.node, self.url)
self.reader.authenticator = self.authenticator
self.open()
try:
while not Thread.aborted():
self.read()
finally:
self.close()
def open(self):
"""
Open the reader.
"""
while not Thread.aborted():
try:
self.reader.open()
break
except Exception:
log.exception(self.getName())
sleep(30)
def close(self):
"""
Close the reader.
"""
try:
self.reader.close()
except Exception:
log.exception(self.getName())
def read(self):
"""
Read and process incoming documents.
"""
try:
wait = self.wait
reader = self.reader
message, document = reader.next(wait)
if message is None:
# wait expired
return
log.debug("{%s} read: %s", self.getName(), document)
self.dispatch(document)
message.ack()
except DocumentError, de:
self.rejected(de.code, de.description, de.document, de.details)
except Exception:
log.exception(self.getName())
sleep(60)
self.close()
self.open()
class ConsumerThread(Thread):
"""
An AMQP (abstract) consumer.
"""
def __init__(self, node, url, wait=3):
"""
:param node: An AMQP queue.
:type node: gofer.messaging.adapter.model.Node
:param url: The broker URL.
:type url: str
:param wait: Number of seconds to wait for a message.
:type wait: int
"""
Thread.__init__(self, name=node.name)
self.url = url
self.node = node
self.wait = wait
self.authenticator = None
self.reader = None
self.setDaemon(True)
def shutdown(self):
"""
Shutdown the consumer.
"""
self.abort()
@released
def run(self):
"""
Main consumer loop.
"""
self.reader = Reader(self.node, self.url)
self.reader.authenticator = self.authenticator
self.open()
try:
while not Thread.aborted():
self.read()
finally:
self.close()
def open(self):
"""
Open the reader.
"""
while not Thread.aborted():
try:
self.reader.open()
break
except Exception:
log.exception(self.getName())
sleep(30)
def close(self):
"""
Close the reader.
"""
try:
self.reader.close()
except Exception:
log.exception(self.getName())
def read(self):
"""
Read and process incoming documents.
"""
try:
wait = self.wait
reader = self.reader
message, document = reader.next(wait)
if message is None:
# wait expired
return
log.debug('{%s} read: %s', self.getName(), document)
self.dispatch(document)
message.ack()
except DocumentError, de:
self.rejected(de.code, de.description, de.document, de.details)
except Exception:
log.exception(self.getName())
sleep(60)
self.close()
self.open()
class ConsumerThread(Thread):
"""
An AMQP (abstract) consumer.
"""
def __init__(self, node, url):
"""
:param node: An AMQP queue.
:type node: gofer.messaging.adapter.model.Node
:param url: The broker URL.
:type url: str
"""
Thread.__init__(self, name=node.name)
self.url = url
self.node = node
self.authenticator = None
self._reader = None
self.setDaemon(True)
def shutdown(self):
"""
Shutdown the consumer.
"""
self.abort()
@released
def run(self):
"""
Main consumer loop.
"""
self._reader = Reader(self.node, self.url)
self._reader.authenticator = self.authenticator
self._open()
try:
while not Thread.aborted():
self._read()
finally:
self._close()
def _open(self):
"""
Open the reader.
"""
while not Thread.aborted():
try:
self._reader.open()
break
except Exception:
log.exception(self.getName())
sleep(60)
def _close(self):
"""
Close the reader.
"""
try:
self._reader.close()
except Exception:
log.exception(self.getName())
def _read(self):
"""
Read and process incoming documents.
"""
try:
message, document = self._reader.next(10)
if message is None:
return
log.debug('{%s} read: %s', self.getName(), document)
self.dispatch(document)
message.ack()
except InvalidDocument, invalid:
self._rejected(invalid.code, invalid.description, invalid.document, invalid.details)
except Exception:
log.exception(self.getName())
sleep(60)
self._close()
self._open()
