def setUp(self):
# Some items to play with.
self.item_a = FakeQueueItem("a")
self.item_b = FakeQueueItem("b")
self.item_c = FakeQueueItem("c")
self.item_d = FakeQueueItem("d")
self.item_e = FakeQueueItem("e")
# And an empty priority queue.
self.queue = PriorityQueue()
def __init__(self, batch_executions=False):
"""Create an executor.
batch_executions (bool): if True, the executor will receive a
list of operations in the queue instead of one operation
at a time.
"""
super(Executor, self).__init__()
self._batch_executions = batch_executions
self._operation_queue = PriorityQueue()
def setUp(self):
# Some items to play with.
self.item_a = FakeQueueItem("a")
self.item_b = FakeQueueItem("b")
self.item_c = FakeQueueItem("c")
self.item_d = FakeQueueItem("d")
self.item_e = FakeQueueItem("e")
# And an empty priority queue.
self.queue = PriorityQueue()
def __init__(self, batch_executions=False):
"""Create an executor.
batch_executions (bool): if True, the executor will receive a
list of operations in the queue instead of one operation
at a time.
"""
super(Executor, self).__init__()
self._batch_executions = batch_executions
self._operation_queue = PriorityQueue()
class Executor(object): # pylint: disable=R0921
"""A class taking care of executing operations.
The class can be configured to receive one operation at a time, or
all available operations in one go.
In the first case, subclasses must implement an execute() method
that takes an operation as input and should perform it. In the
second case, execute() takes a list of operations as input.
"""
def __init__(self, batch_executions=False):
"""Create an executor.
batch_executions (bool): if True, the executor will receive a
list of operations in the queue instead of one operation
at a time.
"""
super(Executor, self).__init__()
self._batch_executions = batch_executions
self._operation_queue = PriorityQueue()
def get_status(self):
"""Return a the status of the queues.
More precisely, a list of entries in the executor's queue. The
first item is the top item, the others are not in order.
return ([QueueEntry]): the list with the queued elements.
"""
return self._operation_queue.get_status()
def enqueue(self, item, priority=None, timestamp=None):
"""Add an item to the queue.
item (QueueItem): the item to add.
priority (int|None) the priority, or None to use default.
timestamp (datetime|None) the timestamp of the first request
for the operation, or None to use now.
return (bool): True if successfully enqueued.
"""
return self._operation_queue.push(item, priority, timestamp)
def dequeue(self, item):
"""Remove an item from the queue.
item (QueueItem): the item to remove.
"""
self._operation_queue.remove(item)
def run(self):
"""Monitor the queue, and dispatch operations when available.
This is an infinite loop that, at each iteration, blocks until
there is at least an element in the queue, and then extracts
the operation(s) and dispatch them to the executor. Any error
during the operation is sent to the logger and then
suppressed, because the loop must go on.
"""
while True:
# Wait for the queue to be non-empty.
to_execute = [self._operation_queue.pop(wait=True)]
if self._batch_executions:
# TODO: shall we yield to other greenlets? I think
# that it is going to be extremely unlikely to have
# more than one operations.
while not self._operation_queue.empty():
to_execute.append(self._operation_queue.pop())
assert len(to_execute) > 0, "Expected at least one element."
if self._batch_executions:
try:
logger.info("Executing operations `%s' and %d more.",
to_execute[0].item,
len(to_execute) - 1)
self.execute(to_execute)
logger.info(
"Operations `%s' and %d more concluded "
"successfully.", to_execute[0].item,
len(to_execute) - 1)
except Exception:
logger.error(
"Unexpected error when executing operation "
"`%s' (and %d more operations).",
to_execute[0].item,
len(to_execute) - 1,
exc_info=True)
else:
try:
logger.info("Executing operation `%s'.",
to_execute[0].item)
self.execute(to_execute[0])
logger.info("Operation `%s' concluded successfully",
to_execute[0].item)
except Exception:
logger.error(
"Unexpected error when executing operation `%s'.",
to_execute[0].item,
exc_info=True)
def execute(self, entry):
"""Perform a single operation.
Must be implemented if batch_execution is false.
entry (QueueEntry|[QueueEntry]): the top element of the queue,
in case batch_executions is false, or the list of all
currently available elements, in case it is true - in any
case, each element contains both the operations and the
info on priority and timestamp, in case we need to
re-enqueue the item.
"""
raise NotImplementedError("Please use a subclass.")
def tearDown(self):
self.queue = PriorityQueue()
class TestPriorityQueue(unittest.TestCase):
def setUp(self):
# Some items to play with.
self.item_a = FakeQueueItem("a")
self.item_b = FakeQueueItem("b")
self.item_c = FakeQueueItem("c")
self.item_d = FakeQueueItem("d")
self.item_e = FakeQueueItem("e")
# And an empty priority queue.
self.queue = PriorityQueue()
def tearDown(self):
self.queue = PriorityQueue()
def test_success(self):
"""Verify a simple success case.
Push three items in the wrong order, and recover them.
"""
self.assertTrue(self.queue._verify())
self.queue.push(self.item_a, PriorityQueue.PRIORITY_LOW)
self.queue.push(self.item_b, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(10))
self.queue.push(self.item_c, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(5))
self.assertTrue(self.queue._verify())
self.assertEqual(self.queue.top().item, self.item_c)
top = self.queue.pop()
self.assertEqual(top.item, self.item_c)
top = self.queue.pop()
self.assertEqual(top.item, self.item_b)
top = self.queue.pop()
self.assertEqual(top.item, self.item_a)
self.assertTrue(self.queue._verify())
with self.assertRaises(LookupError):
self.queue.pop()
def test_set_priority(self):
"""Test that priority get changed and item moved."""
self.queue.push(self.item_a, PriorityQueue.PRIORITY_LOW)
self.queue.push(self.item_b, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(10))
self.queue.push(self.item_c, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(5))
self.queue.set_priority(self.item_a, PriorityQueue.PRIORITY_HIGH)
self.assertTrue(self.queue._verify())
self.assertEqual(self.queue.top().item, self.item_a)
def test_pop_waiting(self):
"""Test that pop with waiting works.
Spawn a greenlet waiting for the queue to become non-empty,
and make sure that it wakes up when a new item arrives. Twice,
to be sure.
"""
def waiting():
obj_read = self.queue.pop(wait=True)
self.assertEqual(obj_read.item, self.item_a)
obj_read = self.queue.pop(wait=True)
self.assertEqual(obj_read.item, self.item_b)
greenlet = gevent.spawn(waiting)
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
self.queue.push(self.item_a)
self.assertTrue(self.queue._verify())
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
# The second time we push on top of the queue, but the first
# item should have already been collected.
self.queue.push(self.item_b, priority=PriorityQueue.PRIORITY_HIGH)
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
self.assertTrue(greenlet.successful())
def test_pop_multiple_waiting(self):
"""Test that pop with waiting works, even with several consumers.
Spawn three greenlets waiting for the queue to become
non-empty, and make sure that they get fed eventually.
"""
def waiting():
obj_read = self.queue.pop(wait=True)
self.assertEqual(obj_read.item, self.item_a)
greenlets = [gevent.spawn(waiting),
gevent.spawn(waiting),
gevent.spawn(waiting)]
gevent.sleep(0.01)
self.queue.push(self.item_a)
self.assertTrue(self.queue._verify())
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
# Now exactly one greenlet should have terminated.
terminated = sum(1 for greenlet in greenlets if greenlet.ready())
successful = sum(1 for greenlet in greenlets if greenlet.successful())
self.assertEqual(terminated, 1)
self.assertEqual(successful, 1)
self.queue.push(self.item_a)
self.assertTrue(self.queue._verify())
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
# Now two.
terminated = sum(1 for greenlet in greenlets if greenlet.ready())
successful = sum(1 for greenlet in greenlets if greenlet.successful())
self.assertEqual(terminated, 2)
self.assertEqual(successful, 2)
self.queue.push(self.item_a)
gevent.sleep(0.01)
# And three.
terminated = sum(1 for greenlet in greenlets if greenlet.ready())
successful = sum(1 for greenlet in greenlets if greenlet.successful())
self.assertEqual(terminated, 3)
self.assertEqual(successful, 3)
def test_remove(self):
"""Test that items get removed."""
self.queue.push(self.item_a, PriorityQueue.PRIORITY_LOW)
self.assertFalse(self.item_b in self.queue)
self.queue.push(self.item_b, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(10))
self.queue.push(self.item_c, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(5))
self.assertTrue(self.item_b in self.queue)
self.queue.remove(self.item_b)
self.assertFalse(self.item_b in self.queue)
self.queue._verify()
class Executor(object): # pylint: disable=R0921
"""A class taking care of executing operations.
The class can be configured to receive one operation at a time, or
all available operations in one go.
In the first case, subclasses must implement an execute() method
that takes an operation as input and should perform it. In the
second case, execute() takes a list of operations as input.
"""
def __init__(self, batch_executions=False):
"""Create an executor.
batch_executions (bool): if True, the executor will receive a
list of operations in the queue instead of one operation
at a time.
"""
super(Executor, self).__init__()
self._batch_executions = batch_executions
self._operation_queue = PriorityQueue()
def __contains__(self, item):
"""Return whether the item is in the queue.
item (QueueItem): the item to look for.
return (bool): whether operation is in the queue.
"""
return item in self._operation_queue
def get_status(self):
"""Return a the status of the queues.
More precisely, a list of entries in the executor's queue. The
first item is the top item, the others are not in order.
return ([QueueEntry]): the list with the queued elements.
"""
return self._operation_queue.get_status()
def enqueue(self, item, priority=None, timestamp=None):
"""Add an item to the queue.
item (QueueItem): the item to add.
priority (int|None) the priority, or None to use default.
timestamp (datetime|None) the timestamp of the first request
for the operation, or None to use now.
return (bool): True if successfully enqueued.
"""
return self._operation_queue.push(item, priority, timestamp)
def dequeue(self, item):
"""Remove an item from the queue.
item (QueueItem): the item to remove.
"""
self._operation_queue.remove(item)
def run(self):
"""Monitor the queue, and dispatch operations when available.
This is an infinite loop that, at each iteration, blocks until
there is at least an element in the queue, and then extracts
the operation(s) and dispatch them to the executor. Any error
during the operation is sent to the logger and then
suppressed, because the loop must go on.
"""
while True:
# Wait for the queue to be non-empty.
to_execute = [self._operation_queue.pop(wait=True)]
if self._batch_executions:
# TODO: shall we yield to other greenlets? I think
# that it is going to be extremely unlikely to have
# more than one operations.
while not self._operation_queue.empty():
to_execute.append(self._operation_queue.pop())
assert len(to_execute) > 0, "Expected at least one element."
if self._batch_executions:
try:
logger.info("Executing operations `%s' and %d more.",
to_execute[0].item, len(to_execute) - 1)
self.execute(to_execute)
logger.info("Operations `%s' and %d more concluded "
"successfully.", to_execute[0].item,
len(to_execute) - 1)
except Exception:
logger.error(
"Unexpected error when executing operation "
"`%s' (and %d more operations).", to_execute[0].item,
len(to_execute) - 1, exc_info=True)
else:
try:
logger.info("Executing operation `%s'.",
to_execute[0].item)
self.execute(to_execute[0])
logger.info("Operation `%s' concluded successfully",
to_execute[0].item)
except Exception:
logger.error(
"Unexpected error when executing operation `%s'.",
to_execute[0].item, exc_info=True)
def execute(self, entry):
"""Perform a single operation.
Must be implemented if batch_execution is false.
entry (QueueEntry|[QueueEntry]): the top element of the queue,
in case batch_executions is false, or the list of all
currently available elements, in case it is true - in any
case, each element contains both the operations and the
info on priority and timestamp, in case we need to
re-enqueue the item.
"""
raise NotImplementedError("Please use a subclass.")
def tearDown(self):
self.queue = PriorityQueue()
class TestPriorityQueue(unittest.TestCase):
def setUp(self):
# Some items to play with.
self.item_a = FakeQueueItem("a")
self.item_b = FakeQueueItem("b")
self.item_c = FakeQueueItem("c")
self.item_d = FakeQueueItem("d")
self.item_e = FakeQueueItem("e")
# And an empty priority queue.
self.queue = PriorityQueue()
def tearDown(self):
self.queue = PriorityQueue()
def test_success(self):
"""Verify a simple success case.
Push three items in the wrong order, and recover them.
"""
self.assertTrue(self.queue._verify())
self.queue.push(self.item_a, PriorityQueue.PRIORITY_LOW)
self.queue.push(self.item_b, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(10))
self.queue.push(self.item_c, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(5))
self.assertTrue(self.queue._verify())
self.assertEqual(self.queue.top().item, self.item_c)
top = self.queue.pop()
self.assertEqual(top.item, self.item_c)
top = self.queue.pop()
self.assertEqual(top.item, self.item_b)
top = self.queue.pop()
self.assertEqual(top.item, self.item_a)
self.assertTrue(self.queue._verify())
with self.assertRaises(LookupError):
self.queue.pop()
def test_set_priority(self):
"""Test that priority get changed and item moved."""
self.queue.push(self.item_a, PriorityQueue.PRIORITY_LOW)
self.queue.push(self.item_b, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(10))
self.queue.push(self.item_c, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(5))
self.queue.set_priority(self.item_a, PriorityQueue.PRIORITY_HIGH)
self.assertTrue(self.queue._verify())
self.assertEqual(self.queue.top().item, self.item_a)
def test_pop_waiting(self):
"""Test that pop with waiting works.
Spawn a greenlet waiting for the queue to become non-empty,
and make sure that it wakes up when a new item arrives. Twice,
to be sure.
"""
def waiting():
obj_read = self.queue.pop(wait=True)
self.assertEqual(obj_read.item, self.item_a)
obj_read = self.queue.pop(wait=True)
self.assertEqual(obj_read.item, self.item_b)
greenlet = gevent.spawn(waiting)
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
self.queue.push(self.item_a)
self.assertTrue(self.queue._verify())
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
# The second time we push on top of the queue, but the first
# item should have already been collected.
self.queue.push(self.item_b, priority=PriorityQueue.PRIORITY_HIGH)
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
self.assertTrue(greenlet.successful())
def test_pop_multiple_waiting(self):
"""Test that pop with waiting works, even with several consumers.
Spawn three greenlets waiting for the queue to become
non-empty, and make sure that they get fed eventually.
"""
def waiting():
obj_read = self.queue.pop(wait=True)
self.assertEqual(obj_read.item, self.item_a)
greenlets = [gevent.spawn(waiting),
gevent.spawn(waiting),
gevent.spawn(waiting)]
gevent.sleep(0.01)
self.queue.push(self.item_a)
self.assertTrue(self.queue._verify())
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
# Now exactly one greenlet should have terminated.
terminated = sum(1 for greenlet in greenlets if greenlet.ready())
successful = sum(1 for greenlet in greenlets if greenlet.successful())
self.assertEqual(terminated, 1)
self.assertEqual(successful, 1)
self.queue.push(self.item_a)
self.assertTrue(self.queue._verify())
gevent.sleep(0.01)
self.assertTrue(self.queue._verify())
# Now two.
terminated = sum(1 for greenlet in greenlets if greenlet.ready())
successful = sum(1 for greenlet in greenlets if greenlet.successful())
self.assertEqual(terminated, 2)
self.assertEqual(successful, 2)
self.queue.push(self.item_a)
gevent.sleep(0.01)
# And three.
terminated = sum(1 for greenlet in greenlets if greenlet.ready())
successful = sum(1 for greenlet in greenlets if greenlet.successful())
self.assertEqual(terminated, 3)
self.assertEqual(successful, 3)
def test_remove(self):
"""Test that items get removed."""
self.queue.push(self.item_a, PriorityQueue.PRIORITY_LOW)
self.assertFalse(self.item_b in self.queue)
self.queue.push(self.item_b, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(10))
self.queue.push(self.item_c, PriorityQueue.PRIORITY_MEDIUM,
timestamp=make_datetime(5))
self.assertTrue(self.item_b in self.queue)
self.queue.remove(self.item_b)
self.assertFalse(self.item_b in self.queue)
self.queue._verify()
class Executor(metaclass=ABCMeta):
"""A class taking care of executing operations.
The class can be configured to receive one operation at a time, or
all available operations in one go.
In the first case, subclasses must implement an execute() method
that takes an operation as input and should perform it. In the
second case, execute() takes a list of operations as input.
"""
def __init__(self, batch_executions=False):
"""Create an executor.
batch_executions (bool): if True, the executor will receive a
list of operations in the queue instead of one operation
at a time.
"""
super().__init__()
self._batch_executions = batch_executions
self._operation_queue = PriorityQueue()
def __contains__(self, item):
"""Return whether the item is in the queue.
item (QueueItem): the item to look for.
return (bool): whether operation is in the queue.
"""
return item in self._operation_queue
def get_status(self):
"""Return a the status of the queues.
More precisely, a list of entries in the executor's queue. The
first item is the top item, the others are not in order.
return ([QueueEntry]): the list with the queued elements.
"""
return self._operation_queue.get_status()
def enqueue(self, item, priority=None, timestamp=None):
"""Add an item to the queue.
item (QueueItem): the item to add.
priority (int|None) the priority, or None to use default.
timestamp (datetime|None) the timestamp of the first request
for the operation, or None to use now.
return (bool): True if successfully enqueued.
"""
return self._operation_queue.push(item, priority, timestamp)
def dequeue(self, item):
"""Remove an item from the queue.
item (QueueItem): the item to remove.
"""
self._operation_queue.remove(item)
def run(self):
"""Monitor the queue, and dispatch operations when available.
This is an infinite loop that, at each iteration, blocks until
there is at least an element in the queue, and then extracts
the operation(s) and dispatch them to the executor. Any error
during the operation is sent to the logger and then
suppressed, because the loop must go on.
"""
while True:
# Wait for the queue to be non-empty.
to_execute = [self._operation_queue.pop(wait=True)]
if self._batch_executions:
max_operations = self.max_operations_per_batch()
while not self._operation_queue.empty() and (
max_operations == 0
or len(to_execute) < max_operations):
to_execute.append(self._operation_queue.pop())
assert len(to_execute) > 0, "Expected at least one element."
if self._batch_executions:
try:
logger.info("Executing operations `%s' and %d more.",
to_execute[0].item,
len(to_execute) - 1)
self.execute(to_execute)
logger.info(
"Operations `%s' and %d more concluded "
"successfully.", to_execute[0].item,
len(to_execute) - 1)
except Exception:
logger.error(
"Unexpected error when executing operation "
"`%s' (and %d more operations).",
to_execute[0].item,
len(to_execute) - 1,
exc_info=True)
else:
try:
logger.info("Executing operation `%s'.",
to_execute[0].item)
self.execute(to_execute[0])
logger.info("Operation `%s' concluded successfully",
to_execute[0].item)
except Exception:
logger.error(
"Unexpected error when executing operation `%s'.",
to_execute[0].item,
exc_info=True)
def max_operations_per_batch(self):
"""Return the maximum number of operations in a batch.
If the service has batch executions, this method returns the
maximum size of a batch (the batch might be smaller if not
enough operations are present in the queue).
return (int): the maximum number of operations, or 0 to
indicate no limits.
"""
return 0
@abstractmethod
def execute(self, entry):
"""Perform a single operation.
Must be implemented if batch_execution is false.
entry (QueueEntry|[QueueEntry]): the top element of the queue,
in case batch_executions is false, or the list of all
currently available elements, in case it is true - in any
case, each element contains both the operations and the
info on priority and timestamp, in case we need to
re-enqueue the item.
"""
pass
