class Executor(object): # pylint: disable=R0921
"""A class taking care of executing operations.
The class can be configured to receive one operation at a time, or
all available operations in one go.
In the first case, subclasses must implement an execute() method
that takes an operation as input and should perform it. In the
second case, execute() takes a list of operations as input.
"""
def __init__(self, batch_executions=False):
"""Create an executor.
batch_executions (bool): if True, the executor will receive a
list of operations in the queue instead of one operation
at a time.
"""
super(Executor, self).__init__()
self._batch_executions = batch_executions
self._operation_queue = PriorityQueue()
def __contains__(self, item):
"""Return whether the item is in the queue.
item (QueueItem): the item to look for.
return (bool): whether operation is in the queue.
"""
return item in self._operation_queue
def get_status(self):
"""Return a the status of the queues.
More precisely, a list of entries in the executor's queue. The
first item is the top item, the others are not in order.
return ([QueueEntry]): the list with the queued elements.
"""
return self._operation_queue.get_status()
def enqueue(self, item, priority=None, timestamp=None):
"""Add an item to the queue.
item (QueueItem): the item to add.
priority (int|None) the priority, or None to use default.
timestamp (datetime|None) the timestamp of the first request
for the operation, or None to use now.
return (bool): True if successfully enqueued.
"""
return self._operation_queue.push(item, priority, timestamp)
def dequeue(self, item):
"""Remove an item from the queue.
item (QueueItem): the item to remove.
"""
self._operation_queue.remove(item)
def run(self):
"""Monitor the queue, and dispatch operations when available.
This is an infinite loop that, at each iteration, blocks until
there is at least an element in the queue, and then extracts
the operation(s) and dispatch them to the executor. Any error
during the operation is sent to the logger and then
suppressed, because the loop must go on.
"""
while True:
# Wait for the queue to be non-empty.
to_execute = [self._operation_queue.pop(wait=True)]
if self._batch_executions:
# TODO: shall we yield to other greenlets? I think
# that it is going to be extremely unlikely to have
# more than one operations.
while not self._operation_queue.empty():
to_execute.append(self._operation_queue.pop())
assert len(to_execute) > 0, "Expected at least one element."
if self._batch_executions:
try:
logger.info("Executing operations `%s' and %d more.",
to_execute[0].item, len(to_execute) - 1)
self.execute(to_execute)
logger.info("Operations `%s' and %d more concluded "
"successfully.", to_execute[0].item,
len(to_execute) - 1)
except Exception:
logger.error(
"Unexpected error when executing operation "
"`%s' (and %d more operations).", to_execute[0].item,
len(to_execute) - 1, exc_info=True)
else:
try:
logger.info("Executing operation `%s'.",
to_execute[0].item)
self.execute(to_execute[0])
logger.info("Operation `%s' concluded successfully",
to_execute[0].item)
except Exception:
logger.error(
"Unexpected error when executing operation `%s'.",
to_execute[0].item, exc_info=True)
def execute(self, entry):
"""Perform a single operation.
Must be implemented if batch_execution is false.
entry (QueueEntry|[QueueEntry]): the top element of the queue,
in case batch_executions is false, or the list of all
currently available elements, in case it is true - in any
case, each element contains both the operations and the
info on priority and timestamp, in case we need to
re-enqueue the item.
"""
raise NotImplementedError("Please use a subclass.")
class Executor(object): # pylint: disable=R0921
"""A class taking care of executing operations.
The class can be configured to receive one operation at a time, or
all available operations in one go.
In the first case, subclasses must implement an execute() method
that takes an operation as input and should perform it. In the
second case, execute() takes a list of operations as input.
"""
def __init__(self, batch_executions=False):
"""Create an executor.
batch_executions (bool): if True, the executor will receive a
list of operations in the queue instead of one operation
at a time.
"""
super(Executor, self).__init__()
self._batch_executions = batch_executions
self._operation_queue = PriorityQueue()
def get_status(self):
"""Return a the status of the queues.
More precisely, a list of entries in the executor's queue. The
first item is the top item, the others are not in order.
return ([QueueEntry]): the list with the queued elements.
"""
return self._operation_queue.get_status()
def enqueue(self, item, priority=None, timestamp=None):
"""Add an item to the queue.
item (QueueItem): the item to add.
priority (int|None) the priority, or None to use default.
timestamp (datetime|None) the timestamp of the first request
for the operation, or None to use now.
return (bool): True if successfully enqueued.
"""
return self._operation_queue.push(item, priority, timestamp)
def dequeue(self, item):
"""Remove an item from the queue.
item (QueueItem): the item to remove.
"""
self._operation_queue.remove(item)
def run(self):
"""Monitor the queue, and dispatch operations when available.
This is an infinite loop that, at each iteration, blocks until
there is at least an element in the queue, and then extracts
the operation(s) and dispatch them to the executor. Any error
during the operation is sent to the logger and then
suppressed, because the loop must go on.
"""
while True:
# Wait for the queue to be non-empty.
to_execute = [self._operation_queue.pop(wait=True)]
if self._batch_executions:
# TODO: shall we yield to other greenlets? I think
# that it is going to be extremely unlikely to have
# more than one operations.
while not self._operation_queue.empty():
to_execute.append(self._operation_queue.pop())
assert len(to_execute) > 0, "Expected at least one element."
if self._batch_executions:
try:
logger.info("Executing operations `%s' and %d more.",
to_execute[0].item,
len(to_execute) - 1)
self.execute(to_execute)
logger.info(
"Operations `%s' and %d more concluded "
"successfully.", to_execute[0].item,
len(to_execute) - 1)
except Exception:
logger.error(
"Unexpected error when executing operation "
"`%s' (and %d more operations).",
to_execute[0].item,
len(to_execute) - 1,
exc_info=True)
else:
try:
logger.info("Executing operation `%s'.",
to_execute[0].item)
self.execute(to_execute[0])
logger.info("Operation `%s' concluded successfully",
to_execute[0].item)
except Exception:
logger.error(
"Unexpected error when executing operation `%s'.",
to_execute[0].item,
exc_info=True)
def execute(self, entry):
"""Perform a single operation.
Must be implemented if batch_execution is false.
entry (QueueEntry|[QueueEntry]): the top element of the queue,
in case batch_executions is false, or the list of all
currently available elements, in case it is true - in any
case, each element contains both the operations and the
info on priority and timestamp, in case we need to
re-enqueue the item.
"""
raise NotImplementedError("Please use a subclass.")
class Executor(metaclass=ABCMeta):
"""A class taking care of executing operations.
The class can be configured to receive one operation at a time, or
all available operations in one go.
In the first case, subclasses must implement an execute() method
that takes an operation as input and should perform it. In the
second case, execute() takes a list of operations as input.
"""
def __init__(self, batch_executions=False):
"""Create an executor.
batch_executions (bool): if True, the executor will receive a
list of operations in the queue instead of one operation
at a time.
"""
super().__init__()
self._batch_executions = batch_executions
self._operation_queue = PriorityQueue()
def __contains__(self, item):
"""Return whether the item is in the queue.
item (QueueItem): the item to look for.
return (bool): whether operation is in the queue.
"""
return item in self._operation_queue
def get_status(self):
"""Return a the status of the queues.
More precisely, a list of entries in the executor's queue. The
first item is the top item, the others are not in order.
return ([QueueEntry]): the list with the queued elements.
"""
return self._operation_queue.get_status()
def enqueue(self, item, priority=None, timestamp=None):
"""Add an item to the queue.
item (QueueItem): the item to add.
priority (int|None) the priority, or None to use default.
timestamp (datetime|None) the timestamp of the first request
for the operation, or None to use now.
return (bool): True if successfully enqueued.
"""
return self._operation_queue.push(item, priority, timestamp)
def dequeue(self, item):
"""Remove an item from the queue.
item (QueueItem): the item to remove.
"""
self._operation_queue.remove(item)
def run(self):
"""Monitor the queue, and dispatch operations when available.
This is an infinite loop that, at each iteration, blocks until
there is at least an element in the queue, and then extracts
the operation(s) and dispatch them to the executor. Any error
during the operation is sent to the logger and then
suppressed, because the loop must go on.
"""
while True:
# Wait for the queue to be non-empty.
to_execute = [self._operation_queue.pop(wait=True)]
if self._batch_executions:
max_operations = self.max_operations_per_batch()
while not self._operation_queue.empty() and (
max_operations == 0
or len(to_execute) < max_operations):
to_execute.append(self._operation_queue.pop())
assert len(to_execute) > 0, "Expected at least one element."
if self._batch_executions:
try:
logger.info("Executing operations `%s' and %d more.",
to_execute[0].item,
len(to_execute) - 1)
self.execute(to_execute)
logger.info(
"Operations `%s' and %d more concluded "
"successfully.", to_execute[0].item,
len(to_execute) - 1)
except Exception:
logger.error(
"Unexpected error when executing operation "
"`%s' (and %d more operations).",
to_execute[0].item,
len(to_execute) - 1,
exc_info=True)
else:
try:
logger.info("Executing operation `%s'.",
to_execute[0].item)
self.execute(to_execute[0])
logger.info("Operation `%s' concluded successfully",
to_execute[0].item)
except Exception:
logger.error(
"Unexpected error when executing operation `%s'.",
to_execute[0].item,
exc_info=True)
def max_operations_per_batch(self):
"""Return the maximum number of operations in a batch.
If the service has batch executions, this method returns the
maximum size of a batch (the batch might be smaller if not
enough operations are present in the queue).
return (int): the maximum number of operations, or 0 to
indicate no limits.
"""
return 0
@abstractmethod
def execute(self, entry):
"""Perform a single operation.
Must be implemented if batch_execution is false.
entry (QueueEntry|[QueueEntry]): the top element of the queue,
in case batch_executions is false, or the list of all
currently available elements, in case it is true - in any
case, each element contains both the operations and the
info on priority and timestamp, in case we need to
re-enqueue the item.
"""
pass
