def PopItem(self):
"""Pops an item off the queue.
If no ZeroMQ socket has been created, one will be created the first
time this method is called.
Raises:
QueueEmpty: If the queue is empty, and no item could be popped within the
queue timeout.
zmq.error.ZMQError: If an error is encountered by ZeroMQ.
"""
logging.debug(u'Pop on {0:s} queue, port {1:d}'.format(
self.name, self.port))
if not self._zmq_socket:
self._CreateZMQSocket()
try:
return self._queue.get(timeout=self._buffer_timeout_seconds)
except Queue.Empty:
return queue.QueueAbort()
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
return queue.QueueAbort()
else:
raise
except KeyboardInterrupt:
self.Close(abort=True)
raise
def _StopExtractionProcesses(self, abort=False):
"""Stops the extraction processes.
Args:
abort: optional boolean to indicate the stop is issued on abort.
"""
self._StopProcessMonitoring()
# Note that multiprocessing.Queue is very sensitive regarding
# blocking on either a get or a put. So we try to prevent using
# any blocking behavior.
if abort:
# Signal all the processes to abort.
self._AbortTerminate()
# Wake the processes to make sure that they are not blocking
# waiting for the queue not to be full.
self._path_spec_queue.Empty()
self.event_object_queue.Empty()
self._parse_error_queue.Empty()
# On abort we need to have the collector process get a SIGTERM first.
self._stop_collector_event.set()
# Wake the processes to make sure that they are not blocking
# waiting for new items.
for _ in range(self._number_of_extraction_workers):
self._path_spec_queue.PushItem(queue.QueueAbort(), block=False)
self.event_object_queue.PushItem(queue.QueueAbort(), block=False)
# TODO: enable this when the parse error queue consumer is operational.
# self._parse_error_queue.PushItem(queue.QueueAbort(), block=False)
# Try terminating the processes in the normal way.
self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT)
if not abort:
# Check if the processes are still alive and terminate them if necessary.
self._AbortTerminate()
self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT)
# Set abort to True to stop queue.join_thread() from blocking.
self._path_spec_queue.Close(abort=True)
self.event_object_queue.Close(abort=True)
self._parse_error_queue.Close(abort=True)
if abort:
# Kill any remaining processes, which can be necessary if
# the collector dies.
self._AbortKill()
def _ZeroMQResponder(self, source_queue, socket, terminate_event):
"""Listens for requests and replies to clients.
Args:
source_queue: The queue to uses to pull items from.
socket: The socket to listen to, and send responses to.
terminate_event: The event that signals that the queue should terminate.
Raises:
QueueEmpty: If a timeout occurs when trying to reply to a request.
zmq.error.ZMQError: If an error occurs in ZeroMQ.
"""
logging.debug(u'ZeroMQ responder thread started')
while not terminate_event.isSet():
try:
# We need to receive a request before we send.
_ = socket.recv()
except zmq.error.Again:
logging.debug(u'{0:s} did not receive a request within the '
u'timeout of {1:d} seconds.'.format(
self.name, self.timeout_seconds))
continue
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
break
else:
raise
try:
if self._closed:
item = source_queue.get_nowait()
else:
item = source_queue.get(True, self._buffer_timeout_seconds)
except Queue.Empty:
item = queue.QueueAbort()
try:
self._zmq_socket.send_pyobj(item)
except zmq.error.Again:
logging.debug(
u'{0:s} could not reply to a request within {1:d} seconds.'
.format(self.name, self.timeout_seconds))
raise errors.QueueEmpty
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
break
else:
raise
socket.close(self._linger_seconds)
def _ZeroMQResponder(self, source_queue, socket, terminate_event):
"""Listens for requests and replies to clients.
Args:
source_queue: The queue to uses to pull items from.
socket: The socket to listen to, and send responses to.
terminate_event: The event that signals that the queue should terminate.
Raises:
zmq.error.ZMQError: If an error is encountered by ZeroMQ.
"""
logging.debug(u'ZeroMQ responder thread started')
while not terminate_event.isSet():
try:
item = socket.recv_pyobj()
except zmq.error.Again:
# No item received within timeout.
item = queue.QueueAbort()
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
break
else:
raise
retries = 0
while retries < self._DOWNSTREAM_QUEUE_MAX_TRIES:
try:
self._queue.put(item, timeout=self._buffer_timeout_seconds)
break
except Queue.Full:
logging.debug(u'Queue {0:s} buffer limit hit.'.format(
self.name))
retries += 1
if retries >= self._DOWNSTREAM_QUEUE_MAX_TRIES:
logging.error(
u'Queue {0:s} unserved for too long, aborting'.
format(self.name))
break
else:
time.sleep(self._DOWNSTREAM_QUEUE_SLEEP_TIME)
continue
logging.info(u'Queue {0:s} responder exiting.'.format(self.name))
def PushItem(self, item, block=True):
"""Push an item on to the queue.
If no ZeroMQ socket has been created, one will be created the first time
this method is called.
Args:
item: The item to push on to the queue.
block: Optional argument to indicate whether the push should be performed
in blocking or non-block mode.
Raises:
KeyboardInterrupt: If the process is sent a KeyboardInterrupt while
pushing an item.
zmq.error.Again: If it was not possible to push the item to the queue
within the timeout.
zmq.error.ZMQError: If a ZeroMQ specific error occurs.
"""
logging.debug(u'Push on {0:s} queue, port {1:d}'.format(
self.name, self.port))
if not self._zmq_socket:
self._CreateZMQSocket()
try:
if block:
self._zmq_socket.send_pyobj(item)
else:
self._zmq_socket.send_pyobj(item, zmq.DONTWAIT)
except zmq.error.Again:
logging.error(u'{0:s} unable to push item, raising.'.format(
self.name))
raise
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
return queue.QueueAbort()
else:
raise
except KeyboardInterrupt:
self.Close(abort=True)
raise
def PopItem(self):
"""Pops an item off the queue.
If no ZeroMQ socket has been created, one will be created the first
time this method is called.
Returns:
The item retrieved from the queue, as a deserialized Python object.
Raises:
KeyboardInterrupt: If the process is sent a KeyboardInterrupt while
popping an item.
QueueEmpty: If the queue is empty, and no item could be popped within the
queue timeout.
zmq.error.ZMQError: If an error occurs in ZeroMQ.
"""
logging.debug(u'Pop on {0:s} queue, port {1:d}'.format(
self.name, self.port))
if not self._zmq_socket:
self._CreateZMQSocket()
try:
self._zmq_socket.send_pyobj(None)
return self._zmq_socket.recv_pyobj()
except zmq.error.Again:
raise errors.QueueEmpty
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(
u'ZMQ syscall interrupted in {0:s}. Queue aborting.'.
format(self.name))
return queue.QueueAbort()
else:
raise
except KeyboardInterrupt:
self.Close(abort=True)
raise