def PopItem(self): """Pops an item off the queue. If no ZeroMQ socket has been created, one will be created the first time this method is called. Raises: QueueEmpty: If the queue is empty, and no item could be popped within the queue timeout. zmq.error.ZMQError: If an error is encountered by ZeroMQ. """ logging.debug(u'Pop on {0:s} queue, port {1:d}'.format( self.name, self.port)) if not self._zmq_socket: self._CreateZMQSocket() try: return self._queue.get(timeout=self._buffer_timeout_seconds) except Queue.Empty: return queue.QueueAbort() except zmq.error.ZMQError as exception: if exception.errno == errno.EINTR: logging.error(u'ZMQ syscall interrupted in {0:s}.'.format( self.name)) return queue.QueueAbort() else: raise except KeyboardInterrupt: self.Close(abort=True) raise
def _StopExtractionProcesses(self, abort=False): """Stops the extraction processes. Args: abort: optional boolean to indicate the stop is issued on abort. """ self._StopProcessMonitoring() # Note that multiprocessing.Queue is very sensitive regarding # blocking on either a get or a put. So we try to prevent using # any blocking behavior. if abort: # Signal all the processes to abort. self._AbortTerminate() # Wake the processes to make sure that they are not blocking # waiting for the queue not to be full. self._path_spec_queue.Empty() self.event_object_queue.Empty() self._parse_error_queue.Empty() # On abort we need to have the collector process get a SIGTERM first. self._stop_collector_event.set() # Wake the processes to make sure that they are not blocking # waiting for new items. for _ in range(self._number_of_extraction_workers): self._path_spec_queue.PushItem(queue.QueueAbort(), block=False) self.event_object_queue.PushItem(queue.QueueAbort(), block=False) # TODO: enable this when the parse error queue consumer is operational. # self._parse_error_queue.PushItem(queue.QueueAbort(), block=False) # Try terminating the processes in the normal way. self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT) if not abort: # Check if the processes are still alive and terminate them if necessary. self._AbortTerminate() self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT) # Set abort to True to stop queue.join_thread() from blocking. self._path_spec_queue.Close(abort=True) self.event_object_queue.Close(abort=True) self._parse_error_queue.Close(abort=True) if abort: # Kill any remaining processes, which can be necessary if # the collector dies. self._AbortKill()
def _ZeroMQResponder(self, source_queue, socket, terminate_event): """Listens for requests and replies to clients. Args: source_queue: The queue to uses to pull items from. socket: The socket to listen to, and send responses to. terminate_event: The event that signals that the queue should terminate. Raises: QueueEmpty: If a timeout occurs when trying to reply to a request. zmq.error.ZMQError: If an error occurs in ZeroMQ. """ logging.debug(u'ZeroMQ responder thread started') while not terminate_event.isSet(): try: # We need to receive a request before we send. _ = socket.recv() except zmq.error.Again: logging.debug(u'{0:s} did not receive a request within the ' u'timeout of {1:d} seconds.'.format( self.name, self.timeout_seconds)) continue except zmq.error.ZMQError as exception: if exception.errno == errno.EINTR: logging.error(u'ZMQ syscall interrupted in {0:s}.'.format( self.name)) break else: raise try: if self._closed: item = source_queue.get_nowait() else: item = source_queue.get(True, self._buffer_timeout_seconds) except Queue.Empty: item = queue.QueueAbort() try: self._zmq_socket.send_pyobj(item) except zmq.error.Again: logging.debug( u'{0:s} could not reply to a request within {1:d} seconds.' .format(self.name, self.timeout_seconds)) raise errors.QueueEmpty except zmq.error.ZMQError as exception: if exception.errno == errno.EINTR: logging.error(u'ZMQ syscall interrupted in {0:s}.'.format( self.name)) break else: raise socket.close(self._linger_seconds)
def _ZeroMQResponder(self, source_queue, socket, terminate_event): """Listens for requests and replies to clients. Args: source_queue: The queue to uses to pull items from. socket: The socket to listen to, and send responses to. terminate_event: The event that signals that the queue should terminate. Raises: zmq.error.ZMQError: If an error is encountered by ZeroMQ. """ logging.debug(u'ZeroMQ responder thread started') while not terminate_event.isSet(): try: item = socket.recv_pyobj() except zmq.error.Again: # No item received within timeout. item = queue.QueueAbort() except zmq.error.ZMQError as exception: if exception.errno == errno.EINTR: logging.error(u'ZMQ syscall interrupted in {0:s}.'.format( self.name)) break else: raise retries = 0 while retries < self._DOWNSTREAM_QUEUE_MAX_TRIES: try: self._queue.put(item, timeout=self._buffer_timeout_seconds) break except Queue.Full: logging.debug(u'Queue {0:s} buffer limit hit.'.format( self.name)) retries += 1 if retries >= self._DOWNSTREAM_QUEUE_MAX_TRIES: logging.error( u'Queue {0:s} unserved for too long, aborting'. format(self.name)) break else: time.sleep(self._DOWNSTREAM_QUEUE_SLEEP_TIME) continue logging.info(u'Queue {0:s} responder exiting.'.format(self.name))
def PushItem(self, item, block=True): """Push an item on to the queue. If no ZeroMQ socket has been created, one will be created the first time this method is called. Args: item: The item to push on to the queue. block: Optional argument to indicate whether the push should be performed in blocking or non-block mode. Raises: KeyboardInterrupt: If the process is sent a KeyboardInterrupt while pushing an item. zmq.error.Again: If it was not possible to push the item to the queue within the timeout. zmq.error.ZMQError: If a ZeroMQ specific error occurs. """ logging.debug(u'Push on {0:s} queue, port {1:d}'.format( self.name, self.port)) if not self._zmq_socket: self._CreateZMQSocket() try: if block: self._zmq_socket.send_pyobj(item) else: self._zmq_socket.send_pyobj(item, zmq.DONTWAIT) except zmq.error.Again: logging.error(u'{0:s} unable to push item, raising.'.format( self.name)) raise except zmq.error.ZMQError as exception: if exception.errno == errno.EINTR: logging.error(u'ZMQ syscall interrupted in {0:s}.'.format( self.name)) return queue.QueueAbort() else: raise except KeyboardInterrupt: self.Close(abort=True) raise
def PopItem(self): """Pops an item off the queue. If no ZeroMQ socket has been created, one will be created the first time this method is called. Returns: The item retrieved from the queue, as a deserialized Python object. Raises: KeyboardInterrupt: If the process is sent a KeyboardInterrupt while popping an item. QueueEmpty: If the queue is empty, and no item could be popped within the queue timeout. zmq.error.ZMQError: If an error occurs in ZeroMQ. """ logging.debug(u'Pop on {0:s} queue, port {1:d}'.format( self.name, self.port)) if not self._zmq_socket: self._CreateZMQSocket() try: self._zmq_socket.send_pyobj(None) return self._zmq_socket.recv_pyobj() except zmq.error.Again: raise errors.QueueEmpty except zmq.error.ZMQError as exception: if exception.errno == errno.EINTR: logging.error( u'ZMQ syscall interrupted in {0:s}. Queue aborting.'. format(self.name)) return queue.QueueAbort() else: raise except KeyboardInterrupt: self.Close(abort=True) raise