예제 #1
0
    def run_forever(self):
        """Keep running (unless we get a Ctrl-C).
        """
        while True:
            message = None
            try:
                self.wait_if_paused()

                message = self.source_queue.get(block=True)
                data = message.payload

                with stats.time_block(self.statsd_client):
                    new_messages = self.handle_data(data)

                # Must be successful if we have reached here.
                stats.mark_successful_job(self.statsd_client)

                self.post_handle_data()

            except KeyboardInterrupt:
                logging.info("Caught Ctrl-C. Byee!")
                # Break out of our loop.
                #
                break

            except:
                # Keep going, but don't ack the message.
                # Also, log the exception.
                logging.exception("Exception handling data")

                if self.handle_exception is not None:
                    self.handle_exception()

                if message:
                    if self.requeue:
                        message.requeue()
                    elif self.reject:
                        message.reject()

                stats.mark_failed_job(self.statsd_client)

            else:
                # Queue up the new messages (if any).
                #
                if new_messages:
                    self.queue_new_messages(new_messages)

                # We're done with the original message.
                #
                message.ack()
예제 #2
0
    def batched_run_forever(self, size, wait_timeout_seconds=5):
        """This will take messages off the queue and put them in a buffer.
        Once the buffer reaches the given size, handle_data is called for the
        entire buffer. (So handle_data must be able to handle a list.)
        If handle_data doesn't throw an exception, all messages are ack'd.
        Otherwise all messages are requeued/rejected.
        """
        buffer = []

        while True:
            new_messages = []
            try:
                self.wait_if_paused()

                queue_was_empty = False
                message = None

                try:
                    # We need to have a timeout. Otherwise if we had no more
                    # messages coming in, but len(buffer) < size, then the
                    # buffer would never get processed!
                    message = self.source_queue.get(block=True, timeout=wait_timeout_seconds)
                except queue.Empty:
                    queue_was_empty = True

                if message:
                    buffer.append(message)

                # We proceed to handle the buffer if
                # 1. it has reached the given size or
                # 2. we drained the queue, but the buffer is smaller than size
                if len(buffer) >= size or (buffer and queue_was_empty):
                    try:
                        with stats.time_block(self.statsd_client):
                            new_messages = self.handle_batch(buffer)

                        stats.mark_successful_job(self.statsd_client)
                        self.post_handle_data()
                    except KeyboardInterrupt as ki:
                        # Raise this for the outer try to handle.
                        raise ki
                    except:
                        logging.exception("Exception handling batch")
                        if self.handle_exception is not None:
                            self.handle_exception()
                        stats.mark_failed_job(self.statsd_client)
                    finally:
                        # If all went well then we have ack'd the messages.
                        # If not, we have requeued or rejected them.
                        # Either way we are done with the buffer.
                        buffer = []

            except KeyboardInterrupt:
                logging.info("Caught Ctrl-C. Byee!")
                break

            except:
                # When could this happen? Perhaps while waiting?
                logging.exception("Exception elsewhere in batched_run_forever")

            else:
                if new_messages:
                    self.queue_new_messages(new_messages)