예제 #1
0
파일: nsq_to_nsq.py 프로젝트: elubow/pynsq
class NSQProxy:
    def __init__(self, topic, nsqds):
        self.topic = topic
        self.writer_pool = HostPool([Writer([nsqd]) for nsqd in nsqds])

    def relay(self, nsq_message):
        nsq_message.enable_async()
        writer = self.writer_pool.get()
        callback = functools.partial(self._on_message_response, nsq_message=nsq_message, writer=writer)
        writer.pub(self.topic, nsq_message.body, callback)

    def _on_message_response(self, conn, data, nsq_message, writer):
        if isinstance(data, Error):
            logging.warning("requeuing message: %s", nsq_message.body)
            self.writer_pool.failed(writer)
            nsq_message.requeue()
        else:
            self.writer_pool.success(writer)
            nsq_message.finish()
예제 #2
0
class NSQProxy:
    def __init__(self, topic, nsqds):
        self.topic = topic
        self.writer_pool = HostPool([Writer([nsqd]) for nsqd in nsqds])

    def relay(self, nsq_message):
        nsq_message.enable_async()
        writer = self.writer_pool.get()
        callback = functools.partial(
            self._on_message_response, nsq_message=nsq_message, writer=writer)
        writer.pub(self.topic, nsq_message.body, callback)

    def _on_message_response(self, conn, data, nsq_message, writer):
        if isinstance(data, Error):
            logging.warning("requeuing message: %s", nsq_message.body)
            self.writer_pool.failed(writer)
            nsq_message.requeue()
        else:
            self.writer_pool.success(writer)
            nsq_message.finish()
예제 #3
0
    def __init__(self, simplequeue_address, all_tasks, max_tries=5, sleeptime_failed_queue=5,
            sleeptime_queue_empty=0.5, sleeptime_requeue=1, requeue_delay=90, mget_items=0,
            failed_count=0, queuename=None, preprocess_method=None, validate_method=None,
            requeue_giveup=None, failed_message_dir=None):
        """
        BaseReader provides a queue that calls each task provided by ``all_tasks`` up to ``max_tries``
        requeueing on any failures with increasing multiples of ``requeue_delay`` between subsequent
        tries of each message.

        ``preprocess_method`` defines an optional method that can alter the message data before
        other task functions are called.
        ``validate_method`` defines an optional method that returns a boolean as to weather or not
        this message should be processed.
        ``all_tasks``  defines the a mapping of tasks and functions that individually will be called
        with the message data.
        ``requeue_giveup`` defines a callback for when a message has been called ``max_tries`` times
        ``failed_message_dir`` defines a directory where failed messages should be written to
        """
        assert isinstance(all_tasks, dict)
        for key, method in all_tasks.items():
            assert callable(method), "key %s must have a callable value" % key
        if preprocess_method:
            assert callable(preprocess_method)
        if validate_method:
            assert callable(validate_method)
        assert isinstance(queuename, (str, unicode))
        assert isinstance(mget_items, int)
        
        if not isinstance(simplequeue_address, HostPool):
            if isinstance(simplequeue_address, (str, unicode)):
                simplequeue_address = [simplequeue_address]
            assert isinstance(simplequeue_address, (list, set, tuple))
            simplequeue_address = HostPool(simplequeue_address)
        
        self.simplequeue_address = simplequeue_address
        self.max_tries = max_tries
        self.requeue_giveup = requeue_giveup
        self.mget_items = mget_items
        self.sleeptime_failed_queue = sleeptime_failed_queue
        self.sleeptime_queue_empty = sleeptime_queue_empty
        self.sleeptime_requeue = sleeptime_requeue
        self.requeue_delay = requeue_delay  # seconds
        ## max delay time is requeue_delay * (max_tries + max_tries-1 + max_tries-2 ... 1)
        self.failed_message_dir = failed_message_dir or tempfile.mkdtemp()
        assert os.access(self.failed_message_dir, os.W_OK)
        self.failed_count = failed_count
        self.queuename = queuename
        self.task_lookup = all_tasks
        self.preprocess_method = preprocess_method
        self.validate_method = validate_method
        self.backoff_timer = dict((k, BackoffTimer.BackoffTimer(0, 120)) for k in self.task_lookup.keys())
        # a default backoff timer
        self.backoff_timer['__preprocess'] = BackoffTimer.BackoffTimer(0, 120)
        self.quit_flag = False
예제 #4
0
 def __init__(self, topic, nsqds):
     self.topic = topic
     self.writer_pool = HostPool([Writer([nsqd]) for nsqd in nsqds])
예제 #5
0
파일: nsq_to_nsq.py 프로젝트: nsqio/pynsq
 def __init__(self, topic, nsqds):
     self.topic = topic
     self.writer_pool = HostPool([Writer([nsqd]) for nsqd in nsqds])