def listen_for_changes():
  wikipedia_exchange = Exchange('wikipedia', type='direct', durable=False)
  queue = Queue('status', wikipedia_exchange, routing_key='status')
  with Connection() as conn:
    with Consumer(conn, queue, callbacks=[handle_message]):
      for _ in eventloop(conn):
        pass
Example #2
0
def listen_for_changes():
    wikipedia_exchange = Exchange('wikipedia', type='direct', durable=False)
    queue = Queue('status', wikipedia_exchange, routing_key='status')
    with Connection() as conn:
        with Consumer(conn, queue, callbacks=[handle_message]):
            for _ in eventloop(conn):
                pass
Example #3
0
def start_filter(app, conn, filter, limit=None, timeout=1.0,
        ack_messages=False, tasks=None, queues=None,
        callback=None, forever=False, on_declare_queue=None,
        consume_from=None, state=None, **kwargs):
    state = state or State()
    queues = prepare_queues(queues)
    if isinstance(tasks, basestring):
        tasks = set(tasks.split(','))
    if tasks is None:
        tasks = set([])

    def update_state(body, message):
        state.count += 1
        if limit and state.count >= limit:
            raise StopFiltering()

    def ack_message(body, message):
        message.ack()

    consumer = app.amqp.TaskConsumer(conn, queues=consume_from)

    if tasks:
        filter = filter_callback(filter, tasks)
        update_state = filter_callback(update_state, tasks)
        ack_message = filter_callback(ack_message, tasks)

    consumer.register_callback(filter)
    consumer.register_callback(update_state)
    if ack_messages:
        consumer.register_callback(ack_message)
    if callback is not None:
        callback = partial(callback, state)
        if tasks:
            callback = filter_callback(callback, tasks)
        consumer.register_callback(callback)

    # declare all queues on the new broker.
    for queue in consumer.queues:
        if queues and queue.name not in queues:
            continue
        if on_declare_queue is not None:
            on_declare_queue(queue)
        try:
            _, mcount, _ = queue(consumer.channel).queue_declare(passive=True)
            if mcount:
                state.total_apx += mcount
        except conn.channel_errors + (StdChannelError, ):
            pass

    # start migrating messages.
    with consumer:
        try:
            for _ in eventloop(conn,  # pragma: no cover
                               timeout=timeout, ignore_timeouts=forever):
                pass
        except socket.timeout:
            pass
        except StopFiltering:
            pass
    return state
Example #4
0
def start_filter(app, conn, filter, limit=None, timeout=1.0,
        ack_messages=False, tasks=None, queues=None,
        callback=None, forever=False, on_declare_queue=None,
        consume_from=None, state=None, **kwargs):
    state = state or State()
    queues = prepare_queues(queues)
    if isinstance(tasks, string_t):
        tasks = set(tasks.split(','))
    if tasks is None:
        tasks = set([])

    def update_state(body, message):
        state.count += 1
        if limit and state.count >= limit:
            raise StopFiltering()

    def ack_message(body, message):
        message.ack()

    consumer = app.amqp.TaskConsumer(conn, queues=consume_from)

    if tasks:
        filter = filter_callback(filter, tasks)
        update_state = filter_callback(update_state, tasks)
        ack_message = filter_callback(ack_message, tasks)

    consumer.register_callback(filter)
    consumer.register_callback(update_state)
    if ack_messages:
        consumer.register_callback(ack_message)
    if callback is not None:
        callback = partial(callback, state)
        if tasks:
            callback = filter_callback(callback, tasks)
        consumer.register_callback(callback)

    # declare all queues on the new broker.
    for queue in consumer.queues:
        if queues and queue.name not in queues:
            continue
        if on_declare_queue is not None:
            on_declare_queue(queue)
        try:
            _, mcount, _ = queue(consumer.channel).queue_declare(passive=True)
            if mcount:
                state.total_apx += mcount
        except conn.channel_errors + (StdChannelError, ):
            pass

    # start migrating messages.
    with consumer:
        try:
            for _ in eventloop(conn,  # pragma: no cover
                               timeout=timeout, ignore_timeouts=forever):
                pass
        except socket.timeout:
            pass
        except StopFiltering:
            pass
    return state
Example #5
0
 def receive_message(self):
     with Consumer(self.connection,
                   on_message=self.on_response,
                   queues=[self.callback_queue], no_ack=True):
         for _ in eventloop(self.connection):
             if self.terminated:
                 break
Example #6
0
def migrate_tasks(source, dest, timeout=1.0, app=None,
        migrate=None, callback=None):
    state = State()
    app = app_or_default(app)

    def update_state(body, message):
        state.count += 1

    producer = app.amqp.TaskProducer(dest)
    if migrate is None:
        migrate = partial(migrate_task, producer)
    consumer = app.amqp.TaskConsumer(source)
    consumer.register_callback(update_state)
    if callback is not None:
        callback = partial(callback, state)
        consumer.register_callback(callback)
    consumer.register_callback(migrate)

    # declare all queues on the new broker.
    for queue in consumer.queues:
        queue(producer.channel).declare()
        try:
            _, mcount, _ = queue(consumer.channel).queue_declare(passive=True)
            if mcount:
                state.total_apx += mcount
        except source.channel_errors + (StdChannelError, ):
            pass

    # start migrating messages.
    with consumer:
        try:
            for _ in eventloop(source, timeout=timeout):  # pragma: no cover
                pass
        except socket.timeout:
            return
Example #7
0
def main(arguments):
    #: By default messages sent to exchanges are persistent (delivery_mode=2),
    #: and queues and exchanges are durable.
    exchange = Exchange('kombu_demo', type='direct')
    queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')

    def pretty(obj):
        return pformat(obj, indent=4)

    #: This is the callback applied when a message is received.
    def handle_message(body, message):
        print('Received message: {0!r}'.format(body))
        print('  properties:\n{0}'.format(pretty(message.properties)))
        print('  delivery_info:\n{0}'.format(pretty(message.delivery_info)))
        message.ack()

    #: Create a connection and a channel.
    #: If hostname, userid, password and virtual_host is not specified
    #: the values below are the default, but listed here so it can
    #: be easily changed.
    with Connection('redis://localhost:6379') as connection:

        #: Create consumer using our callback and queue.
        #: Second argument can also be a list to consume from
        #: any number of queues.
        with Consumer(connection, queue, callbacks=[handle_message]):

            #: Each iteration waits for a single event.  Note that this
            #: event may not be a message, or a message that is to be
            #: delivered to the consumers channel, but any event received
            #: on the connection.
            for _ in eventloop(connection):
                pass
Example #8
0
 def start(self):
     # start migrating messages.
     with self.prepare_consumer(self.create_consumer()):
         try:
             for _ in eventloop(self.conn,  # pragma: no cover
                                timeout=self.timeout,
                                ignore_timeouts=self.forever):
                 pass
         except socket.timeout:
             pass
         except StopFiltering:
             pass
     return self.state
Example #9
0
 def start(self):
     # start migrating messages.
     with self.prepare_consumer(self.create_consumer()):
         try:
             for _ in eventloop(self.conn,  # pragma: no cover
                                timeout=self.timeout,
                                ignore_timeouts=self.forever):
                 pass
         except socket.timeout:
             pass
         except StopFiltering:
             pass
     return self.state
Example #10
0
 def listen(self, callbacks: Optional[list] = None) -> None:
     """
     Start a listener and handle messeages with the callback(s).  If the
     queue does not already exist in the exchange, it will be created.
     """
     with kombu.Connection(self.connection_str) as conn:
         with kombu.Consumer(conn, [self.queue],
                             callbacks=callbacks,
                             no_ack=True):
             try:
                 print("STARTING LISTENER")
                 for _ in kombu.eventloop(conn):
                     pass
             except KeyboardInterrupt:
                 print("\nExiting\n")
Example #11
0
    print('  properties:\n%s' % (pretty(message.properties), ))
    print('  delivery_info:\n%s' % (pretty(message.delivery_info), ))
    message.ack()

#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('pyamqp://*****:*****@localhost:5672//') as connection:
    # The configuration of the message flow is as follows:
    #   gateway_kombu_exchange -> internal_kombu_exchange -> kombu_demo queue
    gateway_exchange = Exchange('gateway_kombu_demo')(connection)
    exchange = Exchange('internal_kombu_demo')(connection)
    gateway_exchange.declare()
    exchange.declare()
    exchange.bind_to(gateway_exchange, routing_key='kombu_demo')

    queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')

    #: Create consumer using our callback and queue.
    #: Second argument can also be a list to consume from
    #: any number of queues.
    with Consumer(connection, queue, callbacks=[handle_message]):

        #: This waits for a single event.  Note that this event may not
        #: be a message, or a message that is to be delivered to the consumers
        #: channel, but any event received on the connection.
        recv = eventloop(connection)
        while True:
            recv.next()
Example #12
0

def pretty(obj):
    return pformat(obj, indent=4)


#: This is the callback applied when a message is received.
def handle_message(body, message):
    print('Received message: {0!r}'.format(body))
    print('  properties:\n{0}'.format(pretty(message.properties)))
    print('  delivery_info:\n{0}'.format(pretty(message.delivery_info)))
    message.ack()

#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('amqp://*****:*****@localhost:5672//') as connection:

    #: Create consumer using our callback and queue.
    #: Second argument can also be a list to consume from
    #: any number of queues.
    with Consumer(connection, queue, callbacks=[handle_message]):

        #: Each iteration waits for a single event.  Note that this
        #: event may not be a message, or a message that is to be
        #: delivered to the consumers channel, but any event received
        #: on the connection.
        for _ in eventloop(connection):
            pass
 def drain_events(self, **kwargs):
     for _ in eventloop(self.connection, **kwargs):
         pass
Example #14
0
exchange = Exchange('kombu_demo', type='direct')
queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')


def pretty(obj):
    return pformat(obj, indent=4)


#: This is the callback applied when a message is received.
def handle_message(body, message):
    print('Received message: %r' % (body, ))
    print('  properties:\n%s' % (pretty(message.properties), ))
    print('  delivery_info:\n%s' % (pretty(message.delivery_info), ))
    message.ack()

#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('amqp://*****:*****@localhost:5672//') as connection:

    #: Create consumer using our callback and queue.
    #: Second argument can also be a list to consume from
    #: any number of queues.
    with Consumer(connection, queue, callbacks=[handle_message]):

        #: This waits for a single event.  Note that this event may not
        #: be a message, or a message that is to be delivered to the consumers
        #: channel, but any event received on the connection.
        eventloop(connection, limit=1, timeout=10.0)
 def drain_events(self, **kwargs):
     for _ in eventloop(self.connection, **kwargs):
         pass
Example #16
0
 def listener(self):
     self.lock_client.add_queue(self.lock_client_q)
     self.lock_client.consume(no_ack=True)
     for _ in eventloop(self.lock_client.connection):
         pass
Example #17
0
 def _consume_messages(self):
     for _ in eventloop(self._lock_manager.connection,
                        timeout=1,
                        ignore_timeouts=True):
         pass
Example #18
0
queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')


def pretty(obj):
    return pformat(obj, indent=4)


#: This is the callback applied when a message is received.
def handle_message(body, message):
    print('Received message: %r' % (body, ))
    print('  properties:\n%s' % (pretty(message.properties), ))
    print('  delivery_info:\n%s' % (pretty(message.delivery_info), ))
    message.ack()


#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('amqp://*****:*****@localhost:5672//') as connection:

    #: Create consumer using our callback and queue.
    #: Second argument can also be a list to consume from
    #: any number of queues.
    with Consumer(connection, queue, callbacks=[handle_message]):

        #: This waits for a single event.  Note that this event may not
        #: be a message, or a message that is to be delivered to the consumers
        #: channel, but any event received on the connection.
        eventloop(connection, limit=1, timeout=10.0)
Example #19
0
def pretty(obj):
    return pformat(obj, indent=4)


#: This is the callback applied when a message is received.
def handle_message(body, message):
    print('Received message: %r' % (body, ))
    print('  properties:\n%s' % (pretty(message.properties), ))
    print('  delivery_info:\n%s' % (pretty(message.delivery_info), ))
    message.ack()


#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('amqp://*****:*****@localhost:5672//') as connection:

    #: Create consumer using our callback and queue.
    #: Second argument can also be a list to consume from
    #: any number of queues.
    with Consumer(connection, queue, callbacks=[handle_message]):

        #: Each iteration waits for a single event.  Note that this
        #: event may not be a message, or a message that is to be
        #: delivered to the consumers channel, but any event received
        #: on the connection.
        for _ in eventloop(connection):
            pass
Example #20
0
def migrate_tasks(source, dest, limit=None, timeout=1.0, ack_messages=False,
        app=None, migrate=migrate_task, tasks=None, queues=None, callback=None,
        forever=False, **kwargs):
    state = State()
    app = app_or_default(app)

    if isinstance(queues, basestring):
        queues = queues.split(',')
    if isinstance(queues, list):
        queues = dict(tuple(islice(cycle(q.split(':')), None, 2))
                        for q in queues)
    if queues is None:
        queues = {}

    if isinstance(tasks, basestring):
        tasks = set(tasks.split(','))
    if tasks is None:
        tasks = set([])

    def update_state(body, message):
        state.count += 1

    def ack_message(body, message):
        message.ack()

    producer = app.amqp.TaskProducer(dest)
    migrate = partial(migrate, producer, queues=queues)
    consumer = app.amqp.TaskConsumer(source)

    if tasks:
        migrate = filter_callback(migrate, tasks)
        update_state = filter_callback(update_state, tasks)
        ack_message = filter_callback(ack_message, tasks)

    consumer.register_callback(migrate)
    consumer.register_callback(update_state)
    if ack_messages:
        consumer.register_callback(ack_message)
    if callback is not None:
        callback = partial(callback, state)
        if tasks:
            callback = filter_callback(callback, tasks)
        consumer.register_callback(callback)

    # declare all queues on the new broker.
    for queue in consumer.queues:
        if queues and queue.name not in queues:
            continue

        new_queue = queue(producer.channel)
        new_queue.name = queues.get(queue.name, queue.name)
        if new_queue.routing_key == queue.name:
            new_queue.routing_key = queues.get(queue.name,
                                               new_queue.routing_key)
        if new_queue.exchange.name == queue.name:
            new_queue.exchange.name = queues.get(queue.name, queue.name)
        new_queue.declare()

        try:
            _, mcount, _ = queue(consumer.channel).queue_declare(passive=True)
            if mcount:
                state.total_apx += mcount
        except source.channel_errors + (StdChannelError, ):
            pass

    # start migrating messages.
    with consumer:
        try:
            for _ in eventloop(source, limit=limit,  # pragma: no cover
                               timeout=timeout, ignore_timeouts=forever):
                pass
        except socket.timeout:
            return