def listen_for_changes():
wikipedia_exchange = Exchange('wikipedia', type='direct', durable=False)
queue = Queue('status', wikipedia_exchange, routing_key='status')
with Connection() as conn:
with Consumer(conn, queue, callbacks=[handle_message]):
for _ in eventloop(conn):
pass
def listen_for_changes():
wikipedia_exchange = Exchange('wikipedia', type='direct', durable=False)
queue = Queue('status', wikipedia_exchange, routing_key='status')
with Connection() as conn:
with Consumer(conn, queue, callbacks=[handle_message]):
for _ in eventloop(conn):
pass
def start_filter(app, conn, filter, limit=None, timeout=1.0,
ack_messages=False, tasks=None, queues=None,
callback=None, forever=False, on_declare_queue=None,
consume_from=None, state=None, **kwargs):
state = state or State()
queues = prepare_queues(queues)
if isinstance(tasks, basestring):
tasks = set(tasks.split(','))
if tasks is None:
tasks = set([])
def update_state(body, message):
state.count += 1
if limit and state.count >= limit:
raise StopFiltering()
def ack_message(body, message):
message.ack()
consumer = app.amqp.TaskConsumer(conn, queues=consume_from)
if tasks:
filter = filter_callback(filter, tasks)
update_state = filter_callback(update_state, tasks)
ack_message = filter_callback(ack_message, tasks)
consumer.register_callback(filter)
consumer.register_callback(update_state)
if ack_messages:
consumer.register_callback(ack_message)
if callback is not None:
callback = partial(callback, state)
if tasks:
callback = filter_callback(callback, tasks)
consumer.register_callback(callback)
# declare all queues on the new broker.
for queue in consumer.queues:
if queues and queue.name not in queues:
continue
if on_declare_queue is not None:
on_declare_queue(queue)
try:
_, mcount, _ = queue(consumer.channel).queue_declare(passive=True)
if mcount:
state.total_apx += mcount
except conn.channel_errors + (StdChannelError, ):
pass
# start migrating messages.
with consumer:
try:
for _ in eventloop(conn, # pragma: no cover
timeout=timeout, ignore_timeouts=forever):
pass
except socket.timeout:
pass
except StopFiltering:
pass
return state
def start_filter(app, conn, filter, limit=None, timeout=1.0,
ack_messages=False, tasks=None, queues=None,
callback=None, forever=False, on_declare_queue=None,
consume_from=None, state=None, **kwargs):
state = state or State()
queues = prepare_queues(queues)
if isinstance(tasks, string_t):
tasks = set(tasks.split(','))
if tasks is None:
tasks = set([])
def update_state(body, message):
state.count += 1
if limit and state.count >= limit:
raise StopFiltering()
def ack_message(body, message):
message.ack()
consumer = app.amqp.TaskConsumer(conn, queues=consume_from)
if tasks:
filter = filter_callback(filter, tasks)
update_state = filter_callback(update_state, tasks)
ack_message = filter_callback(ack_message, tasks)
consumer.register_callback(filter)
consumer.register_callback(update_state)
if ack_messages:
consumer.register_callback(ack_message)
if callback is not None:
callback = partial(callback, state)
if tasks:
callback = filter_callback(callback, tasks)
consumer.register_callback(callback)
# declare all queues on the new broker.
for queue in consumer.queues:
if queues and queue.name not in queues:
continue
if on_declare_queue is not None:
on_declare_queue(queue)
try:
_, mcount, _ = queue(consumer.channel).queue_declare(passive=True)
if mcount:
state.total_apx += mcount
except conn.channel_errors + (StdChannelError, ):
pass
# start migrating messages.
with consumer:
try:
for _ in eventloop(conn, # pragma: no cover
timeout=timeout, ignore_timeouts=forever):
pass
except socket.timeout:
pass
except StopFiltering:
pass
return state
def receive_message(self):
with Consumer(self.connection,
on_message=self.on_response,
queues=[self.callback_queue], no_ack=True):
for _ in eventloop(self.connection):
if self.terminated:
break
def migrate_tasks(source, dest, timeout=1.0, app=None,
migrate=None, callback=None):
state = State()
app = app_or_default(app)
def update_state(body, message):
state.count += 1
producer = app.amqp.TaskProducer(dest)
if migrate is None:
migrate = partial(migrate_task, producer)
consumer = app.amqp.TaskConsumer(source)
consumer.register_callback(update_state)
if callback is not None:
callback = partial(callback, state)
consumer.register_callback(callback)
consumer.register_callback(migrate)
# declare all queues on the new broker.
for queue in consumer.queues:
queue(producer.channel).declare()
try:
_, mcount, _ = queue(consumer.channel).queue_declare(passive=True)
if mcount:
state.total_apx += mcount
except source.channel_errors + (StdChannelError, ):
pass
# start migrating messages.
with consumer:
try:
for _ in eventloop(source, timeout=timeout): # pragma: no cover
pass
except socket.timeout:
return
def main(arguments):
#: By default messages sent to exchanges are persistent (delivery_mode=2),
#: and queues and exchanges are durable.
exchange = Exchange('kombu_demo', type='direct')
queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')
def pretty(obj):
return pformat(obj, indent=4)
#: This is the callback applied when a message is received.
def handle_message(body, message):
print('Received message: {0!r}'.format(body))
print(' properties:\n{0}'.format(pretty(message.properties)))
print(' delivery_info:\n{0}'.format(pretty(message.delivery_info)))
message.ack()
#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('redis://localhost:6379') as connection:
#: Create consumer using our callback and queue.
#: Second argument can also be a list to consume from
#: any number of queues.
with Consumer(connection, queue, callbacks=[handle_message]):
#: Each iteration waits for a single event. Note that this
#: event may not be a message, or a message that is to be
#: delivered to the consumers channel, but any event received
#: on the connection.
for _ in eventloop(connection):
pass
def start(self):
# start migrating messages.
with self.prepare_consumer(self.create_consumer()):
try:
for _ in eventloop(self.conn, # pragma: no cover
timeout=self.timeout,
ignore_timeouts=self.forever):
pass
except socket.timeout:
pass
except StopFiltering:
pass
return self.state
def start(self):
# start migrating messages.
with self.prepare_consumer(self.create_consumer()):
try:
for _ in eventloop(self.conn, # pragma: no cover
timeout=self.timeout,
ignore_timeouts=self.forever):
pass
except socket.timeout:
pass
except StopFiltering:
pass
return self.state
def listen(self, callbacks: Optional[list] = None) -> None:
"""
Start a listener and handle messeages with the callback(s). If the
queue does not already exist in the exchange, it will be created.
"""
with kombu.Connection(self.connection_str) as conn:
with kombu.Consumer(conn, [self.queue],
callbacks=callbacks,
no_ack=True):
try:
print("STARTING LISTENER")
for _ in kombu.eventloop(conn):
pass
except KeyboardInterrupt:
print("\nExiting\n")
print(' properties:\n%s' % (pretty(message.properties), ))
print(' delivery_info:\n%s' % (pretty(message.delivery_info), ))
message.ack()
#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('pyamqp://*****:*****@localhost:5672//') as connection:
# The configuration of the message flow is as follows:
# gateway_kombu_exchange -> internal_kombu_exchange -> kombu_demo queue
gateway_exchange = Exchange('gateway_kombu_demo')(connection)
exchange = Exchange('internal_kombu_demo')(connection)
gateway_exchange.declare()
exchange.declare()
exchange.bind_to(gateway_exchange, routing_key='kombu_demo')
queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')
#: Create consumer using our callback and queue.
#: Second argument can also be a list to consume from
#: any number of queues.
with Consumer(connection, queue, callbacks=[handle_message]):
#: This waits for a single event. Note that this event may not
#: be a message, or a message that is to be delivered to the consumers
#: channel, but any event received on the connection.
recv = eventloop(connection)
while True:
recv.next()
def pretty(obj):
return pformat(obj, indent=4)
#: This is the callback applied when a message is received.
def handle_message(body, message):
print('Received message: {0!r}'.format(body))
print(' properties:\n{0}'.format(pretty(message.properties)))
print(' delivery_info:\n{0}'.format(pretty(message.delivery_info)))
message.ack()
#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('amqp://*****:*****@localhost:5672//') as connection:
#: Create consumer using our callback and queue.
#: Second argument can also be a list to consume from
#: any number of queues.
with Consumer(connection, queue, callbacks=[handle_message]):
#: Each iteration waits for a single event. Note that this
#: event may not be a message, or a message that is to be
#: delivered to the consumers channel, but any event received
#: on the connection.
for _ in eventloop(connection):
pass
def drain_events(self, **kwargs):
for _ in eventloop(self.connection, **kwargs):
pass
exchange = Exchange('kombu_demo', type='direct')
queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')
def pretty(obj):
return pformat(obj, indent=4)
#: This is the callback applied when a message is received.
def handle_message(body, message):
print('Received message: %r' % (body, ))
print(' properties:\n%s' % (pretty(message.properties), ))
print(' delivery_info:\n%s' % (pretty(message.delivery_info), ))
message.ack()
#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('amqp://*****:*****@localhost:5672//') as connection:
#: Create consumer using our callback and queue.
#: Second argument can also be a list to consume from
#: any number of queues.
with Consumer(connection, queue, callbacks=[handle_message]):
#: This waits for a single event. Note that this event may not
#: be a message, or a message that is to be delivered to the consumers
#: channel, but any event received on the connection.
eventloop(connection, limit=1, timeout=10.0)
def drain_events(self, **kwargs):
for _ in eventloop(self.connection, **kwargs):
pass
def listener(self):
self.lock_client.add_queue(self.lock_client_q)
self.lock_client.consume(no_ack=True)
for _ in eventloop(self.lock_client.connection):
pass
def _consume_messages(self):
for _ in eventloop(self._lock_manager.connection,
timeout=1,
ignore_timeouts=True):
pass
queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')
def pretty(obj):
return pformat(obj, indent=4)
#: This is the callback applied when a message is received.
def handle_message(body, message):
print('Received message: %r' % (body, ))
print(' properties:\n%s' % (pretty(message.properties), ))
print(' delivery_info:\n%s' % (pretty(message.delivery_info), ))
message.ack()
#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('amqp://*****:*****@localhost:5672//') as connection:
#: Create consumer using our callback and queue.
#: Second argument can also be a list to consume from
#: any number of queues.
with Consumer(connection, queue, callbacks=[handle_message]):
#: This waits for a single event. Note that this event may not
#: be a message, or a message that is to be delivered to the consumers
#: channel, but any event received on the connection.
eventloop(connection, limit=1, timeout=10.0)
def pretty(obj):
return pformat(obj, indent=4)
#: This is the callback applied when a message is received.
def handle_message(body, message):
print('Received message: %r' % (body, ))
print(' properties:\n%s' % (pretty(message.properties), ))
print(' delivery_info:\n%s' % (pretty(message.delivery_info), ))
message.ack()
#: Create a connection and a channel.
#: If hostname, userid, password and virtual_host is not specified
#: the values below are the default, but listed here so it can
#: be easily changed.
with Connection('amqp://*****:*****@localhost:5672//') as connection:
#: Create consumer using our callback and queue.
#: Second argument can also be a list to consume from
#: any number of queues.
with Consumer(connection, queue, callbacks=[handle_message]):
#: Each iteration waits for a single event. Note that this
#: event may not be a message, or a message that is to be
#: delivered to the consumers channel, but any event received
#: on the connection.
for _ in eventloop(connection):
pass
def migrate_tasks(source, dest, limit=None, timeout=1.0, ack_messages=False,
app=None, migrate=migrate_task, tasks=None, queues=None, callback=None,
forever=False, **kwargs):
state = State()
app = app_or_default(app)
if isinstance(queues, basestring):
queues = queues.split(',')
if isinstance(queues, list):
queues = dict(tuple(islice(cycle(q.split(':')), None, 2))
for q in queues)
if queues is None:
queues = {}
if isinstance(tasks, basestring):
tasks = set(tasks.split(','))
if tasks is None:
tasks = set([])
def update_state(body, message):
state.count += 1
def ack_message(body, message):
message.ack()
producer = app.amqp.TaskProducer(dest)
migrate = partial(migrate, producer, queues=queues)
consumer = app.amqp.TaskConsumer(source)
if tasks:
migrate = filter_callback(migrate, tasks)
update_state = filter_callback(update_state, tasks)
ack_message = filter_callback(ack_message, tasks)
consumer.register_callback(migrate)
consumer.register_callback(update_state)
if ack_messages:
consumer.register_callback(ack_message)
if callback is not None:
callback = partial(callback, state)
if tasks:
callback = filter_callback(callback, tasks)
consumer.register_callback(callback)
# declare all queues on the new broker.
for queue in consumer.queues:
if queues and queue.name not in queues:
continue
new_queue = queue(producer.channel)
new_queue.name = queues.get(queue.name, queue.name)
if new_queue.routing_key == queue.name:
new_queue.routing_key = queues.get(queue.name,
new_queue.routing_key)
if new_queue.exchange.name == queue.name:
new_queue.exchange.name = queues.get(queue.name, queue.name)
new_queue.declare()
try:
_, mcount, _ = queue(consumer.channel).queue_declare(passive=True)
if mcount:
state.total_apx += mcount
except source.channel_errors + (StdChannelError, ):
pass
# start migrating messages.
with consumer:
try:
for _ in eventloop(source, limit=limit, # pragma: no cover
timeout=timeout, ignore_timeouts=forever):
pass
except socket.timeout:
return