def hijack_comment(db_con, connection, hijack_key, comment): correlation_id = uuid() callback_queue = Queue( uuid(), channel=connection.default_channel, durable=False, exclusive=True, auto_delete=True, max_priority=4, consumer_arguments={"x-priority": 4}, ) with connection.Producer() as producer: producer.publish( { "key": hijack_key, "comment": comment }, exchange="", routing_key="db-hijack-comment", retry=True, declare=[callback_queue], reply_to=callback_queue.name, correlation_id=correlation_id, priority=4, serializer="ujson", ) while True: if callback_queue.get(): break time.sleep(0.1) result = hijack_action_test_result(db_con, hijack_key, "comment", comment) assert (result is True ), 'Action "hijack_comment" for hijack id #{0} failed'.format( hijack_key)
def hijack_multiple_action(db_con, connection, hijack_keys, action): correlation_id = uuid() callback_queue = Queue( uuid(), channel=connection.default_channel, durable=False, exclusive=True, auto_delete=True, max_priority=4, consumer_arguments={"x-priority": 4}, ) with connection.Producer() as producer: producer.publish( { "keys": hijack_keys, "action": action }, exchange="", routing_key="db-hijack-multiple-action", retry=True, declare=[callback_queue], reply_to=callback_queue.name, correlation_id=correlation_id, priority=4, serializer="ujson", ) while True: msg = callback_queue.get() if msg: assert ( msg.payload["status"] == "accepted" ), 'Action "{}" for [{}] failed with reason: {}'.format( action, hijack_keys, msg.payload.get("reason", "")) break time.sleep(0.1)
class MqServer(object): """ exchange='E_X7_W2S', queue='Q_X7_W2S',routing_key = 'RK_X7_W2S' """ def __init__(self, callback, kwargs): self.callback = callback if kwargs: self.kwargs = kwargs else: self.kwargs = MqDict def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"): self.conn = BrokerConnection(hostname, userid, password, virtual_host) # define Web2Server exchange exchange = Exchange(self.kwargs["X7_E"], type="direct") self.queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"]) channel = self.conn.channel() consumer = Consumer(channel, self.queue, callbacks=[self.callback]) consumer.consume() def run(self, once=False): if once: self.conn.drain_events() else: while True: self.conn.drain_events() def get(self): message = self.queue.get(block=True) message.ack() return message
def hijack_multiple_action(db_con, connection, hijack_keys, action): correlation_id = uuid() callback_queue = Queue( uuid(), channel=connection.default_channel, durable=False, exclusive=True, auto_delete=True, max_priority=4, consumer_arguments={"x-priority": 4}, ) with connection.Producer() as producer: producer.publish( {"keys": hijack_keys, "action": action}, exchange="", routing_key="db-hijack-multiple-action", retry=True, declare=[callback_queue], reply_to=callback_queue.name, correlation_id=correlation_id, priority=4, ) while True: if callback_queue.get(): break time.sleep(0.1)
def change_conf(connection, new_config, old_config, comment): changes = "".join(difflib.unified_diff(new_config, old_config)) if changes: correlation_id = uuid() callback_queue = Queue( uuid(), channel=connection.default_channel, durable=False, auto_delete=True, max_priority=4, consumer_arguments={"x-priority": 4}, ) with connection.Producer() as producer: producer.publish( { "config": new_config, "comment": comment }, exchange="", routing_key="config-modify-queue", serializer="yaml", retry=True, declare=[callback_queue], reply_to=callback_queue.name, correlation_id=correlation_id, priority=4, ) while True: if callback_queue.get(): break time.sleep(0.1)
def queue_get(self, queue_name, ack=True): queue = Queue(queue_name, channel=self._channel()) msg = queue.get() if not msg: return None if ack: msg.ack() return msg
def test_basic_get(self): chan1 = self.connection.channel() producer = Producer(chan1, self.exchange) producer.publish({"basic.get": "this"}, routing_key="basic_get") chan1.close() chan2 = self.connection.channel() queue = Queue("amqplib_basic_get", self.exchange, "basic_get") queue = queue(chan2) queue.declare() for i in range(50): m = queue.get() if m: break time.sleep(0.1) self.assertEqual(m.payload, {"basic.get": "this"}) chan2.close()
def test_basic_get(self): if not self.verify_alive(): return chan1 = self.connection.channel() producer = Producer(chan1, self.exchange) chan2 = self.connection.channel() queue = Queue(self.P("basic_get"), self.exchange, "basic_get") queue = queue(chan2) queue.declare() producer.publish({"basic.get": "this"}, routing_key="basic_get") chan1.close() for i in range(self.event_loop_max): m = queue.get() if m: break time.sleep(0.1) self.assertEqual(m.payload, {"basic.get": "this"}) chan2.close()
def get_mesage(): exchange = Exchange(name=exchange_name, type='topic', exclusive=False, durable=False, auto_delete=False) queue_name = 'neutron_notifications.info' q = Queue(queue_name, exchange=exchange, routing_key=routing_key, channel=conn.channel()) msg = q.get() if msg is None: print 'No messages' return try: pprint.pprint(json.loads(msg.body), indent=2) except ValueError: print msg.body finally: msg.ack()
def test_basic_get(self): if not self.verify_alive(): return chan1 = self.connection.channel() producer = chan1.Producer(self.exchange) chan2 = self.connection.channel() queue = Queue(self.P('basic_get'), self.exchange, 'basic_get') queue = queue(chan2) queue.declare() producer.publish({'basic.get': 'this'}, routing_key='basic_get') chan1.close() for i in range(self.event_loop_max): m = queue.get() if m: break time.sleep(0.1) self.assertEqual(m.payload, {'basic.get': 'this'}) self.purge([queue.name]) chan2.close()
def config_request_rpc(conn): """ Initial RPC of this service to request the configuration. The RPC is blocked until the configuration service replies back. """ correlation_id = uuid() callback_queue = Queue( uuid(), channel=conn.default_channel, durable=False, auto_delete=True, max_priority=4, consumer_arguments={"x-priority": 4}, ) with conn.Producer() as producer: producer.publish( "", exchange="", routing_key="config-request-queue", reply_to=callback_queue.name, correlation_id=correlation_id, retry=True, declare=[ Queue( "config-request-queue", durable=False, max_priority=4, consumer_arguments={"x-priority": 4}, ), callback_queue, ], priority=4, serializer="ujson", ) while True: if callback_queue.get(): break time.sleep(0.1) print("Config RPC finished")
def load_as_sets(connection): correlation_id = uuid() callback_queue = Queue( uuid(), channel=connection.default_channel, durable=False, exclusive=True, auto_delete=True, max_priority=4, consumer_arguments={"x-priority": 4}, ) with connection.Producer() as producer: producer.publish( {}, exchange="", routing_key="conf-load-as-sets-queue", retry=True, declare=[callback_queue], reply_to=callback_queue.name, correlation_id=correlation_id, priority=4, serializer="ujson", ) while True: m = callback_queue.get() if m: if m.properties["correlation_id"] == correlation_id: r = m.payload if not r["success"]: with open("configs/config.yaml") as f1, open( "configs/config3.yaml" ) as f3: new_data = f3.read() old_data = f1.read() Helper.change_conf( connection, new_data, old_data, "online_as_set_test_failed" ) break time.sleep(0.1)
def test_get(self): b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel()) b.get() assert 'basic_get' in b.channel
def test_get_when_no_m2p(self): chan = Mock() q = Queue('a')(chan) chan.message_to_python = None assert q.get()
def create_queues(self): ''' Creates (or ensure the existence of) the queues that an Interactive task needs. At least: 1) One queue named after the task used for managing Interactions - This queue is used to store single message all the time, that is always consumed and requeued which contains the map of IDs to queue name roots that serves both to communicate the queue name root and serve as a locking mechanism if only one running instance is permitted (one_at_a_time is in force). 2) One queue for instructions (client->task) 3) Optionally one queue for updates (task->client) - We can use the standard Celery update_status mechanism and/or our own queue depending on the configurations: update_via_backend update_via_broker This (create_queues) should be called in the Task/Client before the task is published (i.e. a request sent to the worker to run the task). That way: 1) the queues exist by the time the task is running. 2) the queue name root is known to the client. 3) the existence of the request is known to both sides in case one_at_a_time is in force There is an odd asymmetry in Kombu. We can send messages using only the known exchange and the task_id as a routing key. But to read a queue, we need to know its name. The management queue has a predictable name (that of the task), the instruction and update queues less so. We can name them using the task_id, or sequentially (if use_indexed_queue_names is True, which is cleaner if monitoring the broker as the task_id which is a UUID is a long messy thing to have in a queue name). The task will receive the queue name root as a kwarg "queue_name_root" but only if use_indexed_queue_names is True (it doesn't need a queue_name_root if not, as it can use the default_queue_name_root(). Either way the task can check the management queue to confirm its assigned queue_name_root. Returns: the queue name root used. ''' log.debug(f'Creating Queues for: {self.fullname}') log.debug(f'\tBroker: {current_app.conf.broker_url}') with Connection(current_app.conf.broker_url) as conn: try: # Connection is lazy. Force a connection now. conn.connect() c = conn.connection log.debug(f'\tConnection: {self.connection_names(c)[1]}') # Create a channel on the connection and log it in the RabbitMQ webmonitor format ch = c.channel() log.debug(f'\tChannel: {self.channel_names(ch)[1]}') # Create or get the exchange x = Exchange(self.exchange_name, channel=ch, durable=True) x.declare( ) # Makes the exchange appears on the RabbitMQ web monitor log.debug(f'\tExchange: {x.name}') # Ensure the management queue exists before we do anything else # kill zombies() expects to find it in place. q = Queue(self.name, exchange=x, channel=ch, routing_key=self.management_key(), durable=True) q.declare( ) # Makes the queue appears on the RabbitMQ web monitor log.debug(f'\tManagement Queue: {q.name}') self.management_queue = augment_queue(q) # First configure the queue names if self.use_indexed_queue_names: qname_root = self.first_free_indexed_queue_name_root() else: qname_root = self.default_queue_name_root() self.queue_name_root = qname_root self.set_management_data(qname_root) qnames = self.queue_names(qname_root) # Create the other queues for k, qname in qnames.items(): # Already created the management queue if k != self.management_key(): q = Queue(qname, exchange=x, channel=ch, routing_key=k, durable=True) q.declare( ) # Makes the queue appears on the RabbitMQ web monitor if k == self.instruction_key(): log.debug(f'\tInstruction Queue: {q.name}') self.instruction_queue = augment_queue(q) elif k == self.update_key(): log.debug(f'\tUpdate Queue: {q.name}') self.update_queue = augment_queue(q) # Ensure at least one valid message is on the update queue. # Mainly so that get_updates() can safely assume there is at least one updt_msg = q.get() if updt_msg and isinstance(updt_msg.payload, tuple): # If a valid tuple was on the queue, requeue it # Pretty unlikely seeings we just created the queue! But still... updt_msg.requeue() else: # If an invalid message (not a tuple) was on the queue # just ack() it (to get rid of it) if updt_msg: updt_msg.ack() # Put a valid message on the queue so it's not empty p = Producer(ch, exchange=q.exchange, routing_key=q.routing_key) p.publish(("UPDATE_QUEUE_CREATED", )) except Exception as e: log.error(f'QUEUE CREATION ERROR: {e}') return qname_root
def test_get(self): b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel()) b.get() self.assertIn('basic_get', b.channel)
def test_get_when_no_m2p(self): chan = Mock() q = Queue('a')(chan) chan.message_to_python = None self.assertTrue(q.get())
class DistLock(object): def __init__(self, name): self.name = name self.exchange = Exchange(self.name) self.routing_key = 'lock_routing_' + self.name self.acquire_requests_routing_key = 'acquire_routing_' + self.name self.lock_requests_q = Queue( name='lock_requests_q_' + self.name, exchange=self.exchange, routing_key=self.routing_key, channel=Connection(), ) self.acquire_requests_q = Queue( name='acquire_requests_q_' + self.name, exchange=self.exchange, routing_key=self.acquire_requests_routing_key, channel=Connection(), ) self._lock_manager = Consumer( Connection(), queues=[self.lock_requests_q], on_message=self._on_message, ) self._lock_manager.consume() self.lock_requests_q.declare() self.acquire_requests_q.declare() self.redis_connection = redis.StrictRedis() self.lock_monitor_thread = threading.Thread(target=self._manage_lock) self.consumer_thread = threading.Thread(target=self._consume_messages) self.lock_monitor_thread.daemon = True self.consumer_thread.daemon = True self._start_consumer() def _start_consumer(self): self.lock_requests_q.purge() self.acquire_requests_q.purge() self.redis_connection.set('is_consuming_' + self.name, True) self.redis_connection.delete('current_lock_owner_' + self.name) self.consumer_thread.start() self.lock_monitor_thread.start() def _stop_consumer(self): self.redis_connection.delete('is_consuming_' + self.name) self.redis_connection.delete('current_lock_owner_' + self.name) self.consumer_thread.join() self._lock_manager.connection.release() ## TODO: look this up self.lock_requests_q.purge() self.acquire_requests_q.purge() def _consume_messages(self): for _ in eventloop(self._lock_manager.connection, timeout=1, ignore_timeouts=True): pass def _on_message(self, message): print(message.payload) message.ack() p = Producer( Connection(), exchange=self.exchange, auto_declare=True, ) if message.payload.get('request') == 'RELEASE': # inform the current lock owner that lock has been released p.publish( dict(request='RELEASED', id=message.payload.get('id')), routing_key=message.payload.get('id'), exchange=self.exchange, ) self.redis_connection.delete('current_lock_owner_') else: p.publish(dict(request='ENQUEUE', id=message.payload.get('id')), routing_key=self.acquire_requests_routing_key, exchange=self.exchange) def _manage_lock(self): p = Producer( Connection(), self.exchange, auto_declare=True, ) while self.redis_connection.get('is_consuming_' + self.name): if not self.redis_connection.get('current_lock_owner_' + self.name): # Get next candidate owner from queue message = self.acquire_requests_q.get() if not message: continue print(message.payload) self.redis_connection.set('current_lock_owner_' + self.name, message.payload.get('id')) # Inform the candidate owner that lock has been granted # message not deleted until ack'ed message.ack() p.publish( dict(request='GRANTED', id=message.payload.get('id')), routing_key=message.payload.get('id'), exchange=self.exchange, ) def __del__(self): self._stop_consumer()
class TestVerifyTask(unittest.TestCase): def setUp(self): # Open connection to RabbitMQ self.conn = Connection(config['broker_url']) self.channel = self.conn.channel() # Declare Verify queue q = config['queues']['verify'] self.verifyQ = Queue(q['name'], channel=self.channel, exchange=Exchange(q['name']), routing_key=q['name'], max_priority=q['max_task_priority']) self.verifyQ.declare() # Declare API queue q = config['queues']['api'] self.apiQ = Queue(q['name'], channel=self.channel, exchange=Exchange(q['name']), routing_key=q['name'], max_priority=q['max_task_priority']) self.apiQ.declare() def tearDown(self): # Delete Verify queue self.apiQ.delete() # Delete API queue self.verifyQ.delete() # Close connection self.conn.close() def test_verify(self): data = [{ 'filename': '/var/store/15525119098910.pdf', 'algorithm': 'md5', 'checksum': 'ec4e3b91d2e03fdb17db55ff46da43b2' }, { 'filename': '/var/store/15525119098910.pdf', 'algorithm': 'sha512', 'checksum': 'bc803d8abccf18d89765d6ae9fb7d490ad07f57a48e4987acc1' '73af4e65f143a4d215ffb59e9eebeb03849baab5a6e016e2806' 'a2cd0e84b14c778bdb84afbbf4' }] for i in data: self.assertTrue(path.exists(i['filename'])) # Queues cleanup self.verifyQ.purge() self.apiQ.purge() # Random DFO ID dfo_id = randint(1, 2147483647) # Send task q = config['queues']['verify'] producer = self.conn.Producer() producer.publish( routing_key=q['name'], body=[[dfo_id, i['filename'], 'test', i['algorithm']], {}, {}], headers={ 'task': 'verify_dfo', 'id': str(uuid.uuid1()) }) # Wait for result message for max 5 seconds msg = None wait = 0 while wait <= 5 and msg is None: msg = self.apiQ.get(no_ack=False) if msg is None: sleep(1) wait += 1 # Tests self.assertFalse(msg is None) self.assertTrue(msg.payload[0][0] == dfo_id) self.assertTrue(msg.payload[0][1] == i['algorithm']) self.assertTrue(msg.payload[0][2] == i['checksum'])
""" from kombu import Connection, Exchange, Queue, Consumer, eventloop from pprint import pformat #: By default messages sent to exchanges are persistent (delivery_mode=2), #: and queues and exchanges are durable. exchange = Exchange() connection = Connection('amqp://*****:*****@localhost:5672//') # Create (if necessary) a queue bound to the connection. queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')(connection) queue.declare() def pretty(obj): return pformat(obj, indent=4) #: This is the callback applied when a message is received. def handle_message(body, message): print('Received message: %r' % (body, )) print(' properties:\n%s' % (pretty(message.properties), )) print(' delivery_info:\n%s' % (pretty(message.delivery_info), )) print('') print('headers: \n%s' % message.properties['application_headers']) message.ack() message = queue.get() if message: handle_message(message.body, message)