def hijack_comment(db_con, connection, hijack_key, comment):
correlation_id = uuid()
callback_queue = Queue(
uuid(),
channel=connection.default_channel,
durable=False,
exclusive=True,
auto_delete=True,
max_priority=4,
consumer_arguments={"x-priority": 4},
)
with connection.Producer() as producer:
producer.publish(
{
"key": hijack_key,
"comment": comment
},
exchange="",
routing_key="db-hijack-comment",
retry=True,
declare=[callback_queue],
reply_to=callback_queue.name,
correlation_id=correlation_id,
priority=4,
serializer="ujson",
)
while True:
if callback_queue.get():
break
time.sleep(0.1)
result = hijack_action_test_result(db_con, hijack_key, "comment",
comment)
assert (result is True
), 'Action "hijack_comment" for hijack id #{0} failed'.format(
hijack_key)
def hijack_multiple_action(db_con, connection, hijack_keys, action):
correlation_id = uuid()
callback_queue = Queue(
uuid(),
channel=connection.default_channel,
durable=False,
exclusive=True,
auto_delete=True,
max_priority=4,
consumer_arguments={"x-priority": 4},
)
with connection.Producer() as producer:
producer.publish(
{
"keys": hijack_keys,
"action": action
},
exchange="",
routing_key="db-hijack-multiple-action",
retry=True,
declare=[callback_queue],
reply_to=callback_queue.name,
correlation_id=correlation_id,
priority=4,
serializer="ujson",
)
while True:
msg = callback_queue.get()
if msg:
assert (
msg.payload["status"] == "accepted"
), 'Action "{}" for [{}] failed with reason: {}'.format(
action, hijack_keys, msg.payload.get("reason", ""))
break
time.sleep(0.1)
class MqServer(object):
"""
exchange='E_X7_W2S', queue='Q_X7_W2S',routing_key = 'RK_X7_W2S'
"""
def __init__(self, callback, kwargs):
self.callback = callback
if kwargs:
self.kwargs = kwargs
else:
self.kwargs = MqDict
def connect(self, hostname="localhost", userid="guest", password="******", virtual_host="/"):
self.conn = BrokerConnection(hostname, userid, password, virtual_host)
# define Web2Server exchange
exchange = Exchange(self.kwargs["X7_E"], type="direct")
self.queue = Queue(self.kwargs["X7_Q"], exchange, routing_key=self.kwargs["X7_RK"])
channel = self.conn.channel()
consumer = Consumer(channel, self.queue, callbacks=[self.callback])
consumer.consume()
def run(self, once=False):
if once:
self.conn.drain_events()
else:
while True:
self.conn.drain_events()
def get(self):
message = self.queue.get(block=True)
message.ack()
return message
def hijack_multiple_action(db_con, connection, hijack_keys, action):
correlation_id = uuid()
callback_queue = Queue(
uuid(),
channel=connection.default_channel,
durable=False,
exclusive=True,
auto_delete=True,
max_priority=4,
consumer_arguments={"x-priority": 4},
)
with connection.Producer() as producer:
producer.publish(
{"keys": hijack_keys, "action": action},
exchange="",
routing_key="db-hijack-multiple-action",
retry=True,
declare=[callback_queue],
reply_to=callback_queue.name,
correlation_id=correlation_id,
priority=4,
)
while True:
if callback_queue.get():
break
time.sleep(0.1)
def change_conf(connection, new_config, old_config, comment):
changes = "".join(difflib.unified_diff(new_config, old_config))
if changes:
correlation_id = uuid()
callback_queue = Queue(
uuid(),
channel=connection.default_channel,
durable=False,
auto_delete=True,
max_priority=4,
consumer_arguments={"x-priority": 4},
)
with connection.Producer() as producer:
producer.publish(
{
"config": new_config,
"comment": comment
},
exchange="",
routing_key="config-modify-queue",
serializer="yaml",
retry=True,
declare=[callback_queue],
reply_to=callback_queue.name,
correlation_id=correlation_id,
priority=4,
)
while True:
if callback_queue.get():
break
time.sleep(0.1)
def queue_get(self, queue_name, ack=True):
queue = Queue(queue_name, channel=self._channel())
msg = queue.get()
if not msg:
return None
if ack:
msg.ack()
return msg
def test_basic_get(self):
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
producer.publish({"basic.get": "this"}, routing_key="basic_get")
chan1.close()
chan2 = self.connection.channel()
queue = Queue("amqplib_basic_get", self.exchange, "basic_get")
queue = queue(chan2)
queue.declare()
for i in range(50):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {"basic.get": "this"})
chan2.close()
def test_basic_get(self):
if not self.verify_alive():
return
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
chan2 = self.connection.channel()
queue = Queue(self.P("basic_get"), self.exchange, "basic_get")
queue = queue(chan2)
queue.declare()
producer.publish({"basic.get": "this"}, routing_key="basic_get")
chan1.close()
for i in range(self.event_loop_max):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {"basic.get": "this"})
chan2.close()
def test_basic_get(self):
if not self.verify_alive():
return
chan1 = self.connection.channel()
producer = Producer(chan1, self.exchange)
chan2 = self.connection.channel()
queue = Queue(self.P("basic_get"), self.exchange, "basic_get")
queue = queue(chan2)
queue.declare()
producer.publish({"basic.get": "this"}, routing_key="basic_get")
chan1.close()
for i in range(self.event_loop_max):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {"basic.get": "this"})
chan2.close()
def get_mesage():
exchange = Exchange(name=exchange_name, type='topic',
exclusive=False, durable=False, auto_delete=False)
queue_name = 'neutron_notifications.info'
q = Queue(queue_name, exchange=exchange,
routing_key=routing_key, channel=conn.channel())
msg = q.get()
if msg is None:
print 'No messages'
return
try:
pprint.pprint(json.loads(msg.body), indent=2)
except ValueError:
print msg.body
finally:
msg.ack()
def test_basic_get(self):
if not self.verify_alive():
return
chan1 = self.connection.channel()
producer = chan1.Producer(self.exchange)
chan2 = self.connection.channel()
queue = Queue(self.P('basic_get'), self.exchange, 'basic_get')
queue = queue(chan2)
queue.declare()
producer.publish({'basic.get': 'this'}, routing_key='basic_get')
chan1.close()
for i in range(self.event_loop_max):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {'basic.get': 'this'})
self.purge([queue.name])
chan2.close()
def test_basic_get(self):
if not self.verify_alive():
return
chan1 = self.connection.channel()
producer = chan1.Producer(self.exchange)
chan2 = self.connection.channel()
queue = Queue(self.P('basic_get'), self.exchange, 'basic_get')
queue = queue(chan2)
queue.declare()
producer.publish({'basic.get': 'this'}, routing_key='basic_get')
chan1.close()
for i in range(self.event_loop_max):
m = queue.get()
if m:
break
time.sleep(0.1)
self.assertEqual(m.payload, {'basic.get': 'this'})
self.purge([queue.name])
chan2.close()
def config_request_rpc(conn):
"""
Initial RPC of this service to request the configuration.
The RPC is blocked until the configuration service replies back.
"""
correlation_id = uuid()
callback_queue = Queue(
uuid(),
channel=conn.default_channel,
durable=False,
auto_delete=True,
max_priority=4,
consumer_arguments={"x-priority": 4},
)
with conn.Producer() as producer:
producer.publish(
"",
exchange="",
routing_key="config-request-queue",
reply_to=callback_queue.name,
correlation_id=correlation_id,
retry=True,
declare=[
Queue(
"config-request-queue",
durable=False,
max_priority=4,
consumer_arguments={"x-priority": 4},
),
callback_queue,
],
priority=4,
serializer="ujson",
)
while True:
if callback_queue.get():
break
time.sleep(0.1)
print("Config RPC finished")
def load_as_sets(connection):
correlation_id = uuid()
callback_queue = Queue(
uuid(),
channel=connection.default_channel,
durable=False,
exclusive=True,
auto_delete=True,
max_priority=4,
consumer_arguments={"x-priority": 4},
)
with connection.Producer() as producer:
producer.publish(
{},
exchange="",
routing_key="conf-load-as-sets-queue",
retry=True,
declare=[callback_queue],
reply_to=callback_queue.name,
correlation_id=correlation_id,
priority=4,
serializer="ujson",
)
while True:
m = callback_queue.get()
if m:
if m.properties["correlation_id"] == correlation_id:
r = m.payload
if not r["success"]:
with open("configs/config.yaml") as f1, open(
"configs/config3.yaml"
) as f3:
new_data = f3.read()
old_data = f1.read()
Helper.change_conf(
connection, new_data, old_data, "online_as_set_test_failed"
)
break
time.sleep(0.1)
def test_get(self):
b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel())
b.get()
assert 'basic_get' in b.channel
def test_get_when_no_m2p(self):
chan = Mock()
q = Queue('a')(chan)
chan.message_to_python = None
assert q.get()
def create_queues(self):
'''
Creates (or ensure the existence of) the queues that an Interactive
task needs.
At least:
1) One queue named after the task used for managing Interactions
- This queue is used to store single message all the time,
that is always consumed and requeued which contains the map
of IDs to queue name roots that serves both to communicate
the queue name root and serve as a locking mechanism if only
one running instance is permitted (one_at_a_time is in force).
2) One queue for instructions (client->task)
3) Optionally one queue for updates (task->client)
- We can use the standard Celery update_status mechanism and/or
our own queue depending on the configurations:
update_via_backend
update_via_broker
This (create_queues) should be called in the Task/Client before the task is
published (i.e. a request sent to the worker to run the task). That way:
1) the queues exist by the time the task is running.
2) the queue name root is known to the client.
3) the existence of the request is known to both sides in case one_at_a_time is in force
There is an odd asymmetry in Kombu.
We can send messages using only the known exchange and the task_id as a routing
key. But to read a queue, we need to know its name. The management queue has a
predictable name (that of the task), the instruction and update queues less so.
We can name them using the task_id, or sequentially (if use_indexed_queue_names
is True, which is cleaner if monitoring the broker as the task_id which is a UUID
is a long messy thing to have in a queue name).
The task will receive the queue name root as a kwarg "queue_name_root" but only
if use_indexed_queue_names is True (it doesn't need a queue_name_root if not, as
it can use the default_queue_name_root(). Either way the task can check the
management queue to confirm its assigned queue_name_root.
Returns: the queue name root used.
'''
log.debug(f'Creating Queues for: {self.fullname}')
log.debug(f'\tBroker: {current_app.conf.broker_url}')
with Connection(current_app.conf.broker_url) as conn:
try:
# Connection is lazy. Force a connection now.
conn.connect()
c = conn.connection
log.debug(f'\tConnection: {self.connection_names(c)[1]}')
# Create a channel on the connection and log it in the RabbitMQ webmonitor format
ch = c.channel()
log.debug(f'\tChannel: {self.channel_names(ch)[1]}')
# Create or get the exchange
x = Exchange(self.exchange_name, channel=ch, durable=True)
x.declare(
) # Makes the exchange appears on the RabbitMQ web monitor
log.debug(f'\tExchange: {x.name}')
# Ensure the management queue exists before we do anything else
# kill zombies() expects to find it in place.
q = Queue(self.name,
exchange=x,
channel=ch,
routing_key=self.management_key(),
durable=True)
q.declare(
) # Makes the queue appears on the RabbitMQ web monitor
log.debug(f'\tManagement Queue: {q.name}')
self.management_queue = augment_queue(q)
# First configure the queue names
if self.use_indexed_queue_names:
qname_root = self.first_free_indexed_queue_name_root()
else:
qname_root = self.default_queue_name_root()
self.queue_name_root = qname_root
self.set_management_data(qname_root)
qnames = self.queue_names(qname_root)
# Create the other queues
for k, qname in qnames.items():
# Already created the management queue
if k != self.management_key():
q = Queue(qname,
exchange=x,
channel=ch,
routing_key=k,
durable=True)
q.declare(
) # Makes the queue appears on the RabbitMQ web monitor
if k == self.instruction_key():
log.debug(f'\tInstruction Queue: {q.name}')
self.instruction_queue = augment_queue(q)
elif k == self.update_key():
log.debug(f'\tUpdate Queue: {q.name}')
self.update_queue = augment_queue(q)
# Ensure at least one valid message is on the update queue.
# Mainly so that get_updates() can safely assume there is at least one
updt_msg = q.get()
if updt_msg and isinstance(updt_msg.payload,
tuple):
# If a valid tuple was on the queue, requeue it
# Pretty unlikely seeings we just created the queue! But still...
updt_msg.requeue()
else:
# If an invalid message (not a tuple) was on the queue
# just ack() it (to get rid of it)
if updt_msg:
updt_msg.ack()
# Put a valid message on the queue so it's not empty
p = Producer(ch,
exchange=q.exchange,
routing_key=q.routing_key)
p.publish(("UPDATE_QUEUE_CREATED", ))
except Exception as e:
log.error(f'QUEUE CREATION ERROR: {e}')
return qname_root
def test_get(self):
b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel())
b.get()
self.assertIn('basic_get', b.channel)
def test_get_when_no_m2p(self):
chan = Mock()
q = Queue('a')(chan)
chan.message_to_python = None
self.assertTrue(q.get())
class DistLock(object):
def __init__(self, name):
self.name = name
self.exchange = Exchange(self.name)
self.routing_key = 'lock_routing_' + self.name
self.acquire_requests_routing_key = 'acquire_routing_' + self.name
self.lock_requests_q = Queue(
name='lock_requests_q_' + self.name,
exchange=self.exchange,
routing_key=self.routing_key,
channel=Connection(),
)
self.acquire_requests_q = Queue(
name='acquire_requests_q_' + self.name,
exchange=self.exchange,
routing_key=self.acquire_requests_routing_key,
channel=Connection(),
)
self._lock_manager = Consumer(
Connection(),
queues=[self.lock_requests_q],
on_message=self._on_message,
)
self._lock_manager.consume()
self.lock_requests_q.declare()
self.acquire_requests_q.declare()
self.redis_connection = redis.StrictRedis()
self.lock_monitor_thread = threading.Thread(target=self._manage_lock)
self.consumer_thread = threading.Thread(target=self._consume_messages)
self.lock_monitor_thread.daemon = True
self.consumer_thread.daemon = True
self._start_consumer()
def _start_consumer(self):
self.lock_requests_q.purge()
self.acquire_requests_q.purge()
self.redis_connection.set('is_consuming_' + self.name, True)
self.redis_connection.delete('current_lock_owner_' + self.name)
self.consumer_thread.start()
self.lock_monitor_thread.start()
def _stop_consumer(self):
self.redis_connection.delete('is_consuming_' + self.name)
self.redis_connection.delete('current_lock_owner_' + self.name)
self.consumer_thread.join()
self._lock_manager.connection.release() ## TODO: look this up
self.lock_requests_q.purge()
self.acquire_requests_q.purge()
def _consume_messages(self):
for _ in eventloop(self._lock_manager.connection,
timeout=1,
ignore_timeouts=True):
pass
def _on_message(self, message):
print(message.payload)
message.ack()
p = Producer(
Connection(),
exchange=self.exchange,
auto_declare=True,
)
if message.payload.get('request') == 'RELEASE':
# inform the current lock owner that lock has been released
p.publish(
dict(request='RELEASED', id=message.payload.get('id')),
routing_key=message.payload.get('id'),
exchange=self.exchange,
)
self.redis_connection.delete('current_lock_owner_')
else:
p.publish(dict(request='ENQUEUE', id=message.payload.get('id')),
routing_key=self.acquire_requests_routing_key,
exchange=self.exchange)
def _manage_lock(self):
p = Producer(
Connection(),
self.exchange,
auto_declare=True,
)
while self.redis_connection.get('is_consuming_' + self.name):
if not self.redis_connection.get('current_lock_owner_' +
self.name):
# Get next candidate owner from queue
message = self.acquire_requests_q.get()
if not message:
continue
print(message.payload)
self.redis_connection.set('current_lock_owner_' + self.name,
message.payload.get('id'))
# Inform the candidate owner that lock has been granted
# message not deleted until ack'ed
message.ack()
p.publish(
dict(request='GRANTED', id=message.payload.get('id')),
routing_key=message.payload.get('id'),
exchange=self.exchange,
)
def __del__(self):
self._stop_consumer()
class TestVerifyTask(unittest.TestCase):
def setUp(self):
# Open connection to RabbitMQ
self.conn = Connection(config['broker_url'])
self.channel = self.conn.channel()
# Declare Verify queue
q = config['queues']['verify']
self.verifyQ = Queue(q['name'],
channel=self.channel,
exchange=Exchange(q['name']),
routing_key=q['name'],
max_priority=q['max_task_priority'])
self.verifyQ.declare()
# Declare API queue
q = config['queues']['api']
self.apiQ = Queue(q['name'],
channel=self.channel,
exchange=Exchange(q['name']),
routing_key=q['name'],
max_priority=q['max_task_priority'])
self.apiQ.declare()
def tearDown(self):
# Delete Verify queue
self.apiQ.delete()
# Delete API queue
self.verifyQ.delete()
# Close connection
self.conn.close()
def test_verify(self):
data = [{
'filename': '/var/store/15525119098910.pdf',
'algorithm': 'md5',
'checksum': 'ec4e3b91d2e03fdb17db55ff46da43b2'
}, {
'filename':
'/var/store/15525119098910.pdf',
'algorithm':
'sha512',
'checksum':
'bc803d8abccf18d89765d6ae9fb7d490ad07f57a48e4987acc1'
'73af4e65f143a4d215ffb59e9eebeb03849baab5a6e016e2806'
'a2cd0e84b14c778bdb84afbbf4'
}]
for i in data:
self.assertTrue(path.exists(i['filename']))
# Queues cleanup
self.verifyQ.purge()
self.apiQ.purge()
# Random DFO ID
dfo_id = randint(1, 2147483647)
# Send task
q = config['queues']['verify']
producer = self.conn.Producer()
producer.publish(
routing_key=q['name'],
body=[[dfo_id, i['filename'], 'test', i['algorithm']], {}, {}],
headers={
'task': 'verify_dfo',
'id': str(uuid.uuid1())
})
# Wait for result message for max 5 seconds
msg = None
wait = 0
while wait <= 5 and msg is None:
msg = self.apiQ.get(no_ack=False)
if msg is None:
sleep(1)
wait += 1
# Tests
self.assertFalse(msg is None)
self.assertTrue(msg.payload[0][0] == dfo_id)
self.assertTrue(msg.payload[0][1] == i['algorithm'])
self.assertTrue(msg.payload[0][2] == i['checksum'])
"""
from kombu import Connection, Exchange, Queue, Consumer, eventloop
from pprint import pformat
#: By default messages sent to exchanges are persistent (delivery_mode=2),
#: and queues and exchanges are durable.
exchange = Exchange()
connection = Connection('amqp://*****:*****@localhost:5672//')
# Create (if necessary) a queue bound to the connection.
queue = Queue('kombu_demo', exchange, routing_key='kombu_demo')(connection)
queue.declare()
def pretty(obj):
return pformat(obj, indent=4)
#: This is the callback applied when a message is received.
def handle_message(body, message):
print('Received message: %r' % (body, ))
print(' properties:\n%s' % (pretty(message.properties), ))
print(' delivery_info:\n%s' % (pretty(message.delivery_info), ))
print('')
print('headers: \n%s' % message.properties['application_headers'])
message.ack()
message = queue.get()
if message:
handle_message(message.body, message)
def test_get(self):
b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel())
b.get()
self.assertIn('basic_get', b.channel)
def test_get(self):
b = Queue('foo', self.exchange, 'foo', channel=get_conn().channel())
b.get()
assert 'basic_get' in b.channel
