def publish(self, payload: str, routing_key: str):
try:
with self._rabbit_connection.connection.channel() as channel:
message = rabbitpy.Message(channel=channel,
body_value=bytes(payload, encoding='utf8'),
opinionated=True)
message.publish(exchange=self._exchange,
routing_key=routing_key,
mandatory=True,
immediate=False)
except rabbitpy.exceptions.MessageReturnedException as error:
# In in case no queue was bound to the exchange at the time of publish, create an
# emergency queue to store the message until a consumer creates their own queue and retry publishing
print(f"Failed to send message: {error}")
with self._rabbit_connection.connection.channel() as _channel:
_message = rabbitpy.Message(channel=_channel,
body_value=bytes(payload + " RETRIED", encoding='utf8'),
opinionated=True)
_message.publish(exchange=self._exchange,
routing_key=routing_key,
mandatory=False,
immediate=False)
print("Tried to send it second time...")
def test_publishing_with_unrouted_msg(self):
with self.subTest(
"Publishing non-routable messages with mandatory set"):
'''
The `mandatory` flag is an argument that's passed along with the Basic.Publish RPC command and
tells RabbitMQ that if a message isn't routable, it should send the message back to the publisher via a Basic.Return RPC.
'''
with rabbitpy.Connection(URL) as connection:
with self.assertRaises(
rabbitpy.exceptions.MessageReturnedException
) as assert_raises_context:
with connection.channel() as channel:
exchange = declare_exchange(
channel, 'publishing-with-mandatory-set')
body = 'test'
message = rabbitpy.Message(
channel, body, {
'content_type': 'text/plain',
'timestamp': datetime.datetime.now(),
})
# RabbitMQ can't route the message because there's no queue bound to the exchange and routing key.
message.publish(exchange,
'no-routing-key',
mandatory=True)
mre = assert_raises_context.exception
self.assertIn('NO_ROUTE', str(mre))
self.assertEqual(channel.state_description, 'Closing')
with self.subTest("Using alternate exchanges for unroutable messages"):
with rabbitpy.Connection(URL) as connection:
with connection.channel() as channel:
alternate_exchange = declare_exchange(
channel, 'alternate-exchange', exchange_type='fanout')
exchange = declare_exchange(channel,
'primary-exchange',
exchange_type='topic',
arguments={
'alternate-exchange':
alternate_exchange.name
})
dlq = declare_queue(channel, 'unroutable-messages-queue')
bind(dlq, alternate_exchange, '#')
origin_len = len(dlq)
body = 'test'
message = rabbitpy.Message(channel, body,
properties('text/plain'))
message.publish(exchange, 'no-routing-key',
mandatory=True) # go to dlq
time.sleep(1)
self.assertEqual(len(dlq), origin_len + 1)
message.publish(exchange, 'no-routing-key') # goto dlq
time.sleep(1)
self.assertEqual(len(dlq), origin_len + 2)
def create_msg(body, type):
try:
url = 'amqp://' + setting.MQUSER + ':' + setting.MQPASSWORD + '@' + setting.MQSERVER + ':' + setting.MQPORT + '/%2f'
with rabbitpy.Connection(url) as conn:
with conn.channel() as channel:
exchange = rabbitpy.Exchange(channel=channel,
name=setting.MQEXCHANGENAME,
durable=True)
exchange.declare()
queue = rabbitpy.Queue(channel=channel,
name=setting.MQQUEUENAME,
durable=True)
queue.declare()
# Bind the queue
queue.bind(exchange, setting.MQROUTINGKEY)
message = rabbitpy.Message(
channel, body, {
'content_type': 'text/plain',
'delivery_mode': 2,
'message_type': type
})
message.publish(exchange, setting.MQROUTINGKEY)
except:
pass
def putget(project):
channel = get_channel()
properties = {
'delivery_mode': 2,
'message_id': str(uuid.uuid4())
}
if request.method == 'PUT':
job = request.get_json()
message = rabbitpy.Message(channel, job, properties=properties)
message.publish('', project)
data = job
if request.method == 'GET':
queue = rabbitpy.Queue(channel, project)
try:
message = queue.get()
except rabbitpy.exceptions.AMQPNotFound:
return "", 204
if message is not None:
msg = message.json()
msg.update(message.properties)
data = loads(dumps(msg, default=lambda obj: str(obj)))
message.ack()
else:
return "", 204
return jsonify(data)
def test_reject_msg_by_nack(self):
channel = get_channel(URL)
exchange = declare_exchange(channel, 'test-nack-msgs-exchange')
queue = declare_queue(channel, "test-nack-msgs-queue")
bind(queue, exchange, "my-routing-key")
queue.purge()
random_count = random.randint(5, 10)
for i in range(0, random_count):
message = rabbitpy.Message(channel, f'test maeesge-{i}',
properties())
message.publish(exchange, 'my-routing-key')
consumer = queue.consume()
msg0 = next(consumer)
self.assertEqual(get_unacked_number(queue.name), random_count)
msg0.nack(False) # reject
self.assertEqual(get_unacked_number(queue.name), random_count - 1)
msg1 = next(consumer)
msg1.nack(True) # requeue
requeued_msg_body = msg1.body.decode('utf-8')
self.assertEqual(get_unacked_number(queue.name), random_count - 1)
_msg2 = next(consumer)
msg3 = next(consumer)
msg3.nack(requeue=False, all_previous=True)
self.assertEqual(get_unacked_number(queue.name),
random_count - 1 - 2) # _msg2 and msg3 are unacked
for m in consumer:
if m.body.decode('utf-8') == requeued_msg_body:
self.assertTrue(m.redelivered)
break
else:
self.assertFalse(m.redelivered)
def _publish(self, msg, exchange_name, routing_key, priority=None):
properties = {'delivery_mode': 2}
if priority:
properties['priority'] = priority
message = rabbitpy.Message(self._channel, msg, properties=properties)
message.publish(exchange_name, routing_key)
def test_exception_is_raised(self):
conn = rabbitpy.Connection(os.environ['RABBITMQ_URL'])
channel = conn.channel()
channel.enable_publisher_confirms()
msg = rabbitpy.Message(channel, body_value='hello')
with self.assertRaises(exceptions.AMQPNotFound):
msg.publish(exchange='invalid', mandatory=True)
def put(self, msg, queue):
"""Non-blocking put.
:type msg: T
:type queue: queue
"""
_msg = rabbitpy.Message(self._ch, msg, {})
_msg.publish('', queue.name)
def test_using_dead_letter_exchange(self):
channel = get_channel(URL)
exchange = declare_exchange(channel,
'test-dlx-normal-exchange',
exchange_type='topic')
with self.subTest("Using x-dead-letter-exchange argument"):
dlx = declare_exchange(channel, 'test-dlx-exchange')
dlq = declare_queue(channel, "test-dlx-dlq")
bind(dlq, dlx, "my-routing-key")
dlq.purge()
queue = declare_queue(channel,
"test-dlx-normal-queue",
dead_letter_exchange=dlx.name)
bind(queue, exchange, "my-routing-key")
queue.purge()
###
message = rabbitpy.Message(channel, 'test dlx message',
properties())
message.publish(exchange, 'my-routing-key')
msg = next(queue.consume())
msg.nack(False)
dmsg = next(dlq.consume())
self.assertEqual(msg.body.decode('utf-8'), 'test dlx message')
with self.subTest("Using x-dead-letter-routing-key argument"):
queue = declare_queue(
channel,
"test-dlx-routing-queue",
dead_letter_exchange=exchange.
name, # have to specify dead_letter_exchange along with dead_letter_routing_key
dead_letter_routing_key="dl-routing-key")
bind(queue, exchange, "normal-routing-key")
queue.purge()
dlq = declare_queue(channel, "test-dlx-routing-dlq")
bind(dlq, exchange, "dl-routing-key")
dlq.purge()
rabbitpy.Message(channel, 'test dlx routing message',
properties()).publish(exchange,
'normal-routing-key')
msg = next(queue.consume())
msg.nack(False)
dmsg = next(dlq.consume())
self.assertEqual(msg.body.decode('utf-8'),
'test dlx routing message')
def load_urls(self, urls):
with self.connection.channel() as channel:
for url in urls:
message = rabbitpy.Message(
channel=channel,
body_value=url,
properties={"delivery_mode": 2},
)
message.publish(self.EXCHANGE, self.ROUTING_KEY)
def publisher(conn):
if not conn:
conn = rabbitpy.Connection('amqp://*****:*****@localhost:5672/%2F')
chan = conn.channel()
for i in range(10000):
routing_key = "teste_mira{}".format(randrange(10))
message = rabbitpy.Message(chan, '{}'.format(i))
message.publish('', routing_key)
print("foi")
def request_job(self, payload):
try:
with self._rabbit_connection.connection.channel() as channel:
self.correlation_id = str(uuid.uuid4())
reply_queue = rabbitpy.Queue(
channel=channel,
name='',
durable=True,
auto_delete=True,
message_ttl=5 * 24 * 60 * 60 * 1000 # 5 days
)
reply_queue.declare()
reply_queue.bind(self._exchange, reply_queue.name)
message = rabbitpy.Message(channel=channel,
body_value=bytes(payload, encoding='utf8'),
properties={
'correlation_id': self.correlation_id,
'reply_to': reply_queue.name,
},
opinionated=True)
message.publish(exchange=self._exchange,
routing_key=self._routing_key,
mandatory=True,
immediate=False)
self._consume_reply(reply_queue)
except rabbitpy.exceptions.MessageReturnedException as error:
# In in case no queue was bound to the exchange at the time of publish, create an
# emergency queue to store the message until a consumer creates their own queue and retry publishing
print(f"Failed to send message: {error}")
with self._rabbit_connection.connection.channel() as _channel:
_message = rabbitpy.Message(channel=_channel,
body_value=bytes(payload + " RETRIED", encoding='utf8'),
opinionated=True)
_message.publish(exchange=self._exchange,
routing_key=self._routing_key,
mandatory=False,
immediate=False)
print("Tried to send it second time...")
self._consume_reply(reply_queue)
def moveToTop(config, queue, task_id):
import threading
lock = threading.Lock()
lock.acquire()
rc = 0
try:
with rabbitpy.Connection('amqp://{}:{}@{}:{}/%2F'.format(
g_username, g_password, config["rabbitMQ_address"],
config["rabbitMQ_port"])) as conn:
with conn.channel() as channel:
# dedicated queue would be consumed firstly
q = rabbitpy.Queue(channel, queue)
q.durable = True
q.declare()
channel.enable_publisher_confirms()
_t = []
_r = ''
for i in range(0, q.__len__()):
msg = q.get()
msg.ack()
task = json.loads(msg.body)
if task["task_id"] == task_id:
_r = msg.body
else:
_t.append(msg.body)
_t.insert(0, _r)
for i in _t:
msg = rabbitpy.Message(channel, i)
# Publish the message, looking for the return value to be a bool True/False
if msg.publish("", queue, mandatory=True):
LOGGER.debug(
'Message {} publish confirmed by RabbitMQ'.format(
msg.body))
else:
LOGGER.error(
'Message {} publish not confirmed by RabbitMQ'.
format(msg.body))
rc = -1
except:
LOGGER.error(traceback.format_exc())
rc = -1
finally:
lock.release()
return rc
#print queryTask({"rabbitMQ_address":'127.0.0.1', 'rabbitMQ_port':5672}, '127.0.0.1', 'APL', ['PIT', 'CIT'], ['OTM', 'PV'])
#print queryTask1({"rabbitMQ_address":'10.239.111.152', 'rabbitMQ_port':5672},"10.239.132.227", "APL",["CIT", "PIT"], ["OTM", "PV"])
#a = getDedicatedTaskQueue({"rabbitMQ_address":'10.239.153.126', 'rabbitMQ_port':5672},"10.239.132.227")
#b = getDedicatedTaskQueue_pika({"rabbitMQ_address":'10.239.153.126', 'rabbitMQ_port':5672},"APL_OTM_CIT")
#print(type(b[0]), b)
def setUp(self):
self.connection = rabbitpy.Connection(os.environ['RABBITMQ_URL'])
self.channel = self.connection.channel()
self.queue = rabbitpy.Queue(self.channel, 'redeliver-test')
self.queue.declare()
# Publish the message that will be rejected
message = rabbitpy.Message(self.channel, 'Payload Value')
message.publish('', 'redeliver-test')
# Get and reject the message
msg1 = self.queue.get()
msg1.reject(requeue=True)
def setUp(self):
self.connection = rabbitpy.Connection(os.environ['RABBITMQ_URL'])
self.channel = self.connection.channel()
self.channel.enable_publisher_confirms()
self.exchange = rabbitpy.TopicExchange(self.channel, 'pql-test')
self.exchange.declare()
self.queue = rabbitpy.Queue(self.channel, 'pql-queue')
self.queue.declare()
self.queue.bind(self.exchange, 'test.#')
for iteration in range(0, self.ITERATIONS):
message = rabbitpy.Message(self.channel, str(uuid.uuid4()))
if not message.publish(self.exchange, 'test.publish.pql'):
LOGGER.error('Error publishing message %i', iteration)
def test_delivery_mode_2(self):
with rabbitpy.Connection(URL) as connection:
with connection.channel() as channel:
exchange = declare_exchange(channel,
'test-delivery-mode2-exchange')
queue = declare_queue(channel, "")
origin_len = len(queue)
bind(queue, exchange, "my-routing-key")
message = rabbitpy.Message(channel, 'test', {
**properties(), 'delivery_mode': 2
})
message.publish(exchange, 'my-routing-key')
time.sleep(1)
self.assertEqual(len(queue), origin_len + 1)
def deliver_etl(result):
queue_name = 'ml_events'
with rabbitpy.Connection() as conn:
logger.debug("Connected to RabbitMQ")
with conn.channel() as channel:
channel.enable_publisher_confirms()
queue = rabbitpy.Queue(channel, queue_name)
queue.durable = True
queue.declare()
message = rabbitpy.Message(channel, json.dumps(result))
logger.info("ML Event message pushed to RabbitMQ")
if not message.publish('', queue_name, mandatory=True):
logger.error("Message failed to publish data for stream id: {0}")
def send_return(self, payload: str):
"""
Send return to TaskRequester which contains the results of task.
:param str payload: payload of task response
"""
assert type(
payload
) is str, "payload must be a string or convertible to JSON"
response = rabbitpy.Message(
self.channel,
body_value=payload,
properties={'correlation_id': self.corr_id})
response.publish(exchange='', routing_key=self.replyTo)
def create_message(channel, item, default_serializer='msgpack'):
if isinstance(item, Message):
return rabbitpy.Message(channel,
compress(to_string(item)),
properties=dict(content_type='message/rfc822',
content_encoding='gzip'))
elif default_serializer == 'msgpack':
return rabbitpy.Message(channel,
msgpack.packb(item),
properties=dict(
content_type='application/x-msgpack', ))
elif default_serializer == 'tarball':
return rabbitpy.Message(channel,
item,
properties=dict(
content_type='application/x-tar', ))
elif default_serializer == 'pickle':
return rabbitpy.Message(channel,
compress(cPickle.dumps(item)),
properties=dict(
content_type='application/x-pickle',
content_encoding='gzip'))
else:
return rabbitpy.Message(channel, item)
def test_publisher_confirms(self):
with rabbitpy.Connection(URL) as connection:
with connection.channel() as channel:
exchange = declare_exchange(channel, 'publisher-comfirms')
queue = declare_queue(channel, "test-publisher-confirms")
bind(queue, exchange, "my-routing-key")
# Prior to publishing any messages, a message publisher must issue a Confirm.Select RPC request to RabbitMQ and
# wait for a Confirm.SelectOk response to know that delivery confirmations are enabled
channel.enable_publisher_confirms()
body = 'test'
message = rabbitpy.Message(channel, body,
properties('text/plain'))
ack = message.publish(exchange, 'my-routing-key')
self.assertTrue(ack)
ack = message.publish(exchange, 'no-routing-key')
self.assertTrue(ack) # although no route, still ack
def test_acknowledging_multiple_messages_at_once(self):
channel = get_channel(URL)
exchange = declare_exchange(channel, 'test-ack-multi-msgs-exchange')
queue = declare_queue(channel, "test-ack-multi-msgs-queue")
bind(queue, exchange, "my-routing-key")
origin_len = len(queue)
random_count = random.randint(2, 10)
for i in range(0, random_count):
message = rabbitpy.Message(channel, f'test maeesge-{i}',
properties())
message.publish(exchange, 'my-routing-key')
consumer = queue.consume()
for _ in range(0, random_count - 1):
msg = next(consumer)
msg.ack(all_previous=True)
self.assertEqual(get_unacked_number(queue.name), 1 + origin_len)
def test_prefetching_via_qos_settings(self):
channel = get_channel(URL)
random_prefetch_count = random.randint(1, 10)
print(f"prefetch count: {random_prefetch_count}")
channel.prefetch_count(random_prefetch_count)
exchange = declare_exchange(channel,
'test-prefetching-via-qos-exchange')
queue = declare_queue(channel, "test-prefetching-via-qos-queue")
bind(queue, exchange, "my-routing-key")
for i in range(0, random_prefetch_count * 2):
message = rabbitpy.Message(channel, f'test maeesge-{i}',
properties())
message.publish(exchange, 'my-routing-key')
consumer = queue.consume(
) # can also use queue.consume(prefetch=random_prefetch_count)
next(consumer) # start consuming
time.sleep(1)
self.assertEqual(get_unacked_number(queue.name), random_prefetch_count)
def publish(self,
routing_key: str,
payload: str,
important: bool = True):
"""
Publish of internal event. All internal subscribers will receive it.
Parameters:
:param str routing_key: Routing key for event
:param str payload: Payload of event
:param str important: indicate whether the publishing important or not,
if yes it will set the mandatory publishing Feature
"""
try:
with self.b_rabbit.connection.channel() as channel:
channel.enable_publisher_confirms()
message = rabbitpy.Message(channel=channel,
body_value=dumps(payload))
published = message.publish(exchange=self.exchange,
routing_key=routing_key,
mandatory=important)
if not published:
logger.warning(
f'message sent from: {self.exchange_name} but RabbitMQ indicates Message publishing failure'
)
else:
logger.info(
f'message sent from: {self.exchange_name} and received successfully from RabbitMQ'
)
return published
except rabbitpy.exceptions.MessageReturnedException as e:
logger.error(
f"Because of the Mandatory Publishing, a Consumer Queue must already be binded to the Exchange"
f" to make sure that the published message will be sooner or later consumed and we will not lose it"
f" so make sure that the subscriber Queue is already bounded to the Publisher \n"
f"More Description of the Exception => {e.args}")
except Exception as e:
logger.exception(e.args, exc_info=False)
def test_no_ack_mode(self):
channel = get_channel(URL)
exchange = declare_exchange(channel, 'test-no-ack-mode-exchange')
queue = declare_queue(channel, "test-no-ack-mode-queue")
origin_len = len(queue)
bind(queue, exchange, "my-routing-key")
random_num = random.randint(1, 5)
print(f'random number of msgs: {random_num}, origin_len: {origin_len}')
for i in range(0, random_num):
message = rabbitpy.Message(channel, f'test no-ack maeesge-{i}',
properties())
message.publish(exchange, 'my-routing-key')
time.sleep(1)
self.assertEqual(len(queue), random_num + origin_len)
consumer = queue.consume(no_ack=True)
time.sleep(1)
next(consumer) # start generator (read all messages)
self.assertEqual(len(queue), 0)
self.assertEqual(get_unacked_number(queue.name), 0)
def publish(self, line):
"""Publish the line to RabbitMQ
:param str line:
"""
msg = rabbitpy.Message(self.channel,
line.rstrip('\r\n'),
self.get_properties(),
opinionated=self.args.auto_id)
if self.args.confirm:
try:
if not msg.publish(self.args.exchange, self.args.routing_key):
self.error('Could not confirm delivery of last message')
except exceptions.AMQPNotFound as error:
self.error('Error publishing message: %s', error.message)
else:
msg.publish(self.args.exchange, self.args.routing_key)
self.counter += 1
self.log('Message #{0} published'.format(self.counter))
def setUp(self):
self.connection = rabbitpy.Connection(os.environ['RABBITMQ_URL'])
self.channel = self.connection.channel()
self.exchange = rabbitpy.TopicExchange(self.channel, 'test-pact')
self.exchange.declare()
self.queue = rabbitpy.Queue(self.channel, 'pact-queue')
self.queue.declare()
self.queue.bind(self.exchange, 'test.#')
self.app_id = 'PublishAndConsumeIteratorTest'
self.message_body = b'ABC1234567890'
self.message_type = 'test'
self.msg = rabbitpy.Message(
self.channel, self.message_body, {
'app_id': self.app_id,
'message_id': str(uuid.uuid4()),
'timestamp': int(time.time()),
'message_type': self.message_type
})
self.msg.publish(self.exchange, 'test.publish.consume')
def setUp(self):
self.connection = rabbitpy.Connection(os.environ['RABBITMQ_URL'])
self.channel = self.connection.channel()
self.exchange = rabbitpy.TopicExchange(self.channel, 'test-pagt')
self.exchange.declare()
self.queue = rabbitpy.Queue(self.channel, 'pagt-queue')
self.queue.declare()
self.queue.bind(self.exchange, 'test.#')
self.app_id = 'PublishAndGetTest'
self.message_body = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
self.message_type = 'test'
self.msg = rabbitpy.Message(
self.channel, self.message_body, {
'app_id': self.app_id,
'message_id': str(uuid.uuid4()),
'timestamp': int(time.time()),
'message_type': self.message_type
})
self.msg.publish(self.exchange, 'test.publish.get')
def _send_message(self, channel, queue, content, src_message):
"""
The method used everywhere to send a message in a queue
"""
try:
properties = {
"priority": src_message.properties["priority"],
"headers": {
"source-id": self.queue_name
}
}
reply_message = rabbitpy.Message(channel,
content,
properties=properties)
# Empty exchange, routing key is the queue name
return reply_message.publish("", queue, mandatory=True)
except RabbitpyException:
raise
except:
self.logger.exception("Unable to send a message")
return False
def test_multi_consumers(self):
channel = get_channel(URL)
exchange = declare_exchange(channel, 'test-multi-consumers-exchange')
queue = declare_queue(channel, "test-multi-consumers-queue")
bind(queue, exchange, "my-routing-key")
total = 0
origin_len = len(queue)
random_num = random.randint(1, 10)
def worker1():
nonlocal total
consumer = queue.consume()
for msg in consumer:
print(
f"Get msg in worker1 (consumer tag: {queue.consumer_tag})")
total += 1
msg.ack()
def worker2():
nonlocal total
q = declare_queue(channel, queue.name)
consumer = q.consume()
for msg in consumer:
print(f"Get msg in worker2 (consumer tag: {q.consumer_tag})")
total += 1
msg.ack()
t1 = threading.Thread(target=worker1, name='Worker1')
t2 = threading.Thread(target=worker2, name='Worker2')
t1.setDaemon(True)
t2.setDaemon(True)
t1.start()
t2.start()
for i in range(0, random_num):
message = rabbitpy.Message(channel, f'test maeesge-{i}',
properties())
message.publish(exchange, 'my-routing-key')
time.sleep(1)
self.assertEqual(total, random_num + origin_len)
def _consume(self, channel):
try:
for message in self._queue.consume(prefetch=1):
if self._callback is not None and message is not None:
_job_result = self._callback(message.body)
message.ack()
_return_message = rabbitpy.Message(
channel=channel,
body_value=bytes(_job_result, encoding='utf8'),
properties={
'correlation_id':
message.properties['correlation_id']
},
opinionated=True)
_return_message.publish(
exchange=self._exchange,
routing_key=message.properties['reply_to'],
mandatory=False,
immediate=False)
except rabbitpy.exceptions.RemoteCancellationException:
self._queue.stop_consuming()
return
