async def get():
message_identifier = str(uuid.uuid4())
timestamp = int(time.time() * 1000)
msg = dramatiq.Message(
queue_name="worker_queue",
actor_name="task",
args=({
"task_id": message_identifier,
"user_id": "default"
}, ),
kwargs={},
options={},
message_id=message_identifier,
message_timestamp=timestamp,
)
channel = await get_database()
# channel = await connection.channel()
exchange = await channel.declare_exchange("topic_logs", ExchangeType.TOPIC)
message_body = json.dumps(
dict(
state="queued",
progress=0,
user_id="default",
task_id=message_identifier,
message_type="task",
)).encode()
message = Message(message_body)
await exchange.publish(
message,
routing_key="default.task",
)
return rabbitmq_broker.enqueue(
msg) # .get_result(block=False, timeout=10000)
async def publisher():
channel1 = await self.create_channel(connection=conn,
cleanup=False)
for i in range(messages):
await channel1.default_exchange.publish(
Message(body=str(i).encode()), routing_key=queue.name)
async def test_simple_publish_without_confirm(self):
queue_name = self.get_random_name("test_connection")
routing_key = self.get_random_name()
channel = await self.create_channel(publisher_confirms=False)
exchange = await self.declare_exchange('direct',
auto_delete=True,
channel=channel)
queue = await self.declare_queue(queue_name,
auto_delete=True,
channel=channel)
await queue.bind(exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
result = await exchange.publish(
Message(body, content_type='text/plain', headers={'foo': 'bar'}),
routing_key)
self.assertIsNone(result)
incoming_message = await queue.get(timeout=5)
incoming_message.ack()
self.assertEqual(incoming_message.body, body)
await queue.unbind(exchange, routing_key)
async def test_simple_publish_and_receive_to_bound_exchange(self):
routing_key = self.get_random_name()
src_name = self.get_random_name("source", "exchange")
dest_name = self.get_random_name("destination", "exchange")
channel = await self.create_channel()
src_exchange = await self.declare_exchange(
src_name, auto_delete=True, channel=channel
)
dest_exchange = await self.declare_exchange(
dest_name, auto_delete=True, channel=channel
)
queue = await self.declare_queue(auto_delete=True, channel=channel)
await queue.bind(dest_exchange, routing_key)
await dest_exchange.bind(src_exchange, routing_key)
self.addCleanup(dest_exchange.unbind, src_exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
await src_exchange.publish(
Message(
body, content_type='text/plain',
headers={'foo': 'bar'}
),
routing_key
)
incoming_message = await queue.get(timeout=5)
incoming_message.ack()
self.assertEqual(incoming_message.body, body)
await queue.unbind(dest_exchange, routing_key)
async def test_purge_queue(self):
queue_name = self.get_random_name("test_connection4")
routing_key = self.get_random_name()
channel = await self.create_channel()
exchange = await channel.declare_exchange('direct', auto_delete=True)
queue = await channel.declare_queue(queue_name, auto_delete=True)
await queue.bind(exchange, routing_key)
try:
body = bytes(shortuuid.uuid(), 'utf-8')
await exchange.publish(
Message(
body, content_type='text/plain',
headers={'foo': 'bar'}
),
routing_key
)
await queue.purge()
with pytest.raises(asyncio.TimeoutError):
await queue.get(timeout=1)
except aio_pika.exceptions.QueueEmpty:
await queue.unbind(exchange, routing_key)
await queue.delete()
async def publish(self, content: Any):
if not hasattr(self, "active_exchange"):
raise JSONQueueError("no active exchange set")
message = Message(bytes(json_encode(content), "utf-8"),
content_type="application/json")
await self.active_exchange.publish(message, self.routing_key)
async def test_message_nack(self):
client = await self.create_connection()
queue_name = self.get_random_name("test_nack_queue")
body = uuid.uuid4().bytes
channel = await client.channel()
queue = await channel.declare_queue(queue_name, auto_delete=True)
await channel.default_exchange.publish(
Message(body=body), routing_key=queue_name
)
message = await queue.get() # type: aio_pika.IncomingMessage
self.assertEqual(message.body, body)
message.nack(requeue=True)
message = await queue.get()
self.assertTrue(message.redelivered)
self.assertEqual(message.body, body)
message.ack()
await queue.delete()
await asyncio.wait((client.close(), client.closing), loop=self.loop)
async def rmq_send(message, key):
"""
Создает подключение к RabbitMQ, создает exchange типа direct и отсылает в него id сообщения, хранящегося в БД
:param message:
:param key:
:return None:
"""
# Perform connection
connection = await connect(URL)
# Creating a channel
channel = await connection.channel()
messages_exchange = await channel.declare_exchange(exchange, ExchangeType.DIRECT)
routing_key = key
message_body = str(message.id).encode('utf-8')
rmq_message = Message(message_body, delivery_mode=DeliveryMode.PERSISTENT)
await messages_exchange.publish(rmq_message, routing_key=routing_key)
print(" [x] Sent %r" % rmq_message)
print('rmq message body:', rmq_message.body.decode('utf-8'))
await connection.close()
async def send(self, loop):
# Perform connection
connection = await connect(
"amqp://*****:*****@localhost/",
# "amqp://*****:*****@192.168.0.101/",
loop=loop)
# Creating a channel
channel = await connection.channel()
topic_logs_exchange = await channel.declare_exchange(
'topic_logs', ExchangeType.TOPIC)
routing_key = (sys.argv[1] if len(sys.argv) > 2 else 'anonymous.info')
message_body = (b' '.join(arg.encode() for arg in sys.argv[2:])
or b"Hello World!")
message = Message(message_body, delivery_mode=DeliveryMode.PERSISTENT)
# Sending the message
await topic_logs_exchange.publish(message, routing_key=routing_key)
print(" [x] Sent %r" % message)
await connection.close()
async def main(loop):
# Perform connection
connection = await connect(host='localhost',
login='******',
password='******')
# Creating a channel
channel = await connection.channel()
topic_logs_exchange = await channel.declare_exchange(
'test', ExchangeType.TOPIC)
routing_key = ('1', '2', '3', '4')
message_body = [(f"{i} - Messages", str(randint(1, 4))) for i in range(10)]
for el in message_body:
message = Message(bytes(el[0], encoding='utf-8'),
delivery_mode=DeliveryMode.PERSISTENT)
# Sending the message
await topic_logs_exchange.publish(message, routing_key=el[1])
print(f" [x] Sent Message: {el[0]} | to: {el[1]}")
await connection.close()
async def _on_history_response(self, message: aio_pika.Message):
with message.process(requeue=True):
body = message.body
from_token = message.app_id
correlation_id = message.correlation_id
request_duration = float(
message.headers.get("x-request-duration", "-1"))
logger.debug(
"received message from {}, correlation id: {}, reply_to: {}",
from_token,
correlation_id,
message.reply_to,
)
history_response_pb = history_pb2.HistoryResponse()
history_response_pb.ParseFromString(body)
history_response = HistoryResponse(history_response_pb,
request_duration)
logger.debug("message is an history response")
try:
future = self._request_futures[correlation_id]
future.set_result(history_response)
except (KeyError, asyncio.InvalidStateError):
logger.error(
"received history response with unknown correlation id {} "
"from {}",
correlation_id,
from_token,
)
return
async def _get_message(self,
body,
content_type: str,
type: str,
correlation_id: str,
identifier: str,
reply_to: str = None,
expiration: DateType = None) -> Message:
return Message(
body=body,
content_type=content_type,
type=type,
correlation_id=correlation_id,
reply_to=reply_to,
expiration=expiration,
headers={
'lang': 'py',
'method': identifier,
# 'id': correlation_id,
# 'shadow': shadow,
# 'countdown': countdown,
# 'eta': eta,
# 'group': group_id,
# 'retries': retries,
# 'timelimit': [time_limit, soft_time_limit],
# 'root_id': root_id,
# 'parent_id': parent_id,
'origin': self.get_nodename()
})
async def send_transaction_queue(
rabbitmq: Connection, *,
wallet_id,
amount: Decimal,
transaction_type: Literal["TRANSFER", "REFILL"],
transfer_wallet_id: Optional[int] = None
):
TransactionInputValidate(
transaction_type=transaction_type,
transfer_wallet_id=transfer_wallet_id
)
u = uuid.uuid4()
channel = await rabbitmq.channel()
data = {
"uuid": str(u),
"wallet_id": wallet_id,
"amount": str(amount),
"transaction_type": transaction_type,
"transfer_wallet_id": transfer_wallet_id
}
message = Message(
json.dumps(data).encode("utf-8"),
delivery_mode=DeliveryMode.PERSISTENT
)
await channel.default_exchange.publish(
message, routing_key=settings.RABBITMQ_QUEUE_NAME
)
return u
async def on_message(exchange: Exchange, message: IncomingMessage):
with message.process():
payload = json.loads(message.body.decode())
try:
if payload['type'] == 'signup':
return_data = await signup(payload['data'])
elif payload['type'] == 'login':
return_data = await login(payload['data'])
elif payload['type'] == 'validate':
return_data = await validate(payload['data'])
except ValueError as err:
response = json.dumps({'status': err, 'data': {}}).encode()
else:
if isinstance(return_data, str):
response = json.dumps({
'status': return_data,
'data': {}
}).encode()
else:
response = json.dumps({
'status': 'ok',
'data': return_data
}).encode()
await exchange.publish(Message(body=response,
content_type='application/json',
correlation_id=message.correlation_id),
routing_key=message.reply_to)
def test_big_message(self):
client = yield from self.create_connection()
queue_name = self.get_random_name("test_big")
routing_key = self.get_random_name()
channel = yield from client.channel()
exchange = yield from channel.declare_exchange('direct',
auto_delete=True)
queue = yield from channel.declare_queue(queue_name, auto_delete=True)
yield from queue.bind(exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8') * 9999999
yield from exchange.publish(
Message(body, content_type='text/plain', headers={'foo': 'bar'}),
routing_key)
incoming_message = yield from queue.get(timeout=5)
incoming_message.ack()
self.assertEqual(incoming_message.body, body)
yield from queue.unbind(exchange, routing_key)
yield from queue.delete()
yield from wait((client.close(), client.closing), loop=self.loop)
async def publish(self, message_content, priority=0):
"""publish: Produce Message to Message Broker
- message_content: payload of message to be published
- priority: message priority
"""
try:
self.exchange = await self.channel.declare_exchange(
self.exchange_name, ExchangeType.FANOUT)
for binding_key in self.binding_keys:
message = Message(body=message_content,
delivery_mode=DeliveryMode.NOT_PERSISTENT,
priority=priority)
# logger.debug(
# f'msg Publish: Exchange: {self.exchange_name}, Routing:{binding_key + self.binding_suffix}'
# )
await self.exchange.publish(message,
routing_key=binding_key +
self.binding_suffix)
except aio_pika_exception.AMQPException as e:
logger.error(e)
await self.terminate()
sys.exit(-1)
except Exception as e:
logger.error('Exception during Publishing Message to Broker')
logger.error(e)
await self.terminate()
sys.exit(-1)
async def put(self, message: dict):
message_json = json.dumps(message)
await self._channel.default_exchange.publish(
Message(message_json.encode('utf-8')),
routing_key=self._queue_name)
meter_log.info("BrokerClient publishing message %s to Queue %s" %
(message_json, self._queue_name))
def test_message_copy():
msg1 = Message(
bytes(shortuuid.uuid(), "utf-8"),
content_type="application/json",
content_encoding="text",
timestamp=datetime(2000, 1, 1),
headers={
"h1": "v1",
"h2": "v2"
},
)
msg2 = copy(msg1)
msg1.lock()
assert not msg2.locked
async def publish(self, msg: str, priority: int = 0) -> None:
await self.exchange.publish(Message(
msg.encode(),
priority=priority,
delivery_mode=DeliveryMode.PERSISTENT,
),
routing_key=self.routing_key)
async def main(loop):
connection = await connect_robust("amqp://*****:*****@127.0.0.1/",
loop=loop)
queue_name = "test_queue"
routing_key = "test_queue"
# Creating channel
channel = await connection.channel()
# Declaring exchange
exchange = await channel.declare_exchange('direct', auto_delete=True)
# Declaring queue
queue = await channel.declare_queue(queue_name, auto_delete=True)
# Binding queue
await queue.bind(exchange, routing_key)
await exchange.publish(
Message(bytes('Hello', 'utf-8'),
content_type='text/plain',
headers={'foo': 'bar'}), routing_key)
# Receiving message
incoming_message = await queue.get(timeout=5)
# Confirm message
await incoming_message.ack()
await queue.unbind(exchange, routing_key)
await queue.delete()
await connection.close()
async def test_queue_iterator_close_with_noack(self):
messages = []
queue_name = self.get_random_name("test_queue")
async def task_inner():
nonlocal messages
connection = await self.create_connection()
async with connection:
channel = await connection.channel()
queue = await channel.declare_queue(queue_name)
async with queue.iterator(no_ack=True) as q:
async for message in q:
messages.append(message)
return
connection = await self.create_connection()
channel = await connection.channel()
queue = await channel.declare_queue(queue_name)
await channel.default_exchange.publish(
Message(b'fooz'),
routing_key=queue_name,
)
task = self.loop.create_task(task_inner())
await task
assert messages
assert messages[0].body == b'fooz'
async def discover(self, url: str) -> Dict[str, Dict[str, List[str]]]:
"""
Faktory entrypoint for the discovery process.
:param url: The url to scrape.
:type url: string
"""
self.logger.info(f"Received: {url}")
async with self.http_session.get(url, ssl=False) as resp:
if resp.status >= 400:
self.logger.warn(f"Got error code {resp.status} for {url}")
html_body = await resp.text()
async with self._amqp_pool.acquire() as raw_channel:
raw_exchange = await raw_channel.declare_exchange(
"discovery_raw", ExchangeType.FANOUT
)
message = Message(
json.dumps({"url": url, "html": html_body}).encode("utf-8"),
delivery_mode=DeliveryMode.PERSISTENT,
)
# routing_key has to be present but is irelevant
await raw_exchange.publish(message, routing_key="")
result = self.extract(html_body, url)
self.logger.info(f"Done processing: {url}")
return {"data": result}
async def test_reject_twice(self):
client = await self.create_connection()
queue_name = self.get_random_name("test_connection")
routing_key = self.get_random_name()
channel = await client.channel()
exchange = await channel.declare_exchange('direct', auto_delete=True)
queue = await channel.declare_queue(queue_name, auto_delete=True)
await queue.bind(exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
await exchange.publish(
Message(
body, content_type='text/plain',
headers={'foo': 'bar'}
),
routing_key
)
incoming_message = await queue.get(timeout=5)
incoming_message.reject(requeue=False)
with pytest.raises(MessageProcessError):
incoming_message.reject(requeue=False)
self.assertEqual(incoming_message.body, body)
await queue.unbind(exchange, routing_key)
await queue.delete()
await asyncio.wait((client.close(), client.closing), loop=self.loop)
def test_simple_publish_and_receive(self):
queue_name = self.get_random_name("test_connection")
routing_key = self.get_random_name()
channel = yield from self.create_channel()
exchange = yield from self.declare_exchange('direct',
auto_delete=True,
channel=channel)
queue = yield from self.declare_queue(queue_name,
auto_delete=True,
channel=channel)
yield from queue.bind(exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
result = yield from exchange.publish(
Message(body, content_type='text/plain', headers={'foo': 'bar'}),
routing_key)
self.assertTrue(result)
incoming_message = yield from queue.get(timeout=5)
incoming_message.ack()
self.assertEqual(incoming_message.body, body)
yield from queue.unbind(exchange, routing_key)
async def test_internal_exchange(self):
client = await self.create_connection()
routing_key = self.get_random_name()
exchange_name = self.get_random_name("internal", "exchange")
channel = await client.channel()
exchange = await self.declare_exchange(
exchange_name,
auto_delete=True,
internal=True,
channel=channel
)
queue = await self.declare_queue(auto_delete=True, channel=channel)
await queue.bind(exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
with pytest.raises(ValueError):
f = exchange.publish(
Message(
body, content_type='text/plain',
headers={'foo': 'bar'}
),
routing_key
)
await f
await queue.unbind(exchange, routing_key)
def test_consuming_not_coroutine(self):
client = yield from self.create_connection()
queue_name = self.get_random_name("tc2")
routing_key = self.get_random_name()
channel = yield from client.channel()
exchange = yield from channel.declare_exchange('direct',
auto_delete=True)
queue = yield from channel.declare_queue(queue_name, auto_delete=True)
yield from queue.bind(exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
f = asyncio.Future(loop=self.loop)
def handle(message):
message.ack()
self.assertEqual(message.body, body)
self.assertEqual(message.routing_key, routing_key)
f.set_result(True)
yield from queue.consume(handle)
yield from exchange.publish(
Message(body, content_type='text/plain', headers={'foo': 'bar'}),
routing_key)
if not f.done():
yield from f
yield from queue.unbind(exchange, routing_key)
yield from exchange.delete()
yield from wait((client.close(), client.closing), loop=self.loop)
def test_message_copy(self):
msg1 = Message(
bytes(shortuuid.uuid(), 'utf-8'),
content_type='application/json',
content_encoding='text',
timestamp=datetime(2000, 1, 1),
headers={
'h1': 'v1',
'h2': 'v2'
},
)
msg2 = copy(msg1)
msg1.lock()
self.assertFalse(msg2.locked)
def test_purge_queue(self):
queue_name = self.get_random_name("test_connection4")
routing_key = self.get_random_name()
channel = yield from self.create_channel()
exchange = yield from channel.declare_exchange('direct',
auto_delete=True)
queue = yield from channel.declare_queue(queue_name, auto_delete=True)
yield from queue.bind(exchange, routing_key)
try:
body = bytes(shortuuid.uuid(), 'utf-8')
yield from exchange.publish(
Message(body,
content_type='text/plain',
headers={'foo': 'bar'}), routing_key)
yield from queue.purge()
with self.assertRaises(asyncio.TimeoutError):
yield from queue.get(timeout=1)
except:
yield from queue.unbind(exchange, routing_key)
yield from queue.delete()
async def test_simple_publish_and_receive_delivery_mode_explicitly(self):
queue_name = self.get_random_name("test_connection")
routing_key = self.get_random_name()
channel = await self.create_channel()
exchange = await self.declare_exchange('direct',
auto_delete=True,
channel=channel)
queue = await self.declare_queue(queue_name,
auto_delete=True,
channel=channel)
await queue.bind(exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
await exchange.publish(
Message(body,
content_type='text/plain',
headers={'foo': 'bar'},
delivery_mode=None), routing_key)
incoming_message = await queue.get(timeout=5)
incoming_message.ack()
self.assertEqual(incoming_message.body, body)
await queue.unbind(exchange, routing_key)
async def _republish_message(self, message, attempt_number):
headers = {
**message.headers,
ATTEMPT_NUMBER: attempt_number + 1,
}
new_message = Message(
body=message.body,
headers=headers,
content_type=message.content_type,
content_encoding=message.content_encoding,
delivery_mode=message.delivery_mode,
correlation_id=message.correlation_id,
priority=message.priority,
reply_to=message.reply_to,
expiration=message.expiration,
message_id=message.message_id,
timestamp=message.timestamp,
type=message.type,
app_id=message.app_id,
user_id=message.user_id,
)
await self.exchange.publish(new_message,
message.routing_key,
mandatory=True)
await message.nack(requeue=False)
