def retry_deliveries(self):
"""
Handle [MQTT-4.4.0-1] by resending PUBLISH and PUBREL messages for pending out messages
:return:
"""
self.logger.debug("Begin messages delivery retries")
ack_packets = []
for packet_id in self.session.outgoing_msg:
message = self.session.outgoing_msg[packet_id]
if message.is_new() or message.is_published():
self.logger.debug("Retrying publish message Id=%d", packet_id)
message.publish_packet.dup_flag = True
ack = False
while not ack:
yield from self.outgoing_queue.put(message.publish_packet)
message.retry_publish()
ack = yield from message.wait_acknowledge()
ack_packets.append(packet_id)
if message.is_received():
self.logger.debug("Retrying pubrel message Id=%d", packet_id)
yield from self.outgoing_queue.put(PubrelPacket.build(packet_id))
message.sent_pubrel()
for packet_id in ack_packets:
del self.session.outgoing_msg[packet_id]
self.logger.debug("%d messages redelivered" % len(ack_packets))
self.logger.debug("End messages delivery retries")
def retry_deliveries(self):
"""
Handle [MQTT-4.4.0-1] by resending PUBLISH and PUBREL messages for pending out messages
:return:
"""
self.logger.debug("Begin messages delivery retries")
ack_packets = []
for packet_id in self.session.outgoing_msg:
message = self.session.outgoing_msg[packet_id]
if message.is_new() or message.is_published():
self.logger.debug("Retrying publish message Id=%d", packet_id)
message.publish_packet.dup_flag = True
ack = False
while not ack:
yield from self.outgoing_queue.put(message.publish_packet)
message.retry_publish()
ack = yield from message.wait_acknowledge()
ack_packets.append(packet_id)
if message.is_received():
self.logger.debug("Retrying pubrel message Id=%d", packet_id)
yield from self.outgoing_queue.put(
PubrelPacket.build(packet_id))
message.sent_pubrel()
for packet_id in ack_packets:
del self.session.outgoing_msg[packet_id]
self.logger.debug("%d messages redelivered" % len(ack_packets))
self.logger.debug("End messages delivery retries")
def handle_pubrec(self, pubrec: PubrecPacket):
packet_id = pubrec.variable_header.packet_id
try:
inflight_message = self.session.outgoing_msg[packet_id]
inflight_message.received_pubrec()
yield from self.outgoing_queue.put(PubrelPacket.build(packet_id))
inflight_message.sent_pubrel()
except KeyError as ke:
self.logger.warn("Received PUBREC for unknown pending subscription with Id: %s" % packet_id)
def handle_pubrec(self, pubrec: PubrecPacket):
packet_id = pubrec.variable_header.packet_id
try:
inflight_message = self.session.outgoing_msg[packet_id]
inflight_message.received_pubrec()
yield from self.outgoing_queue.put(PubrelPacket.build(packet_id))
inflight_message.sent_pubrel()
except KeyError as ke:
self.logger.warn(
"Received PUBREC for unknown pending subscription with Id: %s"
% packet_id)
def server_mock(reader, writer):
try:
packet = PublishPacket.build("/topic", b"test_data", rand_packet_id(), False, QOS_2, False)
yield from packet.to_stream(writer)
pubrec = yield from PubrecPacket.from_stream(reader)
self.assertIsNotNone(pubrec)
self.assertEqual(packet.packet_id, pubrec.packet_id)
self.assertIn(packet.packet_id, self.handler._pubrel_waiters)
pubrel = PubrelPacket.build(packet.packet_id)
yield from pubrel.to_stream(writer)
pubcomp = yield from PubcompPacket.from_stream(reader)
self.assertIsNotNone(pubcomp)
self.assertEqual(packet.packet_id, pubcomp.packet_id)
except Exception as ae:
future.set_exception(ae)
def server_mock(reader, writer):
try:
packet = PublishPacket.build('/topic', b'test_data', rand_packet_id(), False, QOS_2, False)
yield from packet.to_stream(writer)
pubrec = yield from PubrecPacket.from_stream(reader)
self.assertIsNotNone(pubrec)
self.assertEqual(packet.packet_id, pubrec.packet_id)
self.assertIn(packet.packet_id, self.handler._pubrel_waiters)
pubrel = PubrelPacket.build(packet.packet_id)
yield from pubrel.to_stream(writer)
pubcomp = yield from PubcompPacket.from_stream(reader)
self.assertIsNotNone(pubcomp)
self.assertEqual(packet.packet_id, pubcomp.packet_id)
except Exception as ae:
future.set_exception(ae)
def server_mock(reader, writer):
try:
packet = yield from PublishPacket.from_stream(reader)
self.assertEqual(packet.topic_name, '/topic')
self.assertEqual(packet.qos, QOS_2)
self.assertIsNotNone(packet.packet_id)
self.assertIn(packet.packet_id, self.session.inflight_out)
self.assertIn(packet.packet_id, self.handler._pubrec_waiters)
pubrec = PubrecPacket.build(packet.packet_id)
yield from pubrec.to_stream(writer)
pubrel = yield from PubrelPacket.from_stream(reader)
self.assertIn(packet.packet_id, self.handler._pubcomp_waiters)
pubcomp = PubcompPacket.build(packet.packet_id)
yield from pubcomp.to_stream(writer)
except Exception as ae:
future.set_exception(ae)
async def server_mock(stream):
try:
packet = PublishPacket.build('/topic', b'test_data',
rand_packet_id(), False, QOS_2,
False)
await packet.to_stream(stream)
pubrec = await PubrecPacket.from_stream(stream)
self.assertIsNotNone(pubrec)
self.assertEqual(packet.packet_id, pubrec.packet_id)
self.assertIn(packet.packet_id, self.handler._pubrel_waiters)
pubrel = PubrelPacket.build(packet.packet_id)
await pubrel.to_stream(stream)
pubcomp = await PubcompPacket.from_stream(stream)
self.assertIsNotNone(pubcomp)
self.assertEqual(packet.packet_id, pubcomp.packet_id)
except Exception as ae:
raise
def server_mock(reader, writer):
try:
packet = yield from PublishPacket.from_stream(reader)
self.assertEquals(packet.topic_name, "/topic")
self.assertEquals(packet.qos, QOS_2)
self.assertIsNotNone(packet.packet_id)
self.assertIn(packet.packet_id, self.session.inflight_out)
self.assertIn(packet.packet_id, self.handler._pubrec_waiters)
pubrec = PubrecPacket.build(packet.packet_id)
yield from pubrec.to_stream(writer)
pubrel = yield from PubrelPacket.from_stream(reader)
self.assertIn(packet.packet_id, self.handler._pubcomp_waiters)
pubcomp = PubcompPacket.build(packet.packet_id)
yield from pubcomp.to_stream(writer)
except Exception as ae:
future.set_exception(ae)
def _inflight_coro(self):
self.logger.debug("Starting in-flight messages polling coro")
while self._running:
self._inflight_ready.set()
yield from asyncio.sleep(self.config["inflight-polling-interval"])
self.logger.debug("in-flight polling coro wake-up")
try:
while not self.incoming_queues[PacketType.PUBACK].empty():
packet = self.incoming_queues[PacketType.PUBACK].get_nowait()
packet_id = packet.variable_header.packet_id
inflight_message = self.inflight_messages.get(packet_id)
inflight_message.acknowledge()
self.logger.debug("Message with packet Id=%s acknowledged" % packet_id)
while not self.incoming_queues[PacketType.PUBREC].empty():
packet = self.incoming_queues[PacketType.PUBREC].get_nowait()
packet_id = packet.variable_header.packet_id
inflight_message = self.inflight_messages.get(packet_id)
inflight_message.receive()
self.logger.debug("Message with packet Id=%s received" % packet_id)
rel_packet = PubrelPacket.build(packet_id)
yield from self.outgoing_queue.put(rel_packet)
inflight_message.release()
self.logger.debug("Message with packet Id=%s released" % packet_id)
while not self.incoming_queues[PacketType.PUBCOMP].empty():
packet = self.incoming_queues[PacketType.PUBCOMP].get_nowait()
packet_id = packet.variable_header.packet_id
inflight_message = self.inflight_messages.get(packet_id)
inflight_message.complete()
self.logger.debug("Message with packet Id=%s completed" % packet_id)
yield from self._inflight_changed.acquire()
self._inflight_changed.notify_all()
self._inflight_changed.release()
except KeyError:
self.logger.warn(
"Received %s for unknown inflight message Id %d" % (packet.fixed_header.packet_type, packet_id)
)
except MachineError as me:
self.logger.warn("Packet type incompatible with message QOS: %s" % me)
self.logger.debug("In-flight messages polling coro stopped")
def test_from_stream(self):
data = b'\x60\x02\x00\x0a'
stream = asyncio.StreamReader(loop=self.loop)
stream.feed_data(data)
message = self.loop.run_until_complete(PubrelPacket.from_stream(stream))
self.assertEqual(message.variable_header.packet_id, 10)
def test_from_stream(self):
data = b'\x60\x02\x00\x0a'
stream = BufferReader(data)
message = self.loop.run_until_complete(PubrelPacket.from_stream(stream))
self.assertEqual(message.variable_header.packet_id, 10)
def test_to_bytes(self):
variable_header = PacketIdVariableHeader(10)
publish = PubrelPacket(variable_header=variable_header)
out = publish.to_bytes()
self.assertEqual(out, b'\x62\x02\x00\x0a')
def test_to_bytes(self):
variable_header = PacketIdVariableHeader(10)
publish = PubrelPacket(variable_header=variable_header)
out = publish.to_bytes()
self.assertEqual(out, b"\x62\x02\x00\x0a")
def test_from_stream(self):
data = b"\x60\x02\x00\x0a"
stream = BufferReader(data)
message = self.loop.run_until_complete(
PubrelPacket.from_stream(stream))
self.assertEqual(message.variable_header.packet_id, 10)
def _handle_qos2_message_flow(self, app_message):
"""
Handle QOS_2 application message acknowledgment
For incoming messages, this method stores the message, sends PUBREC, waits for PUBREL, initiate delivery
and send PUBCOMP
For outgoing messages, this methods sends PUBLISH, waits for PUBREC, discards messages and wait for PUBCOMP
:param app_message:
:return:
"""
assert app_message.qos == QOS_2
if app_message.direction == OUTGOING:
if app_message.pubrel_packet and app_message.pubcomp_packet:
raise HBMQTTException(
"Message '%d' has already been acknowledged" %
app_message.packet_id)
if not app_message.pubrel_packet:
# Store message
if app_message.publish_packet is not None:
# This is a retry flow, no need to store just check the message exists in session
if app_message.packet_id not in self.session.inflight_out:
raise HBMQTTException(
"Unknown inflight message '%d' in session" %
app_message.packet_id)
publish_packet = app_message.build_publish_packet(dup=True)
else:
# Store message in session
self.session.inflight_out[
app_message.packet_id] = app_message
publish_packet = app_message.build_publish_packet()
# Send PUBLISH packet
yield from self._send_packet(publish_packet)
app_message.publish_packet = publish_packet
# Wait PUBREC
if app_message.packet_id in self._pubrec_waiters:
# PUBREC waiter already exists for this packet ID
message = "Can't add PUBREC waiter, a waiter already exists for message Id '%s'" \
% app_message.packet_id
self.logger.warning(message)
raise HBMQTTException(message)
waiter = asyncio.Future(loop=self._loop)
self._pubrec_waiters[app_message.packet_id] = waiter
yield from waiter
del self._pubrec_waiters[app_message.packet_id]
app_message.pubrec_packet = waiter.result()
if not app_message.pubcomp_packet:
# Send pubrel
app_message.pubrel_packet = PubrelPacket.build(
app_message.packet_id)
yield from self._send_packet(app_message.pubrel_packet)
# Wait for PUBCOMP
waiter = asyncio.Future(loop=self._loop)
self._pubcomp_waiters[app_message.packet_id] = waiter
yield from waiter
del self._pubcomp_waiters[app_message.packet_id]
app_message.pubcomp_packet = waiter.result()
# Discard inflight message
del self.session.inflight_out[app_message.packet_id]
elif app_message.direction == INCOMING:
self.session.inflight_in[app_message.packet_id] = app_message
# Send pubrec
pubrec_packet = PubrecPacket.build(app_message.packet_id)
yield from self._send_packet(pubrec_packet)
app_message.pubrec_packet = pubrec_packet
# Wait PUBREL
if app_message.packet_id in self._pubrel_waiters and not self._pubrel_waiters[
app_message.packet_id].done():
# PUBREL waiter already exists for this packet ID
message = "A waiter already exists for message Id '%s', canceling it" \
% app_message.packet_id
self.logger.warning(message)
self._pubrel_waiters[app_message.packet_id].cancel()
try:
waiter = asyncio.Future(loop=self._loop)
self._pubrel_waiters[app_message.packet_id] = waiter
yield from waiter
del self._pubrel_waiters[app_message.packet_id]
app_message.pubrel_packet = waiter.result()
# Initiate delivery and discard message
yield from self.session.delivered_message_queue.put(
app_message)
del self.session.inflight_in[app_message.packet_id]
# Send pubcomp
pubcomp_packet = PubcompPacket.build(app_message.packet_id)
yield from self._send_packet(pubcomp_packet)
app_message.pubcomp_packet = pubcomp_packet
except asyncio.CancelledError:
self.logger.debug("Message flow cancelled")
def _handle_qos2_message_flow(self, app_message):
"""
Handle QOS_2 application message acknowledgment
For incoming messages, this method stores the message, sends PUBREC, waits for PUBREL, initiate delivery
and send PUBCOMP
For outgoing messages, this methods sends PUBLISH, waits for PUBREC, discards messages and wait for PUBCOMP
:param app_message:
:return:
"""
assert app_message.qos == QOS_2
if app_message.direction == OUTGOING:
if app_message.pubrel_packet and app_message.pubcomp_packet:
raise HBMQTTException("Message '%d' has already been acknowledged" % app_message.packet_id)
if not app_message.pubrel_packet:
# Store message
if app_message.publish_packet is not None:
# This is a retry flow, no need to store just check the message exists in session
if app_message.packet_id not in self.session.inflight_out:
raise HBMQTTException("Unknown inflight message '%d' in session" % app_message.packet_id)
publish_packet = app_message.build_publish_packet(dup=True)
else:
# Store message in session
self.session.inflight_out[app_message.packet_id] = app_message
publish_packet = app_message.build_publish_packet()
# Send PUBLISH packet
yield from self._send_packet(publish_packet)
app_message.publish_packet = publish_packet
# Wait PUBREC
if app_message.packet_id in self._pubrec_waiters:
# PUBREC waiter already exists for this packet ID
message = "Can't add PUBREC waiter, a waiter already exists for message Id '%s'" \
% app_message.packet_id
self.logger.warning(message)
raise HBMQTTException(message)
waiter = asyncio.Future(loop=self._loop)
self._pubrec_waiters[app_message.packet_id] = waiter
yield from waiter
del self._pubrec_waiters[app_message.packet_id]
app_message.pubrec_packet = waiter.result()
if not app_message.pubcomp_packet:
# Send pubrel
app_message.pubrel_packet = PubrelPacket.build(app_message.packet_id)
yield from self._send_packet(app_message.pubrel_packet)
# Wait for PUBCOMP
waiter = asyncio.Future(loop=self._loop)
self._pubcomp_waiters[app_message.packet_id] = waiter
yield from waiter
del self._pubcomp_waiters[app_message.packet_id]
app_message.pubcomp_packet = waiter.result()
# Discard inflight message
del self.session.inflight_out[app_message.packet_id]
elif app_message.direction == INCOMING:
self.session.inflight_in[app_message.packet_id] = app_message
# Send pubrec
pubrec_packet = PubrecPacket.build(app_message.packet_id)
yield from self._send_packet(pubrec_packet)
app_message.pubrec_packet = pubrec_packet
# Wait PUBREL
if app_message.packet_id in self._pubrel_waiters and not self._pubrel_waiters[app_message.packet_id].done():
# PUBREL waiter already exists for this packet ID
message = "Can't add PUBREL waiter, a waiter already exists for message Id '%s'" \
% app_message.packet_id
self.logger.warning(message)
raise HBMQTTException(message)
waiter = asyncio.Future(loop=self._loop)
self._pubrel_waiters[app_message.packet_id] = waiter
yield from waiter
del self._pubrel_waiters[app_message.packet_id]
app_message.pubrel_packet = waiter.result()
# Initiate delivery and discard message
yield from self.session.delivered_message_queue.put(app_message)
del self.session.inflight_in[app_message.packet_id]
# Send pubcomp
pubcomp_packet = PubcompPacket.build(app_message.packet_id)
yield from self._send_packet(pubcomp_packet)
app_message.pubcomp_packet = pubcomp_packet