class LayeredTwistedIOLoop(TwistedIOLoop):
"""Layers a TwistedIOLoop on top of a TornadoReactor on a PollIOLoop.
This is of course silly, but is useful for testing purposes to make
sure we're implementing both sides of the various interfaces
correctly. In some tests another TornadoReactor is layered on top
of the whole stack.
"""
def initialize(self, **kwargs):
self.real_io_loop = PollIOLoop(make_current=False) # type: ignore
reactor = self.real_io_loop.run_sync(gen.coroutine(TornadoReactor))
super(LayeredTwistedIOLoop, self).initialize(reactor=reactor, **kwargs)
self.add_callback(self.make_current)
def close(self, all_fds=False):
super(LayeredTwistedIOLoop, self).close(all_fds=all_fds)
# HACK: This is the same thing that test_class.unbuildReactor does.
for reader in self.reactor._internalReaders:
self.reactor.removeReader(reader)
reader.connectionLost(None)
self.real_io_loop.close(all_fds=all_fds)
def stop(self):
# One of twisted's tests fails if I don't delay crash()
# until the reactor has started, but if I move this to
# TwistedIOLoop then the tests fail when I'm *not* running
# tornado-on-twisted-on-tornado. I'm clearly missing something
# about the startup/crash semantics, but since stop and crash
# are really only used in tests it doesn't really matter.
def f():
self.reactor.crash()
# Become current again on restart. This is needed to
# override real_io_loop's claim to being the current loop.
self.add_callback(self.make_current)
self.reactor.callWhenRunning(f)
class LayeredTwistedIOLoop(TwistedIOLoop):
"""Layers a TwistedIOLoop on top of a TornadoReactor on a PollIOLoop.
This is of course silly, but is useful for testing purposes to make
sure we're implementing both sides of the various interfaces
correctly. In some tests another TornadoReactor is layered on top
of the whole stack.
"""
def initialize(self, **kwargs):
self.real_io_loop = PollIOLoop(make_current=False) # type: ignore
reactor = self.real_io_loop.run_sync(gen.coroutine(TornadoReactor))
super(LayeredTwistedIOLoop, self).initialize(reactor=reactor, **kwargs)
self.add_callback(self.make_current)
def close(self, all_fds=False):
super(LayeredTwistedIOLoop, self).close(all_fds=all_fds)
# HACK: This is the same thing that test_class.unbuildReactor does.
for reader in self.reactor._internalReaders:
self.reactor.removeReader(reader)
reader.connectionLost(None)
self.real_io_loop.close(all_fds=all_fds)
def stop(self):
# One of twisted's tests fails if I don't delay crash()
# until the reactor has started, but if I move this to
# TwistedIOLoop then the tests fail when I'm *not* running
# tornado-on-twisted-on-tornado. I'm clearly missing something
# about the startup/crash semantics, but since stop and crash
# are really only used in tests it doesn't really matter.
def f():
self.reactor.crash()
# Become current again on restart. This is needed to
# override real_io_loop's claim to being the current loop.
self.add_callback(self.make_current)
self.reactor.callWhenRunning(f)
def install():
"""DEPRECATED
pyzmq 17 no longer needs any special integration for tornado.
"""
_deprecated()
PollIOLoop.configure(ZMQIOLoop)
def current(*args, **kwargs):
"""Returns the current thread’s IOLoop.
"""
# install ZMQIOLoop as the active IOLoop implementation
# when using tornado 3
if tornado_version >= (3,):
PollIOLoop.configure(ZMQIOLoop)
return PollIOLoop.current(*args, **kwargs)
def current(*args, **kwargs):
"""Returns the current threads IOLoop.
"""
#
# install PuritanicalIOLoop as the active IOLoop implementation
#
PollIOLoop.configure(PuritanicalIOLoop)
return PollIOLoop.current(*args, **kwargs)
def run(self):
while True:
try:
context, packet = self.queue.get()
buff = str(packet)
PollIOLoop.instance().add_callback(self.ioloop_callback, context, buff)
finally:
self.queue.task_done()
def reset_timeout(self):
if not self._keep_alive: # dont set timeout if keep_alive is zero that be set by client
return
if self._timeout_handle != None:
PollIOLoop.instance().remove_timeout(self._timeout_handle)
self._timeout_handler = None
self._timeout_handle = PollIOLoop.instance().call_later(1.5*self._keep_alive, self.keep_alive_timeout)
logging.info(" reset_timeout --> device_id: %s" % (self._client_id))
return
def current(cls, *args, **kwargs):
"""Returns the current thread’s IOLoop.
"""
# install ZMQIOLoop as the active IOLoop implementation
# when using tornado 3
PollIOLoop.configure(cls)
_deprecated()
loop = PollIOLoop.current(*args, **kwargs)
return loop
def close_callback(self):
PollIOLoop.instance().remove_timeout(self._timeout_handle)
if self._connect_state == mq_cs_connected:
logging.info(" close_callback --> device_id: %s " %
(self._client_id))
self._ppmtdb.work_queue.put(
(self._ppmtdb.set_inactive_cnn,
(self, ))) # put a set inactive task into queue
return
def reset_timeout(self):
if not self._keep_alive: # dont set timeout if keep_alive is zero that be set by client
return
if self._timeout_handle != None:
PollIOLoop.instance().remove_timeout(self._timeout_handle)
self._timeout_handler = None
self._timeout_handle = PollIOLoop.instance().call_later(
1.5 * self._keep_alive, self.keep_alive_timeout)
logging.info(" reset_timeout --> device_id: %s" % (self._client_id))
return
def mqtt3_send_publish(self, context, topic, qos, message):
logging.info("sended pub --> topic: %s device_id: %s " % (topic, context._client_id))
packet = bytearray()
packet.extend(struct.pack("!B", MQTT3_PUBLISH | (qos << 1)))
topic_len = len(topic)
msg_len = 0 if message == None else len(message)
remain_len = 2 + topic_len + msg_len
if qos:
remain_len += 2 # for mid
self._pack_remain_len(packet, remain_len)
fmt = "!H" + str(topic_len) + "s"
# topic_len + topic
packet.extend(struct.pack(fmt, topic_len, topic))
if qos: # for mid
context._out_mid += 1
if 65536 == context._out_mid:
context._out_mid = 1
packet.extend(struct.pack("!H", context._out_mid))
if msg_len: # for payload
fmt = "!" + str(msg_len) + "s"
packet.extend(struct.pack(fmt, message))
mid = context._out_mid
wait_state = yvmq_msg_state.invalid
if qos == 1:
wait_state = yvmq_msg_state.wait_puback
elif qos == 2:
wait_state = yvmq_msg_state.wait_pubrec
if wait_state:
# self.inc_in_flight_o()
# context._wait_pub_queue.put( (mid, wait_state, packet) )
context._wait_pub_queue.put(
(mid, wait_state, packet, topic, qos)
) # modify on June 14, 2014 for PPMESSAGE just send onece. Ben Ning
# PollIOLoop.instance().add_callback(context.wait_pub_queue_handler_callback, MQTT3_PUBLISH)
PollIOLoop.instance().add_callback(context.wait_pub_queue_handler_callback)
else:
self.send_queue.put((context, packet)) # put a send packet task into send_queue
self.sys_info._sent_publish_messages += 1
def instance(*args, **kwargs):
"""Returns a global `IOLoop` instance.
Most applications have a single, global `IOLoop` running on the
main thread. Use this method to get this instance from
another thread. To get the current thread's `IOLoop`, use `current()`.
"""
#
# install PuritanicalIOLoop as the active IOLoop implementation
#
PollIOLoop.configure(PuritanicalIOLoop)
return PollIOLoop.instance(*args, **kwargs)
def instance(*args, **kwargs):
"""Returns a global `IOLoop` instance.
Most applications have a single, global `IOLoop` running on the
main thread. Use this method to get this instance from
another thread. To get the current thread's `IOLoop`, use `current()`.
"""
# install ZMQIOLoop as the active IOLoop implementation
# when using tornado 3
if tornado_version >= (3,):
PollIOLoop.configure(ZMQIOLoop)
return PollIOLoop.instance(*args, **kwargs)
def current(cls, *args, **kwargs):
"""Returns the current thread’s IOLoop.
"""
# install ZMQIOLoop as the active IOLoop implementation
# when using tornado 3
if tornado_version >= (3,):
PollIOLoop.configure(cls)
loop = PollIOLoop.current(*args, **kwargs)
if not isinstance(loop, cls):
warnings.warn("IOLoop.current expected instance of %r, got %r" % (cls, loop),
RuntimeWarning, stacklevel=2,
)
return loop
def instance(cls, *args, **kwargs):
"""Returns a global `IOLoop` instance.
Most applications have a single, global `IOLoop` running on the
main thread. Use this method to get this instance from
another thread. To get the current thread's `IOLoop`, use `current()`.
"""
# install ZMQIOLoop as the active IOLoop implementation
# when using tornado 3
if tornado_version >= (3,):
PollIOLoop.configure(cls)
loop = PollIOLoop.instance(*args, **kwargs)
if not isinstance(loop, cls):
warnings.warn("IOLoop.current expected instance of %r, got %r" % (cls, loop),
RuntimeWarning, stacklevel=2,
)
return loop
def update_sysinfo_callback(self):
# self.recount_cnns()
self.sys_info.recalculate()
self.work_queue.put((self.update_sys_info_topic, ()))
self.update_sysinfo_timeout = PollIOLoop.instance().add_timeout(
self.sys_info._timestamp + 15, self.update_sysinfo_callback
)
def add_cnn(self, context, clean=True):
if not context:
return
if clean:
if context._client_id in self.cnns:
old_context = self.cnns[context._client_id].context
if context is not old_context:
PollIOLoop.instance().add_callback(old_context.close_for_reconnect, context, clean)
if not self.cnns[context._client_id].active:
self.sys_info._active_clients += 1
self.sys_info._inactive_clients -= 1
self.cnns[context._client_id].active = True
self.cnns[context._client_id].clear()
self.cnns[context._client_id].context = context
self.cnns[context._client_id].cnn_flag = context._cnn_flag
else:
self.sys_info._total_clients += 1
self.sys_info._active_clients += 1
self.cnns[context._client_id] = mqtt3cnn(context)
context._wait_pub_queue = self.cnns[context._client_id].wait_pub_queue
else:
if context._client_id in self.cnns:
old_context = self.cnns[context._client_id].context
if context is not old_context:
PollIOLoop.instance().add_callback(old_context.close_for_reconnect, context, clean)
self.cnns[context._client_id].context = context
self.cnns[context._client_id].cnn_flag = context._cnn_flag
if not self.cnns[context._client_id].active:
self.cnns[context._client_id].active = True
self.sys_info._active_clients += 1
self.sys_info._inactive_clients -= 1
context._wait_pub_queue = self.cnns[context._client_id].wait_pub_queue
else:
self.cnns[context._client_id] = mqtt3cnn(context)
self.sys_info._active_clients += 1
self.sys_info._total_clients += 1
context._wait_pub_queue = self.cnns[context._client_id].wait_pub_queue
def wait_pub_completed_timeout(self):
if self._curr_pub_msg != None:
return
logging.info ("wait_pub_completed_timeout: resend_times ( %d) " % self._curr_pub_msg.resend_times)
if self._curr_pub_msg.resend_times == 5:
self.close()
else:
self._curr_pub_msg.resend()
self._curr_pub_msg.timeout = PollIOLoop.instance().call_later(7, self.wait_pub_completed_timeout)
return
def ppmt_main(self):
#print "Mqtt3 Broker start ......"
logging.info("Mqtt3 Broker start ......")
work_thread = worker(gMQbase.work_queue)
send_thread = sender(gMQbase.send_queue)
work_thread.start()
send_thread.start()
server = mqtt3server()
server.listen(1883)
PollIOLoop.instance().start()
send_thread.join()
work_thread.join()
gMQbase.work_queue.join()
gMQbase.send_queue.join()
return
def __init__(self):
self.pub_root = mqtt3pub("/")
self.sub_root = mqtt3sub("/")
self.cnns = {} # { client_id: mqtt3cnn }
self.level = 0
self.work_queue = Queue()
self.send_queue = Queue()
self.sys_info = sys_stat()
self.update_sysinfo_timeout = PollIOLoop.instance().add_timeout(
self.sys_info._timestamp + 15, self.update_sysinfo_callback
)
def initialize(self, **kwargs):
PollIOLoop.initialize(self, impl=DebugPoller(), **kwargs)
def instance():
PollIOLoop.configure(DebugLoop)
return PollIOLoop.instance()
def __init__(self, stream, ppmtdb=gMQbase, auth=gAuth):
self._ppmtdb = ppmtdb
self._stream = stream
self._auth = auth
self._sock = None
self._message_retry = 20
self._last_retry_check = 0
self._client_id = None
self._will_topic = None
self._will_message = None
self._user_name = None
self._password = None
self._connect_state = mq_cs_new
# self._total_received_bytes = 0
# self._total_sent_bytes = 0
self._pub_acl = None
self._sub_acl = None
# self._semaphore = threading.BoundedSemaphore(1)
# self.res_queue = Queue() # result queue for worker thread
self._wait_pub_queue = Queue(
) # wait to pubish with qos 1 or qos 2 message
self._curr_pub_msg = None # current publish message what waiting to be completed
''' for incoming packet from client '''
self._msgtype = 0
self._dup = False
self._out_mid = 0
self._last_mid = 0
self._qos = 0
self._retain = False
self._username = ""
self._password = ""
self._buff = B""
self._fixed_header_byte1 = 0
self._len_bytes_count = 0
self._error_code = 0
self._keep_alive = 60
self._timeout_handle = PollIOLoop.instance().call_later(
1.5 * self._keep_alive, self.keep_alive_timeout)
self._stream.set_close_callback(self.close_callback)
# for remaining len
self._multiplier = 1
self._remaining_len = 0
self._cnn_flag = 0
self.handler = {
MQTT3_CNNECT: lambda: self.mqtt3_connect_handler(),
# MQTT3_CONNACK: lambda: self.mqtt3_connack_handler(),
MQTT3_PUBLISH: lambda: self.mqtt3_publish_handler(),
MQTT3_PUBACK: lambda: self.mqtt3_puback_handler(),
MQTT3_PUBREC: lambda: self.mqtt3_pubrec_handler(),
MQTT3_PUBREL: lambda: self.mqtt3_pubrel_handler(),
MQTT3_PUBCOMP: lambda: self.mqtt3_pubcomp_handler(),
MQTT3_SUBSCRIBE: lambda: self.mqtt3_subscribe_handler(),
# MQTT3_SUBACK: lambda: self.suback_handler(),
MQTT3_UNSUBSCRIBE: lambda: self.mqtt3_unsubscribe_handler(),
# MQTT3_UNSUBACK: lambda: self.unsuback_handler(),
MQTT3_PINGREQ: lambda: self.mqtt3_pingreq_handler(),
# MQTT3_PINGRESP: lambda: self.pingresp_handler(),
MQTT3_DISCONNCT: lambda: self.mqtt3_disconnect_handler()
}
self.mqtt3_recv_packet()
def initialize(self, **kwargs):
self.real_io_loop = PollIOLoop(make_current=False) # type: ignore
reactor = self.real_io_loop.run_sync(gen.coroutine(TornadoReactor))
super(LayeredTwistedIOLoop, self).initialize(reactor=reactor, **kwargs)
self.add_callback(self.make_current)
def close_callback(self):
PollIOLoop.instance().remove_timeout(self._timeout_handle)
if self._connect_state == mq_cs_connected:
logging.info(" close_callback --> device_id: %s " % ( self._client_id))
self._ppmtdb.work_queue.put( (self._ppmtdb.set_inactive_cnn, (self,)) ) # put a set inactive task into queue
return
def __init__(self, stream, ppmtdb=gMQbase, auth=gAuth):
self._ppmtdb = ppmtdb
self._stream = stream
self._auth = auth
self._sock = None
self._message_retry = 20
self._last_retry_check = 0
self._client_id = None
self._will_topic = None
self._will_message = None
self._user_name = None
self._password = None
self._connect_state = mq_cs_new
# self._total_received_bytes = 0
# self._total_sent_bytes = 0
self._pub_acl = None
self._sub_acl = None
# self._semaphore = threading.BoundedSemaphore(1)
# self.res_queue = Queue() # result queue for worker thread
self._wait_pub_queue = Queue() # wait to pubish with qos 1 or qos 2 message
self._curr_pub_msg = None # current publish message what waiting to be completed
''' for incoming packet from client '''
self._msgtype = 0
self._dup = False
self._out_mid = 0
self._last_mid = 0
self._qos = 0
self._retain = False
self._username = ""
self._password = ""
self._buff = B""
self._fixed_header_byte1 = 0
self._len_bytes_count = 0
self._error_code = 0
self._keep_alive = 60
self._timeout_handle = PollIOLoop.instance().call_later(1.5 * self._keep_alive, self.keep_alive_timeout)
self._stream.set_close_callback(self.close_callback)
# for remaining len
self._multiplier = 1
self._remaining_len = 0
self._cnn_flag = 0
self.handler = {
MQTT3_CNNECT: lambda:self.mqtt3_connect_handler(),
# MQTT3_CONNACK: lambda: self.mqtt3_connack_handler(),
MQTT3_PUBLISH: lambda: self.mqtt3_publish_handler(),
MQTT3_PUBACK: lambda: self.mqtt3_puback_handler(),
MQTT3_PUBREC: lambda: self.mqtt3_pubrec_handler(),
MQTT3_PUBREL: lambda: self.mqtt3_pubrel_handler(),
MQTT3_PUBCOMP: lambda: self.mqtt3_pubcomp_handler(),
MQTT3_SUBSCRIBE: lambda: self.mqtt3_subscribe_handler(),
# MQTT3_SUBACK: lambda: self.suback_handler(),
MQTT3_UNSUBSCRIBE: lambda: self.mqtt3_unsubscribe_handler(),
# MQTT3_UNSUBACK: lambda: self.unsuback_handler(),
MQTT3_PINGREQ: lambda: self.mqtt3_pingreq_handler(),
# MQTT3_PINGRESP: lambda: self.pingresp_handler(),
MQTT3_DISCONNCT: lambda: self.mqtt3_disconnect_handler()
}
self.mqtt3_recv_packet()
def resend_message(self):
PollIOLoop.instance().add_callback(self.context.wait_pub_queue_handler_callback)
return
