class LayeredTwistedIOLoop(TwistedIOLoop): """Layers a TwistedIOLoop on top of a TornadoReactor on a PollIOLoop. This is of course silly, but is useful for testing purposes to make sure we're implementing both sides of the various interfaces correctly. In some tests another TornadoReactor is layered on top of the whole stack. """ def initialize(self, **kwargs): self.real_io_loop = PollIOLoop(make_current=False) # type: ignore reactor = self.real_io_loop.run_sync(gen.coroutine(TornadoReactor)) super(LayeredTwistedIOLoop, self).initialize(reactor=reactor, **kwargs) self.add_callback(self.make_current) def close(self, all_fds=False): super(LayeredTwistedIOLoop, self).close(all_fds=all_fds) # HACK: This is the same thing that test_class.unbuildReactor does. for reader in self.reactor._internalReaders: self.reactor.removeReader(reader) reader.connectionLost(None) self.real_io_loop.close(all_fds=all_fds) def stop(self): # One of twisted's tests fails if I don't delay crash() # until the reactor has started, but if I move this to # TwistedIOLoop then the tests fail when I'm *not* running # tornado-on-twisted-on-tornado. I'm clearly missing something # about the startup/crash semantics, but since stop and crash # are really only used in tests it doesn't really matter. def f(): self.reactor.crash() # Become current again on restart. This is needed to # override real_io_loop's claim to being the current loop. self.add_callback(self.make_current) self.reactor.callWhenRunning(f)
def install(): """DEPRECATED pyzmq 17 no longer needs any special integration for tornado. """ _deprecated() PollIOLoop.configure(ZMQIOLoop)
def current(*args, **kwargs): """Returns the current thread’s IOLoop. """ # install ZMQIOLoop as the active IOLoop implementation # when using tornado 3 if tornado_version >= (3,): PollIOLoop.configure(ZMQIOLoop) return PollIOLoop.current(*args, **kwargs)
def current(*args, **kwargs): """Returns the current threads IOLoop. """ # # install PuritanicalIOLoop as the active IOLoop implementation # PollIOLoop.configure(PuritanicalIOLoop) return PollIOLoop.current(*args, **kwargs)
def run(self): while True: try: context, packet = self.queue.get() buff = str(packet) PollIOLoop.instance().add_callback(self.ioloop_callback, context, buff) finally: self.queue.task_done()
def reset_timeout(self): if not self._keep_alive: # dont set timeout if keep_alive is zero that be set by client return if self._timeout_handle != None: PollIOLoop.instance().remove_timeout(self._timeout_handle) self._timeout_handler = None self._timeout_handle = PollIOLoop.instance().call_later(1.5*self._keep_alive, self.keep_alive_timeout) logging.info(" reset_timeout --> device_id: %s" % (self._client_id)) return
def current(cls, *args, **kwargs): """Returns the current thread’s IOLoop. """ # install ZMQIOLoop as the active IOLoop implementation # when using tornado 3 PollIOLoop.configure(cls) _deprecated() loop = PollIOLoop.current(*args, **kwargs) return loop
def close_callback(self): PollIOLoop.instance().remove_timeout(self._timeout_handle) if self._connect_state == mq_cs_connected: logging.info(" close_callback --> device_id: %s " % (self._client_id)) self._ppmtdb.work_queue.put( (self._ppmtdb.set_inactive_cnn, (self, ))) # put a set inactive task into queue return
def reset_timeout(self): if not self._keep_alive: # dont set timeout if keep_alive is zero that be set by client return if self._timeout_handle != None: PollIOLoop.instance().remove_timeout(self._timeout_handle) self._timeout_handler = None self._timeout_handle = PollIOLoop.instance().call_later( 1.5 * self._keep_alive, self.keep_alive_timeout) logging.info(" reset_timeout --> device_id: %s" % (self._client_id)) return
def mqtt3_send_publish(self, context, topic, qos, message): logging.info("sended pub --> topic: %s device_id: %s " % (topic, context._client_id)) packet = bytearray() packet.extend(struct.pack("!B", MQTT3_PUBLISH | (qos << 1))) topic_len = len(topic) msg_len = 0 if message == None else len(message) remain_len = 2 + topic_len + msg_len if qos: remain_len += 2 # for mid self._pack_remain_len(packet, remain_len) fmt = "!H" + str(topic_len) + "s" # topic_len + topic packet.extend(struct.pack(fmt, topic_len, topic)) if qos: # for mid context._out_mid += 1 if 65536 == context._out_mid: context._out_mid = 1 packet.extend(struct.pack("!H", context._out_mid)) if msg_len: # for payload fmt = "!" + str(msg_len) + "s" packet.extend(struct.pack(fmt, message)) mid = context._out_mid wait_state = yvmq_msg_state.invalid if qos == 1: wait_state = yvmq_msg_state.wait_puback elif qos == 2: wait_state = yvmq_msg_state.wait_pubrec if wait_state: # self.inc_in_flight_o() # context._wait_pub_queue.put( (mid, wait_state, packet) ) context._wait_pub_queue.put( (mid, wait_state, packet, topic, qos) ) # modify on June 14, 2014 for PPMESSAGE just send onece. Ben Ning # PollIOLoop.instance().add_callback(context.wait_pub_queue_handler_callback, MQTT3_PUBLISH) PollIOLoop.instance().add_callback(context.wait_pub_queue_handler_callback) else: self.send_queue.put((context, packet)) # put a send packet task into send_queue self.sys_info._sent_publish_messages += 1
def instance(*args, **kwargs): """Returns a global `IOLoop` instance. Most applications have a single, global `IOLoop` running on the main thread. Use this method to get this instance from another thread. To get the current thread's `IOLoop`, use `current()`. """ # # install PuritanicalIOLoop as the active IOLoop implementation # PollIOLoop.configure(PuritanicalIOLoop) return PollIOLoop.instance(*args, **kwargs)
def instance(*args, **kwargs): """Returns a global `IOLoop` instance. Most applications have a single, global `IOLoop` running on the main thread. Use this method to get this instance from another thread. To get the current thread's `IOLoop`, use `current()`. """ # install ZMQIOLoop as the active IOLoop implementation # when using tornado 3 if tornado_version >= (3,): PollIOLoop.configure(ZMQIOLoop) return PollIOLoop.instance(*args, **kwargs)
def current(cls, *args, **kwargs): """Returns the current thread’s IOLoop. """ # install ZMQIOLoop as the active IOLoop implementation # when using tornado 3 if tornado_version >= (3,): PollIOLoop.configure(cls) loop = PollIOLoop.current(*args, **kwargs) if not isinstance(loop, cls): warnings.warn("IOLoop.current expected instance of %r, got %r" % (cls, loop), RuntimeWarning, stacklevel=2, ) return loop
def instance(cls, *args, **kwargs): """Returns a global `IOLoop` instance. Most applications have a single, global `IOLoop` running on the main thread. Use this method to get this instance from another thread. To get the current thread's `IOLoop`, use `current()`. """ # install ZMQIOLoop as the active IOLoop implementation # when using tornado 3 if tornado_version >= (3,): PollIOLoop.configure(cls) loop = PollIOLoop.instance(*args, **kwargs) if not isinstance(loop, cls): warnings.warn("IOLoop.current expected instance of %r, got %r" % (cls, loop), RuntimeWarning, stacklevel=2, ) return loop
def update_sysinfo_callback(self): # self.recount_cnns() self.sys_info.recalculate() self.work_queue.put((self.update_sys_info_topic, ())) self.update_sysinfo_timeout = PollIOLoop.instance().add_timeout( self.sys_info._timestamp + 15, self.update_sysinfo_callback )
def add_cnn(self, context, clean=True): if not context: return if clean: if context._client_id in self.cnns: old_context = self.cnns[context._client_id].context if context is not old_context: PollIOLoop.instance().add_callback(old_context.close_for_reconnect, context, clean) if not self.cnns[context._client_id].active: self.sys_info._active_clients += 1 self.sys_info._inactive_clients -= 1 self.cnns[context._client_id].active = True self.cnns[context._client_id].clear() self.cnns[context._client_id].context = context self.cnns[context._client_id].cnn_flag = context._cnn_flag else: self.sys_info._total_clients += 1 self.sys_info._active_clients += 1 self.cnns[context._client_id] = mqtt3cnn(context) context._wait_pub_queue = self.cnns[context._client_id].wait_pub_queue else: if context._client_id in self.cnns: old_context = self.cnns[context._client_id].context if context is not old_context: PollIOLoop.instance().add_callback(old_context.close_for_reconnect, context, clean) self.cnns[context._client_id].context = context self.cnns[context._client_id].cnn_flag = context._cnn_flag if not self.cnns[context._client_id].active: self.cnns[context._client_id].active = True self.sys_info._active_clients += 1 self.sys_info._inactive_clients -= 1 context._wait_pub_queue = self.cnns[context._client_id].wait_pub_queue else: self.cnns[context._client_id] = mqtt3cnn(context) self.sys_info._active_clients += 1 self.sys_info._total_clients += 1 context._wait_pub_queue = self.cnns[context._client_id].wait_pub_queue
def wait_pub_completed_timeout(self): if self._curr_pub_msg != None: return logging.info ("wait_pub_completed_timeout: resend_times ( %d) " % self._curr_pub_msg.resend_times) if self._curr_pub_msg.resend_times == 5: self.close() else: self._curr_pub_msg.resend() self._curr_pub_msg.timeout = PollIOLoop.instance().call_later(7, self.wait_pub_completed_timeout) return
def ppmt_main(self): #print "Mqtt3 Broker start ......" logging.info("Mqtt3 Broker start ......") work_thread = worker(gMQbase.work_queue) send_thread = sender(gMQbase.send_queue) work_thread.start() send_thread.start() server = mqtt3server() server.listen(1883) PollIOLoop.instance().start() send_thread.join() work_thread.join() gMQbase.work_queue.join() gMQbase.send_queue.join() return
def __init__(self): self.pub_root = mqtt3pub("/") self.sub_root = mqtt3sub("/") self.cnns = {} # { client_id: mqtt3cnn } self.level = 0 self.work_queue = Queue() self.send_queue = Queue() self.sys_info = sys_stat() self.update_sysinfo_timeout = PollIOLoop.instance().add_timeout( self.sys_info._timestamp + 15, self.update_sysinfo_callback )
def initialize(self, **kwargs): PollIOLoop.initialize(self, impl=DebugPoller(), **kwargs)
def instance(): PollIOLoop.configure(DebugLoop) return PollIOLoop.instance()
def __init__(self, stream, ppmtdb=gMQbase, auth=gAuth): self._ppmtdb = ppmtdb self._stream = stream self._auth = auth self._sock = None self._message_retry = 20 self._last_retry_check = 0 self._client_id = None self._will_topic = None self._will_message = None self._user_name = None self._password = None self._connect_state = mq_cs_new # self._total_received_bytes = 0 # self._total_sent_bytes = 0 self._pub_acl = None self._sub_acl = None # self._semaphore = threading.BoundedSemaphore(1) # self.res_queue = Queue() # result queue for worker thread self._wait_pub_queue = Queue( ) # wait to pubish with qos 1 or qos 2 message self._curr_pub_msg = None # current publish message what waiting to be completed ''' for incoming packet from client ''' self._msgtype = 0 self._dup = False self._out_mid = 0 self._last_mid = 0 self._qos = 0 self._retain = False self._username = "" self._password = "" self._buff = B"" self._fixed_header_byte1 = 0 self._len_bytes_count = 0 self._error_code = 0 self._keep_alive = 60 self._timeout_handle = PollIOLoop.instance().call_later( 1.5 * self._keep_alive, self.keep_alive_timeout) self._stream.set_close_callback(self.close_callback) # for remaining len self._multiplier = 1 self._remaining_len = 0 self._cnn_flag = 0 self.handler = { MQTT3_CNNECT: lambda: self.mqtt3_connect_handler(), # MQTT3_CONNACK: lambda: self.mqtt3_connack_handler(), MQTT3_PUBLISH: lambda: self.mqtt3_publish_handler(), MQTT3_PUBACK: lambda: self.mqtt3_puback_handler(), MQTT3_PUBREC: lambda: self.mqtt3_pubrec_handler(), MQTT3_PUBREL: lambda: self.mqtt3_pubrel_handler(), MQTT3_PUBCOMP: lambda: self.mqtt3_pubcomp_handler(), MQTT3_SUBSCRIBE: lambda: self.mqtt3_subscribe_handler(), # MQTT3_SUBACK: lambda: self.suback_handler(), MQTT3_UNSUBSCRIBE: lambda: self.mqtt3_unsubscribe_handler(), # MQTT3_UNSUBACK: lambda: self.unsuback_handler(), MQTT3_PINGREQ: lambda: self.mqtt3_pingreq_handler(), # MQTT3_PINGRESP: lambda: self.pingresp_handler(), MQTT3_DISCONNCT: lambda: self.mqtt3_disconnect_handler() } self.mqtt3_recv_packet()
def initialize(self, **kwargs): self.real_io_loop = PollIOLoop(make_current=False) # type: ignore reactor = self.real_io_loop.run_sync(gen.coroutine(TornadoReactor)) super(LayeredTwistedIOLoop, self).initialize(reactor=reactor, **kwargs) self.add_callback(self.make_current)
def close_callback(self): PollIOLoop.instance().remove_timeout(self._timeout_handle) if self._connect_state == mq_cs_connected: logging.info(" close_callback --> device_id: %s " % ( self._client_id)) self._ppmtdb.work_queue.put( (self._ppmtdb.set_inactive_cnn, (self,)) ) # put a set inactive task into queue return
def __init__(self, stream, ppmtdb=gMQbase, auth=gAuth): self._ppmtdb = ppmtdb self._stream = stream self._auth = auth self._sock = None self._message_retry = 20 self._last_retry_check = 0 self._client_id = None self._will_topic = None self._will_message = None self._user_name = None self._password = None self._connect_state = mq_cs_new # self._total_received_bytes = 0 # self._total_sent_bytes = 0 self._pub_acl = None self._sub_acl = None # self._semaphore = threading.BoundedSemaphore(1) # self.res_queue = Queue() # result queue for worker thread self._wait_pub_queue = Queue() # wait to pubish with qos 1 or qos 2 message self._curr_pub_msg = None # current publish message what waiting to be completed ''' for incoming packet from client ''' self._msgtype = 0 self._dup = False self._out_mid = 0 self._last_mid = 0 self._qos = 0 self._retain = False self._username = "" self._password = "" self._buff = B"" self._fixed_header_byte1 = 0 self._len_bytes_count = 0 self._error_code = 0 self._keep_alive = 60 self._timeout_handle = PollIOLoop.instance().call_later(1.5 * self._keep_alive, self.keep_alive_timeout) self._stream.set_close_callback(self.close_callback) # for remaining len self._multiplier = 1 self._remaining_len = 0 self._cnn_flag = 0 self.handler = { MQTT3_CNNECT: lambda:self.mqtt3_connect_handler(), # MQTT3_CONNACK: lambda: self.mqtt3_connack_handler(), MQTT3_PUBLISH: lambda: self.mqtt3_publish_handler(), MQTT3_PUBACK: lambda: self.mqtt3_puback_handler(), MQTT3_PUBREC: lambda: self.mqtt3_pubrec_handler(), MQTT3_PUBREL: lambda: self.mqtt3_pubrel_handler(), MQTT3_PUBCOMP: lambda: self.mqtt3_pubcomp_handler(), MQTT3_SUBSCRIBE: lambda: self.mqtt3_subscribe_handler(), # MQTT3_SUBACK: lambda: self.suback_handler(), MQTT3_UNSUBSCRIBE: lambda: self.mqtt3_unsubscribe_handler(), # MQTT3_UNSUBACK: lambda: self.unsuback_handler(), MQTT3_PINGREQ: lambda: self.mqtt3_pingreq_handler(), # MQTT3_PINGRESP: lambda: self.pingresp_handler(), MQTT3_DISCONNCT: lambda: self.mqtt3_disconnect_handler() } self.mqtt3_recv_packet()
def resend_message(self): PollIOLoop.instance().add_callback(self.context.wait_pub_queue_handler_callback) return