class Mqtt(Connector): def __init__(self, config, asm=None): self.name = "mqtt" self.config = config self.asm = asm self.default_room = "MyDefaultRoom" self.client = None async def connect(self): self.client = MQTTClient(self.asm.name) self.client.on_message = on_message self.client.set_auth_credentials( os.getenv('MQTT_USER', "arcus"), os.getenv('MQTT_PASSWORD', "arcusarcus")) await self.client.connect(os.getenv('MQTT_HOST', "mqtt"), 1883, keepalive=60, version=MQTTv311) _LOGGER.info("Connected to MQTT") async def listen(self): self.client.subscribe(self.asm.name + "/#", qos=1) stop = asyncio.Event() await stop.wait() @register_event(Message) async def respond(self, message): self.client.publish(self.asm.name, 'Message payload', response_topic='RESPONSE/TOPIC') async def disconnect(self): # Disconnect from the service await self.client.disconnect()
async def main(broker_host, username, password): client = MQTTClient("client-id-mitch", user_property=('hello', 'there')) # client = MQTTClient("client-id-mitch") client.on_connect = on_connect client.on_message = on_message client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe client.set_auth_credentials(username, password.encode()) await client.connect(host=broker_host, port=1883) # await client.connect() print("connected, now ready to send...") data = f"This is a test! {str(time.time())}" hash = hashlib.sha256(data.encode()).hexdigest() client.publish('test/time1', "hello test/time1..", qos=1, message_expiry_interval=5, content_type="json", response_topic='RESPONSE/TOPIC2', user_property=[('hash', hash), ('time', str(time.time()))]) client.publish('test/time2', "hello test/time2..", qos=1, message_expiry_interval=5, content_type="json", response_topic='RESPONSE/TOPIC', user_property=[('hash', hash), ('time', str(time.time()))]) await STOP.wait() await client.disconnect()
async def connect(self): try: print( '""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""' ) print('Attempting MQTT connection...') print('MQTT host : ', self.broker_host) print('MQTT user : '******'MQTT error, restarting in 8s...') await asyncio.sleep(8) await self.connect()
async def main(broker_host, token): client = MQTTClient(__name__) client.on_connect = on_connect client.on_disconnect = on_disconnect client.on_message = on_message client.on_subscribe = on_subscribe if token: client.set_auth_credentials(token) log.debug(f'Connecting to {broker_host}') await client.connect(broker_host) def on_app_changed(status, payload): client.publish(BASETOPIC + "/app", payload) def on_volume_changed(status, payload): client.publish(BASETOPIC + "/volume", payload) lg = LGClient() lg.ac.subscribe_get_current(on_app_changed) lg.mc.subscribe_get_volume(on_volume_changed) # Periodic heartbeat from the bridge def heartbeat(): while True: client.publish(f"{BASETOPIC}/heartbeat/{client._client_id}", str(datetime.datetime.now())) time.sleep(1) threading.Thread(target=heartbeat, daemon=True).start() await stopEvent.wait() await client.disconnect()
class MQTTCli: def __init__(self, loop: asyncio.AbstractEventLoop, device_registry): self.loop = loop self.device_registry = device_registry self.client = MQTTClient("client-id") self.client.on_connect = self.on_connect self.client.on_message = self.on_message self.client.on_disconnect = self.on_disconnect self.client.on_subscribe = self.on_subscribe async def connect(): user = "******" pw = "oochi8eengehuYohgeBu1foobooceeZ7to5ieng7pis8saephaetah0hoaphiK3F" broker_host = "192.168.50.95" self.client.set_auth_credentials(user, pw) await self.client.connect(broker_host) def on_connect(client, flags, rc, properties): print('Connected') self.client.subscribe('home-assistant/command', qos=0) def on_message(client, topic, payload, qos, properties): print('RECV MSG:', payload) device_registry.bluetooth_devices.send_message(payload, True, False) publish('home-assistant/response', payload, qos=1) def on_disconnect(client, packet, exc=None): print('Disconnected') def on_subscribe(client, mid, qos, properties): print('SUBSCRIBED') def ask_exit(*args): STOP.set()
async def main(broker_host, username, password): client = MQTTClient(client_id="pub-client-id", receive_maximum=24000) client.on_disconnect = on_disconnect client.set_auth_credentials(username=username, password=password) await client.connect(broker_host) for i in range(10000): client.publish(message_or_topic='TEST/T1', payload=str(i), qos=1, retain=True, message_expiry_interval=60)
def prepare_client(token): CLIENT_ID = create_uid() client = MQTTClient(CLIENT_ID) client.on_connect = on_connect client.on_message = on_message client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe client.set_auth_credentials(token, None) return client
async def main(): client = MQTTClient(mqtt_client_id, session_expiry_interval=86400 * 10, clean_session=False) client.on_connect = on_connect client.on_message = on_message client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe client.set_auth_credentials(username, None) await client.connect(broker_host) await STOP.wait() await client.disconnect()
async def main(broker_host, username, password): client = MQTTClient(client_id="sub-client-id", receive_maximum=24000, clean_session=False, session_expiry_interval=60) client.on_connect = on_connect client.on_message = on_message client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe client.set_auth_credentials(username=username, password=password) await client.connect(broker_host) await STOP.wait() await client.disconnect()
async def main(broker_host, token): client = MQTTClient("vscode-client") client.on_connect = on_connect client.on_message = on_message client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe client.set_auth_credentials(token, None) await client.connect(broker_host) # client.publish('TEST/TIME', str(time.time()), qos=0) await STOP.wait() await client.disconnect()
async def main(broker_host): client = MQTTClient("client-id") client.on_connect = on_connect client.on_message = on_message client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe client.set_auth_credentials('admin', 'admin') await client.connect(broker_host, 8080, keepalive=60, version=MQTTv50) #client.publish('TEST/TIME', str(time.time()), qos=1) time.sleep(30) await STOP.wait() await client.disconnect()
async def main(): client = MQTTClient('flespi-examples-mqtt-client-python', clean_session=True) client.on_connect = on_connect client.on_message = on_message client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe # see https://flespi.com/kb/tokens-access-keys-to-flespi-platform to read about flespi tokens client.set_auth_credentials( 'FlespiToken {}'.format(os.environ.get("FlespiToken")), None) print('mqtt client created, connecting...') await client.connect('mqtt.flespi.io', port=8883, ssl=True) await STOP.wait() await client.disconnect() print('disconnected')
async def main(broker_host, token): client_id = create_uid() client = MQTTClient(client_id) client.on_connect = on_connect client.on_message = on_message client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe client.set_auth_credentials(token, None) await client.connect(broker_host) # Watchdog print out online_clients every 5s t = threading.Thread(target=watchdog_clients) t.setDaemon(True) t.start() await STOP.wait() await client.disconnect()
async def main(broker_host, token): global con con = pymysql.connect(host=mysql_host, user=mysql_user, passwd=mysql_passwd, db=mysql_db, autocommit=True) client = MQTTClient('message_listener') client.on_connect = on_connect client.on_message = on_message client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe client.set_auth_credentials(token, None) await client.connect(broker_host) await STOP.wait() await client.disconnect()
async def main(loop, broker_host, token): engine = create_engine(SQLALCHEMY_DATABASE_URI) conn = engine.connect() Session = sessionmaker(bind=engine) session = Session() client = MQTTClient("client-id") client.on_connect = on_connect client.on_message = partial(on_message, conn, session) client.on_disconnect = on_disconnect client.on_subscribe = on_subscribe client.set_auth_credentials(token, None) await client.connect(broker_host) await STOP.wait() await client.disconnect()
async def main(broker_host, token, loop): global MQTTClient, QueuePersister, CustomMicroUser, logging, after_get, STOP, asyncio logging.info("Client ID %s", os.environ.get('MQTT_CLIENT_ID')) client = MQTTClient(os.environ.get('MQTT_CLIENT_ID')) client.set_auth_credentials(token, None) await client.connect(broker_host) queuePersist = QueuePersister(client, loop) users_to_get = [] for i in range(1, 10000): users_to_get.append(asyncio.create_task(queuePersist.get(CustomMicroUser(pk=1), after_get))) await asyncio.gather( *users_to_get ) await STOP.wait() await client.disconnect()
class MQTTClient(Client): ''' Client for subscribing to a broker and pipe incomming messages to configured producer. ''' def __init__( self, **kwargs: Any, ) -> None: super().__init__(**kwargs) self._inner_client = GMQTTClient(None) self._inner_client.on_connect = self.on_connect self._inner_client.on_message = self.on_message self._inner_client.on_disconnect = self.on_disconnect self._inner_client.on_subscribe = self.on_subscribe self._inner_client.pipe_message = self.pipe_message if not self.username is None: self._inner_client.set_auth_credentials(self.username, self.password) async def connect(self, topics: Tuple[str, int]) -> None: ''' Connects to broker. ''' LOG.debug('Connecting to MQTT broker.') try: await self._inner_client.connect(self.uri, 1883, keepalive=60, version=MQTTv311) subscriptions = [Subscription(t[0], qos=t[1]) for t in topics] self._inner_client.subscribe(subscriptions, subscription_identifier=1) await self.producer.connect() self.connected = True except: self.connected = False async def disconnect(self) -> None: await self._inner_client.disconnect() await self.producer.disconnect() self.connected = False
class Deconz2acp(): ################### # Sync class init ################### def __init__(self, settings): self.settings = settings print("{} Deconz2acp __init__() DEBUG={}".format( self.ts_string(), DEBUG), file=sys.stderr, flush=True) ##################################### # Signal handler for SIGINT, SIGTERM ##################################### def ask_exit(self, *args): self.STOP.set() ##################################### # Return current timestamp as string ##################################### def ts_string(self): return '{:.6f}'.format(time.time()) ############################################################### # Async initialization ############################################################### async def start(self, zigbee_data): print("{} Deconz2acp start()".format(self.ts_string()), file=sys.stderr, flush=True) self.zigbee_data = zigbee_data # Define async events for exit and reload (will set via signals) self.STOP = asyncio.Event() self.RELOAD = asyncio.Event() # connect output MQTT broker await self.connect_output_mqtt() # Connect input WebSocket asyncio.ensure_future(self.subscribe_input_ws()) # Connect to input websocket async def subscribe_input_ws(self): ws_url = self.settings["input_ws"]["url"] connected = False while True: try: async with websockets.connect(ws_url) as ws: connected = True print("{} Deconz2acp connected to {}".format( self.ts_string(), ws_url), flush=True) while connected: try: if DEBUG: print("{} Deconz2acp awaiting msg from {}". format(self.ts_string(), ws_url), flush=True) # Here we await & receive any websocket message msg = await ws.recv() if DEBUG: pretty_msg = json.dumps(json.loads(msg), indent=4) print( "{} Deconz2acp msg received from {}:\n{}". format(self.ts_string(), ws_url, pretty_msg), flush=True) #debug we're stuffing in a fake "zigbee" topic self.handle_input_message(msg) except websockets.exceptions.ConnectionClosedError: connected = False print("{} Deconz2acp disconnected from {}".format( self.ts_string(), ws_url), flush=True) print("{} Deconz2acp websocket read loop ended".format( self.ts_string()), flush=True) except ConnectionRefusedError: print("{} Deconz2acp websocket connection refused from {}". format(self.ts_string(), ws_url), flush=True) await asyncio.sleep(2) # sleep 2 seconds and retry print("{} Deconz2acp websocket connect loop ended".format( self.ts_string()), flush=True) async def connect_output_mqtt(self): self.output_client = MQTTClient(None) # auto-generate client id self.output_client.on_connect = self.output_on_connect self.output_client.on_message = self.output_on_message self.output_client.on_disconnect = self.output_on_disconnect self.output_client.on_subscribe = self.output_on_subscribe user = self.settings["output_mqtt"]["user"] password = self.settings["output_mqtt"]["password"] host = self.settings["output_mqtt"]["host"] port = self.settings["output_mqtt"]["port"] self.output_client.set_auth_credentials(user, password) try: await self.output_client.connect(host, port, keepalive=60, version=MQTTv311) except Exception as e: if hasattr(e, 'args') and e.args[0] == 5: print( "{}\033[1;31m FAIL: Connect output_mqtt auth (as {} )\033[0;0m" .format(self.ts_string(), user), file=sys.stderr, flush=True) else: print( "{}\033[1;31m FAIL gmqtt connect exception\n{}\n{}\033[0;0m" .format(self.ts_string(), e), file=sys.stderr, flush=True) self.ask_exit() ############################################################### # Sensor data message handler ############################################################### def handle_input_message(self, msg_bytes): msg_dict = json.loads(msg_bytes) # Add required zigbee properties by updating msg_dict # send_data will be True if zigbee_data.decode decides this message should be sent via MQTT. send_data = self.zigbee_data.handle_ws_message(msg_dict) if send_data: topic = "" if "acp_id" in msg_dict: topic += msg_dict["acp_id"] self.send_output_message(topic, msg_dict) else: print("{} Incoming message not sent to MQTT\n{}\n".format( self.ts_string(), msg_bytes), flush=True) def send_output_message(self, topic, msg_dict): msg_bytes = json.dumps(msg_dict) #print("publishing {}".format(msg_bytes), flush=True) output_topic = self.settings["output_mqtt"]["topic_prefix"] + topic if DEBUG: pretty_msg = json.dumps(msg_dict, indent=4) print("{} MQTT publish disabled by DEBUG setting:\n{}".format( self.ts_string(), pretty_msg), flush=True) else: self.output_client.publish(output_topic, msg_bytes, qos=0) ############################################################### # WS INPUT ############################################################### def input_ws_connected(self, uri): print('{} INPUT Connected to {}'.format(self.ts_string(), uri), flush=True) ############################################################### # MQTT OUTPUT ############################################################### def output_on_connect(self, client, flags, rc, properties): print('{} OUTPUT Connected to {} as {}'.format( self.ts_string(), self.settings["output_mqtt"]["host"], self.settings["output_mqtt"]["user"]), flush=True) def output_on_disconnect(self, client, packet, exc=None): print("{} OUTPUT Disconnected\n".format(self.ts_string()), file=sys.stderr, flush=True) # These GMQTT methods here for completeness although not used def output_on_message(self, client, topic, msg_bytes, qos, properties): print('OUTPUT RECV MSG?:', msg_bytes, flush=True) def output_on_subscribe(self, client, mid, qos, properties): print('OUTPUT SUBSCRIBED? to {}', flush=True) ############################################################### # CLEANUP on EXIT SIGNAL (SIGINT or SIGTERM) ############################################################### async def finish(self): await self.STOP.wait() print("\n{} Deconz2acp interrupted - disconnecting\n".format( self.ts_string()), file=sys.stderr, flush=True) await self.output_client.disconnect()
class MQTTClient(Entity): """ A helper class for MQTT. Handles all the connection details. Returned to the library or module that calls self._MQTTYombo.new(). .. code-block:: python self.my_mqtt = self._MQTT.new(on_message_callback=self.mqtt_incoming, client_id="my_client_name") self.my_mqtt.subscribe("yombo/devices/+/get") # subscribe to a topic. + is a wildcard for a single section. """ def __init__(self, parent, hostname: Optional[str] = None, port: Optional[int] = None, username: Optional[str] = None, password: Optional[str] = None, use_ssl: Optional[bool] = None, version: Optional[str] = None, keepalive: Optional[int] = None, session_expiry: Optional[int] = None, receive_maximum: Optional[int] = None, user_property: Optional[Union[tuple, List[tuple]]] = None, last_will: Optional = None, maximum_packet_size: Optional[int] = None, on_message_callback: Callable = None, subscribe_callback: Callable = None, unsubscribe_callback: Callable = None, connected_callback: Optional[Callable] = None, disconnected_callback: Optional[Callable] = None, error_callback: Optional[Callable] = None, client_id: Optional[str] = None, password2: Optional[str] = None): """ Creates a new client connection to an MQTT broker. :param parent: A reference to the MQTT library. :param hostname: IP address or hostname to connect to. :param port: Port number to connect to. :param username: Username to connect as. Use "" to not use a username & password. :param password: Password to to connect with. Use "" to not use a password. :param use_ssl: Use SSL when attempting to connect to server, default is True. :param version: MQTT version to use, default: MQTTv50. Other: MQTTv311 :param keepalive: How often the connection should be checked that it's still alive. :param session_expiry: How many seconds the session should be valid. Defaults to 0. :param receive_maximum: The Client uses this value to limit the number of QoS 1 and QoS 2 publications that it is willing to process concurrently. :param user_property: Connection user_property. A tuple or list of tuples. :param last_will: Last will message generated by 'will()'. :param maximum_packet_size: The maximum size the mqtt payload should be, in size. :param on_message_callback: (required) method - Method to send messages to. :param connected_callback: method - If you want a function called when connected to server. :param disconnected_callback: method - If you want a function called when disconnected from server. :param subscribe_callback: method - This method will be called when successfully subscribed to topic. :param unsubscribe_callback: method - This method will be called when successfully unsubscribed from topic. :param error_callback: method - A function to call if something goes wrong. :param client_id: (default - random) - A client id to use for logging. :param password2: A second password to try. Used by MQTTYombo. :return: """ self._Entity_type: str = "MQTTClient" self._Entity_label_attribute: str = "client_id" super().__init__(parent) self.connected = False self.incoming_duplicates = deque([], 150) self.send_queue = deque() self.subscriptions = {} self.unsubscriptions = {} self.topics = {} # Store topics to resubscribe to self.hostname = hostname self.port = port self.username = username self.password = password self.password2 = password2 self.use_ssl = use_ssl self.version = version self.keepalive = keepalive self.session_expiry = session_expiry self.receive_maximum = receive_maximum self.user_property = user_property self.last_will = last_will self.maximum_packet_size = maximum_packet_size self.on_message_callback = on_message_callback self.subscribe_callback = subscribe_callback self.unsubscribe_callback = unsubscribe_callback self.connected_callback = connected_callback self.disconnected_callback = disconnected_callback self.error_callback = error_callback self.client_id = client_id client_options = { "receive_maximum": receive_maximum, "session_expiry_interval": session_expiry, "maximum_packet_size": maximum_packet_size, "user_property": user_property, } self.client = QClient( client_id, **{k: v for k, v in client_options.items() if v is not None}) self.client.set_auth_credentials(username, password.encode()) self.client.on_connect = self.on_connect self.client.on_message = self.on_message_callback self.client.on_disconnect = self.on_disconnect self.client.on_subscribe = self.on_subscribe @inlineCallbacks def connect(self): """Connects to the mqtt broker.""" d = self.as_deferred(self.do_connect()) yield d def as_deferred(self, f): return Deferred.fromFuture(asyncio.ensure_future(f)) async def do_connect(self): """Connects to the mqtt broker.""" await asyncio.create_task( self.client.connect(host=self.hostname, port=self.port)) def on_connect(self, client, flags, rc, properties): """Received a message.""" self.connected = True # Do subscribes for topic, kwargs in self.subscriptions.items(): self.client.subscribe(topic, **kwargs) for topic, kwargs in self.unsubscriptions.items(): self.client.unsubscribe(topic, **kwargs) # Do messages for message in self.send_queue: self.client.publish(message["topic"], **message["kwargs"]) if callable(self.connected_callback): self.connected_callback(properties=properties) def on_disconnect(self, client, packet, exc=None): """Disconnected notification.""" self.connected = False if callable(self.disconnected_callback): self.disconnected_callback(client=client, packet=packet) def on_message(self, client, topic, body, qos, properties): """Received a message.""" if callable(self.on_message_callback): self.on_message_callback(client=client, topic=topic, body=body, qos=qos, properties=properties) def on_subscribe(self, client, mid, qos, properties): """Received subscribe confirmation.""" if callable(self.subscribe_callback): self.subscribe_callback(client=client, mid=mid, qos=qos, properties=properties) def on_unsubscribe(self, client, mid, qos): """Received unsubscribe confirmation.""" if callable(self.unsubscribe_callback): self.unsubscribe_callback(client=client, mid=mid, qos=qos) def subscribe(self, topic: str, **kwargs): """ Subscribe to a topic. :param topic: :param kwargs: :return: """ if "qos" not in kwargs: kwargs["qos"] = 1 if self.session_expiry == 0: self.subscriptions[topic] = kwargs if self.connected is True: self.client.subscribe(topic, **kwargs) def unsubscribe(self, topic: str, **kwargs): """ Unsubscribe from topic. :param topic: Topic to unsubscribe from. :param kwargs: :return: """ if "qos" not in kwargs: kwargs["qos"] = 1 if self.connected is True: self.client.unsubscribe(topic, **kwargs) if self.session_expiry == 0: self.unsubscriptions[topic] = kwargs def publish(self, topic: str, message: Optional[str] = None, qos: Optional[int] = None, **kwargs): """ Publish a message to the MQTT broker. If not connected yet, will hold in a queue for later. :param topic: Topic to publish too. :param message: Message to send. :param qos: quality of service. :param kwargs: Any additional items to send to the qmqtt publish command. :return: """ if qos is None: qos = 1 if self.connected is True: self.client.publish(topic, payload=message, qos=qos, **kwargs) else: kwargs["message"] = message kwargs["qos"] = qos self.send_queue.append({"topic": topic, "kwargs": kwargs})
class MQTT: """MQTT Class""" def __init__(self): # Secret can be set in either an environment variable (used first) or config.json self.MQTT_USERNAME = getenv("MQTT_USER") self.MQTT_PASS = getenv("MQTT_KEY") # The connection handle for making calls self.__client = None self.last_sent_time = dict() self.mqtt_cooldown = dict() self.hostname = "mqtt" self.client_id = getenv("WEB_HOSTNAME") + "-twitchbot-" + str( datetime.now().timestamp()) # Default to not connected self.MQTT_CONNECTION_STATE = False self.ATTN_ENABLE = True self.Topics = MqttTopics async def connect_to_mqtt(self): """Connect to MQTT Server""" # Create an instance of the REST client. if self.MQTT_USERNAME is None or self.MQTT_PASS is None: print("MQTT keys not found, aborting connection") return False try: print("Atempting to connect to MQTT as " + self.MQTT_USERNAME) self.__client = Client(client_id=self.client_id) self.__client.set_auth_credentials(self.MQTT_USERNAME, self.MQTT_PASS) await self.__client.connect(self.hostname, port=1883) print("Connected to MQTT") self.MQTT_CONNECTION_STATE = True return True except Exception as e: print("Failed to connect to MQTT, disabling it") print(e) self.MQTT_CONNECTION_STATE = False return False def get_cooldown(self, feed: str): """ Returns the cooldown Loads it from the database, or returns 0 if one is not set. """ if feed not in self.mqtt_cooldown: q = session.query(Settings.value).filter( Settings.key == f"mqtt_cooldown_{feed}").one_or_none() if q is None: # Value wasn't in the database, lets insert it. insert = Settings(key=f"mqtt_cooldown_{feed}", value=0) session.add(insert) self.mqtt_cooldown[feed] = 0 else: self.mqtt_cooldown[feed] = int(q[0]) return self.mqtt_cooldown[feed] def set_cooldown(self, feed: str, cooldown: int) -> None: """ Sets the MQTT cooldown Updates or inserts the value into the database Exception handling should be done in the calling function """ q = session.query(Settings.id).filter( Settings.key == f"mqtt_cooldown_{feed}").one_or_none() if q is None: # Value wasn't in the database, lets insert it. insert = Settings(key=f"mqtt_cooldown_{feed}", value=cooldown) session.add(insert) self.mqtt_cooldown[feed] = cooldown else: session.query(Settings).filter( Settings.key == f"mqtt_cooldown_{feed}").update( {"value": cooldown}) self.mqtt_cooldown[feed] = cooldown session.commit() async def send(self, feed, value: Union[str, int] = 1, retain: bool = False): """Send to an MQTT topic""" last_sent = self.last_sent_time.get(feed, datetime.min) cooldown = self.get_cooldown(feed) now = datetime.now() if (last_sent + timedelta(seconds=cooldown)) > now: print( f"MQTT {feed} on cooldown for {(last_sent + timedelta(seconds=cooldown)) - now}." ) return False if self.MQTT_CONNECTION_STATE is False: try: if not await self.connect_to_mqtt(): return False except Exception as e: print(e) return False try: self.__client.publish(feed, value, retain=retain) self.last_sent_time[feed] = now return True except Exception as e: print(e) return False
class LinkGMQTT(object): def __init__(self, settings=None): print("LinkGMQTT __init__()") self.settings = settings self.client = MQTTClient(None) # None => autogenerated client id self.client.on_connect = self.on_connect self.client.on_message = self.on_message self.client.on_disconnect = self.on_disconnect self.client.on_subscribe = self.on_subscribe self.subscription_queue = asyncio.Queue() print("LinkGMQTT __init__ completed") async def start(self, server_settings): """ Connects to broker """ print('LinkGMQTT.start() connecting as user {}'.format( server_settings["user"])) self.client.set_auth_credentials(server_settings["user"], server_settings["password"]) try: await self.client.connect(server_settings["host"], keepalive=60, version=MQTTv311) except Exception as e: print("LinkGMQTT connect exception: {}".format(e)) return print('LinkGMQTT.start() connected {}'.format(server_settings["host"])) async def put(self, sensor_id, event): """ Sends sensor_id/event to MQTT broker. sensor_id is string, used as MQTT topic event is dictionary which will be converted to bytes for MQTT message """ #print('LinkGMQTT.put() sending {}'.format(sensor_id)) message = json.dumps(event) self.client.publish(sensor_id, message, qos=0) print("LinkGMQTT.put() published {} {}".format(sensor_id, message)) async def subscribe(self, subscribe_settings): """ Subscribes to sensor events. """ try: self.client.subscribe(subscribe_settings["topic"], qos=0) except Exception as e: print("LinkGMQTT subscribe exception: {}".format(e)) return print("LinkGMQTT.subscribed() {}".format(subscribe_settings["topic"])) async def get(self): print("LinkGMQTT get requested from client, awaiting queue") message = await self.subscription_queue.get() print("LinkGMQTT get returned from queue") return message def on_connect(self, client, flags, rc, properties): print('LinkGMQTT Connected') def on_message(self, client, topic, payload, qos, properties): print('LinkGMQTT RECV MSG:', topic, payload) message = payload.decode('utf-8') message_dict = {} try: message_dict = json.loads(message) except JSONDecodeError: message_dict["message"] = message print("remote_sensors() json msg error: {} => {}".format( topic, message)) message_dict["topic"] = topic self.subscription_queue.put_nowait(message_dict) def on_disconnect(self, client, packet, exc=None): print('LinkGMQTT Disconnected') def on_subscribe(self, client, mid, qos, properties): print('LinkGMQTT Subscribed') async def finish(self): await self.client.disconnect()
class MqttClient: def __init__(self, host: str, port: int, user: str, password: typing.Optional[str] = None): self.subscriptions: typing.Dict[str, typing.List[ValueCallback]] = {} self.host = host self.port = port self.client = Client('sorokdva-dialogs') self.client.set_auth_credentials(user, password) self.client.on_connect = self._on_connect self.client.on_message = self._on_message async def _on_message( self, client: Client, topic: str, payload: bytes, qos, properties, ) -> constants.PubRecReasonCode: log = logging.getLogger('mqtt') futures = [] value = payload.decode() for cb in self.subscriptions.get(topic, []): log.info('passing (%r, %r) to %s', topic, value, cb) futures.append(cb(topic, value)) if futures: await asyncio.wait(futures, return_when=asyncio.ALL_COMPLETED) return constants.PubRecReasonCode.SUCCESS def _on_connect(self, client: Client, flags: int, result: int, properties) -> None: # FIXME make base path configurable self.client.subscribe('/devices/#') def subscribe(self, topic: str, callback: ValueCallback) -> None: self.subscriptions.setdefault(topic, []).append(callback) def send(self, topic: str, message): self.client.publish(topic, message) async def run(self): await self.client.connect(self.host, self.port, version=constants.MQTTv311, keepalive=30) while True: self.client.publish('smarthome', b'ping') await asyncio.sleep(10) @classmethod def from_config(cls, cfg: dict) -> "MqttClient": return cls( host=cfg.get('host', 'localhost'), port=cfg.get('port', 1883), user=cfg.get('login', ''), password=cfg.get('password', None), )
class MQTTPublisher: # pylint: disable=too-many-instance-attributes _APP_NAME = "call_detector" _LOGGER = logging.getLogger(f"{__name__}.{__qualname__}") _UPDATE_INTERVAL = 60 def __init__( # pylint: disable=too-many-arguments self, queue, host="localhost", port=8333, username=None, password=None, ssl=False, retry=False, topic=f"call_detector/{socket.gethostname()}", ): self._client = MQTTClient(self._APP_NAME) if username is not None: self._client.set_auth_credentials(username, password) self._host = host self._port = port self._queue = queue self._topic = topic self._retry = retry self._ssl = ssl self._state = {"call": False} async def run(self): self._LOGGER.info("Running.") while True: try: await self._client.connect(self._host, port=self._port, ssl=self._ssl, version=MQTTv311, keepalive=10) self._LOGGER.info("Connected.") break except Exception: # pylint: disable=broad-except if not self._retry: raise self._LOGGER.exception("Error occured during connecting") await asyncio.sleep(5) while True: try: try: with async_timeout.timeout(self._UPDATE_INTERVAL): msg = await self._queue.get() self._update_state(msg) except asyncio.exceptions.TimeoutError: pass await self._publish_state() except Exception: # pylint: disable=broad-except if not self._retry: raise self._LOGGER.exception("Error occured during publishing") await asyncio.sleep(5) def _update_state(self, msg): del self._state["call"] self._state[msg["source"]] = msg["apps"] apps = reduce(lambda a, b: a + len(b), self._state.values(), 0) self._state["call"] = apps > 0 self._LOGGER.info("State updated: %s", self._state) @throttle(0.5) async def _publish_state(self): self._LOGGER.info("Publishing state %s to topic %s", self._state, self._topic) self._client.publish( self._topic, json.dumps(self._state), qos=1, )
class DecoderManager(): ################### # Sync class init ################### def __init__(self): print("DecoderManager __init__", flush=True) print("{} acp_decoders started\n".format(self.ts_string()), file=sys.stderr, flush=True) self.settings = {} self.settings["decoders"] = [] ##################################### # Signal handler for SIGINT, SIGTERM ##################################### def ask_exit(self, *args): self.STOP.set() ##################################### # Signal handler for SIGALRM ##################################### def reload(self, *args): self.load_decoders_file() ##################################### # Return current timestamp as string ##################################### def ts_string(self): return '{:.6f}'.format(time.time()) ############################################################### # Async initialization ############################################################### async def start(self): # Define async events for exit and reload (will set via signals) self.STOP = asyncio.Event() self.RELOAD = asyncio.Event() # load settings.json into self.settings self.read_settings() # Connect input and output MQTT brokers (which can be same or different) await self.connect_input_mqtt() # debug testing timeout, disabling start of publisher await self.connect_output_mqtt() async def connect_input_mqtt(self): self.input_client = MQTTClient(None) # auto-generate client id self.input_client.on_connect = self.input_on_connect self.input_client.on_message = self.input_on_message self.input_client.on_disconnect = self.input_on_disconnect self.input_client.on_subscribe = self.input_on_subscribe user = self.settings["input_mqtt"]["user"] password = self.settings["input_mqtt"]["password"] host = self.settings["input_mqtt"]["host"] port = self.settings["input_mqtt"]["port"] self.input_client.set_auth_credentials(user, password) await self.input_client.connect(host, port, keepalive=20, version=MQTTv311) async def connect_output_mqtt(self): self.output_client = MQTTClient(None) # auto-generate client id self.output_client.on_connect = self.output_on_connect self.output_client.on_message = self.output_on_message self.output_client.on_disconnect = self.output_on_disconnect self.output_client.on_subscribe = self.output_on_subscribe user = self.settings["output_mqtt"]["user"] password = self.settings["output_mqtt"]["password"] host = self.settings["output_mqtt"]["host"] port = self.settings["output_mqtt"]["port"] self.output_client.set_auth_credentials(user, password) await self.output_client.connect(host, port, keepalive=60, version=MQTTv311) ############################################################### # Settings, including loading enabled decoders # # Builds self.settings from file "settings.json" # Then loads decoders listed in the setting "decoders_file" ############################################################### def read_settings(self): with open('settings.json', 'r') as sf: settings_data = sf.read() # parse file self.settings = json.loads(settings_data) self.load_decoders_file() def load_decoders_file(self): # getting settings filename for decoders list (json) decoders_file = self.settings["decoders_file"] # read the json file with open(decoders_file, 'r') as df: decoders_data = df.read() # parse to a python dictionary decoders_obj = json.loads(decoders_data) # store the new list of decoders as settings["decoders"] self.settings["decoders"] = decoders_obj["decoders"] # import/reload the decoders self.import_decoders(self.settings["decoders"]) # import a list of decoder names def import_decoders(self, new_decoders): self.decoders = [] for decoder_name in new_decoders: self.import_decoder(decoder_name) # import a decoder, given name # Will add { "name": , "decoder": } to self.decoders list def import_decoder(self, decoder_name): print("loading Decoder {}".format(decoder_name), flush=True) module_name = 'decoders.' + decoder_name # A new module can be imported with importlib.import_module() # BUT an already loaded module must use importlib.reload for update to work. if module_name in sys.modules: module = sys.modules[module_name] importlib.reload(module) else: module = importlib.import_module(module_name) # now we have the refreshed/new module, so put Decoder on list self.decoders decoder = module.Decoder(self.settings) print(" loaded Decoder {}".format(decoder_name), flush=True) self.decoders.append({"name": decoder_name, "decoder": decoder}) ############################################################### # Sensor data message handler ############################################################### def handle_input_message(self, topic, msg_bytes): acp_ts = self.ts_string() msg_is_decoded = False for decoder in self.decoders: if decoder["decoder"].test(topic, msg_bytes): decoded = decoder["decoder"].decode(topic, msg_bytes) # If no acp_ts from decoder, insert from server time if not "acp_ts" in decoded: decoded["acp_ts"] = acp_ts print("{} {} decoded by {}".format(acp_ts, decoded["acp_id"], decoder["name"]), flush=True) #debug testing timeout, disabled send: #self.send_output_message(topic, decoded) msg_is_decoded = True break # terminate the loop through decoders when first is found if msg_is_decoded: self.send_output_message(topic, decoded) else: print("{} Incoming message not decoded\n{}\n".format( acp_ts, msg_bytes), flush=True) def send_output_message(self, topic, decoded): msg_bytes = json.dumps(decoded) #print("publishing {}".format(msg_bytes), flush=True) output_topic = self.settings["output_mqtt"]["topic_prefix"] + topic self.output_client.publish(output_topic, msg_bytes, qos=0) ############################################################### # MQTT INPUT ############################################################### def input_on_connect(self, client, flags, rc, properties): print('INPUT Connected to {} as {}'.format( self.settings["input_mqtt"]["host"], self.settings["input_mqtt"]["user"]), flush=True) client.subscribe('#', qos=0) def input_on_message(self, client, topic, msg_bytes, qos, properties): # IMPORTANT! We avoid a loop by ignoring input messages with the output prefix if not topic.startswith(self.settings["output_mqtt"]["topic_prefix"]): if DEBUG: print('INPUT RECV MSG:', msg_bytes, flush=True) self.handle_input_message(topic, msg_bytes) else: if DEBUG: print('INPUT RECV COOKED MSG SKIPPED', flush=True) def input_on_disconnect(self, client, packet, exc=None): print('INPUT Disconnected', flush=True) print("{} INPUT Disconnected\n".format(self.ts_string()), file=sys.stderr, flush=True) def input_on_subscribe(self, client, mid, qos, properties): print('INPUT SUBSCRIBED to {}'.format( self.settings["input_mqtt"]["topic"]), flush=True) ############################################################### # MQTT OUTPUT ############################################################### def output_on_connect(self, client, flags, rc, properties): print('OUTPUT Connected to {} as {}'.format( self.settings["output_mqtt"]["host"], self.settings["output_mqtt"]["user"]), flush=True) def output_on_disconnect(self, client, packet, exc=None): print('OUTPUT Disconnected', flush=True) print("{} OUTPUT Disconnected\n".format(self.ts_string()), file=sys.stderr, flush=True) # These GMQTT methods here for completeness although not used def output_on_message(self, client, topic, msg_bytes, qos, properties): print('OUTPUT RECV MSG?:', msg_bytes, flush=True) def output_on_subscribe(self, client, mid, qos, properties): print('OUTPUT SUBSCRIBED? to {}', flush=True) ############################################################### # CLEANUP on EXIT SIGNAL (SIGINT or SIGTERM) ############################################################### async def finish(self): await self.STOP.wait() print("\nDecoderManager interrupted, closing MQTT clients", flush=True) print("{} DecoderManager interrupted - disconnecting\n".format( self.ts_string()), file=sys.stderr, flush=True) await self.input_client.disconnect() await self.output_client.disconnect()
class GmqttClient(Client): counter = 0 def __init__(self, config: BrokerConfig): self.config: BrokerConfig = config self.client = None self.thread = None self.all_listeners: Dict[str, Subscriber] = {} self.active_listeners: Set = set() self.loop = None self.ready = threading.Event() @property def is_connected(self): return self.client.is_connected def publish(self, topic: str, payload): # At most once (0) # At least once (1) # Exactly once (2). self.ready.wait() assert self.client, 'you need to activate connection' self.client.publish(topic, payload, qos=0, message_expiry_interval=10) def subscribe(self, topic: str, listener: Subscriber): if self.all_listeners.get(topic, None): raise Exception(f'Topic {topic} already registered') self.all_listeners[topic] = listener self.ready.wait() self.PROCESS.set() def unsubscribe(self, topic): listener = self.all_listeners.get(topic, None) if not listener: raise Exception(f'Topic {topic} was not registered') self.all_listeners.pop(topic) self.active_listeners.discard(topic) def connect(self) -> Client: if self.client: return self self.thread = Thread(target=self._loop, daemon=True) self.thread.start() # print('ready received') return self def _loop(self): try: self.loop = asyncio.new_event_loop() self.loop.run_until_complete(self._connect()) # asyncio.run(self._connect()) except KeyboardInterrupt: if self.client: self.client.disconnect() # print("Received exit, exiting") async def _connect(self): GmqttClient.counter += 1 client_id = f'client-id/{platform.node()}/pid_{os.getpid()}/{uuid.getnode()}/{GmqttClient.counter}' self.client = MQTTClient(client_id) self.PROCESS = asyncio.Event() self.client.on_connect = self.on_connect self.client.on_message = self.on_message self.client.on_disconnect = self.on_disconnect self.client.on_subscribe = self.on_subscribe # print('set') self.client.set_auth_credentials(self.config.username, self.config.password) while True: try: await self.client.connect(self.config.hostname, port=self.config.port, ssl=self.config.ssl, version=self.config.mqtt_version) break except Exception as ex: print('connect failed', ex) time.sleep(1) self.ready.set() while True: await self.PROCESS.wait() self.PROCESS.clear() # print('process signaled') self._sync_subscriptions() # await self.STOP.wait() await self.client.disconnect() def on_connect(self, client, flags, rc, properties): self.connected = True print('Connected', id(self)) self.PROCESS.set() self._sync_subscriptions() def _sync_subscriptions(self): if not self.client.is_connected: return unregistered = self.all_listeners.keys() - self.active_listeners for topic in unregistered: self.client.subscribe(topic, qos=1) self.active_listeners.add(topic) def _no_subscriber(self, message): print('RECV MSG with no subscriber:', message.topic, message.payload) def on_message(self, client, topic, payload, qos, properties): s = self.all_listeners.get(topic, self._no_subscriber) s(topic, payload) def on_disconnect(self, client, packet, exc=None): self.connected = False self.active_listeners.clear() print('Disconnected', id(self)) self.PROCESS.set() def on_subscribe(self, client, mid, qos, *args, **kwargs): # print('SUBSCRIBED') pass
class EcovacsMqtt: """Handle mqtt connections.""" def __init__(self, *, continent: str, country: str): self._subscribers: MutableMapping[str, VacuumBot] = {} self._port = 443 self._hostname = f"mq-{continent}.ecouser.net" if country.lower() == "cn": self._hostname = "mq.ecouser.net" self._client: Optional[Client] = None self._received_set_commands: MutableMapping[str, SetCommand] = TTLCache( maxsize=60 * 60, ttl=60) # pylint: disable=unused-argument async def _on_message(client: Client, topic: str, payload: bytes, qos: int, properties: Dict) -> None: _LOGGER.debug("Got message: topic=%s; payload=%s;", topic, payload.decode()) topic_split = topic.split("/") if topic.startswith("iot/atr"): await self._handle_atr(topic_split, payload) elif topic.startswith("iot/p2p"): self._handle_p2p(topic_split, payload) else: _LOGGER.debug("Got unsupported topic: %s", topic) self.__on_message = _on_message async def initialize(self, auth: RequestAuth) -> None: """Initialize MQTT.""" if self._client is not None: self.disconnect() client_id = f"{auth.user_id}@ecouser/{auth.resource}" self._client = Client(client_id) self._client.on_message = self.__on_message self._client.set_auth_credentials(auth.user_id, auth.token) ssl_ctx = ssl.create_default_context() ssl_ctx.check_hostname = False ssl_ctx.verify_mode = ssl.CERT_NONE await self._client.connect(self._hostname, self._port, ssl=ssl_ctx, version=MQTTv311) async def subscribe(self, vacuum_bot: VacuumBot) -> None: """Subscribe for messages for given vacuum.""" if self._client is None: raise NotInitializedError vacuum = vacuum_bot.vacuum self._client.subscribe(_get_subscriptions(vacuum)) self._subscribers[vacuum.did] = vacuum_bot def unsubscribe(self, vacuum_bot: VacuumBot) -> None: """Unsubscribe given vacuum.""" vacuum = vacuum_bot.vacuum if self._subscribers.pop(vacuum.did, None) and self._client: for subscription in _get_subscriptions(vacuum): self._client.unsubscribe(subscription.topic) def disconnect(self) -> None: """Disconnect from MQTT.""" if self._client: self._client.disconnect() self._subscribers.clear() async def _handle_atr(self, topic_split: List[str], payload: bytes) -> None: try: bot = self._subscribers.get(topic_split[3]) if bot: data = json.loads(payload) await bot.handle(topic_split[2], data) except Exception: # pylint: disable=broad-except _LOGGER.error("An exception occurred during handling atr message", exc_info=True) def _handle_p2p(self, topic_split: List[str], payload: bytes) -> None: try: command_name = topic_split[2] if command_name not in SET_COMMAND_NAMES: # command doesn't need special treatment or is not supported yet return is_request = topic_split[9] == "q" request_id = topic_split[10] if is_request: payload_json = json.loads(payload) try: data = payload_json["body"]["data"] except KeyError: _LOGGER.warning( "Could not parse p2p payload: topic=%s; payload=%s", "/".join(topic_split), payload_json, ) return command_class = COMMANDS.get(command_name) if command_class and issubclass(command_class, SetCommand): self._received_set_commands[request_id] = command_class( **data) else: command = self._received_set_commands.get(request_id, None) if not command: _LOGGER.debug( "Response to setCommand came in probably to late. requestId=%s, commandName=%s", request_id, command_name, ) return bot = self._subscribers.get(topic_split[3]) if bot: data = json.loads(payload) if command.handle(bot.events, data) and isinstance( command.args, dict): command.get_command.handle(bot.events, command.args) except Exception: # pylint: disable=broad-except _LOGGER.error("An exception occurred during handling p2p message", exc_info=True)
class MyMQTT(object): def __init__( self, broker: str, port: int = 1883, keepalive: int = 60, auth: List[str] = [None, None], event_functions=None, client_id: str = "client_id", event_loop=None, **kwargs, ) -> None: default_events = { "on_connect": self.on_connect, "on_message": self.on_message, "on_disconnect": self.on_disconnect, "on_subscribe": self.on_subscribe, } if event_functions is None: event_functions = {} self.event_functions = default_events self.event_functions.update(event_functions) # self.client = MQTTClient(client_id) self.auth = auth self.broker = broker self.port = port self.keepalive = keepalive self.stop = asyncio.Event() if event_loop is None: loop = asyncio.get_event_loop() else: loop = event_loop loop.add_signal_handler(signal.SIGINT, self.ask_exit) loop.add_signal_handler(signal.SIGTERM, self.ask_exit) # passing any extra kwargs to the call to be used for e.g. on_connect self.client = MQTTClient(client_id, **kwargs) for _ in [k for k, v in self.event_functions.items() if v is None]: logger.warning(f"mqtt no function assigned to {_}") self.event_functions.pop(k) for k, v in self.event_functions.items(): setattr(self.client, k, v) if any(self.auth): self.client.set_auth_credentials(*self.auth) loop.create_task(self.start(self.broker)) async def start(self, broker): logger.info("starting mqtt") try: await self.client.connect( broker, port=self.port, keepalive=self.keepalive, version=MQTTv311 ) except OSError as e: logger.error(f"unable to connect to mqtt with following error {e}") else: await self.stop.wait() await self.client.disconnect() def publish(self, topic, message, qos=1): self.client.publish(topic, message, qos=qos) def on_connect(self, client, flags, rc, properties): logger.info("MQTT Connected") client.subscribe("#") def on_message(self, client, topic, payload, qos, properties): logger.info(f"MQTT RECV MSG: {topic} {payload}") def on_disconnect(self, client, packet, exc=None): logger.info("MQTT Disconnected") @staticmethod def on_subscribe(*args): logger.info("MQTT SUBSCRIBED") def ask_exit(self, *args): asyncio.Event().set()
class Pipeline: """Object detection and tracking pipeline""" def __init__(self, args): self.args = args # Initialise camera & camera viewport self.init_camera() # Initialise output self.init_output(self.args.output) # Initialise object detector (for some reason it has to happen # here & not within detect_objects(), or else the inference engine # gets upset and starts throwing NaNs at me. Thanks, Python.) self.object_detector = SSD_MOBILENET(wanted_label='person', model_file=self.args.model, label_file=self.args.labels, num_threads=self.args.num_threads, edgetpu=self.args.edgetpu) # Initialise feature encoder if self.args.encoder_model is None: model_filename = '{}/mars-64x32x3.pb'.format( self.args.deepsorthome) else: model_filename = self.args.encoder_model self.encoder = gdet.create_box_encoder( model_filename, batch_size=self.args.encoder_batch_size) self.background_subtraction = not self.args.disable_background_subtraction # Initialise tracker nn_budget = None metric = nn_matching.NearestNeighborDistanceMetric( "cosine", self.args.max_cosine_distance, nn_budget) self.tracker = Tracker(metric, max_iou_distance=self.args.max_iou_distance, max_age=self.args.max_age) # Initialise database self.db = {} self.delcount = 0 self.intcount = 0 self.poscount = 0 self.negcount = 0 self.mqtt = None self.topic = self.args.mqtt_topic self.mqtt_acp_id = self.args.mqtt_acp_id self.heartbeat_delay_secs = self.args.heartbeat_delay_secs self.loop = asyncio.get_event_loop() async def init_mqtt(self): if self.args.mqtt_broker is not None: self.mqtt = MQTTClient('deepdish') if self.args.mqtt_user is not None: self.mqtt.set_auth_credentials(self.args.mqtt_user, self.args.mqtt_pass) await self.mqtt.connect(self.args.mqtt_broker) if self.topic is None: self.topic = 'default/topic' def init_camera(self): self.input = self.args.input if self.input is None: self.input = self.args.camera # Allow live camera frames to be dropped self.everyframe = None else: # Capture every frame from the video file self.everyframe = asyncio.Event() self.cap = cv2.VideoCapture(self.input) # Configure the 'counting line' in the camera viewport if self.args.line is None: w, h = self.args.camera_width, self.args.camera_height self.countline = np.array([[w / 2, 0], [w / 2, h]], dtype=int) else: self.countline = np.array(list( map(int, self.args.line.strip().split(','))), dtype=int).reshape(2, 2) self.cameracountline = self.countline.astype(float) def init_output(self, output): self.color_mode = None # fixme fourcc = cv2.VideoWriter_fourcc(*'MP4V') fps = self.cap.get(cv2.CAP_PROP_FPS) (w, h) = (self.args.camera_width, self.args.camera_height) self.backbuf = Image.new("RGBA", (w, h), (0, 0, 0, 0)) self.draw = ImageDraw.Draw(self.backbuf) self.output = cv2.VideoWriter(self.args.output, fourcc, fps, (w, h)) if self.args.no_framebuffer: self.framebufdev = None else: self.framebufdev = self.args.framebuffer fbX = self.framebufdev[-3:] vsizefile = '/sys/class/graphics/{}/virtual_size'.format(fbX) if not os.path.exists( self.framebufdev) or not os.path.exists(vsizefile): #raise Error('Invalid framebuffer device: {}'.format(self.framebufdev)) print('Invalid framebuffer device: {}'.format( self.framebufdev)) self.framebufdev = None if self.framebufdev is not None: (w, h) = (self.args.framebuffer_width, self.args.framebuffer_height) if w is None or h is None: nums = re.findall('(.*),(.*)', open(vsizefile).read())[0] if w is None: w = int(nums[0]) if h is None: h = int(nums[1]) self.framebufres = (w, h) def read_frame(self): ret, frame = self.cap.read() return (frame, time()) def shutdown(self): self.running = False for p in asyncio.Task.all_tasks(): p.cancel() async def capture(self, q): try: with concurrent.futures.ThreadPoolExecutor() as pool: while self.running: frame = None # Fetch next frame (frame, t_frame) = await self.loop.run_in_executor( pool, self.read_frame) if frame is None: print('No more frames.') self.shutdown() break if self.args.camera_flip: # If we need to flip the image vertically frame = cv2.flip(frame, 0) # Ensure frame is proper size frame = cv2.resize( frame, (self.args.camera_width, self.args.camera_height)) await q.put((frame, t_frame, time())) # If we are ensuring every frame is processed then wait for # synchronising event to be triggered if self.everyframe is not None: await self.everyframe.wait() self.everyframe.clear() finally: self.cap.release() def run_object_detector(self, image): t1 = time() boxes = self.object_detector.detect_image(image) t2 = time() return (boxes, t2 - t1) async def detect_objects(self, q_in, q_out): # Initialise background subtractor backSub = cv2.createBackgroundSubtractorMOG2() frameCount = 0 with concurrent.futures.ThreadPoolExecutor() as pool: while self.running: frameCount += 1 # Obtain next video frame (frame, t_frame, t_prev) = await q_in.get() t_frame_recv = time() # Apply background subtraction to find image-mask of areas of motion if self.background_subtraction: fgMask = backSub.apply(frame) # Convert to PIL Image image = Image.fromarray( cv2.cvtColor(frame, cv2.COLOR_BGRA2RGBA)) t_backsub = time() # Run object detection engine within a Thread Pool (boxes0, delta_t) = await self.loop.run_in_executor( pool, self.run_object_detector, image) # Filter object detection boxes, including only those with areas of motion t1 = time() boxes = [] max_x, max_y = self.args.camera_width, self.args.camera_height for (x, y, w, h) in boxes0: if np.any(np.isnan(boxes0)): # Drop any rubbish results continue x, y = int(np.clip(x, 0, max_x)), int(np.clip(y, 0, max_y)) w, h = int(np.clip(w, 0, max_x - x)), int( np.clip(h, 0, max_y - y)) # Check if the box is almost as large as the camera viewport if w * h > 0.9 * max_x * max_y: # reject as spurious continue # Check if the box includes any detected motion if not self.background_subtraction or np.any( fgMask[x:x + w, y:y + h]): boxes.append((x, y, w, h)) t2 = time() # Send results to next step in pipeline elements = [ FrameInfo(t_frame, frameCount), CameraImage(image), CameraCountLine(self.cameracountline), TimingInfo('Frame processing latency', 'fram', t_prev - t_frame), TimingInfo('Frame / Q1 item received latency', 'q1', t_frame_recv - t_prev), TimingInfo('Background subtraction latency', 'bsub', t_backsub - t_frame_recv), TimingInfo('Object detection latency', 'objd', delta_t + (t2 - t1)) ] await q_out.put((frame, boxes, elements, time())) async def encode_features(self, q_in, q_out): with concurrent.futures.ThreadPoolExecutor() as pool: while self.running: # Obtain next video frame and object detection boxes (frame, boxes, elements, t_prev) = await q_in.get() t1 = time() # Run feature encoder within a Thread Pool features = await self.loop.run_in_executor( pool, self.encoder, frame, boxes) t2 = time() # Build list of 'Detection' objects and send them to next step in pipeline detections = [ Detection(bbox, 1.0, feature) for bbox, feature in zip(boxes, features) ] elements.append( TimingInfo('Q1 / Q2 latency', 'q2', (t1 - t_prev))) elements.append( TimingInfo('Feature encoder latency', 'feat', (t2 - t1))) await q_out.put((detections, elements, time())) async def track_objects(self, q_in, q_out): while self.running: (detections, elements, t_prev) = await q_in.get() t1 = time() boxes = np.array([d.tlwh for d in detections]) scores = np.array([d.confidence for d in detections]) indices = preprocessing.non_max_suppression( boxes, self.args.nms_max_overlap, scores) detections = [detections[i] for i in indices] self.tracker.predict() self.tracker.update(detections) t2 = time() elements.append(TimingInfo('Q2 / Q3 latency', 'q3', (t1 - t_prev))) elements.append(TimingInfo('Tracker latency', 'trak', (t2 - t1))) await q_out.put((detections, elements, time())) async def process_results(self, q_in, q_out): while self.running: (detections, elements, t_prev) = await (q_in.get()) t1 = time() for track in self.tracker.deleted_tracks: i = track.track_id if track.is_deleted(): self.check_deleted_track(track.track_id) for track in self.tracker.tracks: i = track.track_id if not track.is_confirmed() or track.time_since_update > 1: continue if i not in self.db: self.db[i] = [] bbox = track.to_tlbr() # Find the bottom-centre of the bounding box & add it to the tracking database bottomCentre = np.array([(bbox[0] + bbox[2]) / 2.0, bbox[3]]) self.db[i].append(bottomCentre) if len(self.db[i]) > 1: # If we have more than one datapoint for this tracked object pts = (np.array(self.db[i]).reshape( (-1, 1, 2))).reshape(-1) elements.append(TrackedPath(pts)) p1 = self.cameracountline[0] q1 = self.cameracountline[1] p2 = np.array(self.db[i][-1]) q2 = np.array(self.db[i][-2]) cp = np.cross(q1 - p1, q2 - p2) if intersection(p1, q1, p2, q2): self.intcount += 1 print( "track_id={} just intersected camera countline; cross-prod={}; intcount={}" .format(i, cp, self.intcount)) elements.append(TrackedPathIntersection(pts[-4:])) if cp >= 0: self.poscount += 1 crossing_type = 'pos' else: self.negcount += 1 crossing_type = 'neg' await self.publish_crossing_event_to_mqtt( elements, crossing_type) elements.append(TrackedObject(bbox, str(track.track_id))) for det in detections: bbox = det.to_tlbr() elements.append(DetectedObject(bbox)) elements.append(CountingStats(self.negcount, self.poscount)) t2 = time() elements.append(TimingInfo('Q3 / Q4 latency', 'q4', (t1 - t_prev))) elements.append( TimingInfo('Results processing latency', 'proc', (t2 - t1))) await q_out.put((elements, time())) async def publish_crossing_event_to_mqtt(self, elements, crossing_type): if self.mqtt is not None: for e in elements: if isinstance(e, FrameInfo): t_frame = e.t_frame break payload = json.dumps({ 'acp_ts': str(t_frame), 'acp_id': self.mqtt_acp_id, 'acp_event': 'crossing', 'acp_event_value': crossing_type, 'poscount': self.poscount, 'negcount': self.negcount, 'diff': self.poscount - self.negcount }) self.mqtt.publish(self.topic, payload) async def periodic_mqtt_heartbeat(self): if self.mqtt is not None: while True: payload = json.dumps({ 'acp_ts': str(time()), 'acp_id': self.mqtt_acp_id, 'poscount': self.poscount, 'negcount': self.negcount, 'diff': self.poscount - self.negcount }) self.mqtt.publish(self.topic, payload) await asyncio.sleep(self.heartbeat_delay_secs) async def graphical_output(self, render: RenderInfo, elements, output_wh: (int, int)): (output_w, output_h) = output_wh # Clear screen self.draw.rectangle([0, 0, output_w, output_h], fill=0, outline=0) # Sort elements by display priority elements.sort(key=lambda e: e.priority) # Draw elements for e in elements: if hasattr(e, 'do_render'): e.do_render(render) # Copy backbuf to output backarray = np.array(self.backbuf) if self.color_mode is not None: outputbgra = cv2.cvtColor(backarray, self.color_mode) else: outputbgra = backarray outputrgb = cv2.cvtColor(outputbgra, cv2.COLOR_BGRA2RGB) if self.output is not None: self.output.write(outputrgb) if self.framebufdev is not None: outputrgba = cv2.cvtColor(outputbgra, cv2.COLOR_BGRA2RGBA) outputfbuf = cv2.resize(outputrgba, self.framebufres) try: with open(self.framebufdev, 'wb') as buf: buf.write(outputfbuf) except: print( 'failed to write to framebuffer device {} ...disabling it.' .format(self.framebufdev)) self.framebufdev = None await streaminfo.set_frame(outputrgb) #cv2.imshow('main', outputrgb) def text_output(self, handle, elements): # Sort elements by priority elements.sort(key=lambda e: e.priority) for e in elements: if hasattr(e, 'do_text'): e.do_text(handle, elements) async def render_output(self, q_in): (output_w, output_h) = (self.args.camera_width, self.args.camera_height) ratio = 1 #fixme render = RenderInfo(ratio, FontLib(output_w), self.draw, self.backbuf) try: while self.running: (elements, t_prev) = await q_in.get() t1 = time() await self.graphical_output(render, elements, (output_w, output_h)) for e in elements: if isinstance(e, FrameInfo): t_frame = e.t_frame break elements.append( TimingInfo('Q4 / Q5 latency', 'q5', t1 - t_prev)) elements.append( TimingInfo('Graphical display latency', 'disp', time() - t1)) t_sum = 0 for e in elements: if isinstance(e, TimingInfo): t_sum += e.delta_t elements.append(TimingInfo('Latency sum', 'sum', t_sum)) t_e2e = time() - t_frame elements.append(TimingInfo('End to end latency', 'e2e', t_e2e)) elements.append(TimingInfo('Missing', 'miss', t_e2e - t_sum)) self.text_output(sys.stdout, elements) if self.everyframe is not None: # Notify other side that this frame is completely processed self.everyframe.set() finally: self.output.release() def check_deleted_track(self, i): if i in self.db and len(self.db[i]) > 1: if any_intersection(self.cameracountline[0], self.cameracountline[1], np.array(self.db[i])): self.delcount += 1 print("delcount={}".format(self.delcount)) self.db[i] = [] async def start(self): self.running = True cameraQueue = FreshQueue() objectQueue = asyncio.Queue(maxsize=1) detectionQueue = asyncio.Queue(maxsize=1) resultQueue = asyncio.Queue(maxsize=1) drawQueue = asyncio.Queue(maxsize=1) asyncio.ensure_future(self.render_output(drawQueue)) asyncio.ensure_future(self.process_results(resultQueue, drawQueue)) asyncio.ensure_future(self.track_objects(detectionQueue, resultQueue)) asyncio.ensure_future(self.encode_features(objectQueue, detectionQueue)) asyncio.ensure_future(self.detect_objects(cameraQueue, objectQueue)) await self.capture(cameraQueue) self.running = False
class GMQTT_Client(MQTT_Base): def __init__(self, mqtt_settings): MQTT_Base.__init__(self, mqtt_settings) self.mqtt_client = None def connect(self): MQTT_Base.connect(self) self.mqtt_client = MQTTClient( 'gmqtt' #self.mqtt_settings["MQTT_CLIENT_ID"] ) self.mqtt_client.on_connect = self._on_connect self.mqtt_client.on_message = self._on_message self.mqtt_client.on_disconnect = self._on_disconnect if self.mqtt_settings["MQTT_USERNAME"]: self.mqtt_client.set_auth_credentials( self.mqtt_settings["MQTT_USERNAME"], self.mqtt_settings["MQTT_PASSWORD"], ) def start(): try: logger.warning('Connecting to MQTT') asyncio.set_event_loop(self.event_loop) # self.event_loop.run_until_complete( # self.mqtt_client.connect(self.mqtt_settings["MQTT_BROKER"], self.mqtt_settings["MQTT_PORT"],keepalive=self.mqtt_settings["MQTT_KEEPALIVE"], version=MQTTv311) # ) logger.warning('Looping forever') self.event_loop.run_forever() logger.warning('Event loop stopped') #self.session.close() except Exception as e: logger.error('Error in event loop {}'.format(e)) self.event_loop = asyncio.new_event_loop() logger.warning("Starting MQTT thread") self._ws_thread = threading.Thread(target=start, args=()) self._ws_thread.daemon = True self._ws_thread.start() future = asyncio.run_coroutine_threadsafe( self.mqtt_client.connect( self.mqtt_settings["MQTT_BROKER"], self.mqtt_settings["MQTT_PORT"], keepalive=self.mqtt_settings["MQTT_KEEPALIVE"], version=MQTTv311), self.event_loop) def publish(self, topic, payload, retain, qos): MQTT_Base.publish(self, topic, payload, retain, qos) if self.mqtt_connected is True: wrapped = functools.partial(self.mqtt_client.publish, topic, payload, retain=retain, qos=qos) self.event_loop.call_soon_threadsafe(wrapped) else: logger.warning( "Device MQTT publish NOT CONNECTED: {}, retain {}, qos {}, payload: {}" .format(topic, retain, qos, payload)) # future = asyncio.run_coroutine_threadsafe( # self.mqtt_client.publish(topic, payload, retain=retain, qos=qos), # self.event_loop # ) def subscribe(self, topic, qos): # subclass to provide MQTT_Base.subscribe(self, topic, qos) self.mqtt_client.subscribe(topic, qos) def unsubscribe(self, topic): # subclass to provide MQTT_Base.unsubscribe(self, topic) self.mqtt_client.unsubscribe(topic) def set_will(self, will, topic, retain, qos): MQTT_Base.set_will(self, will, topic, retain, qos) #self.mqtt_client.will_set(will, topic, retain, qos) def _on_connect(self, client, flags, rc, properties): logger.info("MQTT On Connect: {}".format(rc)) self.mqtt_connected = rc == 0 def _on_message(self, client, topic, payload, qos, properties): #topic = msg.topic #payload = msg.payload.decode("utf-8") MQTT_Base._on_message(self, topic, payload, False, qos) def _on_disconnect(self, client, packet, exc=None): self.mqtt_connected = False # note, change this uses the property setter, do not really need to catch this in the base class logger.warning("MQTT Disconnection {} {} {}".format( client, packet, exc)) MQTT_Base._on_disconnect(self, 0)
class GMQTTConnector(BaseConnector): """GMQTTConnector uses gmqtt library for connectors running over MQTT. """ def __init__(self, host, port, subscribe_topic, publish_topic, **kwargs): self.host = host self.port = port # topics self.subscribe_topic = subscribe_topic self.publish_topic = publish_topic # connection self.connection_id = uuid.uuid4().hex[:8] self.is_connected = False self.client = MQTTClient(self.connection_id) # callbacks self.client.on_connect = self.on_connect self.client.on_message = self.on_message self.client.on_disconnect = self.on_disconnect self.client.on_subscribe = self.on_subscribe self.STOP = asyncio.Event() # options self.ack_topic = kwargs.get('ack_topic') self.enable_ssl = kwargs.get('enable_ssl', False) self.enable_auth = kwargs.get('enable_auth', False) self.username = kwargs.get('username') self.password = kwargs.get('password') self.client_cert = kwargs.get('client_cert') self.client_key = kwargs.get('client_key') self.qos = kwargs.get('qos', 2) def get_connection_details(self): """get_connection_details returns the details about the current MQTT connection. """ return dict( connection_id=self.connection_id, host=self.host, port=self.port, is_connected=self.is_connected, subscribe_topic=self.subscribe_topic, publish_topic=self.publish_topic ) def on_connect(self, *args): """on_connect is a callback that gets exectued after the connection is made. Arguments: client {MQTTClient} -- gmqtt.MQTTClient flags {int} -- connection flags rc {int} -- connection result code properties {dict} -- config of the current connection """ logger.info("Connected with result code %s", str(args[2])) # Subscribing in on_connect() means that if we lose the connection and # reconnect then subscriptions will be renewed. # client.subscribe("$SYS/#", qos=0) if isinstance(self.subscribe_topic, str): self.client.subscribe(self.subscribe_topic, qos=self.qos) elif isinstance(self.subscribe_topic, list): for topic in self.subscribe_topic: self.client.subscribe(topic, qos=self.qos) else: logger.warning('subscribe_topic is either None or an unknown data type.' ' Currently subscribed to 0 topics.') async def on_message(self, *args): """on_message callback gets executed when the connection receives a message. Arguments: client {MQTTClient} -- gmqtt.MQTTClient topic {string} -- topic from which message was received payload {bytes} -- actual message bytes received qos {string} -- message QOS level (0,1,2) properties {dict} -- message properties """ logger.info("%s %s", args[1], str(args[2])) return 0 @staticmethod def on_disconnect(*args): """on_disconnect is a callback that gets executed after a disconnection occurs""" logger.info('Disconnected') @staticmethod def on_subscribe(*args): """on_subscribe is a callback that gets executed after a subscription is succesful""" logger.info('Subscribed') def ask_exit(self): """sets the STOP variable so that a signal gets sent to disconnect the client """ self.STOP.set() async def start(self): """starts initiates the connnection with the broker Raises: DestinationNotAvailable: If broker is not available ConnectionFailed: If connection failed due to any other reason """ try: conn_kwargs = dict(host=self.host, port=self.port) if self.enable_auth: self.client.set_auth_credentials(self.username, self.password) if self.enable_ssl: assert self.client_cert and self.client_key, \ "Cannot enable ssl without specifying client_cert and client_key" ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) ssl_context.load_cert_chain(self.client_cert, keyfile=self.client_key) conn_kwargs.update(dict(ssl=ssl_context)) await self.client.connect(**conn_kwargs) self.is_connected = True except ConnectionRefusedError as e: # raising from None suppresses the exception chain raise DestinationNotAvailable( f'Connection Failed: Error connecting to' f' {self.host}:{self.port} - {e}' ) from None except Exception as e: raise ConnectionFailed(e) async def publish(self, *args, **kwargs): """publishes the message to the topic using client.publish""" self.client.publish(*args, **kwargs) async def stop(self): """force stop the connection with the MQTT broker.""" await self.client.disconnect() self.is_connected = False