def unsubscribe(self, mqttclient: MqttClient) -> None: super().unsubscribe(mqttclient) mqttclient.unsubscribe(self.stateTopic) mqttclient.unsubscribe(self.heatingTopic) mqttclient.unsubscribe(self.setpointTopic) mqttclient.unsubscribe(self.ackAlarmTopic) for topic in self.tempReferences: mqttclient.unsubscribe(topic)
class MqttPlugin(plugins.SimplePlugin): """ Plugin that listens to MQTT topics and publishes the payload 'unmodified' to a channel on the CherryPy bus. The cherrypy channel name is the same as the MQTT topic Requires PAHO """ def __init__( self, bus: wspbus, broker: str, port: int, topic_list: Union[str, List[str]] ) -> None: """ Setup the plugin :param bus: Cherrypy internal Bus :param broker: Mqtt broker :param port: Port of the Mqtt broker :param topic_list: topic to subscribe """ # Cherrypy plugins.SimplePlugin doesn't accept the super().__init__() # Inside cherrypy's docs it's like this # https://docs.cherrypy.org/en/latest/extend.html#create-a-plugin plugins.SimplePlugin.__init__(self, bus) self.broker = broker self.port = port self.topic_list = topic_list self.client = Client(client_id=f"Catalog{randrange(1, 100000)}") self.client.on_message = Bus(bus).my_on_message def start(self): self.bus.log("Setup mqttcherrypy") self.client.connect(self.broker, self.port) self.bus.log(f"Connected to broker: {self.broker} port: {self.port})") self.client.loop_start() self.client.subscribe(self.topic_list) self.bus.log(f"Subscribed to {self.topic_list}") def stop(self): self.bus.log("Shut down mqttcherrypy") self.client.unsubscribe(self.topic_list) self.bus.log(f"Unsubscribed from {self.topic_list}") self.client.loop_stop(force=True) self.client.disconnect() self.bus.log(f"Disconnected from: {self.broker} port: {self.port}")
def unsubscribe(self, topics): """ Unsubscribe of a given topic """ if not isinstance(topics, list): topics = [topics] for topic in topics: rc, mid = Mosquitto.unsubscribe(self, topic) self._subscriptions[mid] = topic self.log(logging.INFO, "Sent unsubscription request of topic %s" % topic)
class Cep2Zigbee2mqttClient: """ This class implements a simple zigbee2mqtt client. By default it subscribes to all events of the default topic (zigbee2mqtt/#). No methods for explicitly publishing to zigbee2mqtt are provided, since the class can provide higher level abstraction methods for this. An example implemented example is this class' check_health(). Since all events from zigbee2mqtt are subscribed, the events filtering and management are up to the class user. For that, a callback can be set in the initializer (on_message_clbk) for processing the received messages. This callback is blocking, i.e. once the subscriber receives an event and invokes the callback, no new events will be processed. Careful should be taken with methods that might take too much time to process the events or that might eventually block (for example, sending an event to another service). """ ROOT_TOPIC = "zigbee2mqtt/#" def __init__(self, host: str, on_message_clbk: Callable[[Optional[Cep2Zigbee2mqttMessage]], None], port: int = 1883, topics: List[str] = [ROOT_TOPIC]): """ Class initializer where the MQTT broker's host and port can be set, the list of topics to subscribe and a callback to handle events from zigbee2mqtt. Args: host (str): string with the hostname, or IP address, of the MQTT broker. on_message_clbk (Callable[[Zigbee2mqttMessage], None]): a function that is called when a message is received from zigbee2mqtt. This returns None if the port (int): network port of the MQTT broker. Defaults to 1883. topics (List[str], optional): a list of topics that the client will subscribe to. Defaults to ["zigbee2mqtt/#"]. """ self.__client = MqttClient() self.__client.on_connect = self.__on_connect self.__client.on_disconnect = self.__on_disconnect self.__client.on_message = self.__on_message self.__connected = False self.__events_queue = Queue() self.__host = host self.__on_message_clbk = on_message_clbk self.__port = port self.__stop_worker = Event() self.__subscriber_thread = Thread(target=self.__worker, daemon=True) self.__topics = topics def connect(self) -> None: """ Connects to the MQTT broker specified in the initializer. This is a blocking function. """ # In the client is already connected then stop here. if self.__connected: return # Connect to the host given in initializer. self.__client.connect(self.__host, self.__port) self.__client.loop_start() # Subscribe to all topics given in the initializer. for t in self.__topics: self.__client.subscribe(t) # Start the subscriber thread. self.__subscriber_thread.start() def change_state(self, device_id: str, state: str) -> None: if not self.__connected: raise RuntimeError("The client is not connected. Connect first.") self.__client.publish(topic=f"zigbee2mqtt/{device_id}/set", payload=json.dumps({"state": f"{state}"})) def check_health(self) -> str: """ Allows to check whether zigbee2mqtt is healthy, i.e. the service is running properly. Refer to zigbee2mqtt for more information. This is a blocking function that waits for a response to the health request. Returns: A string with a description of zigbee2mqtt's health. This can be 'ok' or 'fail'. """ health_status = "fail" health_response_received = Event() # This function will run the subscriber on a thread. The subscriber must be started first, # so that the health_check message is received, even if the broker does not have message # persistence active. This should also avoid that messages with QoS 0 are not received. # Also, if the subscriber is started after, it could happen the message to be received by # another subscriber and never by this one. # More information can be found in # https://pagefault.blog/2020/02/05/how-to-set-up-persistent-storage-for-mosquitto-mqtt-broker def health_check_subscriber(): message = subscribe.simple(hostname=self.__host, port=self.__port, topics="zigbee2mqtt/bridge/response/health_check") if message: # Decode and parse JSON payload. payload = message.payload.decode("utf-8") health = json.loads(payload) # The nonlocal keyword is used to set the variable health_status that is not defined # in this function's scope. For more information got to # https://www.programiz.com/python-programming/global-local-nonlocal-variables nonlocal health_status health_status = health.get("status", "fail") # Set the flag so that the function exits. health_response_received.set() # Start a thread with the subscriber that will wait for the response of the health_check. Thread(target=health_check_subscriber, daemon=True).start() # Wait for the subscriber to establish a connection with the broker. sleep(.5) # Publish the health_check request. publish.single(hostname=self.__host, port=self.__port, topic="zigbee2mqtt/bridge/request/health_check") # Wait until the response is received. If it is not received within 5 seconds, then return # the default state: "fail". health_response_received.wait(timeout=5) return health_status def disconnect(self) -> None: """ Disconnects from the MQTT broker. """ self.__stop_worker.set() self.__client.loop_stop() # Unsubscribe from all topics given in the initializer. for t in self.__topics: self.__client.unsubscribe(t) self.__client.disconnect() def __on_connect(self, client, userdata, flags, rc) -> None: """ Callback invoked when a connection with the MQTT broker is established. Refer to paho-mqtt documentation for more information on this callback: https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#callbacks """ # Set connected flag to true. This is later used if multiple calls to connect are made. This # way the user does not need to very if the client is connected. self.__connected = True print("MQTT client connected") def __on_disconnect(self, client, userdata, rc) -> None: """ Callback invoked when the client disconnects from the MQTT broker occurs. Refer to paho-mqtt documentation for more information on this callback: https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#callbacks """ # Set connected flag to false. This is later used if multiple calls to connect are made. # This way the user does not need to very if the client is connected. self.__connected = False print("MQTT client disconnected") def __on_message(self, client, userdata, message: MQTTMessage) -> None: """ Callback invoked when a message has been received on a topic that the client subscribed. Refer to paho-mqtt documentation for more information on this callback: https://www.eclipse.org/paho/index.php?page=clients/python/docs/index.php#callbacks """ # Push a message to the queue. This will later be processed by the worker. self.__events_queue.put(message) def __worker(self) -> None: """ This method pulls zigbee2mqtt messages from the queue of received messages, pushed when a message is received, i.e. by the __on_message() callback. This method will be stopped when the instance of zigbee2mqttClient disconnects, i.e. disconnect() is called and sets the __stop_worker event. """ while not self.__stop_worker.is_set(): try: # Pull a message from the queue. message = self.__events_queue.get(timeout=1) except Empty: # This exception is raised when the queue pull times out. Ignore it and retry a new # pull. pass else: # If a message was successfully pulled from the queue, then process it. # NOTE: this else condition is part of the try and it is executed when the action # inside the try does not throws and exception. # The decode() transforms a byte array into a string, following the utf-8 encoding. if message: self.__on_message_clbk(Cep2Zigbee2mqttMessage.parse(message.topic, message.payload.decode("utf-8")))
class Mqtt(): def __init__(self, app=None): self.app = app self.client = Client() self.refresh_time = 1.0 self.topics = [] self.mqtt_thread = Thread(target=self._loop_forever) self.mqtt_thread.daemon = True if app is not None: self.init_app(app) def init_app(self, app): self.refresh_time = app.config.get('MQTT_REFRESH_TIME', 1.0) self._connect( username=app.config.get('MQTT_USERNAME'), password=app.config.get('MQTT_PASSWORD'), broker_url=app.config.get('MQTT_BROKER_URL', 'localhost'), broker_port=app.config.get('MQTT_BROKER_PORT', 1883) ) def _connect(self, username, password, broker_url, broker_port): if not self.mqtt_thread.is_alive(): if username is not None: self.client.username_pw_set(username, password) res = self.client.connect(broker_url, broker_port) if res == 0: self.mqtt_thread.start() def _disconnect(self): self.client.disconnect() def _loop_forever(self): while True: time.sleep(self.refresh_time) self.client.loop(timeout=1.0, max_packets=1) def on_topic(self, topic: str): """ Decorator to add a callback function that is called when a certain topic has been published. The callback function is expected to have the following form: `handle_topic(client, userdata, message)` :parameter topic: a string specifying the subscription topic to subscribe to **Example usage:** :: @on_topic('home/mytopic') def handle_mytopic(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): self.client.message_callback_add(topic, handler) return handler return decorator def subscribe(self, topic: str, qos: int=0): """ Subscribe to a certain topic. :param topic: a string specifying the subscription topic to subscribe to. :param qos: the desired quality of service level for the subscription. Defaults to 0. A topic is a UTF-8 string, which is used by the broker to filter messages for each connected client. A topic consists of one or more topic levels. Each topic level is separated by a forward slash (topic level separator). **Topic example:** `myhome/groundfloor/livingroom/temperature` """ # TODO: add support for list of topics # don't subscribe if already subscribed if topic in self.topics: return # try to subscribe result, mid = self.client.subscribe(topic, qos) # if successful add to topics if result == MQTT_ERR_SUCCESS: self.topics.append(topic) return result def unsubscribe(self, topic: str): """ Unsubscribe from a single topic. :param topic: a single string that is the subscription topic to unsubscribe from """ # don't unsubscribe if not in topics if topic not in self.topics: return result, mid = self.client.unsubscribe(topic) # if successful remove from topics if result == MQTT_ERR_SUCCESS: self.topics.remove(topic) return result def unsubscribe_all(self): """ Unsubscribe from all topics. """ topics = self.topics[:] for topic in topics: self.unsubscribe(topic) def publish(self, topic: str, payload: bytes=None, qos: int=0, retain: bool=False) -> Tuple[int, int]: """ Send a message to the broker. :param topic: the topic that the message should be published on :param payload: the actual message to send. If not given, or set to None a zero length message will be used. Passing an int or float will result in the payload being converted to a string representing that number. If you wish to send a true int/float, use struct.pack() to create the payload you require. :param qos: the quality of service level to use :param retain: if set to True, the message will be set as the "last known good"/retained message for the topic :returns: Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN if the client is not currently connected. mid is the message ID for the publish request. """ return self.client.publish(topic, payload, qos, retain) def on_message(self): """ Decorator to handle all messages that have been subscribed. **Example Usage:** :: @mqtt.on_message() def handle_messages(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): self.client.on_message = handler return handler return decorator
class MqttConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() self.__gateway = gateway # Reference to TB Gateway self._connector_type = connector_type # Should be "mqtt" self.config = config # mqtt.json contents self.__log = log self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__subscribes_sent = {} # Extract main sections from configuration --------------------------------------------------------------------- self.__broker = config.get('broker') self.__mapping = [] self.__server_side_rpc = [] self.__connect_requests = [] self.__disconnect_requests = [] self.__attribute_requests = [] self.__attribute_updates = [] mandatory_keys = { "mapping": ['topicFilter', 'converter'], "serverSideRpc": [ 'deviceNameFilter', 'methodFilter', 'requestTopicExpression', 'valueExpression' ], "connectRequests": ['topicFilter'], "disconnectRequests": ['topicFilter'], "attributeRequests": ['topicFilter', 'topicExpression', 'valueExpression'], "attributeUpdates": [ 'deviceNameFilter', 'attributeFilter', 'topicExpression', 'valueExpression' ] } # Mappings, i.e., telemetry/attributes-push handlers provided by user via configuration file self.load_handlers('mapping', mandatory_keys['mapping'], self.__mapping) # RPCs, i.e., remote procedure calls (ThingsBoard towards devices) handlers self.load_handlers('serverSideRpc', mandatory_keys['serverSideRpc'], self.__server_side_rpc) # Connect requests, i.e., telling ThingsBoard that a device is online even if it does not post telemetry self.load_handlers('connectRequests', mandatory_keys['connectRequests'], self.__connect_requests) # Disconnect requests, i.e., telling ThingsBoard that a device is offline even if keep-alive has not expired yet self.load_handlers('disconnectRequests', mandatory_keys['disconnectRequests'], self.__disconnect_requests) # Shared attributes direct requests, i.e., asking ThingsBoard for some shared attribute value self.load_handlers('attributeRequests', mandatory_keys['attributeRequests'], self.__attribute_requests) # Attributes updates requests, i.e., asking ThingsBoard to send updates about an attribute self.load_handlers('attributeUpdates', mandatory_keys['attributeUpdates'], self.__attribute_updates) # Setup topic substitution lists for each class of handlers ---------------------------------------------------- self.__mapping_sub_topics = {} self.__connect_requests_sub_topics = {} self.__disconnect_requests_sub_topics = {} self.__attribute_requests_sub_topics = {} # Set up external MQTT broker connection ----------------------------------------------------------------------- client_id = self.__broker.get( "clientId", ''.join(random.choice(string.ascii_lowercase) for _ in range(23))) self._client = Client(client_id) self.setName( config.get( "name", self.__broker.get( "name", 'Mqtt Broker ' + ''.join( random.choice(string.ascii_lowercase) for _ in range(5))))) if "username" in self.__broker["security"]: self._client.username_pw_set(self.__broker["security"]["username"], self.__broker["security"]["password"]) if "caCert" in self.__broker["security"] \ or self.__broker["security"].get("type", "none").lower() == "tls": ca_cert = self.__broker["security"].get("caCert") private_key = self.__broker["security"].get("privateKey") cert = self.__broker["security"].get("cert") if ca_cert is None: self._client.tls_set_context( ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) else: try: self._client.tls_set(ca_certs=ca_cert, certfile=cert, keyfile=private_key, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) except Exception as e: self.__log.error( "Cannot setup connection to broker %s using SSL. " "Please check your configuration.\nError: ", self.get_name()) self.__log.exception(e) if self.__broker["security"].get("insecure", False): self._client.tls_insecure_set(True) else: self._client.tls_insecure_set(False) # Set up external MQTT broker callbacks ------------------------------------------------------------------------ self._client.on_connect = self._on_connect self._client.on_message = self._on_message self._client.on_subscribe = self._on_subscribe self._client.on_disconnect = self._on_disconnect # self._client.on_log = self._on_log # Set up lifecycle flags --------------------------------------------------------------------------------------- self._connected = False self.__stopped = False self.daemon = True self.__msg_queue = Queue() self.__workers_thread_pool = [] self.__max_msg_number_for_worker = config.get( 'maxMessageNumberPerWorker', 10) self.__max_number_of_workers = config.get('maxNumberOfWorkers', 100) def load_handlers(self, handler_flavor, mandatory_keys, accepted_handlers_list): if handler_flavor not in self.config: self.__log.error("'%s' section missing from configuration", handler_flavor) else: for handler in self.config.get(handler_flavor): discard = False for key in mandatory_keys: if key not in handler: # Will report all missing fields to user before discarding the entry => no break here discard = True self.__log.error( "Mandatory key '%s' missing from %s handler: %s", key, handler_flavor, simplejson.dumps(handler)) else: self.__log.debug( "Mandatory key '%s' found in %s handler: %s", key, handler_flavor, simplejson.dumps(handler)) if discard: self.__log.error( "%s handler is missing some mandatory keys => rejected: %s", handler_flavor, simplejson.dumps(handler)) else: accepted_handlers_list.append(handler) self.__log.debug( "%s handler has all mandatory keys => accepted: %s", handler_flavor, simplejson.dumps(handler)) self.__log.info("Number of accepted %s handlers: %d", handler_flavor, len(accepted_handlers_list)) self.__log.info( "Number of rejected %s handlers: %d", handler_flavor, len(self.config.get(handler_flavor)) - len(accepted_handlers_list)) def is_connected(self): return self._connected def open(self): self.__stopped = False self.start() def run(self): try: self.__connect() except Exception as e: self.__log.exception(e) try: self.close() except Exception as e: self.__log.exception(e) while True: if self.__stopped: break elif not self._connected: self.__connect() self.__threads_manager() sleep(.2) def __connect(self): while not self._connected and not self.__stopped: try: self._client.connect(self.__broker['host'], self.__broker.get('port', 1883)) self._client.loop_start() if not self._connected: sleep(1) except ConnectionRefusedError as e: self.__log.error(e) sleep(10) def close(self): self.__stopped = True try: self._client.disconnect() except Exception as e: log.exception(e) self._client.loop_stop() self.__log.info('%s has been stopped.', self.get_name()) def get_name(self): return self.name def __subscribe(self, topic, qos): message = self._client.subscribe(topic, qos) try: self.__subscribes_sent[message[1]] = topic except Exception as e: self.__log.exception(e) def _on_connect(self, client, userdata, flags, result_code, *extra_params): result_codes = { 1: "incorrect protocol version", 2: "invalid client identifier", 3: "server unavailable", 4: "bad username or password", 5: "not authorised", } if result_code == 0: self._connected = True self.__log.info('%s connected to %s:%s - successfully.', self.get_name(), self.__broker["host"], self.__broker.get("port", "1883")) self.__log.debug( "Client %s, userdata %s, flags %s, extra_params %s", str(client), str(userdata), str(flags), extra_params) self.__mapping_sub_topics = {} # Setup data upload requests handling ---------------------------------------------------------------------- for mapping in self.__mapping: try: # Load converter for this mapping entry ------------------------------------------------------------ # mappings are guaranteed to have topicFilter and converter fields. See __init__ default_converter_class_name = "JsonMqttUplinkConverter" # Get converter class from "extension" parameter or default converter converter_class_name = mapping["converter"].get( "extension", default_converter_class_name) # Find and load required class module = TBModuleLoader.import_module( self._connector_type, converter_class_name) if module: self.__log.debug('Converter %s for topic %s - found!', converter_class_name, mapping["topicFilter"]) converter = module(mapping) else: self.__log.error("Cannot find converter for %s topic", mapping["topicFilter"]) continue # Setup regexp topic acceptance list --------------------------------------------------------------- regex_topic = TBUtility.topic_to_regex( mapping["topicFilter"]) # There may be more than one converter per topic, so I'm using vectors if not self.__mapping_sub_topics.get(regex_topic): self.__mapping_sub_topics[regex_topic] = [] self.__mapping_sub_topics[regex_topic].append(converter) # Subscribe to appropriate topic ------------------------------------------------------------------- self.__subscribe(mapping["topicFilter"], mapping.get("subscriptionQos", 1)) self.__log.info('Connector "%s" subscribe to %s', self.get_name(), TBUtility.regex_to_topic(regex_topic)) except Exception as e: self.__log.exception(e) # Setup connection requests handling ----------------------------------------------------------------------- for request in [ entry for entry in self.__connect_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 1)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__connect_requests_sub_topics[topic_filter] = request # Setup disconnection requests handling -------------------------------------------------------------------- for request in [ entry for entry in self.__disconnect_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 1)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__disconnect_requests_sub_topics[topic_filter] = request # Setup attributes requests handling ----------------------------------------------------------------------- for request in [ entry for entry in self.__attribute_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 1)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__attribute_requests_sub_topics[topic_filter] = request else: if result_code in result_codes: self.__log.error("%s connection FAIL with error %s %s!", self.get_name(), result_code, result_codes[result_code]) else: self.__log.error("%s connection FAIL with unknown error!", self.get_name()) def _on_disconnect(self, *args): self._connected = False self.__log.debug('"%s" was disconnected. %s', self.get_name(), str(args)) def _on_log(self, *args): self.__log.debug(args) def _on_subscribe(self, _, __, mid, granted_qos, *args): log.info(args) try: if granted_qos[0] == 128: self.__log.error( '"%s" subscription failed to topic %s subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) else: self.__log.info( '"%s" subscription success to topic %s, subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) except Exception as e: self.__log.exception(e) # Success or not, remove this topic from the list of pending subscription requests if self.__subscribes_sent.get(mid) is not None: del self.__subscribes_sent[mid] def put_data_to_convert(self, converter, message, content) -> bool: if not self.__msg_queue.full(): self.__msg_queue.put((converter.convert, message.topic, content), True, 100) return True return False def _save_converted_msg(self, topic, data): self.__gateway.send_to_storage(self.name, data) self.statistics['MessagesSent'] += 1 self.__log.debug("Successfully converted message from topic %s", topic) def __threads_manager(self): if len(self.__workers_thread_pool) == 0: worker = MqttConnector.ConverterWorker("Main", self.__msg_queue, self._save_converted_msg) self.__workers_thread_pool.append(worker) worker.start() number_of_needed_threads = round( self.__msg_queue.qsize() / self.__max_msg_number_for_worker, 0) threads_count = len(self.__workers_thread_pool) if number_of_needed_threads > threads_count < self.__max_number_of_workers: thread = MqttConnector.ConverterWorker( "Worker " + ''.join( random.choice(string.ascii_lowercase) for _ in range(5)), self.__msg_queue, self._save_converted_msg) self.__workers_thread_pool.append(thread) thread.start() elif number_of_needed_threads < threads_count and threads_count > 1: worker: MqttConnector.ConverterWorker = self.__workers_thread_pool[ -1] if not worker.in_progress: worker.stopped = True self.__workers_thread_pool.remove(worker) def _on_message(self, client, userdata, message): self.statistics['MessagesReceived'] += 1 content = TBUtility.decode(message) # Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" --------------------------- topic_handlers = [ regex for regex in self.__mapping_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: # Note: every topic may be associated to one or more converter. This means that a single MQTT message # may produce more than one message towards ThingsBoard. This also means that I cannot return after # the first successful conversion: I got to use all the available ones. # I will use a flag to understand whether at least one converter succeeded request_handled = False for topic in topic_handlers: available_converters = self.__mapping_sub_topics[topic] for converter in available_converters: try: if isinstance(content, list): for item in content: request_handled = self.put_data_to_convert( converter, message, item) if not request_handled: self.__log.error( 'Cannot find converter for the topic:"%s"! Client: %s, User data: %s', message.topic, str(client), str(userdata)) else: request_handled = self.put_data_to_convert( converter, message, content) except Exception as e: log.exception(e) if not request_handled: self.__log.error( 'Cannot find converter for the topic:"%s"! Client: %s, User data: %s', message.topic, str(client), str(userdata)) # Note: if I'm in this branch, this was for sure a telemetry/attribute push message # => Execution must end here both in case of failure and success return None # Check if message topic exists in connection handlers "i.e., I'm connecting a device" ------------------------- topic_handlers = [ regex for regex in self.__connect_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: for topic in topic_handlers: handler = self.__connect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search( handler["deviceTypeTopicExpression"], message.topic) found_device_type = device_type_match.group( 0) if device_type_match is not None else handler[ "deviceTypeTopicExpression"] elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from connection request") continue # Note: device must be added even if it is already known locally: else ThingsBoard # will not send RPCs and attribute updates self.__log.info("Connecting device %s of type %s", found_device_name, found_device_type) self.__gateway.add_device(found_device_name, {"connector": self}, device_type=found_device_type) # Note: if I'm in this branch, this was for sure a connection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in disconnection handlers "i.e., I'm disconnecting a device" ------------------- topic_handlers = [ regex for regex in self.__disconnect_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: for topic in topic_handlers: handler = self.__disconnect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search( handler["deviceTypeTopicExpression"], message.topic) if device_type_match is not None: found_device_type = device_type_match.group(0) elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from disconnection request") continue if found_device_name in self.__gateway.get_devices(): self.__log.info("Disconnecting device %s of type %s", found_device_name, found_device_type) self.__gateway.del_device(found_device_name) else: self.__log.info("Device %s was not connected", found_device_name) break # Note: if I'm in this branch, this was for sure a disconnection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in attribute request handlers "i.e., I'm asking for a shared attribute" -------- topic_handlers = [ regex for regex in self.__attribute_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: try: for topic in topic_handlers: handler = self.__attribute_requests_sub_topics[topic] found_device_name = None found_attribute_name = None # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get attribute name, either from topic or from content if handler.get("attributeNameTopicExpression"): attribute_name_match = search( handler["attributeNameTopicExpression"], message.topic) if attribute_name_match is not None: found_attribute_name = attribute_name_match.group( 0) elif handler.get("attributeNameJsonExpression"): found_attribute_name = TBUtility.get_value( handler["attributeNameJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from attribute request") continue if found_attribute_name is None: self.__log.error( "Attribute name missing from attribute request") continue self.__log.info("Will retrieve attribute %s of %s", found_attribute_name, found_device_name) self.__gateway.tb_client.client.gw_request_shared_attributes( found_device_name, [found_attribute_name], lambda data, *args: self.notify_attribute( data, found_attribute_name, handler.get("topicExpression"), handler.get("valueExpression"), handler.get('retain', False))) break except Exception as e: log.exception(e) # Note: if I'm in this branch, this was for sure an attribute request message # => Execution must end here both in case of failure and success return None # Check if message topic exists in RPC handlers ---------------------------------------------------------------- # The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is if self.__gateway.is_rpc_in_progress(message.topic): log.info("RPC response arrived. Forwarding it to thingsboard.") self.__gateway.rpc_with_reply_processing(message.topic, content) return None self.__log.debug( "Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"", message.topic, content) def notify_attribute(self, incoming_data, attribute_name, topic_expression, value_expression, retain): if incoming_data.get("device") is None or incoming_data.get( "value") is None: return device_name = incoming_data.get("device") attribute_value = incoming_data.get("value") topic = topic_expression \ .replace("${deviceName}", str(device_name)) \ .replace("${attributeKey}", str(attribute_name)) data = value_expression.replace("${attributeKey}", str(attribute_name)) \ .replace("${attributeValue}", str(attribute_value)) self._client.publish(topic, data, retain=retain).wait_for_publish() def on_attributes_update(self, content): if self.__attribute_updates: for attribute_update in self.__attribute_updates: if match(attribute_update["deviceNameFilter"], content["device"]): for attribute_key in content["data"]: if match(attribute_update["attributeFilter"], attribute_key): try: topic = attribute_update["topicExpression"] \ .replace("${deviceName}", str(content["device"])) \ .replace("${attributeKey}", str(attribute_key)) \ .replace("${attributeValue}", str(content["data"][attribute_key])) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e try: data = attribute_update["valueExpression"] \ .replace("${attributeKey}", str(attribute_key)) \ .replace("${attributeValue}", str(content["data"][attribute_key])) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e self._client.publish( topic, data, retain=attribute_update.get( 'retain', False)).wait_for_publish() self.__log.debug( "Attribute Update data: %s for device %s to topic: %s", data, content["device"], topic) else: self.__log.error( "Cannot find attributeName by filter in message with data: %s", content) else: self.__log.error( "Cannot find deviceName by filter in message with data: %s", content) else: self.__log.error("Attribute updates config not found.") def server_side_rpc_handler(self, content): self.__log.info("Incoming server-side RPC: %s", content) # Check whether one of my RPC handlers can handle this request for rpc_config in self.__server_side_rpc: if search(rpc_config["deviceNameFilter"], content["device"]) \ and search(rpc_config["methodFilter"], content["data"]["method"]) is not None: # This handler seems able to handle the request self.__log.info("Candidate RPC handler found") expects_response = rpc_config.get("responseTopicExpression") defines_timeout = rpc_config.get("responseTimeout") # 2-way RPC setup if expects_response and defines_timeout: expected_response_topic = rpc_config["responseTopicExpression"] \ .replace("${deviceName}", str(content["device"])) \ .replace("${methodName}", str(content["data"]["method"])) \ .replace("${requestId}", str(content["data"]["id"])) expected_response_topic = TBUtility.replace_params_tags( expected_response_topic, content) timeout = time() * 1000 + rpc_config.get("responseTimeout") # Start listenting on the response topic self.__log.info("Subscribing to: %s", expected_response_topic) self.__subscribe(expected_response_topic, rpc_config.get("responseTopicQoS", 1)) # Wait for subscription to be carried out sub_response_timeout = 10 while expected_response_topic in self.__subscribes_sent.values( ): sub_response_timeout -= 1 sleep(0.1) if sub_response_timeout == 0: break # Ask the gateway to enqueue this as an RPC response self.__gateway.register_rpc_request_timeout( content, timeout, expected_response_topic, self.rpc_cancel_processing) # Wait for RPC to be successfully enqueued, which never fails. while self.__gateway.is_rpc_in_progress( expected_response_topic): sleep(0.1) elif expects_response and not defines_timeout: self.__log.info( "2-way RPC without timeout: treating as 1-way") # Actually reach out for the device request_topic: str = rpc_config.get("requestTopicExpression") \ .replace("${deviceName}", str(content["device"])) \ .replace("${methodName}", str(content["data"]["method"])) \ .replace("${requestId}", str(content["data"]["id"])) request_topic = TBUtility.replace_params_tags( request_topic, content) data_to_send_tags = TBUtility.get_values( rpc_config.get('valueExpression'), content['data'], 'params', get_tag=True) data_to_send_values = TBUtility.get_values( rpc_config.get('valueExpression'), content['data'], 'params', expression_instead_none=True) data_to_send = rpc_config.get('valueExpression') for (tag, value) in zip(data_to_send_tags, data_to_send_values): data_to_send = data_to_send.replace( '${' + tag + '}', str(value)) try: self.__log.info("Publishing to: %s with data %s", request_topic, data_to_send) self._client.publish(request_topic, data_to_send, retain=rpc_config.get( 'retain', False)) if not expects_response or not defines_timeout: self.__log.info( "One-way RPC: sending ack to ThingsBoard immediately" ) self.__gateway.send_rpc_reply( device=content["device"], req_id=content["data"]["id"], success_sent=True) # Everything went out smoothly: RPC is served return except Exception as e: self.__log.exception(e) self.__log.error("RPC not handled: %s", content) def rpc_cancel_processing(self, topic): log.info("RPC canceled or terminated. Unsubscribing from %s", topic) self._client.unsubscribe(topic) class ConverterWorker(Thread): def __init__(self, name, incoming_queue, send_result): super().__init__() self.stopped = False self.setName(name) self.setDaemon(True) self.__msg_queue = incoming_queue self.in_progress = False self.__send_result = send_result def run(self): while not self.stopped: if not self.__msg_queue.empty(): self.in_progress = True convert_function, config, incoming_data = self.__msg_queue.get( True, 100) converted_data = convert_function(config, incoming_data) log.debug(converted_data) self.__send_result(config, converted_data) self.in_progress = False else: sleep(.2)
class Mqtt(): """Main Mqtt class. :param app: flask application object :param connect_async: if True then connect_aync will be used to connect to MQTT broker :param mqtt_logging: if True then messages from MQTT client will be logged """ def __init__(self, app=None, connect_async=False, mqtt_logging=False): # type: (Flask, bool, bool) -> None self.app = app self._connect_async = connect_async # type: bool self._connect_handler = None # type: Optional[Callable] self._disconnect_handler = None # type: Optional[Callable] self.topics = {} # type: Dict[str, TopicQos] self.connected = False self.client = Client() if mqtt_logging: self.client.enable_logger(logger) if app is not None: self.init_app(app) def init_app(self, app): # type: (Flask) -> None """Init the Flask-MQTT addon.""" self.client_id = app.config.get("MQTT_CLIENT_ID", "") self.clean_session = app.config.get("MQTT_CLEAN_SESSION", True) if isinstance(self.client_id, unicode): self.client._client_id = self.client_id.encode('utf-8') else: self.client._client_id = self.client_id self.client._clean_session = self.clean_session self.client._transport = app.config.get("MQTT_TRANSPORT", "tcp").lower() self.client._protocol = app.config.get("MQTT_PROTOCOL_VERSION", MQTTv311) self.client.on_connect = self._handle_connect self.client.on_disconnect = self._handle_disconnect self.username = app.config.get("MQTT_USERNAME") self.password = app.config.get("MQTT_PASSWORD") self.broker_url = app.config.get("MQTT_BROKER_URL", "localhost") self.broker_port = app.config.get("MQTT_BROKER_PORT", 1883) self.tls_enabled = app.config.get("MQTT_TLS_ENABLED", False) self.keepalive = app.config.get("MQTT_KEEPALIVE", 60) self.last_will_topic = app.config.get("MQTT_LAST_WILL_TOPIC") self.last_will_message = app.config.get("MQTT_LAST_WILL_MESSAGE") self.last_will_qos = app.config.get("MQTT_LAST_WILL_QOS", 0) self.last_will_retain = app.config.get("MQTT_LAST_WILL_RETAIN", False) if self.tls_enabled: self.tls_ca_certs = app.config["MQTT_TLS_CA_CERTS"] self.tls_certfile = app.config.get("MQTT_TLS_CERTFILE") self.tls_keyfile = app.config.get("MQTT_TLS_KEYFILE") self.tls_cert_reqs = app.config.get("MQTT_TLS_CERT_REQS", ssl.CERT_REQUIRED) self.tls_version = app.config.get("MQTT_TLS_VERSION", ssl.PROTOCOL_TLSv1) self.tls_ciphers = app.config.get("MQTT_TLS_CIPHERS") self.tls_insecure = app.config.get("MQTT_TLS_INSECURE", False) # set last will message if self.last_will_topic is not None: self.client.will_set( self.last_will_topic, self.last_will_message, self.last_will_qos, self.last_will_retain, ) self._connect() def _connect(self): # type: () -> None if self.username is not None: self.client.username_pw_set(self.username, self.password) # security if self.tls_enabled: self.client.tls_set( ca_certs=self.tls_ca_certs, certfile=self.tls_certfile, keyfile=self.tls_keyfile, cert_reqs=self.tls_cert_reqs, tls_version=self.tls_version, ciphers=self.tls_ciphers, ) if self.tls_insecure: self.client.tls_insecure_set(self.tls_insecure) if self._connect_async: # if connect_async is used self.client.connect_async(self.broker_url, self.broker_port, keepalive=self.keepalive) else: res = self.client.connect(self.broker_url, self.broker_port, keepalive=self.keepalive) if res == 0: logger.debug("Connected client '{0}' to broker {1}:{2}".format( self.client_id, self.broker_url, self.broker_port)) else: logger.error( "Could not connect to MQTT Broker, Error Code: {0}".format( res)) self.client.loop_start() def _disconnect(self): # type: () -> None self.client.loop_stop() self.client.disconnect() logger.debug('Disconnected from Broker') def _handle_connect(self, client, userdata, flags, rc): # type: (Client, Any, Dict, int) -> None if rc == MQTT_ERR_SUCCESS: self.connected = True for key, item in self.topics.items(): self.client.subscribe(topic=item.topic, qos=item.qos) if self._connect_handler is not None: self._connect_handler(client, userdata, flags, rc) def _handle_disconnect(self, client, userdata, rc): # type: (str, Any, int) -> None self.connected = False if self._disconnect_handler is not None: self._disconnect_handler() def on_topic(self, topic): # type: (str) -> Callable """Decorator. Decorator to add a callback function that is called when a certain topic has been published. The callback function is expected to have the following form: `handle_topic(client, userdata, message)` :parameter topic: a string specifying the subscription topic to subscribe to The topic still needs to be subscribed via mqtt.subscribe() before the callback function can be used to handle a certain topic. This way it is possible to subscribe and unsubscribe during runtime. **Example usage:**:: app = Flask(__name__) mqtt = Mqtt(app) mqtt.subscribe('home/mytopic') @mqtt.on_topic('home/mytopic') def handle_mytopic(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): # type: (Callable[[str], None]) -> Callable[[str], None] self.client.message_callback_add(topic, handler) return handler return decorator def subscribe(self, topic, qos=0): # type: (str, int) -> Tuple[int, int] """ Subscribe to a certain topic. :param topic: a string specifying the subscription topic to subscribe to. :param qos: the desired quality of service level for the subscription. Defaults to 0. :rtype: (int, int) :result: (result, mid) A topic is a UTF-8 string, which is used by the broker to filter messages for each connected client. A topic consists of one or more topic levels. Each topic level is separated by a forward slash (topic level separator). The function returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not currently connected. mid is the message ID for the subscribe request. The mid value can be used to track the subscribe request by checking against the mid argument in the on_subscribe() callback if it is defined. **Topic example:** `myhome/groundfloor/livingroom/temperature` """ # TODO: add support for list of topics # don't subscribe if already subscribed # try to subscribe result, mid = self.client.subscribe(topic=topic, qos=qos) # if successful add to topics if result == MQTT_ERR_SUCCESS: self.topics[topic] = TopicQos(topic=topic, qos=qos) logger.debug('Subscribed to topic: {0}, qos: {1}'.format( topic, qos)) else: logger.error('Error {0} subscribing to topic: {1}'.format( result, topic)) return (result, mid) def unsubscribe(self, topic): # type: (str) -> Optional[Tuple[int, int]] """ Unsubscribe from a single topic. :param topic: a single string that is the subscription topic to unsubscribe from :rtype: (int, int) :result: (result, mid) Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not currently connected. mid is the message ID for the unsubscribe request. The mid value can be used to track the unsubscribe request by checking against the mid argument in the on_unsubscribe() callback if it is defined. """ # don't unsubscribe if not in topics if topic in self.topics: result, mid = self.client.unsubscribe(topic) if result == MQTT_ERR_SUCCESS: self.topics.pop(topic) logger.debug('Unsubscribed from topic: {0}'.format(topic)) else: logger.debug('Error {0} unsubscribing from topic: {1}'.format( result, topic)) # if successful remove from topics return result, mid return None def unsubscribe_all(self): # type: () -> None """Unsubscribe from all topics.""" topics = list(self.topics.keys()) for topic in topics: self.unsubscribe(topic) def publish(self, topic, payload=None, qos=0, retain=False): # type: (str, bytes, int, bool) -> Tuple[int, int] """ Send a message to the broker. :param topic: the topic that the message should be published on :param payload: the actual message to send. If not given, or set to None a zero length message will be used. Passing an int or float will result in the payload being converted to a string representing that number. If you wish to send a true int/float, use struct.pack() to create the payload you require. :param qos: the quality of service level to use :param retain: if set to True, the message will be set as the "last known good"/retained message for the topic :returns: Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN if the client is not currently connected. mid is the message ID for the publish request. """ if not self.connected: self.client.reconnect() result, mid = self.client.publish(topic, payload, qos, retain) if result == MQTT_ERR_SUCCESS: logger.debug('Published topic {0}: {1}'.format(topic, payload)) else: logger.error('Error {0} publishing topic {1}'.format( result, topic)) return (result, mid) def on_connect(self): # type: () -> Callable """Decorator. Decorator to handle the event when the broker responds to a connection request. Only the last decorated function will be called. """ def decorator(handler): # type: (Callable) -> Callable self._connect_handler = handler return handler return decorator def on_disconnect(self): # type: () -> Callable """Decorator. Decorator to handle the event when client disconnects from broker. Only the last decorated function will be called. """ def decorator(handler): # type: (Callable) -> Callable self._disconnect_handler = handler return handler return decorator def on_message(self): # type: () -> Callable """Decorator. Decorator to handle all messages that have been subscribed and that are not handled via the `on_message` decorator. **Note:** Unlike as written in the paho mqtt documentation this callback will not be called if there exists an topic-specific callback added by the `on_topic` decorator. **Example Usage:**:: @mqtt.on_message() def handle_messages(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_message = handler return handler return decorator def on_publish(self): # type: () -> Callable """Decorator. Decorator to handle all messages that have been published by the client. **Example Usage:**:: @mqtt.on_publish() def handle_publish(client, userdata, mid): print('Published message with mid {}.' .format(mid)) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_publish = handler return handler return decorator def on_subscribe(self): # type: () -> Callable """Decorate a callback function to handle subscritions. **Usage:**:: @mqtt.on_subscribe() def handle_subscribe(client, userdata, mid, granted_qos): print('Subscription id {} granted with qos {}.' .format(mid, granted_qos)) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_subscribe = handler return handler return decorator def on_unsubscribe(self): # type: () -> Callable """Decorate a callback funtion to handle unsubscribtions. **Usage:**:: @mqtt.unsubscribe() def handle_unsubscribe(client, userdata, mid) print('Unsubscribed from topic (id: {})' .format(mid)') """ def decorator(handler): # type: (Callable) -> Callable self.client.on_unsubscribe = handler return handler return decorator def on_log(self): # type: () -> Callable """Decorate a callback function to handle MQTT logging. **Example Usage:** :: @mqtt.on_log() def handle_logging(client, userdata, level, buf): print(client, userdata, level, buf) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_log = handler return handler return decorator
class MqttConnection: def __init__(self, ip, port, username, password, connection_callback): self.logger = logging.getLogger("mqtt") self.mqtt = Client() if username is not None: self.mqtt.username_pw_set(username, password) self.mqtt.on_connect = self._on_connect self.mqtt.on_message = self._on_message self.ip = ip self.port = port self.connection_callback = connection_callback self.queue = [] def _on_connect(self, client, userdata, flags, rc): """ The callback for when the client receives a CONNACK response from the server. """ self.logger.debug("connected to mqtt with result code %d" % rc) # subscribing in on_connect() means that if we lose the connection and # reconnect then subscriptions will be renewed. self.connection_callback.on_mqtt_connected(self) if len(self.queue) > 0: self.logger.debug("found %d queued messages" % len(self.queue)) for msg in self.queue: self._internal_send_message(msg[0], msg[1], False) self.queue.clear() self.logger.debug("handled all queued messages") def _on_message(self, client, userdata, msg): """ The callback for when a PUBLISH message is received from the server. """ self.logger.debug("received mqtt publish of %s with data \"%s\"" % (msg.topic, msg.payload)) self.connection_callback.on_mqtt_message_received(msg.topic, msg.payload) def send_message(self, topic, payload): return self._internal_send_message(topic, payload, True) def subscribe(self, topic): self.logger.debug("subscribing to topic %s" % topic) result = self.mqtt.subscribe(topic) if result[0] == MQTT_ERR_NO_CONN: self.logger.warning("no connection while trying to subscribe to topic %s" % topic) return False return result[0] == MQTT_ERR_SUCCESS def unsubscribe(self, topic): self.logger.debug("unsubscribing from topic %s" % topic) result = self.mqtt.unsubscribe(topic) if result[0] == MQTT_ERR_NO_CONN: self.logger.warning("no connection while trying to unsubscribe from topic %s" % topic) return False return result[0] == MQTT_ERR_SUCCESS def _internal_send_message(self, topic, payload, queue): self.logger.debug("sending topic %s with value \"%s\"" % (topic, payload)) result = self.mqtt.publish(topic, payload, retain=True) if result == MQTT_ERR_NO_CONN and queue: self.logger.debug("no connection, saving message with topic %s to queue" % topic) self.queue.append([topic, payload]) elif result[0] != MQTT_ERR_SUCCESS: self.logger.warn("failed sending message %s, mqtt error %s" % (topic, result)) return False return True def start_connection(self): try: self.mqtt.connect(self.ip, self.port) except ConnectionError: self.logger.exception("failed connecting to mqtt") return False self.mqtt.loop_start() return True def stop_connection(self): self.mqtt.disconnect() self.mqtt.loop_stop()
class Service: """Service that sends emails""" _mqtt_client: Dict[str, Client] = {} _service_list: Dict[str, dict] = {} _broker: DefaultDict[str, set] = defaultdict(set) _broker_port: Dict[str, int] = {} _topic: DefaultDict[str, set] = defaultdict(set) _update_thread: Timer = None alarm_user_topic = "labsw4/arduino/contacted/user" _user_list: Dict[str, dict] = {} # Email from_email = "******" #E-mail (Gmail) associated to the service password = "******" #Password of this e-mail subject = "Temperature alarm" signature = "\n\nBest regards,\n\nSmart Home - IoT distributed platform developers" # signature block def __init__(self): """ Instantiate the service """ self.service = Client(client_id=f"EMailService{randrange(1, 100000)}") self.service.on_message = partial(self.my_on_message, email_service=self.service) self.user_lock = Lock() self.service_lock = Lock() def _update(self, service_list: List[dict]): """ Update reserved dicts :param service_list: service list to add """ for alarm in service_list: service = alarm["serviceID"] broker = alarm["end_points"]["MQTT"]["broker"]["ip"] port = alarm["end_points"]["MQTT"]["broker"]["port"] topics = { topic for topic in alarm["end_points"]["MQTT"]["subscribe"] if "alarm_temperature" in topic } self._service_list[service] = { "ip": broker, "topics": topics, } self._broker_port[broker] = port self._broker[broker].update({service}) self._topic[broker].update(topics) print(f"[{time.ctime()}] SERVICE {service} ONLINE") def setup(self, first_time: bool = True): """ Setup the service """ try: print( f"[{time.ctime()}] EXTRACT info about all the services registered" ) result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/services/all" ) while result.status_code != 200: print( f"[{time.ctime()}] WARNING no service registered found, retrying after 30 seconds" ) time.sleep(30) result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}" f"/catalog/services/all") print(f"[{time.ctime()}] SERVICES found") data = json.loads(result.content.decode()) print( f"[{time.ctime()}] EXTRACT from the services list info about Alarm_Temperature" ) service_list = [alarm for alarm in data if find_service(alarm)] if len(service_list) == 0: print( f"[{time.ctime()}] No ServiceTemperatureAlarm found... retrying in 10 seconds" ) time.sleep(10) self.setup(first_time) return print( f"[{time.ctime()}] EXTRACT info about all the users registered" ) result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/users/all" ) user_list = {} if result.status_code == 200: data = json.loads(result.content.decode()) user_list = { user["userID"]: { user["email_addresses"][email] for email in user["email_addresses"] } for user in data } except KeyboardInterrupt: print(f"[{time.ctime()}] EXIT") if first_time: sys.exit() return if first_time: # Connect the service self.service.connect(host=SERVICE_BROKER_PORT["ip"], port=SERVICE_BROKER_PORT["port"]) print(f"[{time.ctime()}] SERVICE CONNECTED to " f"broker: {SERVICE_BROKER_PORT['ip']} " f"on port: port={SERVICE_BROKER_PORT['port']}") # Update registered services and users and subscribe to all the topics with self.service_lock: self._update(service_list) self.subscribe(service_list) with self.user_lock: self._user_list.update(user_list) # Register inside the catalog print( f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}") requests.post( f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services', data=SERVICE_INFO, headers={"Content-Type": "application/json"}) def start(self): """ Start the service. """ # Setup the service self.setup() # Start all the external mqtt_clients for broker in self._mqtt_client: self._mqtt_client[broker].loop_start() # Schedule the update of the registration self._update_thread = Timer(60, self.update_registration) self._update_thread.start() try: # Run the service forever self.service.loop_forever() except KeyboardInterrupt: self.stop() def my_on_message(self, client: Client, userdata: Any, msg: MQTTMessage, email_service: Client = None): """ Check if alarm is true. If so, send emails. :param client: MQTT client :param userdata: They could be any type :param msg: MQTT message :param email_service: Service Client """ data = json.loads(msg.payload.decode()) if data["alarm"]: message_body = f"{data['device']} is out of range of good functioning" else: return with self.user_lock: for user_id in self._user_list: for email in self._user_list[user_id]: msg = MIMEMultipart() msg["From"] = f"Smart Home - IoT distributed platform <{self.from_email}>" msg["To"] = email msg["Subject"] = self.subject msg.attach(MIMEText(message_body + self.signature, 'plain')) server = smtplib.SMTP("smtp.gmail.com", 587) server.starttls() server.login(self.from_email, self.password) text = msg.as_string() server.sendmail(self.from_email, email, text) server.quit() print(f"[{time.ctime()}] EMAIL SENT TO {email}") email_service.publish(self.alarm_user_topic, payload=json.dumps({ "userID": [user_id for user_id in self._user_list] })) print( f"[{time.ctime()}] EMAILS SENT TO USERS: {[user_id for user_id in self._user_list]}" ) def update_registration(self): """ Update service registration to the catalog """ print(f"[{time.ctime()}] EXTRACT info about all the users registered") result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/users/all" ) user_list = {} if result.status_code == 200: data = json.loads(result.content.decode()) user_list = { user["userID"]: { user["email_addresses"][email] for email in user["email_addresses"] } for user in data } with self.user_lock: self._user_list.update(user_list) print( f"[{time.ctime()}] EXTRACT info about all the services registered") result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/services/all" ) # No device found if result.status_code != 200: self.reset() return data = json.loads(result.content.decode()) service_list = [alarm for alarm in data if find_service(alarm)] if len(service_list) == 0: self.reset() return # New Devices Found new_services = {alarm["serviceID"] for alarm in service_list} # Old devices list old_services = set(self._service_list.keys()) # Check if there are update to do if old_services == new_services: # No update, ping the catalog print( f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}" ) requests.post( f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services', data=SERVICE_INFO, headers={"Content-Type": "application/json"}) self._update_thread = Timer(60, self.update_registration) self._update_thread.start() return with self.service_lock: # Delete inactive devices delete_list = { alarm: self._service_list[alarm] for alarm in self._service_list if alarm not in new_services } # Unsubscribe from devices self.unsubscribe(delete_list) # Update registered devices and subscribe new_services = [ alarm for alarm in service_list if alarm["serviceID"] not in old_services ] self._update(new_services) self.subscribe(new_services) self._update_thread = Timer(60, self.update_registration) self._update_thread.start() def reset(self): """ Reset the service """ with self.service_lock: self._clear() self.setup(first_time=False) # Start all the external mqtt_clients for broker in self._mqtt_client: self._mqtt_client[broker].loop_start() self._update_thread = Timer(60, self.update_registration) self._update_thread.start() def unsubscribe(self, service_list: dict): """ Unsubscribe from services :param service_list: service list """ for alarm in service_list: print(f"[{time.ctime()}] SERVICE {alarm} OFFLINE") broker = service_list[alarm]["ip"] topics = service_list[alarm]["topics"] if broker == SERVICE_BROKER_PORT["ip"]: for topic in topics: self.service.unsubscribe(topic) print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}") else: for topic in topics: self._mqtt_client[broker].unsubscribe(topic) print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}") # Update topics self._topic[broker] = self._topic[broker].difference(topics) # Disconnect from external broker if len(self._topic[broker]) == 0: if broker != SERVICE_BROKER_PORT["ip"]: self._mqtt_client[broker].disconnect() self._mqtt_client[broker].loop_stop() print( f"[{time.ctime()}] DISCONNECTED from broker: {broker}") del self._mqtt_client[broker] # Clean del self._topic[broker] del self._broker_port[broker] # Delete device del self._service_list[alarm] self._broker[broker].discard(alarm) def subscribe(self, service_list: List[dict]): """ Subscribe to services :param service_list: list of services """ for alarm in service_list: broker = alarm["end_points"]["MQTT"]["broker"]["ip"] topics = { topic for topic in alarm["end_points"]["MQTT"]["subscribe"] } # Subscribe to topics if broker == SERVICE_BROKER_PORT["ip"]: for topic in topics: self.service.subscribe(topic) print(f"[{time.ctime()}] SUBSCRIBED to : {topic}") # Connect to external brokers else: if broker in self._mqtt_client: # Connection already made for topic in topics: self._mqtt_client[broker].subscribe(topic) print(f"[{time.ctime()}] SUBSCRIBED to : {topic}") else: # Connect to the broker self._mqtt_client[broker] = Client( f"EmailAlarmService{randrange(1, 1000000)}") self._mqtt_client[broker].on_message = partial( self.my_on_message, email_service=self.service) self._mqtt_client[broker].connect( host=broker, port=self._broker_port[broker]) for topic in topics: self._mqtt_client[broker].subscribe(topic) print(f"[{time.ctime()}] SUBSCRIBED to : {topic}") def _clear(self): """ Clear all the data stored """ # Unsubscribe from all topics self.unsubscribe(self._service_list.copy()) # Disconnect from all the external brokers for broker in self._mqtt_client: self._mqtt_client[broker].disconnect() self._mqtt_client[broker].loop_stop() print(f"[{time.ctime()}] DISCONNECTED from broker: {broker}") # Clear self._mqtt_client.clear() self._service_list.clear() self._broker.clear() self._broker_port.clear() self._topic.clear() def stop(self): """ Stop the service """ # Stop the schedule self._update_thread.cancel() # Wait for the threads to finish if self._update_thread.is_alive(): self._update_thread.join() # Disconnect the clients print(f"[{time.ctime()}] SHUTTING DOWN") for client in self._mqtt_client: self._mqtt_client[client].disconnect() self._mqtt_client[client].loop_stop() # Disconnect the service self.service.disconnect() print(f"[{time.ctime()}] EXIT")
class Bettor: def __init__(self, name, money, broker='localhost', auth=None, interact=False): self.name = name self.lock = Lock() self.money = money self.client = Client() self.interest_objects = [] self.interact = interact self.lst = [] self.broker = broker self.auth = auth if auth != None: (usr, pwd) = self.auth self.client.username_pw_set(usr, pwd) self.client.connect(broker) def on_message(self, client, userdata, msg): if msg.topic == 'Available-items': self.lst = msg.payload self.lst = self.lst.decode("utf-8").split() if self.interact == False: self.interesting_items() else: ''' Manage bid in terms of available money. Mutual exclusion to ensure money limitations between all bid processes. ''' (current_bet, bid_owner) = msg.payload.decode("utf-8").split() current_bet = int(current_bet) if self.name != bid_owner: with self.lock: if current_bet > self.money: print('Limit reached for', self.name, ' for item ', msg.topic.replace('results/', '')) self.client.unsubscribe(msg.topic) self.interest_objects.remove( msg.topic.replace('results/', '')) else: bet = current_bet + random.randint(1, 50) topic = msg.topic.replace('results', 'items') print(self.name, 'bets', bet, 'for', topic) self.client.publish(topic, str(bet) + ' ' + str(self.name)) self.money -= bet def bet_process(self): self.client.on_message = self.on_message for it in self.interest_objects: self.client.subscribe('results/' + str(it)) self.client.loop_forever() def interesting_items(self): n = random.randint(1, len(self.lst)) self.interest_objects = random.sample(self.lst, n) print(self.name, 'will play for', self.interest_objects) Process(target=self.bet_process, args=()).start() def get_items(self): self.client.subscribe('Available-items') self.client.on_message = self.on_message self.client.loop_forever() def parse_input_int(self, a): return a.isdigit() and int(a) > 0 and int(a) <= self.money def get_input(self): print('Insert cuantity to bid (you have ', self.money, '): ') value = input() while not self.parse_input_int(value): print( 'Please, insert a valid number, or check if you have enough money' ) value = input('Insert cuantity to bid: ') return value def bet_process_interact(self): (usr, pwd) = self.auth alive = True while alive: cuantity = self.get_input() publish.single('items/' + str(self.interest_objects), str(cuantity) + ' ' + str(self.name), hostname=self.broker, auth={ 'username': usr, 'password': pwd }) self.money -= int(cuantity) alive = self.money != 0 print('Out of money, you are out!') def parse_list(self, elem): return str(elem) in self.lst def start(self): if self.interact: print('Interact mode: make sure you have more than one bidder ') print('(if you do not want to compete against yourself)... ') print('maybe it does not even work :)') self.client.loop_start() (usr, pwd) = self.auth available = subscribe.simple('Available-items', hostname=self.broker, auth={ 'username': usr, 'password': pwd }) objs = available.payload.decode("utf-8") print('This are the available objects: ', objs) self.lst = objs.split() in_obj = input('Wich object do you want to bid for? ') while not self.parse_list(in_obj): in_obj = input('Wich object do you want to bid for? ') self.interest_objects = str(in_obj) self.bet_process_interact() else: Process(target=self.get_items, args=()).start()
class MqttConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() self.__log = log self.config = config self._connector_type = connector_type self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__gateway = gateway self.__broker = config.get('broker') self.__mapping = config.get('mapping') self.__server_side_rpc = config.get('serverSideRpc', []) self.__service_config = { "connectRequests": [], "disconnectRequests": [] } self.__attribute_updates = config.get("attributeUpdates") self.__get_service_config(config) self.__sub_topics = {} client_id = ''.join( random.choice(string.ascii_lowercase) for _ in range(23)) self._client = Client(client_id) self.setName( config.get( "name", self.__broker.get( "name", 'Mqtt Broker ' + ''.join( random.choice(string.ascii_lowercase) for _ in range(5))))) if "username" in self.__broker["security"]: self._client.username_pw_set(self.__broker["security"]["username"], self.__broker["security"]["password"]) if "caCert" in self.__broker["security"] or self.__broker[ "security"].get("type", "none").lower() == "tls": ca_cert = self.__broker["security"].get("caCert") private_key = self.__broker["security"].get("privateKey") cert = self.__broker["security"].get("cert") if ca_cert is None: self._client.tls_set_context( ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) else: try: self._client.tls_set(ca_certs=ca_cert, certfile=cert, keyfile=private_key, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) except Exception as e: self.__log.error( "Cannot setup connection to broker %s using SSL. Please check your configuration.\nError: ", self.get_name()) self.__log.exception(e) self._client.tls_insecure_set(False) self._client.on_connect = self._on_connect self._client.on_message = self._on_message self._client.on_subscribe = self._on_subscribe self.__subscribes_sent = {} # For logging the subscriptions self._client.on_disconnect = self._on_disconnect # self._client.on_log = self._on_log self._connected = False self.__stopped = False self.daemon = True def is_connected(self): return self._connected def open(self): self.__stopped = False self.start() def run(self): try: while not self._connected and not self.__stopped: try: self._client.connect(self.__broker['host'], self.__broker.get('port', 1883)) self._client.loop_start() if not self._connected: time.sleep(1) except Exception as e: self.__log.exception(e) time.sleep(10) except Exception as e: self.__log.exception(e) try: self.close() except Exception as e: self.__log.exception(e) while True: if self.__stopped: break time.sleep(.01) def close(self): self.__stopped = True try: self._client.disconnect() except Exception as e: log.exception(e) self._client.loop_stop() self.__log.info('%s has been stopped.', self.get_name()) def get_name(self): return self.name def __subscribe(self, topic): message = self._client.subscribe(topic) try: self.__subscribes_sent[message[1]] = topic except Exception as e: self.__log.exception(e) def _on_connect(self, client, userdata, flags, result_code, *extra_params): result_codes = { 1: "incorrect protocol version", 2: "invalid client identifier", 3: "server unavailable", 4: "bad username or password", 5: "not authorised", } if result_code == 0: self._connected = True self.__log.info('%s connected to %s:%s - successfully.', self.get_name(), self.__broker["host"], self.__broker.get("port", "1883")) self.__log.debug( "Client %s, userdata %s, flags %s, extra_params %s", str(client), str(userdata), str(flags), extra_params) for mapping in self.__mapping: try: converter = None if mapping["converter"]["type"] == "custom": module = TBUtility.check_and_import( self._connector_type, mapping["converter"]["extension"]) if module is not None: self.__log.debug( 'Custom converter for topic %s - found!', mapping["topicFilter"]) converter = module(mapping) else: self.__log.error( "\n\nCannot find extension module for %s topic.\nPlease check your configuration.\n", mapping["topicFilter"]) else: converter = JsonMqttUplinkConverter(mapping) if converter is not None: regex_topic = TBUtility.topic_to_regex( mapping.get("topicFilter")) if not self.__sub_topics.get(regex_topic): self.__sub_topics[regex_topic] = [] self.__sub_topics[regex_topic].append( {converter: None}) # self._client.subscribe(TBUtility.regex_to_topic(regex_topic)) self.__subscribe(mapping["topicFilter"]) self.__log.info('Connector "%s" subscribe to %s', self.get_name(), TBUtility.regex_to_topic(regex_topic)) else: self.__log.error("Cannot find converter for %s topic", mapping["topicFilter"]) except Exception as e: self.__log.exception(e) try: for request in self.__service_config: if self.__service_config.get(request) is not None: for request_config in self.__service_config.get( request): self.__subscribe(request_config["topicFilter"]) except Exception as e: self.__log.error(e) else: if result_code in result_codes: self.__log.error("%s connection FAIL with error %s %s!", self.get_name(), result_code, result_codes[result_code]) else: self.__log.error("%s connection FAIL with unknown error!", self.get_name()) def _on_disconnect(self, *args): self.__log.debug('"%s" was disconnected. %s', self.get_name(), str(args)) def _on_log(self, *args): self.__log.debug(args) def _on_subscribe(self, _, __, mid, granted_qos): try: if granted_qos[0] == 128: self.__log.error( '"%s" subscription failed to topic %s subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) else: self.__log.info( '"%s" subscription success to topic %s, subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) if self.__subscribes_sent.get(mid) is not None: del self.__subscribes_sent[mid] except Exception as e: self.__log.exception(e) def __get_service_config(self, config): for service_config in self.__service_config: if config.get(service_config): self.__service_config[service_config] = config[service_config] def _on_message(self, client, userdata, message): self.statistics['MessagesReceived'] += 1 content = TBUtility.decode(message) # Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" regex_topic = [ regex for regex in self.__sub_topics if fullmatch(regex, message.topic) ] if regex_topic: try: for regex in regex_topic: if self.__sub_topics.get(regex): for converter_value in range( len(self.__sub_topics.get(regex))): if self.__sub_topics[regex][converter_value]: for converter in self.__sub_topics.get( regex)[converter_value]: converted_content = converter.convert( message.topic, content) if converted_content: try: self.__sub_topics[regex][ converter_value][ converter] = converted_content except Exception as e: self.__log.exception(e) self.__gateway.send_to_storage( self.name, converted_content) self.statistics['MessagesSent'] += 1 else: continue else: self.__log.error( 'Cannot find converter for the topic:"%s"! Client: %s, User data: %s', message.topic, str(client), str(userdata)) return None except Exception as e: log.exception(e) return None # Check if message topic is matched by an existing connection request handler if self.__service_config.get("connectRequests"): for request in self.__service_config["connectRequests"]: # Check that the current connection request handler defines a topic filter (mandatory) if request.get("topicFilter") is None: continue found_device_name = None found_device_type = 'default' # Extract device name and type from regexps, if any. # This cannot be postponed because message topic may contain wildcards if request.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( request["deviceNameJsonExpression"], content) if request.get("deviceNameTopicExpression"): device_name_expression = request[ "deviceNameTopicExpression"] device_name_match = search(device_name_expression, message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) if request.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( request["deviceTypeJsonExpression"], content) if request.get("deviceTypeTopicExpression"): device_type_expression = request[ "deviceTypeTopicExpression"] found_device_type = search(device_type_expression, message.topic) # Check if request topic matches with message topic before of after regexp substitution if message.topic not in request.get("topicFilter"): sub_topic = message.topic # Substitute device name (if defined) in topic if found_device_name is not None: sub_topic = sub(found_device_name, "+", sub_topic) # Substitute device type in topic sub_topic = sub(found_device_type, "+", sub_topic) # If topic still not matches, this is not the correct handler if sub_topic not in request.get("topicFilter"): continue # I'm now sure that this message must be handled by this connection request handler if found_device_name is None: self.__log.error( "Device name missing from connection request") return None # Note: device must be added even if it is already known locally: else ThingsBoard # will not send RPCs and attribute updates self.__gateway.add_device(found_device_name, {"connector": self}, device_type=found_device_type) return None # Check if message topic is matched by an existing disconnection request handler if self.__service_config.get("disconnectRequests"): for request in self.__service_config["disconnectRequests"]: # Check that the current disconnection request handler defines a topic filter (mandatory) if request.get("topicFilter") is None: continue found_device_name = None found_device_type = 'default' # Extract device name and type from regexps, if any. # This cannot be postponed because message topic may contain wildcards if request.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( request["deviceNameJsonExpression"], content) if request.get("deviceNameTopicExpression"): device_name_expression = request[ "deviceNameTopicExpression"] device_name_match = search(device_name_expression, message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) if request.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( request["deviceTypeJsonExpression"], content) if request.get("deviceTypeTopicExpression"): device_type_expression = request[ "deviceTypeTopicExpression"] found_device_type = search(device_type_expression, message.topic) # Check if request topic matches with message topic before of after regexp substitution if message.topic not in request.get("topicFilter"): sub_topic = message.topic # Substitute device name (if defined) in topic if found_device_name is not None: sub_topic = sub(found_device_name, "+", sub_topic) # Substitute device type in topic sub_topic = sub(found_device_type, "+", sub_topic) # If topic still not matches, this is not the correct handler if sub_topic not in request.get("topicFilter"): continue # I'm now sure that this message must be handled by this connection request handler if found_device_name is None: self.__log.error( "Device name missing from disconnection request") return None if found_device_name in self.__gateway.get_devices(): self.__log.info("Device %s of type %s disconnected", found_device_name, found_device_type) self.__gateway.del_device(found_device_name) else: self.__log.info("Device %s is already disconnected", found_device_name) return None if message.topic in self.__gateway.rpc_requests_in_progress: self.__gateway.rpc_with_reply_processing(message.topic, content) else: self.__log.debug( "Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"", message.topic, content) def on_attributes_update(self, content): if self.__attribute_updates: for attribute_update in self.__attribute_updates: if match(attribute_update["deviceNameFilter"], content["device"]): for attribute_key in content["data"]: if match(attribute_update["attributeFilter"], attribute_key): try: topic = attribute_update["topicExpression"]\ .replace("${deviceName}", content["device"])\ .replace("${attributeKey}", attribute_key)\ .replace("${attributeValue}", content["data"][attribute_key]) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e try: data = attribute_update["valueExpression"]\ .replace("${attributeKey}", attribute_key)\ .replace("${attributeValue}", content["data"][attribute_key]) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e self._client.publish(topic, data).wait_for_publish() self.__log.debug( "Attribute Update data: %s for device %s to topic: %s", data, content["device"], topic) else: self.__log.error( "Cannot find attributeName by filter in message with data: %s", content) else: self.__log.error( "Cannot find deviceName by filter in message with data: %s", content) else: self.__log.error("Attribute updates config not found.") def server_side_rpc_handler(self, content): for rpc_config in self.__server_side_rpc: if search(rpc_config["deviceNameFilter"], content["device"]) \ and search(rpc_config["methodFilter"], content["data"]["method"]) is not None: # Subscribe to RPC response topic if rpc_config.get("responseTopicExpression"): topic_for_subscribe = rpc_config["responseTopicExpression"] \ .replace("${deviceName}", content["device"]) \ .replace("${methodName}", content["data"]["method"]) \ .replace("${requestId}", str(content["data"]["id"])) \ .replace("${params}", content["data"]["params"]) if rpc_config.get("responseTimeout"): timeout = time.time() * 1000 + rpc_config.get( "responseTimeout") self.__gateway.register_rpc_request_timeout( content, timeout, topic_for_subscribe, self.rpc_cancel_processing) # Maybe we need to wait for the command to execute successfully before publishing the request. self._client.subscribe(topic_for_subscribe) else: self.__log.error( "Not found RPC response timeout in config, sending without waiting for response" ) # Publish RPC request if rpc_config.get("requestTopicExpression") is not None\ and rpc_config.get("valueExpression"): topic = rpc_config.get("requestTopicExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) data_to_send = rpc_config.get("valueExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) try: self._client.publish(topic, data_to_send) self.__log.debug( "Send RPC with no response request to topic: %s with data %s", topic, data_to_send) if rpc_config.get("responseTopicExpression") is None: self.__gateway.send_rpc_reply( device=content["device"], req_id=content["data"]["id"], success_sent=True) except Exception as e: self.__log.exception(e) def rpc_cancel_processing(self, topic): self._client.unsubscribe(topic)
class MqttConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() self.__gateway = gateway # Reference to TB Gateway self._connector_type = connector_type # Should be "mqtt" self.config = config # mqtt.json contents self.__log = log self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__subscribes_sent = {} # Extract main sections from configuration --------------------------------------------------------------------- self.__broker = config.get('broker') self.__mapping = [] self.__server_side_rpc = [] self.__connect_requests = [] self.__disconnect_requests = [] self.__attribute_requests = [] self.__attribute_updates = [] mandatory_keys = { "mapping": ['topicFilter', 'converter'], "serverSideRpc": ['deviceNameFilter', 'methodFilter'], "connectRequests": ['topicFilter'], "disconnectRequests": ['topicFilter'], "attributeRequests": ['topicFilter', 'topicExpression', 'valueExpression'], "attributeUpdates": [ 'deviceNameFilter', 'attributeFilter', 'topicExpression', 'valueExpression' ] } # Mappings, i.e., telemetry/attributes-push handlers provided by user via configuration file self.load_handlers('mapping', mandatory_keys['mapping'], self.__mapping) # RPCs, i.e., remote procedure calls (ThingsBoard towards devices) handlers self.load_handlers('serverSideRpc', mandatory_keys['serverSideRpc'], self.__server_side_rpc) # Connect requests, i.e., telling ThingsBoard that a device is online even if it does not post telemetry self.load_handlers('connectRequests', mandatory_keys['connectRequests'], self.__connect_requests) # Disconnect requests, i.e., telling ThingsBoard that a device is offline even if keep-alive has not expired yet self.load_handlers('disconnectRequests', mandatory_keys['disconnectRequests'], self.__disconnect_requests) # Shared attributes direct requests, i.e., asking ThingsBoard for some shared attribute value self.load_handlers('attributeRequests', mandatory_keys['attributeRequests'], self.__attribute_requests) # Attributes updates requests, i.e., asking ThingsBoard to send updates about an attribute self.load_handlers('attributeUpdates', mandatory_keys['attributeUpdates'], self.__attribute_updates) # Setup topic substitution lists for each class of handlers ---------------------------------------------------- self.__mapping_sub_topics = {} self.__connect_requests_sub_topics = {} self.__disconnect_requests_sub_topics = {} self.__attribute_requests_sub_topics = {} # Set up external MQTT broker connection ----------------------------------------------------------------------- client_id = ''.join( random.choice(string.ascii_lowercase) for _ in range(23)) self._client = Client(client_id) self.setName( config.get( "name", self.__broker.get( "name", 'Mqtt Broker ' + ''.join( random.choice(string.ascii_lowercase) for _ in range(5))))) if "username" in self.__broker["security"]: self._client.username_pw_set(self.__broker["security"]["username"], self.__broker["security"]["password"]) if "caCert" in self.__broker["security"] \ or self.__broker["security"].get("type", "none").lower() == "tls": ca_cert = self.__broker["security"].get("caCert") private_key = self.__broker["security"].get("privateKey") cert = self.__broker["security"].get("cert") if ca_cert is None: self._client.tls_set_context( ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) else: try: self._client.tls_set(ca_certs=ca_cert, certfile=cert, keyfile=private_key, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) except Exception as e: self.__log.error( "Cannot setup connection to broker %s using SSL. " "Please check your configuration.\nError: ", self.get_name()) self.__log.exception(e) self._client.tls_insecure_set(False) # Set up external MQTT broker callbacks ------------------------------------------------------------------------ self._client.on_connect = self._on_connect self._client.on_message = self._on_message self._client.on_subscribe = self._on_subscribe self._client.on_disconnect = self._on_disconnect # self._client.on_log = self._on_log # Set up lifecycle flags --------------------------------------------------------------------------------------- self._connected = False self.__stopped = False self.daemon = True def load_handlers(self, handler_flavor, mandatory_keys, accepted_handlers_list): if handler_flavor not in self.config: self.__log.error("'%s' section missing from configuration", handler_flavor) else: for handler in self.config.get(handler_flavor): discard = False for key in mandatory_keys: if key not in handler: # Will report all missing fields to user before discarding the entry => no break here discard = True self.__log.error( "Mandatory key '%s' missing from %s handler: %s", key, handler_flavor, json.dumps(handler)) else: self.__log.debug( "Mandatory key '%s' found in %s handler: %s", key, handler_flavor, json.dumps(handler)) if discard: self.__log.error( "%s handler is missing some mandatory keys => rejected: %s", handler_flavor, json.dumps(handler)) else: accepted_handlers_list.append(handler) self.__log.debug( "%s handler has all mandatory keys => accepted: %s", handler_flavor, json.dumps(handler)) self.__log.info("Number of accepted %s handlers: %d", handler_flavor, len(accepted_handlers_list)) self.__log.info( "Number of rejected %s handlers: %d", handler_flavor, len(self.config.get(handler_flavor)) - len(accepted_handlers_list)) def is_connected(self): return self._connected def open(self): self.__stopped = False self.start() def run(self): try: while not self._connected and not self.__stopped: try: self._client.connect(self.__broker['host'], self.__broker.get('port', 1883)) self._client.loop_start() if not self._connected: time.sleep(1) except Exception as e: self.__log.exception(e) time.sleep(10) except Exception as e: self.__log.exception(e) try: self.close() except Exception as e: self.__log.exception(e) while True: if self.__stopped: break time.sleep(.01) def close(self): self.__stopped = True try: self._client.disconnect() except Exception as e: log.exception(e) self._client.loop_stop() self.__log.info('%s has been stopped.', self.get_name()) def get_name(self): return self.name def __subscribe(self, topic, qos): message = self._client.subscribe(topic, qos) try: self.__subscribes_sent[message[1]] = topic except Exception as e: self.__log.exception(e) def _on_connect(self, client, userdata, flags, result_code, *extra_params): result_codes = { 1: "incorrect protocol version", 2: "invalid client identifier", 3: "server unavailable", 4: "bad username or password", 5: "not authorised", } if result_code == 0: self._connected = True self.__log.info('%s connected to %s:%s - successfully.', self.get_name(), self.__broker["host"], self.__broker.get("port", "1883")) self.__log.debug( "Client %s, userdata %s, flags %s, extra_params %s", str(client), str(userdata), str(flags), extra_params) # Setup data upload requests handling ---------------------------------------------------------------------- for mapping in self.__mapping: try: converter = None # Load converter for this mapping entry ------------------------------------------------------------ # mappings are guaranteed to have topicFilter and converter fields. See __init__ converter_type = mapping["converter"]["type"] converter_extension = mapping["converter"]["extension"] if converter_type: if converter_extension: module = TBUtility.check_and_import( self._connector_type, converter_extension) if module: self.__log.debug( 'Custom converter for topic %s - found!', mapping["topicFilter"]) converter = module(mapping) else: self.__log.error( "\n\nCannot find extension module for %s topic." "\nPlease check your configuration.\n", mapping["topicFilter"]) else: converter = JsonMqttUplinkConverter(mapping) if converter is None: self.__log.error("Cannot find converter for %s topic", mapping["topicFilter"]) continue # Setup regexp topic acceptance list --------------------------------------------------------------- regex_topic = TBUtility.topic_to_regex( mapping["topicFilter"]) # There may be more than one converter per topic, so I'm using vectors if not self.__mapping_sub_topics.get(regex_topic): self.__mapping_sub_topics[regex_topic] = [] self.__mapping_sub_topics[regex_topic].append(converter) # Subscribe to appropriate topic ------------------------------------------------------------------- self.__subscribe(mapping["topicFilter"], mapping.get("subscriptionQos", 0)) self.__log.info('Connector "%s" subscribe to %s', self.get_name(), TBUtility.regex_to_topic(regex_topic)) except Exception as e: self.__log.exception(e) # Setup connection requests handling ----------------------------------------------------------------------- for request in [ entry for entry in self.__connect_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 0)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__connect_requests_sub_topics[topic_filter] = request # Setup disconnection requests handling -------------------------------------------------------------------- for request in [ entry for entry in self.__disconnect_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 0)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__disconnect_requests_sub_topics[topic_filter] = request # Setup attributes requests handling ----------------------------------------------------------------------- for request in [ entry for entry in self.__attribute_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 0)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__attribute_requests_sub_topics[topic_filter] = request else: if result_code in result_codes: self.__log.error("%s connection FAIL with error %s %s!", self.get_name(), result_code, result_codes[result_code]) else: self.__log.error("%s connection FAIL with unknown error!", self.get_name()) def _on_disconnect(self, *args): self.__log.debug('"%s" was disconnected. %s', self.get_name(), str(args)) def _on_log(self, *args): self.__log.debug(args) def _on_subscribe(self, _, __, mid, granted_qos): try: if granted_qos[0] == 128: self.__log.error( '"%s" subscription failed to topic %s subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) else: self.__log.info( '"%s" subscription success to topic %s, subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) if self.__subscribes_sent.get(mid) is not None: del self.__subscribes_sent[mid] except Exception as e: self.__log.exception(e) def _on_message(self, client, userdata, message): self.statistics['MessagesReceived'] += 1 content = TBUtility.decode(message) # Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" --------------------------- topic_handlers = [ regex for regex in self.__mapping_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: # Note: every topic may be associated to one or more converter. This means that a single MQTT message # may produce more than one message towards ThingsBoard. This also means that I cannot return after # the first successful conversion: I got to use all the available ones. # I will use a flag to understand whether at least one converter succeeded request_handled = False for topic in topic_handlers: available_converters = self.__mapping_sub_topics[topic] for converter in available_converters: converted_content = converter.convert( message.topic, content) if converted_content: request_handled = True self.__gateway.send_to_storage(self.name, converted_content) self.statistics['MessagesSent'] += 1 self.__log.info( "Successfully converted message from topic %s", message.topic) else: continue if not request_handled: self.__log.error( 'Cannot find converter for the topic:"%s"! Client: %s, User data: %s', message.topic, str(client), str(userdata)) # Note: if I'm in this branch, this was for sure a telemetry/attribute push message # => Execution must end here both in case of failure and success return None # Check if message topic exists in connection handlers "i.e., I'm connecting a device" ------------------------- topic_handlers = [ regex for regex in self.__connect_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: for topic in topic_handlers: handler = self.__connect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search( handler["deviceTypeTopicExpression"], message.topic) if device_type_match is not None: found_device_type = device_type_match.group(0) elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from connection request") continue # Note: device must be added even if it is already known locally: else ThingsBoard # will not send RPCs and attribute updates self.__log.info("Connecting device %s of type %s", found_device_name, found_device_type) self.__gateway.add_device(found_device_name, {"connector": self}, device_type=found_device_type) # Note: if I'm in this branch, this was for sure a connection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in disconnection handlers "i.e., I'm disconnecting a device" ------------------- topic_handlers = [ regex for regex in self.__disconnect_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: for topic in topic_handlers: handler = self.__disconnect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search( handler["deviceTypeTopicExpression"], message.topic) if device_type_match is not None: found_device_type = device_type_match.group(0) elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from disconnection request") continue if found_device_name in self.__gateway.get_devices(): self.__log.info("Disconnecting device %s of type %s", found_device_name, found_device_type) self.__gateway.del_device(found_device_name) else: self.__log.info("Device %s was not connected", found_device_name) break # Note: if I'm in this branch, this was for sure a disconnection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in attribute request handlers "i.e., I'm asking for a shared attribute" -------- topic_handlers = [ regex for regex in self.__attribute_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: try: for topic in topic_handlers: handler = self.__attribute_requests_sub_topics[topic] found_device_name = None found_attribute_name = None # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get attribute name, either from topic or from content if handler.get("attributeNameTopicExpression"): attribute_name_match = search( handler["attributeNameTopicExpression"], message.topic) if attribute_name_match is not None: found_attribute_name = attribute_name_match.group( 0) elif handler.get("attributeNameJsonExpression"): found_attribute_name = TBUtility.get_value( handler["attributeNameJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from attribute request") continue if found_attribute_name is None: self.__log.error( "Attribute name missing from attribute request") continue self.__log.info("Will retrieve attribute %s of %s", found_attribute_name, found_device_name) self.__gateway.tb_client.client.gw_request_shared_attributes( found_device_name, [found_attribute_name], lambda data, *args: self.notify_attribute( data, found_attribute_name, handler.get("topicExpression"), handler.get("valueExpression"))) break except Exception as e: log.exception(e) # Note: if I'm in this branch, this was for sure an attribute request message # => Execution must end here both in case of failure and success return None # Check if message topic exists in RPC handlers ---------------------------------------------------------------- # The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is if message.topic in self.__gateway.rpc_requests_in_progress: self.__gateway.rpc_with_reply_processing(message.topic, content) return None self.__log.debug( "Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"", message.topic, content) def notify_attribute(self, incoming_data, attribute_name, topic_expression, value_expression): if incoming_data.get("device") is None or incoming_data.get( "value") is None: return device_name = incoming_data.get("device") attribute_value = incoming_data.get("value") topic = topic_expression \ .replace("${deviceName}", device_name) \ .replace("${attributeKey}", attribute_name) data = value_expression.replace("${attributeKey}", attribute_name) \ .replace("${attributeValue}", attribute_value) self._client.publish(topic, data).wait_for_publish() def on_attributes_update(self, content): if self.__attribute_updates: for attribute_update in self.__attribute_updates: if match(attribute_update["deviceNameFilter"], content["device"]): for attribute_key in content["data"]: if match(attribute_update["attributeFilter"], attribute_key): try: topic = attribute_update["topicExpression"]\ .replace("${deviceName}", content["device"])\ .replace("${attributeKey}", attribute_key)\ .replace("${attributeValue}", content["data"][attribute_key]) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e try: data = attribute_update["valueExpression"]\ .replace("${attributeKey}", attribute_key)\ .replace("${attributeValue}", content["data"][attribute_key]) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e self._client.publish(topic, data).wait_for_publish() self.__log.debug( "Attribute Update data: %s for device %s to topic: %s", data, content["device"], topic) else: self.__log.error( "Cannot find attributeName by filter in message with data: %s", content) else: self.__log.error( "Cannot find deviceName by filter in message with data: %s", content) else: self.__log.error("Attribute updates config not found.") def server_side_rpc_handler(self, content): for rpc_config in self.__server_side_rpc: if search(rpc_config["deviceNameFilter"], content["device"]) \ and search(rpc_config["methodFilter"], content["data"]["method"]) is not None: # Subscribe to RPC response topic if rpc_config.get("responseTopicExpression"): topic_for_subscribe = rpc_config["responseTopicExpression"] \ .replace("${deviceName}", content["device"]) \ .replace("${methodName}", content["data"]["method"]) \ .replace("${requestId}", str(content["data"]["id"])) \ .replace("${params}", content["data"]["params"]) if rpc_config.get("responseTimeout"): timeout = time.time() * 1000 + rpc_config.get( "responseTimeout") self.__gateway.register_rpc_request_timeout( content, timeout, topic_for_subscribe, self.rpc_cancel_processing) # Maybe we need to wait for the command to execute successfully before publishing the request. self._client.subscribe(topic_for_subscribe) else: self.__log.error( "Not found RPC response timeout in config, sending without waiting for response" ) # Publish RPC request if rpc_config.get("requestTopicExpression") is not None\ and rpc_config.get("valueExpression"): topic = rpc_config.get("requestTopicExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) data_to_send = rpc_config.get("valueExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) try: self._client.publish(topic, data_to_send) self.__log.debug( "Send RPC with no response request to topic: %s with data %s", topic, data_to_send) if rpc_config.get("responseTopicExpression") is None: self.__gateway.send_rpc_reply( device=content["device"], req_id=content["data"]["id"], success_sent=True) except Exception as e: self.__log.exception(e) def rpc_cancel_processing(self, topic): self._client.unsubscribe(topic)
class Judge: max_time = 0 elapsed_time = 0 initial_time = 0 lock = Lock() lst = [] # List of available objects lst_results = Array('i', 0) # Max value for each object lst_names = [] # Name of winner for each object def __init__(self, max_timeout, broker='localhost', auth=None): self.max_time = max_timeout self.auth = auth self.client = Client() if auth != None: (usr, pwd) = auth self.client.username_pw_set(usr, pwd) self.client.connect(broker) def on_message(self, client, userdata, msg): if msg.topic == 'Available-items': ''' This msg contains all available objects. Once received, update the self list to contain them, then create the results list for keeping maximum value for each object. ''' self.lst = msg.payload self.lst = self.lst.decode("utf-8") self.lst = list(map(int, self.lst.split())) self.lst_results = Array('i', len(msg.payload)) print('Available objects', self.lst) for i in range(len(self.lst)): self.lst_names.append('') Process(target=self.collect, args=()).start() time.sleep(2 * random.random()) for i in range(1, len(self.lst) + 1): self.client.publish('results/' + str(i), str(0) + ' ' + str('None')) self.initial_time = time.time() self.elapsed_time = time.time() - self.initial_time else: with self.lock: time.sleep(random.random()) self.elapsed_time = time.time() - self.initial_time if self.elapsed_time < self.max_time: self.update_value(msg) else: if len(self.lst_names) != 0: print('Timeout reached, no more bids are accepted') for i in self.lst: top = 'results/' + str(i) own = str(self.lst_names[i - 1]) client.publish(top, 'Winner is ' + own) self.client.unsubscribe('items/#') self.lst_names = [] def update_value(self, msg): ''' Manage every msg received: 1- update the current value for each object (mutex with self.lock) 2- publish the value in the corresponding topic ''' item_topic = msg.topic (rest, bid_owner) = msg.payload.decode("utf-8").split() result_topic = item_topic.replace('items', 'results') item_pos = int(item_topic.replace('items/', '')) - 1 item_value = int(rest) last_value = self.lst_results[item_pos] if last_value < item_value: self.lst_results[item_pos] = item_value self.lst_names[item_pos] = bid_owner current_value = self.lst_results[item_pos] self.client.publish( result_topic, str(current_value) + ' ' + str(self.lst_names[item_pos])) print('New winner for', str(msg.topic).replace('items/', 'item '), ':', self.lst_names[item_pos]) def self_process(self, topic): self.client.subscribe('items/' + str(topic)) self.client.on_message = self.on_message self.client.loop_forever() def collect(self): print('Waitting users to bid ...') for it in self.lst: self.client.subscribe('items/' + str(it)) self.client.on_message = self.on_message self.client.loop_forever() def get_items(self): self.client.subscribe('Available-items') self.client.on_message = self.on_message self.client.loop_forever() def start(self): self.get_items()
class MQTTBus(BaseBus): class MQTTPubSub(BasePubSub): def __init__(self, channel, bus, handler): try: from Queue import Queue except: from queue import Queue self.bus = bus self.channel = channel self.handler = handler self.queue = Queue() listened = False subscribed = True def listen(self): self.listened = True while self.listened: data = self.queue.get() yield data def close(self): self.listened = False self.subscribed = False def _handle_data(self, data): if self.handler: self.handler(data) elif self.listened: self.queue.put(data) connected = False subscriber_map = {} def __init__(self, host='127.0.0.1', port=1883, username=None, password=None, channel_prefix=None): import uuid from paho.mqtt.client import Client self.host = host self.port = port self.username = username self.password = password self.channel_prefix = channel_prefix clientid = str(uuid.uuid4()) self.mqttclient = Client(client_id=clientid) if self.username: self.mqttclient.username_pw_set(self.username, self.password or None) self.mqttclient.on_connect = self._on_connect self.mqttclient.on_message = self._on_message self._connect() def _connect(self): import threading if self.username: self.mqttclient.username_pw_set(self.username, self.password) self.mqttclient.connect(self.host, self.port) th = threading.Thread(target=self._forloop) th.setDaemon(True) th.start() def _disconnect(self): self.mqttclient.disconnect() def subscriber_with_handler(self, channel, handler): ful_channel = self._get_ful_channel(channel) with self.lock: if ful_channel not in self.subscriber_map: self.mqttclient.subscribe(topic=ful_channel) self.subscriber_map[ful_channel] = { 'mpss': [], } mps = MQTTBus.MQTTPubSub(channel, self, handler) self.subscriber_map[ful_channel]['mpss'].append(mps) return mps def subscriber(self, channel): return self.subscriber_with_handler(channel, None) def publish(self, channel, data): return self.mqttclient.publish(self._get_ful_channel(channel), payload=encode_data(data)) def _forloop(self): self.mqttclient.loop_forever() def _on_connect(self, client, userdata, flags, rc): # print("_on_connect", client, userdata, flags, rc) self.connected = True def _on_message(self, client, userdata, msg): # print("_on_message", client, userdata, msg.topic, msg.payload) try: ful_channel = msg.topic data = decode_data(msg.payload.decode('utf-8')) if ful_channel in self.subscriber_map: changed = True for mps in self.subscriber_map[ful_channel]['mpss']: if mps.subscribed: mps._handle_data(data) else: changed = True if changed: with self.lock: mpss = filter(lambda mps: mps.subscribed, self.subscriber_map[ful_channel]['mpss']) if not mpss: self.mqttclient.unsubscribe(topic=ful_channel) del self.subscriber_map[ful_channel] else: self.subscriber_map[ful_channel]['mpss'] = mpss except: import traceback traceback.print_exc()
class Bridge(object): WEBSOCKETS = "websocket" SSE = "sse" def __init__(self, args, ioloop, dynamic_subscriptions): """ parse config values and setup Routes. """ self.mqtt_topics = [] try: self.mqtt_host = args["mqtt-to-server"]["broker"]["host"] self.mqtt_port = args["mqtt-to-server"]["broker"]["port"] self.bridge_port = args["server-to-client"]["port"] self.stream_protocol = args["server-to-client"]["protocol"] logger.info("Using protocol %s" % self.stream_protocol.lower()) if self.stream_protocol.lower( ) != "websocket" and self.stream_protocol.lower() != "sse": raise ConfigException("Invalid protocol") self.dynamic_subscriptions = dynamic_subscriptions if not self.dynamic_subscriptions: self.mqtt_topics = args["mqtt-to-server"]["topics"] except KeyError as e: raise ConfigException("Error when accessing field", e) from e logger.info("connecting to mqtt") self.topic_dict = {} self.ioloop = ioloop self.mqtt_client = Client() self.mqtt_client.on_message = self.on_mqtt_message self.mqtt_client.on_connect = self.on_mqtt_connect self.mqtt_client.connect_async(host=self.mqtt_host, port=self.mqtt_port) self.mqtt_client.loop_start() self._app = web.Application([ (r'.*', WebsocketHandler if self.stream_protocol == "websocket" else ServeSideEventsHandler, dict(parent=self)), ], debug=True) def get_app(self): return self._app def get_port(self): return self.bridge_port async def parse_req_path(self, req_path): candidate_path = req_path if len(candidate_path) is 1 and candidate_path[0] is "/": return "#" if candidate_path[len(req_path) - 1] is "/": candidate_path = candidate_path + "#" if candidate_path[0] == "/": candidate_path = candidate_path[1:] return candidate_path def on_mqtt_message(self, client, userdata, message): logger.debug("received message on topic %s" % message.topic) async def socket_write_message(self, socket, message): try: await socket.write_message(json.dumps(message)) except Exception as e: logger.error(e) try: await socket.write_message(message) except Exception as e: logger.error(e) def append_dynamic(self, topic): logger.info("adding dynamic subscription for %s " % topic) self.message_callback_add_with_sub_topic(topic, dynamic=True) def remove_dynamic(self, topic): logger.info("removing dynamic subscription for %s " % topic) self.topic_dict.pop(topic, None) self.mqtt_client.unsubscribe(topic) def message_callback_add_with_sub_topic(self, sub_topic, dynamic): logger.info("adding callback for mqtt topic: %s" % sub_topic) def message_callback(client, userdata, message): logger.debug( "Recieved Mqtt Message on %s as result of subscription on %s" % (message.topic, sub_topic)) if sub_topic is not message.topic: if message.topic not in self.topic_dict[sub_topic]["matches"]: self.topic_dict[sub_topic]["matches"].append(message.topic) for topic in self.topic_dict: if topic == message.topic: for socket in self.topic_dict[topic]["sockets"]: logger.debug("sending to socket:") logger.debug(socket) self.ioloop.add_callback( self.socket_write_message, socket=socket, message={ "topic": message.topic, "payload": message.payload.decode('utf-8') }) elif message.topic in self.topic_dict[topic]["matches"]: for socket in self.topic_dict[topic]["sockets"]: logger.debug("sending to socket:") logger.debug(socket) self.ioloop.add_callback( self.socket_write_message, socket=socket, message={ "topic": message.topic, "payload": message.payload.decode('utf-8') }) if sub_topic not in self.topic_dict: self.mqtt_client.message_callback_add(sub_topic, message_callback) self.topic_dict[sub_topic] = { "matches": [], "sockets": [], "dynamic": dynamic } self.mqtt_client.subscribe(sub_topic) def on_mqtt_connect(self, client, userdata, flags, rc): logger.info("mqtt connected to broker %s:%s" % (self.mqtt_host, str(self.mqtt_port))) for topic in self.mqtt_topics: self.message_callback_add_with_sub_topic(topic, dynamic=False) async def mqtt_disconnect(self): t = Thread(target=self.mqtt_client.disconnect, daemon=True) t.start() self.mqtt_client.loop_stop()
class Service: """ Service that publish informations about whether or not the devices are in expected range of good functioning """ _mqtt_client: Dict[str, Client] = {} _device_list: Dict[str, dict] = {} _broker: DefaultDict[str, set] = defaultdict(set) _broker_port: Dict[str, int] = {} _topic: DefaultDict[str, set] = defaultdict(set) _update_thread: Timer = None alarm_topic = "labsw3/arduino/alarm" def __init__(self): """ Instantiate the service """ self.service = Client(client_id=f"AlarmService{randrange(1, 100000)}") self.service.on_message = partial(self.my_on_message, led=self.service, service=self.service) self.lock = Lock() self.device_lock = Lock() def _update(self, device_list: List[dict]): """ Update reserved dicts :param device_list: device list to add """ for arduino in device_list: device = arduino["deviceID"] broker = arduino["end_points"]["MQTT"]["ip"] port = arduino["end_points"]["MQTT"]["port"] topics = { topic for topic in arduino["end_points"]["MQTT"]["end_points"] ["subscribe"] if "temp" in topic } led_topics = { topic for topic in arduino["end_points"]["MQTT"]["end_points"] ["publish"] if "led" in topic } self._device_list[device] = { "ip": broker, "temperature_topics": topics, "led_topics": led_topics } self._broker_port[broker] = port self._broker[broker].update({device}) self._topic[broker].update(topics) print(f"[{time.ctime()}] DEVICE {device} CONNECTED") def setup(self, first_time: bool = True): """ Setup the service """ try: print( f"[{time.ctime()}] EXTRACT info about all the devices registered" ) result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/devices/all" ) while result.status_code != 200: print( f"[{time.ctime()}] WARNING no device registered found, retrying after 30 seconds" ) time.sleep(30) result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}" f"/catalog/devices/all") print(f"[{time.ctime()}] DEVICES found") data = json.loads(result.content.decode()) print( f"[{time.ctime()}] EXTRACT from the devices list info about ArduinoYUN" ) device_list = [device for device in data if find_arduino(device)] if len(device_list) == 0: print( f"[{time.ctime()}] No ArduinoYUN found... retrying in 10 seconds" ) time.sleep(10) self.setup(first_time) return except KeyboardInterrupt: print(f"[{time.ctime()}] EXIT") if first_time: sys.exit() return if first_time: # Connect the service self.service.connect(host=SERVICE_BROKER_PORT["ip"], port=SERVICE_BROKER_PORT["port"]) print(f"[{time.ctime()}] SERVICE CONNECTED to " f"broker: {SERVICE_BROKER_PORT['ip']} " f"on port: port={SERVICE_BROKER_PORT['port']}") # Update registered devices and subscribe to all the topics with self.device_lock: self._update(device_list) self.subscribe(device_list) # Register inside the catalog print( f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}") requests.post( f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services', data=SERVICE_INFO, headers={"Content-Type": "application/json"}) def start(self): """ Start the service. """ # Setup the service self.setup() # Start all the external mqtt_clients for broker in self._mqtt_client: self._mqtt_client[broker].loop_start() # Schedule the update of the registration self._update_thread = Timer(60, self.update_registration) self._update_thread.start() try: # Run the service forever self.service.loop_forever() except KeyboardInterrupt: self.stop() def my_on_message(self, client: Client, userdata: Any, msg: MQTTMessage, led: Client = None, service: Client = None): """ Check if the temperature is good. In case of bad values, turn on a led and send the alarm :param client: MQTT client :param userdata: They could be any type :param msg: MQTT message :param led: Led Client :param service: Service Client """ def _device_led(): """ Find the arduino that generates the temperature telemetry and the topic to control the led """ with self.device_lock: for device in self._device_list: if msg.topic in self._device_list[device][ "temperature_topics"]: return device, self._device_list[device]["led_topics"] data = json.loads(msg.payload.decode()) arduino, led_topics = _device_led() with self.lock: if RANGE["min"] < data["v"] < RANGE["max"]: for topic in led_topics: led.publish(topic, payload=json.dumps({ "n": "led", "v": 0, "u": None })) print( f"[{time.ctime()}] PUBLISHING Alarm status on topic: {self.alarm_topic}" ) service.publish(self.alarm_topic, payload=json.dumps({ "device": arduino, "alarm": False })) else: for topic in led_topics: led.publish(topic, payload=json.dumps({ "n": "led", "v": 1, "u": None })) print( f"[{time.ctime()}] PUBLISHING Alarm status on topic: {self.alarm_topic}" ) service.publish(self.alarm_topic, payload=json.dumps({ "device": arduino, "alarm": True })) def update_registration(self): """ Update service registration to the catalog """ print( f"[{time.ctime()}] EXTRACT info about all the devices registered") result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/devices/all" ) # No device found if result.status_code != 200: self.reset() return data = json.loads(result.content.decode()) device_list = [device for device in data if find_arduino(device)] if len(device_list) == 0: self.reset() return # New Devices Found new_devices = {arduino["deviceID"] for arduino in device_list} # Old devices list old_devices = set(self._device_list.keys()) # Check if there are updates to do if old_devices == new_devices: # No update, ping the catalog print( f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}" ) requests.post( f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services', data=SERVICE_INFO, headers={"Content-Type": "application/json"}) self._update_thread = Timer(60, self.update_registration) self._update_thread.start() return with self.device_lock: # Delete inactive devices delete_list = { arduino: self._device_list[arduino] for arduino in self._device_list if arduino not in new_devices } # Unsubscribe from devices self.unsubscribe(delete_list) # Update registered devices and subscribe new_devices = [ arduino for arduino in device_list if arduino["deviceID"] not in old_devices ] self._update(new_devices) self.subscribe(new_devices) self._update_thread = Timer(60, self.update_registration) self._update_thread.start() def reset(self): """ Reset the service """ with self.device_lock: self._clear() self.setup(first_time=False) # Start all the external mqtt_clients for broker in self._mqtt_client: self._mqtt_client[broker].loop_start() self._update_thread = Timer(60, self.update_registration) self._update_thread.start() def unsubscribe(self, device_list: dict): """ Unsubscribe from devices :param device_list: device list """ for arduino in device_list: print(f"[{time.ctime()}] DEVICE {arduino} DISCONNECTED") broker = device_list[arduino]["ip"] topics = device_list[arduino]["temperature_topics"] if broker == SERVICE_BROKER_PORT["ip"]: for topic in topics: self.service.unsubscribe(topic) print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}") else: for topic in topics: self._mqtt_client[broker].unsubscribe(topic) print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}") # Update topics self._topic[broker] = self._topic[broker].difference(topics) # Disconnect from external broker if len(self._topic[broker]) == 0: if broker != SERVICE_BROKER_PORT["ip"]: self._mqtt_client[broker].disconnect() self._mqtt_client[broker].loop_stop() print( f"[{time.ctime()}] DISCONNECTED from broker: {broker}") del self._mqtt_client[broker] # Clean del self._topic[broker] del self._broker_port[broker] # Delete device del self._device_list[arduino] self._broker[broker].discard(arduino) def subscribe(self, device_list: List[dict]): """ Subscribe to all the devices registered :param device_list: list of device """ for arduino in device_list: broker = arduino["end_points"]["MQTT"]["ip"] topics = { topic for topic in arduino["end_points"]["MQTT"]["end_points"] ["subscribe"] if "temp" in topic } # Subscribe to topics if broker == SERVICE_BROKER_PORT["ip"]: for topic in topics: self.service.subscribe(topic) print(f"[{time.ctime()}] SUBSCRIBED to : {topic}") # Connect to external brokers else: if broker in self._mqtt_client: # Connection already made for topic in topics: self._mqtt_client[broker].subscribe(topic) print(f"[{time.ctime()}] SUBSCRIBED to : {topic}") else: # Connect to the broker self._mqtt_client[broker] = Client( f"TemperatureMeanService{randrange(1, 1000000)}") self._mqtt_client[broker].on_message = partial( self.my_on_message, led=self._mqtt_client[broker], service=self.service) self._mqtt_client[broker].connect( host=broker, port=self._broker_port[broker]) for topic in topics: self._mqtt_client[broker].subscribe(topic) print(f"[{time.ctime()}] SUBSCRIBED to : {topic}") def _clear(self): """ Clear all the data stored """ # Unsubscribe from all topics self.unsubscribe(self._device_list.copy()) # Disconnect from all the external brokers for broker in self._mqtt_client: self._mqtt_client[broker].disconnect() self._mqtt_client[broker].loop_stop() print(f"[{time.ctime()}] DISCONNECTED from broker: {broker}") # Clear self._mqtt_client.clear() self._device_list.clear() self._broker.clear() self._broker_port.clear() self._topic.clear() def stop(self): """ Stop the service """ # Stop the schedule self._update_thread.cancel() # Wait for the threads to finish if self._update_thread.is_alive(): self._update_thread.join() # Disconnect the clients print(f"[{time.ctime()}] SHUTTING DOWN") for client in self._mqtt_client: self._mqtt_client[client].disconnect() self._mqtt_client[client].loop_stop() # Disconnect the service self.service.disconnect() print(f"[{time.ctime()}] EXIT")
class Mqtt(Abstract): def __init__(self, service_info=None): self.connected = False self.client = None self.queue = Queue() super().__init__(service_info) def start(self): self.connect() self.receive_msgs() def stop(self): self.stop_receiving_msgs() self.disconnect() def add_stream(self, stream): self.streams[stream.name] = stream def remove_stream(self, name): if name in self.streams.keys(): self.streams.pop(name) def remove_all_streams(self): self.streams = {} def is_connected(self): if self.client and self.connected: return True return False def connect(self): if not self.client: host = self.service_info['host'] port = self.service_info['port'] self.client = Client() self.client.on_message = self.on_mqtt_msg self.client.on_connect = self.on_connect self.client.on_disconnect = self.on_disconnect self.client.connect(host, port, 60) def on_connect(self, client, userdata, flags, rc): self.connected = True def disconnect(self): if self.client: self.client.disconnect() def on_disconnect(self, client, userdata, rc): self.client = None self.connected = False def send_msg(self, msg, tx_info=None): if self.client: topic = None if tx_info: topic = tx_info['topic'] elif self.service_info: topic = self.service_info['publish']['default_topic'] if self.client and topic: self.client.publish(topic, msg) else: raise MqttError('Cannot send message without a client') def receive_msgs(self): topics = self.service_info['subscribe']['topics'] if self.client: for t in topics: self.client.subscribe(t, 0) sub_type = self.service_info['subscribe']['type'] if sub_type == 'blocking': self.client.loop_forever() elif sub_type == 'unblocking': self.client.loop_start() elif sub_type == 'polled': self.client.loop(self.service_info['subscribe']['loop_time']) else: self.client.loop(.1) else: raise MqttError('Cannot receive message without a client') def stop_receiving_msgs(self): topics = self.service_info['subscribe']['topics'] if self.client: for t in topics: self.client.unsubscribe(t) self.client.loop_stop() def handle_msg(self): msg = self.queue.get() print(f'Received: {msg["topic"]}::{msg["msg"]}') data = super().handle_msg(msg) for k, s in self.streams.items(): events = [] for d in data: timestamp = d.timestamp sample = getattr(d, s.handle) events.append(Event(timestamp, sample)) s.handle_msg(events) def on_mqtt_msg(self, client, userdata, msg): payload = {} payload['topic'] = msg.topic payload['msg'] = msg.payload self.queue.put(payload) self.handle_msg()
def unsubscribe(self, mqttclient: MqttClient): mqttclient.unsubscribe(self.updatedDeviceTopic) with self.devicesLock: for device in self.devices.values(): device.unsubscribe(mqttclient)
class MqttClient(): default_config = { "host": "api.raptorbox.eu", "port": 1883, "reconnect_min_delay": 1, "reconnect_max_delay": 127, } def __init__(self, config): ''' Initialize information for the client provide configurations as parameter, if None provided the configurations would be defaults see MqttClient.default_config for configurations format ''' self.config = MqttClient.default_config if config: for k, v in config.iteritems(): self.config[k] = v self.mqtt_client = Client() self.mqtt_client.username_pw_set(config["username"], config["password"]) if "reconnect_min_delay" in config and "reconnect_max_delay" in config: self.mqtt_client.reconnect_delay_set( config["reconnect_min_delay"], config["reconnect_max_delay"], ) # self.connection_thread = None # self.mqtt_client.tls_set() def connect(self): #TODO consider srv field in dns ''' connect the client to mqtt server with configurations provided by constructor and starts listening for topics ''' self.mqtt_client.connect(self.config["host"], self.config["port"]) # self.mqtt_client.loop_forever() self.mqtt_client.loop_start() def subscribe(self, topic, callback): ''' register 'callback' to "topic". Every message will be passed to callback in order to be executed ''' logging.debug("subscribing to topic: {}".format(topic)) self.mqtt_client.subscribe(topic) self.mqtt_client.message_callback_add(topic, callback) def unsubscribe(self, topic): ''' unsubscribe to topic topic could be either a string or al list of strings containing to topics to unsubscribe to. Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not currently connected. mid is the message ID for the unsubscribe request. The mid value can be used to track the unsubscribe request by checking against the mid argument in the on_unsubscribe() callback if it is defined. ''' return self.mqtt_client.unsubscribe(topic) def disconnect(self): ''' Disconnects from the server ''' self.mqtt_client.disconnect() def reconnect(self): ''' Reconnects after a disconnection, this could be called after connection only ''' self.mqtt_client.reconnect()
class Mqtt(): def __init__(self, app=None): # type: (Flask) -> None self.app = app self.client = Client() self.client.on_connect = self._handle_connect self.client.on_disconnect = self._handle_disconnect self.topics = [] # type: List[str] self.connected = False if app is not None: self.init_app(app) def init_app(self, app): # type: (Flask) -> None self.username = app.config.get('MQTT_USERNAME') self.password = app.config.get('MQTT_PASSWORD') self.broker_url = app.config.get('MQTT_BROKER_URL', 'localhost') self.broker_port = app.config.get('MQTT_BROKER_PORT', 1883) self.tls_enabled = app.config.get('MQTT_TLS_ENABLED', False) self.keepalive = app.config.get('MQTT_KEEPALIVE', 60) self.last_will_topic = app.config.get('MQTT_LAST_WILL_TOPIC') self.last_will_message = app.config.get('MQTT_LAST_WILL_MESSAGE') self.last_will_qos = app.config.get('MQTT_LAST_WILL_QOS', 0) self.last_will_retain = app.config.get('MQTT_LAST_WILL_RETAIN', False) if self.tls_enabled: self.tls_ca_certs = app.config['MQTT_TLS_CA_CERTS'] self.tls_certfile = app.config.get('MQTT_TLS_CERTFILE') self.tls_keyfile = app.config.get('MQTT_TLS_KEYFILE') self.tls_cert_reqs = app.config.get('MQTT_TLS_CERT_REQS', ssl.CERT_REQUIRED) self.tls_version = app.config.get('MQTT_TLS_VERSION', ssl.PROTOCOL_TLSv1) self.tls_ciphers = app.config.get('MQTT_TLS_CIPHERS') self.tls_insecure = app.config.get('MQTT_TLS_INSECURE', False) # set last will message if self.last_will_topic is not None: self.client.will_set(self.last_will_topic, self.last_will_message, self.last_will_qos, self.last_will_retain) self.app = app self._connect() def _connect(self): # type: () -> None if self.username is not None: self.client.username_pw_set(self.username, self.password) # security if self.tls_enabled: if self.tls_insecure: self.client.tls_insecure_set(self.tls_insecure) self.client.tls_set( ca_certs=self.tls_ca_certs, certfile=self.tls_certfile, keyfile=self.tls_keyfile, cert_reqs=self.tls_cert_reqs, tls_version=self.tls_version, ciphers=self.tls_ciphers, ) self.client.loop_start() res = self.client.connect(self.broker_url, self.broker_port, keepalive=self.keepalive) def _disconnect(self): # type: () -> None self.client.loop_stop() self.client.disconnect() def _handle_connect(self, client, userdata, flags, rc): # type: (Client, Any, Dict, int) -> None if rc == MQTT_ERR_SUCCESS: self.connected = True for topic in self.topics: self.client.subscribe(topic) def _handle_disconnect(self, client, userdata, rc): # type: (str, Any, int) -> None self.connected = False def on_topic(self, topic): # type: (str) -> Callable """ Decorator to add a callback function that is called when a certain topic has been published. The callback function is expected to have the following form: `handle_topic(client, userdata, message)` :parameter topic: a string specifying the subscription topic to subscribe to The topic still needs to be subscribed via mqtt.subscribe() before the callback function can be used to handle a certain topic. This way it is possible to subscribe and unsubscribe during runtime. **Example usage:**:: app = Flask(__name__) mqtt = Mqtt(app) mqtt.subscribe('home/mytopic') @mqtt.on_topic('home/mytopic') def handle_mytopic(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): # type: (Callable[[str], None]) -> Callable[[str], None] self.client.message_callback_add(topic, handler) return handler return decorator def subscribe(self, topic, qos=0): # type: (str, int) -> tuple(int, int) """ Subscribe to a certain topic. :param topic: a string specifying the subscription topic to subscribe to. :param qos: the desired quality of service level for the subscription. Defaults to 0. :rtype: (int, int) :result: (result, mid) A topic is a UTF-8 string, which is used by the broker to filter messages for each connected client. A topic consists of one or more topic levels. Each topic level is separated by a forward slash (topic level separator). The function returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not currently connected. mid is the message ID for the subscribe request. The mid value can be used to track the subscribe request by checking against the mid argument in the on_subscribe() callback if it is defined. **Topic example:** `myhome/groundfloor/livingroom/temperature` """ # TODO: add support for list of topics # don't subscribe if already subscribed # try to subscribe result, mid = self.client.subscribe(topic, qos) # if successful add to topics if result == MQTT_ERR_SUCCESS: if topic not in self.topics: self.topics.append(topic) return (result, mid) def unsubscribe(self, topic): # type: (str) -> tuple(int, int) """ Unsubscribe from a single topic. :param topic: a single string that is the subscription topic to unsubscribe from :rtype: (int, int) :result: (result, mid) Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not currently connected. mid is the message ID for the unsubscribe request. The mid value can be used to track the unsubscribe request by checking against the mid argument in the on_unsubscribe() callback if it is defined. """ # don't unsubscribe if not in topics if topic not in self.topics: return result, mid = self.client.unsubscribe(topic) # if successful remove from topics if result == MQTT_ERR_SUCCESS: self.topics.remove(topic) return result, mid def unsubscribe_all(self): # type: () -> None """ Unsubscribe from all topics. """ topics = self.topics[:] for topic in topics: self.unsubscribe(topic) def publish(self, topic, payload=None, qos=0, retain=False): # type: (str, bytes, int, bool) -> Tuple[int, int] """ Send a message to the broker. :param topic: the topic that the message should be published on :param payload: the actual message to send. If not given, or set to None a zero length message will be used. Passing an int or float will result in the payload being converted to a string representing that number. If you wish to send a true int/float, use struct.pack() to create the payload you require. :param qos: the quality of service level to use :param retain: if set to True, the message will be set as the "last known good"/retained message for the topic :returns: Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN if the client is not currently connected. mid is the message ID for the publish request. """ if not self.connected: self.client.reconnect() return self.client.publish(topic, payload, qos, retain) def on_message(self): # type: () -> Callable """ Decorator to handle all messages that have been subscribed and that are not handled via the `on_message` decorator. **Note:** Unlike as written in the paho mqtt documentation this callback will not be called if there exists an topic-specific callback added by the `on_topic` decorator. **Example Usage:**:: @mqtt.on_message() def handle_messages(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_message = handler return handler return decorator def on_publish(self): """ Decorator to handle all messages that have been published by the client. **Example Usage:**:: @mqtt.on_publish() def handle_publish(client, userdata, mid): print('Published message with mid {}.' .format(mid)) """ def decorator(handler): self.client.on_publish = handler return handler return decorator def on_subscribe(self): """ Decorator to handle subscribe callbacks. **Usage:**:: @mqtt.on_subscribe() def handle_subscribe(client, userdata, mid, granted_qos): print('Subscription id {} granted with qos {}.' .format(mid, granted_qos)) """ def decorator(handler): self.client.on_subscribe = handler return handler return decorator def on_unsubscribe(self): """ Decorator to handle unsubscribe callbacks. **Usage:**:: @mqtt.unsubscribe() def handle_unsubscribe(client, userdata, mid) print('Unsubscribed from topic (id: {})' .format(mid)') """ def decorator(handler): self.client.on_unsubscribe = handler return handler return decorator def on_log(self): # type: () -> Callable """ Decorator to handle MQTT logging. **Example Usage:** :: @mqtt.on_log() def handle_logging(client, userdata, level, buf): print(client, userdata, level, buf) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_log = handler return handler return decorator
def unsubscribe(self, mqttclient: MqttClient) -> None: super().unsubscribe(mqttclient) mqttclient.unsubscribe(self.stateTopic)
class MqttClient: """A wrapper around the paho mqtt client specifically for device automation bus This wrapper is mean to handle connect/disconnect as well as sending and receiving messages. Clients can register handlers to be called when certain messages are received. Clients also use this class to publish messages onto the bus. """ class MqttClientEvents: """Container for MqttClient thread events""" def __init__(self): self.connected_event = Event() self.disconnected_event = Event() self.apps_discovered_event = Event() self.capabilities_discovered_event = Event() def __init__(self, host, port): self.host = host self.port = port self.mqtt_client_events = MqttClient.MqttClientEvents() self.thread = None self.client = Client(userdata=self.mqtt_client_events) self.app_registry = {} self.app_registry_timer = None self.app_registry_timer_lock = Lock() self.capabilities_registry = {} self.capabilities_registry_timer = None self.capabilities_registry_timer_lock = Lock() self.topic_handlers = [{ 'topic': 'apps/+', 'regex': 'apps/[^/]*', 'handler': self.on_app_message }, { 'topic': 'platform/+', 'regex': 'platform/[^/]*', 'handler': self.on_app_message }, { 'topic': 'platform/telemetry/monitor/#', 'regex': 'platform/telemetry/monitor/.*', 'handler': self.on_monitor_message }] self.logger = logging.getLogger(__name__) def on_connect(self, client, mqtt_client_events, flags, rc): """Set the connected state and subscribe to existing known topics.""" del mqtt_client_events, flags, rc # Delete unused parameters to prevent warnings. for topic_handler in self.topic_handlers: client.subscribe(topic_handler["topic"]) self.mqtt_client_events.connected_event.set() self.mqtt_client_events.disconnected_event.clear() def on_disconnect(self, client, user_data, rc): """Set the disconnected state.""" del client, user_data, rc # Delete unused parameters to prevent warnings. self.mqtt_client_events.connected_event.clear() self.mqtt_client_events.disconnected_event.set() def on_message(self, client, user_data, packet): """The main message dispatcher This function is called for all messages on all registered topics. It handles dispatching messages to the registered handlers. """ del client, user_data # Delete unused parameters to prevent warnings. try: if packet: self.logger.info("topic: " + packet.topic) # Message payload can come in as a char array instead of string. # Normalize it to a string for easier access by handlers. if type(packet.payload) is str: message = packet.payload else: message = packet.payload.decode("utf-8") self.logger.info("message: " + message) # Since this function receives messages for all topics, use the specified regex # to call the correct handler(s). for topic_handler in self.topic_handlers: if re.fullmatch(topic_handler["regex"], packet.topic): topic_handler["handler"](packet.topic, message) # Since paho eats all the exceptions, catch them here beforehand # and make sure a stack trace is printed. except Exception: traceback.print_exc() raise def on_app_message(self, topic, message): """Local handler for handling registration of media apps. This function receives a message for each registered media application. Since these messages are published as retained, normally this will be called with all the messages in on batch. To know that we have received all of the initial batch, use a simple timeout to measure how long it has been since the last message. When enough time goes by, allow callers to get the list of apps from the registry. """ app = json.loads(message) app["app_id"] = os.path.basename(topic) self.app_registry.update({app["name"]: app}) self.app_registry_timer_lock.acquire(True) if self.app_registry_timer: self.app_registry_timer.cancel() while self.app_registry_timer.is_alive(): pass self.app_registry_timer = Timer( 1.0, self.mqtt_client_events.apps_discovered_event.set).start() self.app_registry_timer_lock.release() def on_platform_message(self, topic, message): """Local handler for handling platform messages This function receives a message for each registered platform capability. """ capability = json.loads(message) capability_id = os.path.basename(topic) self.capabilities_registry.update({capability_id: capability}) self.capabilities_registry_timer_lock.acquire(True) if self.capabilities_registry_timer: self.capabilities_registry_timer.cancel() while self.capabilities_registry_timer.is_alive(): pass self.capabilities_registry_timer = Timer( 1.0, self.mqtt_client_events.capabilities_discovered_event.set).start() self.capabilities_registry_timer_lock.release() def on_monitor_message(self, topic, message): """Local handler for handling process monitor messages """ message = json.loads(message) def get_discovered_apps(self): """Method to get the discovered apps. TODO: Get the app registry out of here into it's own class. """ self.mqtt_client_events.apps_discovered_event.wait() self.app_registry_timer_lock.acquire(True) discovered_apps = self.app_registry.copy().values() self.app_registry_timer_lock.release() return discovered_apps def get_discovered_capabilities(self): """Method to get the discovered platform capabilities. TODO: Get the registry out of here into it's own class. """ self.mqtt_client_events.capabilities_discovered_event.wait() self.capabilities_registry_timer_lock.acquire(True) discovered_capabilities = self.capabilities_registry.copy().values() self.capabilities_registry_timer_lock.release() return discovered_capabilities def _start(self): """Private start method that does th actual work of starting the client in a new thread.""" self.client.enable_logger(self.logger) self.client.on_connect = self.on_connect self.client.on_message = self.on_message self.client.on_disconnect = self.on_disconnect self.client.connect(self.host, self.port) self.client.loop_start() self.mqtt_client_events.connected_event.wait(15.0) if not self.mqtt_client_events.connected_event.is_set(): raise Exception("Connect timed out after 15 seconds.") self.mqtt_client_events.disconnected_event.wait() # yes, forever def start(self): """Public start method used by callers to start the client.""" self.thread = Thread(target=self._start) self.thread.start() def stop(self): """Stop method used to shutdown the client.""" for topic_handler in self.topic_handlers: self.client.unsubscribe(topic_handler["topic"]) self.client.disconnect() self.mqtt_client_events.apps_discovered_event.set() def publish(self, topic, message): """Publish a message to the specified topic.""" self.logger.info("publish: Sending '{}' to topic '{}'".format( message, topic)) self.client.publish(topic, message) def subscribe(self, topic, handler, regex=None): """Add a handler for a particular topic The handler is a function that will be called when a message is received on the specified topic. An optional regex can be provided to filter the topic even more. """ if not regex: regex = topic self.topic_handlers.append({ 'topic': topic, 'handler': handler, 'regex': regex }) self.client.subscribe(topic)
class MqttBus(IBus): """ Args transport from pusher to subscribers via MQTT """ _subscribers = dict() def __init__(self, host=None, *args, publish_time_out=0.05, publish_waiting=0.1, mid_max_len=64, qos=2, retain=True, **kwargs): super().__init__() if host is None: host = 'localhost' self.client = Client(*args, **kwargs) self.client.connect(host) self.client.loop_start() self.client.on_publish = self._on_publish self._received_messages = [] self.mid_max_len = mid_max_len self.publish_time_out = publish_time_out self.publish_waiting = publish_waiting self.qos = qos self.retain = retain @staticmethod def _loads(payload): try: return pickle.loads(payload, fix_imports=True, encoding="utf-8", errors="strict") except Exception as e: log.error('mqtt loads error:' + str(e)) return None @staticmethod def _dumps(data): try: return pickle.dumps(data, protocol=3, fix_imports=True) except Exception as e: log.error('mqtt dumps error:' + str(e)) return None def _on_publish(self, client, userdata, mid): if len(self._received_messages) >= self.mid_max_len: self._received_messages.pop(0) self._received_messages.append(mid) else: self._received_messages.append(mid) def _on_message(self, client, userdata, msg): args, kwargs = self._loads(msg.payload) if self._subscribers.get(msg.topic): for fn in self._subscribers[msg.topic]: fn(*args, **kwargs) else: log.warning( 'mqtt Callback _subscribers is not set for topic={}'.format( msg.topic)) def subscribe(self, topic, fn=None): """ Add function fn to subscribers list """ if fn: assert hasattr(fn, '__call__') self.client.subscribe(topic, qos=self.qos) self.client.message_callback_add( topic, lambda client, userdata, msg: self._on_message( client, userdata, msg)) if self._subscribers.get(topic) is None: self._subscribers[topic] = [fn] log.info('mqtt Function {} subscribed to topic "{}"'.format( fn.__name__, topic)) else: self._subscribers[topic].append(fn) log.info('mqtt Function {} subscribed to topic "{}"'.format( fn.__name__, topic)) else: self.client.message_callback_remove(topic) self.client.unsubscribe(topic) def push(self, topic, *args, **kwargs): """ Call all the subscribers """ _args = [arg for arg in args if isinstance(arg, (float, int, str))] _kwargs = { key: arg for key, arg in kwargs.items() if isinstance(arg, (float, int, str)) } if _args or _kwargs: log.debug( 'pushed topic "{}" with args {} and kwargs {} and {}'.format( topic, str(_args), str(_kwargs), str(set(kwargs.keys()) - set(_kwargs.keys())))) send_data = self._dumps((args, kwargs)) log.info('mqtt Pushing data with size={} to topic={}...'.format( len(send_data), topic)) rc, mid = self.client.publish(topic, send_data, qos=self.qos, retain=self.retain) publish_time = time.time() while mid not in self._received_messages: time.sleep(self.publish_time_out) if (time.time() - publish_time) >= self.publish_waiting: break if rc == 0: log.info('mqtt Pushed data to topic={} with rc {}'.format( topic, rc)) else: log.warning('mqtt Not pushed data to topic={} with rc {}'.format( topic, rc))
class JMSClient(object): """Class JMSClient """ _mh = None _client = None _host = None _port = None _user = None _passw = None _verbose = None _is_connected = None _messages = [] def __init__(self, verbose=False): """Class constructor Called when the object is initialized Args: verbose (bool): verbose mode """ try: self._mh = MasterHead.get_head() self._client = Client() self._client.on_message = self._on_message self._verbose = verbose if (self._verbose): self._client.on_log = self._on_log except MQTTException as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) @property def client(self): """ MQTT client property getter """ return self._client @property def host(self): """ server host property getter """ return self._host @property def port(self): """ server port property getter """ return self._port @property def user(self): """ username property getter """ return self._user @property def passw(self): """ user password property getter """ return self._passw @property def verbose(self): """ verbose property getter """ return self._verbose @property def is_connected(self): """ is_connected property getter """ return self._is_connected def _on_log(self, client, obj, level, string): """ Callback for on_log event """ print(string) def _on_message(self, client, obj, msg): """ Callback for on_message event """ self._messages.append(msg.payload.decode()) def connect(self, host, port=1883, user=None, passw=None, timeout=10): """Method connects to server Args: host (str): hostname port (str): port user (str): username passw (str): password timeout (int): timeout Returns: bool: result Raises: event: jms_before_connect event: jms_after_connected """ try: msg = 'host:{0}, port:{1}, user:{2}, passw:{3}, timeout:{4}'.format( host, port, user, passw, timeout) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_connecting', msg), self._mh.fromhere()) ev = event.Event( 'jms_before_connect', host, port, user, passw, timeout) if (self._mh.fire_event(ev) > 0): host = ev.argv(0) port = ev.argv(1) user = ev.argv(2) passw = ev.argv(3) timeout = ev.argv(4) self._host = host self._port = port self._user = user self._passw = passw if (ev.will_run_default()): if (self._user != None): self._client.username_pw_set(self._user, self._passw) setdefaulttimeout(timeout) self._client.connect(self._host, self._port) self._is_connected = True self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_connected'), self._mh.fromhere()) ev = event.Event('jms_after_connect') self._mh.fire_event(ev) return True except (MQTTException, error, ValueError) as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) return False def disconnect(self): """Method disconnects from server Args: none Returns: bool: result """ try: self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_disconnecting'), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_jms_not_connected'), self._mh.fromhere()) return False else: self._client.disconnect() self._is_connected = False self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_disconnected'), self._mh.fromhere()) return True except (MQTTException, error, ValueError) as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) return False def send(self, destination_name, message): """Method sends message Args: destination_name (str): topic name message (str): message Returns: bool: result Raises: event: jms_before_send event: jms_after_send """ try: msg = 'destination_name:{0}, message:{1}'.format( destination_name, message) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_sending_msg', msg), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_jms_not_connected'), self._mh.fromhere()) return False ev = event.Event('jms_before_send', destination_name, message) if (self._mh.fire_event(ev) > 0): destination_name = ev.argv(0) message = ev.argv(1) if (ev.will_run_default()): res, id = self._client.publish(destination_name, message) if (res != 0): self._mh.demsg('htk_on_error', self._mh._trn.msg( 'htk_jms_sending_error'), self._mh.fromhere()) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_msg_sent'), self._mh.fromhere()) ev = event.Event('jms_after_send') self._mh.fire_event(ev) return True except (MQTTException, error, ValueError) as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) return False def receive(self, destination_name, cnt=1, timeout=10): """Method receives messages Args: destination_name (str): queue name cnt (int): count of messages timeout (int): timeout to receive message Returns: list: messages Raises: event: jms_before_receive event: jms_after_receive """ try: msg = 'destination_name:{0}, count:{1}'.format( destination_name, cnt) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_receiving_msg', msg), self._mh.fromhere()) if (not self._is_connected): self._mh.demsg('htk_on_warning', self._mh._trn.msg( 'htk_jms_not_connected'), self._mh.fromhere()) return None ev = event.Event('jms_before_receive', destination_name, cnt) if (self._mh.fire_event(ev) > 0): destination_name = ev.argv(0) cnt = ev.argv(1) if (ev.will_run_default()): res, id = self._client.subscribe(destination_name) if (res != 0): self._mh.demsg('htk_on_error', self._mh._trn.msg( 'htk_jms_sending_error'), self._mh.fromhere()) return None res = 0 cnt_before = 0 start = time() while (res == 0): res = self._client.loop() cnt_after = len(self._messages) if (cnt_after > cnt_before and cnt_after < cnt): cnt_before = cnt_after elif (cnt_after == cnt or time() > start + timeout): res = -1 messages = self._messages self._client.unsubscribe(destination_name) self._messages = [] self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_jms_msg_received', len(messages)), self._mh.fromhere()) ev = event.Event('jms_after_receive') self._mh.fire_event(ev) return messages except (MQTTException, error, ValueError) as ex: self._mh.demsg('htk_on_error', ex, self._mh.fromhere()) return None
class BroadcastBridge: """ Main Class of the Broadcast Bridge """ def __init__(self, broker_host: str="localhost", broker_port: int=1883, http_host="localhost", http_port=8080, mqtt_prefix='broadcast_bridge/', persistence_file=None): self.mqtt_client = Client() self.mqtt_client.connect(broker_host, broker_port) self.mqtt_client.loop_start() self.mqtt_client.on_message = self.__on_mqtt_message self.__mqtt_prefix = mqtt_prefix self.http_port = http_port self.http_host = http_host self.available_modules = {} self.instances = {} self.log = logging.getLogger('mqttbb') self.log.setLevel(logging.DEBUG) self.__persistence_file = persistence_file self.__read_persistence_file() self.__init_http_server() def __read_persistence_file(self): if self.__persistence_file is None: return try: persistence = json.load(open(self.__persistence_file, 'r')) for instance in persistence: if instance['module'] not in self.available_modules: success = self.add_module(instance['module']) if not success: raise PersistenceException('Module "{}" is needed for load persistence but ' 'seems not to be installed.' .format(instance['module'])) self.add_module_instance(instance['module'], instance['config'], instance['short_description'], instance['uuid']) except FileNotFoundError: return def __update_persistence_file(self): if self.__persistence_file is None: return instances = [] for i, instance in self.instances.items(): instances.append({ 'module': instance['module'], 'config': instance['instance'].config_values, 'uuid': i, 'short_description': instance['short_description'], }) json.dump(instances, open(self.__persistence_file, 'w+')) def __on_mqtt_message(self, client, user_data, message: MQTTMessage): """ receives MQTT message and forward it to the right module :param client: :param user_data: :param message: :return: """ if message.topic.startswith(self.__mqtt_prefix): last = message.topic[len(self.__mqtt_prefix):] match = re.match(r'^(.*)/(.*)$', last) if match.group(1) in self.instances: t = threading.Thread(target=self.instances[match.group(1)]['instance'].on_message, args=(match.group(2), message.payload)) t.start() def add_module(self, name): """ Add module to bridge and check if the given module exists and is compatible. :param name: name of the module :return: Success state """ try: module = __import__(name) module_class = module.Module if module_class == BaseModule: self.log.error('The module class seems to be a copy of base Module in "{}"'.format(name)) if not issubclass(module.Module, BaseModule): self.log.error('Given module "{}" is not valid: Class Module is not inherit from mqttbb.modules.Module' .format(name)) for config in module_class.config: if 'name' not in config: self.log.error('Config definition error, missing name in module "{}"'.format(name)) return False if 'title' not in config: config['title'] = config['name'] if 'type' not in config: self.log.error('Config definition error, missing type in module "{}"'.format(name)) return False if name in self.available_modules: self.log.warning('Module already loaded.') return False self.available_modules[name] = module_class return True except ModuleNotFoundError: self.log.error('Module not found "{}"'.format(name)) except AttributeError: self.log.error('Given module "{}" is not valid: No class "Module" found'.format(name)) return False def add_module_instance(self, name, config, short_description="", instance_id=None): """ Add module instance to broadcast bridge :param name: name of the module :param config: config object for the module :param short_description: short description :param instance_id: id of the instance if already exists :return: void """ if not isinstance(config, dict): raise TypeError("config must be dict") if name not in self.available_modules: raise ModuleNotFoundError() if instance_id is None: instance_id = str(uuid.uuid4()) if instance_id in self.instances: raise InstanceAlreadyExists('Instance id is already registered.') def on_path_register(path): full_path = "{}{}/{}".format(self.__mqtt_prefix, instance_id, path) self.mqtt_client.subscribe(full_path) self.log.debug('Module "{}" registered topic "{}"'.format(name, full_path)) def on_publish(path, payload, retain=False): full_path = "{}{}/{}".format(self.__mqtt_prefix, instance_id, path) self.mqtt_client.publish(full_path, payload, retain=retain) self.instances[instance_id] = { 'instance': self.available_modules[name](config, on_path_register=on_path_register, on_publish=on_publish), 'short_description': short_description, 'module': name } self.__update_mqtt_service_meta() def remove_module_instance(self, uid): if uid not in self.instances: return instance = self.instances[uid]['instance'] for path in instance.paths: full_path = "{}{}/{}".format(self.__mqtt_prefix, uid, path) self.mqtt_client.unsubscribe(full_path) instance.shutdown() del self.instances[uid] self.__update_mqtt_service_meta() def __update_mqtt_service_meta(self): instances = [] for i, instance in self.instances.items(): instances.append({ 'id': i, 'name': instance['instance'].name, 'short_description':instance['short_description'] }) self.mqtt_client.publish('{}meta'.format(self.__mqtt_prefix), json.dumps(instances), retain=True) self.__update_persistence_file() def __init_http_server(self): """ initialize http server (creating routes and config) :return: void """ bootstrap_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'static/bootstrap/dist/css/bootstrap.min.css') template_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'static/templates/') view = functools.partial(bottle.jinja2_view, template_lookup=[template_dir]) @bottle.get('/css/bootstrap') def bootstrap(): return bottle.static_file(os.path.basename(bootstrap_path), os.path.dirname(bootstrap_path)) @bottle.get('/css/bootstrap.min.css.map') def bootstrap_map(): return bottle.static_file(os.path.basename(bootstrap_path)+'.map', os.path.dirname(bootstrap_path)) @bottle.get('/') @view('overview.j2', ) def index(): available_modules = [] for i, module in self.available_modules.items(): available_modules.append({ 'name': module.name, 'id': i, }) instances = [] for i, instance in self.instances.items(): instances.append({ 'short_description': instance['short_description'], 'uid': i, 'module': instance['module'], 'module_name': instance['instance'].name, }) return {'available_modules': available_modules, 'instances': instances} @bottle.get('/info/<uid>') @view('info.j2') def info(uid): if uid not in self.instances: bottle.redirect('/') instance = self.instances[uid] return { 'short_description': instance['short_description'], 'uid': uid, 'module': instance['module'], 'module_name': instance['instance'].name, 'config': instance['instance'].config, 'config_values': instance['instance'].config_values, 'paths': instance['instance'].paths.values(), 'path_prefix': '{}{}/'.format(self.__mqtt_prefix, uid), } @bottle.route('/delete/<uid>', ['GET', 'POST']) @view('delete.j2') def delete(uid): if uid not in self.instances: bottle.redirect('/') return if bottle.request.method == 'POST': self.remove_module_instance(uid) bottle.redirect('/') return instance = self.instances[uid] return { 'short_description': instance['short_description'], 'uid': uid, 'module': instance['module'], 'module_name': instance['instance'].name, } @bottle.get('/add', ['GET', 'POST']) @view('form.j2') def add(): errors = {} module = bottle.request.GET.get('module_id') if module is None or module not in self.available_modules: return bottle.redirect('/') config = self.available_modules[module].config config_values = {} short_description = "" if bottle.request.method == 'POST': form_dict = dict(bottle.request.forms) short_description = form_dict['short_description'] if short_description == '': errors['short_description'] = 'Angabe erforderlich' for element in config: if element['name'] in form_dict: config_values[element['name']] = form_dict[element['name']] try: if len(errors) == 0: self.add_module_instance(module, form_dict, short_description=short_description) bottle.redirect('/') except ConfigError as e: errors[e.config_name] = e.message return {'errors': errors, 'config': config, 'module': module, 'values': config_values, 'short_description': short_description} def http_loop(self): """ start http server as blocking loop :return: """ bottle.run(host=self.http_host, port=self.http_port)
class MqttConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() self.__log = log self.config = config self.__connector_type = connector_type self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__gateway = gateway self.__broker = config.get('broker') self.__mapping = config.get('mapping') self.__server_side_rpc = config.get('serverSideRpc') self.__service_config = { "connectRequests": None, "disconnectRequests": None } self.__attribute_updates = [] self.__get_service_config(config) self.__sub_topics = {} client_id = ''.join( random.choice(string.ascii_lowercase) for _ in range(23)) self._client = Client(client_id) self.setName( config.get( "name", self.__broker.get( "name", 'Mqtt Broker ' + ''.join( random.choice(string.ascii_lowercase) for _ in range(5))))) if "username" in self.__broker["security"]: self._client.username_pw_set(self.__broker["security"]["username"], self.__broker["security"]["password"]) if "caCert" in self.__broker["security"] or self.__broker[ "security"].get("type", "none").lower() == "tls": ca_cert = self.__broker["security"].get("caCert") private_key = self.__broker["security"].get("privateKey") cert = self.__broker["security"].get("cert") if ca_cert is None: self._client.tls_set_context( ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) else: try: self._client.tls_set(ca_certs=ca_cert, certfile=cert, keyfile=private_key, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) except Exception as e: self.__log.error( "Cannot setup connection to broker %s using SSL. Please check your configuration.\nError: %s", self.get_name(), e) self._client.tls_insecure_set(False) self._client.on_connect = self._on_connect self._client.on_message = self._on_message self._client.on_subscribe = self._on_subscribe self.__subscribes_sent = {} # For logging the subscriptions self._client.on_disconnect = self._on_disconnect self._client.on_log = self._on_log self._connected = False self.__stopped = False self.daemon = True def is_connected(self): return self._connected def open(self): self.__stopped = False self.start() def run(self): try: while not self._connected and not self.__stopped: try: self._client.connect(self.__broker['host'], self.__broker.get('port', 1883)) self._client.loop_start() if not self._connected: time.sleep(1) except Exception as e: self.__log.exception(e) time.sleep(10) except Exception as e: self.__log.exception(e) try: self.close() except Exception as e: self.__log.exception(e) while True: if self.__stopped: break else: time.sleep(1) def close(self): self.__stopped = True try: self._client.disconnect() except Exception as e: log.exception(e) self._client.loop_stop() self.__log.info('%s has been stopped.', self.get_name()) def get_name(self): return self.name def __subscribe(self, topic): message = self._client.subscribe(topic) try: self.__subscribes_sent[message[1]] = topic except Exception as e: self.__log.exception(e) def _on_connect(self, client, userdata, flags, rc, *extra_params): result_codes = { 1: "incorrect protocol version", 2: "invalid client identifier", 3: "server unavailable", 4: "bad username or password", 5: "not authorised", } if rc == 0: self._connected = True self.__log.info('%s connected to %s:%s - successfully.', self.get_name(), self.__broker["host"], self.__broker.get("port", "1883")) for mapping in self.__mapping: try: converter = None if mapping["converter"]["type"] == "custom": try: module = TBUtility.check_and_import( self.__connector_type, mapping["converter"]["extension"]) if module is not None: self.__log.debug( 'Custom converter for topic %s - found!', mapping["topicFilter"]) converter = module(mapping) else: self.__log.error( "\n\nCannot find extension module for %s topic.\n\Please check your configuration.\n", mapping["topicFilter"]) except Exception as e: self.__log.exception(e) else: converter = JsonMqttUplinkConverter(mapping) if converter is not None: regex_topic = TBUtility.topic_to_regex( mapping.get("topicFilter")) if not self.__sub_topics.get(regex_topic): self.__sub_topics[regex_topic] = [] self.__sub_topics[regex_topic].append( {converter: None}) # self._client.subscribe(TBUtility.regex_to_topic(regex_topic)) self.__subscribe(mapping["topicFilter"]) self.__log.info('Connector "%s" subscribe to %s', self.get_name(), TBUtility.regex_to_topic(regex_topic)) else: self.__log.error("Cannot find converter for %s topic", mapping["topicFilter"]) except Exception as e: self.__log.exception(e) try: for request in self.__service_config: if self.__service_config.get(request) is not None: for request_config in self.__service_config.get( request): self.__subscribe(request_config["topicFilter"]) except Exception as e: self.__log.error(e) else: if rc in result_codes: self.__log.error("%s connection FAIL with error %s %s!", self.get_name(), rc, result_codes[rc]) else: self.__log.error("%s connection FAIL with unknown error!", self.get_name()) def _on_disconnect(self, *args): self.__log.debug('"%s" was disconnected.', self.get_name()) def _on_log(self, *args): self.__log.debug(args) # pass def _on_subscribe(self, client, userdata, mid, granted_qos): try: if granted_qos[0] == 128: self.__log.error( '"%s" subscription failed to topic %s subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) else: self.__log.info( '"%s" subscription success to topic %s, subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) if self.__subscribes_sent.get(mid) is not None: del self.__subscribes_sent[mid] except Exception as e: self.__log.exception(e) def __get_service_config(self, config): for service_config in self.__service_config: if service_config != "attributeUpdates" and config.get( service_config): self.__service_config[service_config] = config[service_config] else: self.__attribute_updates = config[service_config] def _on_message(self, client, userdata, message): self.statistics['MessagesReceived'] += 1 content = TBUtility.decode(message) regex_topic = [ regex for regex in self.__sub_topics if fullmatch(regex, message.topic) ] if regex_topic: try: for regex in regex_topic: if self.__sub_topics.get(regex): for converter_value in range( len(self.__sub_topics.get(regex))): if self.__sub_topics[regex][converter_value]: for converter in self.__sub_topics.get( regex)[converter_value]: converted_content = converter.convert( message.topic, content) if converted_content: try: self.__sub_topics[regex][ converter_value][ converter] = converted_content except Exception as e: self.__log.exception(e) self.__gateway.send_to_storage( self.name, converted_content) self.statistics['MessagesSent'] += 1 else: continue else: self.__log.error( 'Cannot find converter for topic:"%s"!', message.topic) return except Exception as e: log.exception(e) return elif self.__service_config.get("connectRequests"): connect_requests = [ connect_request for connect_request in self.__service_config.get("connectRequests") ] if connect_requests: for request in connect_requests: if request.get("topicFilter"): if message.topic in request.get("topicFilter") or\ (request.get("deviceNameTopicExpression") is not None and search(request.get("deviceNameTopicExpression"), message.topic)): founded_device_name = None if request.get("deviceNameJsonExpression"): founded_device_name = TBUtility.get_value( request["deviceNameJsonExpression"], content) if request.get("deviceNameTopicExpression"): device_name_expression = request[ "deviceNameTopicExpression"] founded_device_name = search( device_name_expression, message.topic) if founded_device_name is not None and founded_device_name not in self.__gateway.get_devices( ): self.__gateway.add_device( founded_device_name, {"connector": self}) else: self.__log.error( "Cannot find connect request for device from message from topic: %s and with data: %s", message.topic, content) else: self.__log.error( "\"topicFilter\" in connect requests config not found." ) else: self.__log.error("Connection requests in config not found.") elif self.__service_config.get("disconnectRequests") is not None: disconnect_requests = [ disconnect_request for disconnect_request in self.__service_config.get("disconnectRequests") ] if disconnect_requests: for request in disconnect_requests: if request.get("topicFilter") is not None: if message.topic in request.get("topicFilter") or\ (request.get("deviceNameTopicExpression") is not None and search(request.get("deviceNameTopicExpression"), message.topic)): founded_device_name = None if request.get("deviceNameJsonExpression"): founded_device_name = TBUtility.get_value( request["deviceNameJsonExpression"], content) if request.get("deviceNameTopicExpression"): device_name_expression = request[ "deviceNameTopicExpression"] founded_device_name = search( device_name_expression, message.topic) if founded_device_name is not None and founded_device_name in self.__gateway.get_devices( ): self.__gateway.del_device(founded_device_name) else: self.__log.error( "Cannot find connect request for device from message from topic: %s and with data: %s", message.topic, content) else: self.__log.error( "\"topicFilter\" in connect requests config not found." ) else: self.__log.error("Disconnection requests in config not found.") elif message.topic in self.__gateway.rpc_requests_in_progress: self.__gateway.rpc_with_reply_processing(message.topic, content) else: self.__log.debug( "Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"", message.topic, content) def on_attributes_update(self, content): attribute_updates_config = [ update for update in self.__attribute_updates ] if attribute_updates_config: for attribute_update in attribute_updates_config: if match(attribute_update["deviceNameFilter"], content["device"]) and \ content["data"].get(attribute_update["attributeFilter"]): topic = attribute_update["topicExpression"]\ .replace("${deviceName}", content["device"])\ .replace("${attributeKey}", attribute_update["attributeFilter"])\ .replace("${attributeValue}", content["data"][attribute_update["attributeFilter"]]) data = '' try: data = attribute_update["valueExpression"]\ .replace("${attributeKey}", attribute_update["attributeFilter"])\ .replace("${attributeValue}", content["data"][attribute_update["attributeFilter"]]) except Exception as e: self.__log.error(e) self._client.publish(topic, data).wait_for_publish() self.__log.debug( "Attribute Update data: %s for device %s to topic: %s", data, content["device"], topic) else: self.__log.error( "Not found deviceName by filter in message or attributeFilter in message with data: %s", content) else: self.__log.error("Attribute updates config not found.") def server_side_rpc_handler(self, content): for rpc_config in self.__server_side_rpc: if search(rpc_config["deviceNameFilter"], content["device"]) \ and search(rpc_config["methodFilter"], content["data"]["method"]) is not None: # Subscribe to RPC response topic if rpc_config.get("responseTopicExpression"): topic_for_subscribe = rpc_config["responseTopicExpression"] \ .replace("${deviceName}", content["device"]) \ .replace("${methodName}", content["data"]["method"]) \ .replace("${requestId}", str(content["data"]["id"])) \ .replace("${params}", content["data"]["params"]) if rpc_config.get("responseTimeout"): timeout = time.time() * 1000 + rpc_config.get( "responseTimeout") self.__gateway.register_rpc_request_timeout( content, timeout, topic_for_subscribe, self.rpc_cancel_processing) # Maybe we need to wait for the command to execute successfully before publishing the request. self._client.subscribe(topic_for_subscribe) else: self.__log.error( "Not found RPC response timeout in config, sending without waiting for response" ) # Publish RPC request if rpc_config.get("requestTopicExpression") is not None\ and rpc_config.get("valueExpression"): topic = rpc_config.get("requestTopicExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) data_to_send = rpc_config.get("valueExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) try: self._client.publish(topic, data_to_send) self.__log.debug( "Send RPC with no response request to topic: %s with data %s", topic, data_to_send) if rpc_config.get("responseTopicExpression") is None: self.__gateway.send_rpc_reply( device=content["device"], req_id=content["data"]["id"], success_sent=True) except Exception as e: self.__log.exception(e) def rpc_cancel_processing(self, topic): self._client.unsubscribe(topic)
class Service: """Service that sends messages via Telegram bot""" _mqtt_client: Dict[str, Client] = {} _service_list: Dict[str, dict] = {} _broker: DefaultDict[str, set] = defaultdict(set) _broker_port: Dict[str, int] = {} _topic: DefaultDict[str, set] = defaultdict(set) _update_thread: Timer = None chat_id_topic = "labsw4/telegram/user/chat_id" _user_list: Dict[str, dict] = {} # Telegram bot: Bot = Bot(token=TELEGRAM_TOKEN) chat_id_list = set() def __init__(self): """ Instantiate the service """ self.service = Client( client_id=f"TelegramService{randrange(1, 100000)}") self.service.on_message = self.my_on_message self.char_id_lock = Lock() self.service_lock = Lock() def _update(self, service_list: List[dict]): """ Update reserved dicts :param service_list: service list to add """ for alarm in service_list: service = alarm["serviceID"] broker = alarm["end_points"]["MQTT"]["broker"]["ip"] port = alarm["end_points"]["MQTT"]["broker"]["port"] topics = { topic for topic in alarm["end_points"]["MQTT"]["subscribe"] if "alarm_temperature" in topic } self._service_list[service] = {"ip": broker, "topics": topics} self._broker_port[broker] = port self._broker[broker].update({service}) self._topic[broker].update(topics) print(f"[{time.ctime()}] SERVICE {service} ONLINE") def setup(self, first_time: bool = True): """ Setup the service """ try: print( f"[{time.ctime()}] EXTRACT info about all the services registered" ) result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/services/all" ) while result.status_code != 200: print( f"[{time.ctime()}] WARNING no service registered found, retrying after 30 seconds" ) time.sleep(30) result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}" f"/catalog/services/all") print(f"[{time.ctime()}] SERVICES found") data = json.loads(result.content.decode()) print( f"[{time.ctime()}] EXTRACT from the services list info about Alarm_Temperature" ) service_list = [alarm for alarm in data if find_service(alarm)] if len(service_list) == 0: print( f"[{time.ctime()}] No ServiceTemperatureAlarm found... retrying in 10 seconds" ) time.sleep(10) self.setup(first_time) return except KeyboardInterrupt: print(f"[{time.ctime()}] EXIT") if first_time: sys.exit() return if first_time: # Connect the service self.service.connect(host=SERVICE_BROKER_PORT["ip"], port=SERVICE_BROKER_PORT["port"]) print(f"[{time.ctime()}] SERVICE CONNECTED to " f"broker: {SERVICE_BROKER_PORT['ip']} " f"on port: port={SERVICE_BROKER_PORT['port']}") self.service.subscribe(self.chat_id_topic) # Update registered services and subscribe to all the topics with self.service_lock: self._update(service_list) self.subscribe(service_list) # Register inside the catalog print( f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}") requests.post( f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services', data=SERVICE_INFO, headers={"Content-Type": "application/json"}) def start(self): """ Start the service. """ # Setup the service self.setup() # Start all the external mqtt_clients for broker in self._mqtt_client: self._mqtt_client[broker].loop_start() # Schedule the update of the registration self._update_thread = Timer(60, self.update_registration) self._update_thread.start() try: # Run the service forever self.service.loop_forever() except KeyboardInterrupt: self.stop() def my_on_message(self, client: Client, userdata: Any, msg: MQTTMessage): """ Receive new chat ids. Receive alarm status and check if it is true. If so, send messages via Telegram Bot. :param client: MQTT client :param userdata: They could be any type :param msg: MQTT message """ data = json.loads(msg.payload.decode()) if msg.topic == self.chat_id_topic: with self.char_id_lock: self.chat_id_list.add(data["chat_id"]) return if data["alarm"]: message_body = f"{data['device']} is out of range of good functioning{SIGNATURE}" with self.char_id_lock: for chat_id in self.chat_id_list: self.bot.sendMessage(chat_id=chat_id, text=message_body) print( f"[{time.ctime()}] TELEGRAM MESSAGE SENT TO CHAT_ID: {chat_id}" ) else: return def update_registration(self): """ Update service registration to the catalog """ print( f"[{time.ctime()}] EXTRACT info about all the services registered") result: requests.Response = requests.get( url= f"http://{CATALOG_IP_PORT['ip']}:{CATALOG_IP_PORT['port']}/catalog/services/all" ) # No device found if result.status_code != 200: self.reset() return data = json.loads(result.content.decode()) service_list = [alarm for alarm in data if find_service(alarm)] if len(service_list) == 0: self.reset() return # New Devices Found new_services = {alarm["serviceID"] for alarm in service_list} # Old devices list old_services = set(self._service_list.keys()) # Check if there are update to do if old_services == new_services: # No update, ping the catalog print( f"[{time.ctime()}] PING the Catalog on : {CATALOG_IP_PORT['ip']}" ) requests.post( f'http://{CATALOG_IP_PORT["ip"]}:{CATALOG_IP_PORT["port"]}/catalog/services', data=SERVICE_INFO, headers={"Content-Type": "application/json"}) self._update_thread = Timer(60, self.update_registration) self._update_thread.start() return with self.service_lock: # Delete inactive devices delete_list = { alarm: self._service_list[alarm] for alarm in self._service_list if alarm not in new_services } # Unsubscribe from devices self.unsubscribe(delete_list) # Update registered devices and subscribe new_services = [ alarm for alarm in service_list if alarm["serviceID"] not in old_services ] self._update(new_services) self.subscribe(new_services) self._update_thread = Timer(60, self.update_registration) self._update_thread.start() def reset(self): """ Reset the service """ with self.service_lock: self._clear() self.setup(first_time=False) # Start all the external mqtt_clients for broker in self._mqtt_client: self._mqtt_client[broker].loop_start() self._update_thread = Timer(60, self.update_registration) self._update_thread.start() def unsubscribe(self, service_list: dict): """ Unsubscribe from services :param service_list: service list """ for alarm in service_list: print(f"[{time.ctime()}] SERVICE {alarm} OFFLINE") broker = service_list[alarm]["ip"] topics = service_list[alarm]["topics"] if broker == SERVICE_BROKER_PORT["ip"]: for topic in topics: self.service.unsubscribe(topic) print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}") else: for topic in topics: self._mqtt_client[broker].unsubscribe(topic) print(f"[{time.ctime()}] UNSUBSCRIBED from : {topic}") # Update topics self._topic[broker] = self._topic[broker].difference(topics) # Disconnect from external broker if len(self._topic[broker]) == 0: if broker != SERVICE_BROKER_PORT["ip"]: self._mqtt_client[broker].disconnect() self._mqtt_client[broker].loop_stop() print( f"[{time.ctime()}] DISCONNECTED from broker: {broker}") del self._mqtt_client[broker] # Clean del self._topic[broker] del self._broker_port[broker] # Delete device del self._service_list[alarm] self._broker[broker].discard(alarm) def subscribe(self, service_list: List[dict]): """ Subscribe to services :param service_list: list of services """ for alarm in service_list: broker = alarm["end_points"]["MQTT"]["broker"]["ip"] topics = { topic for topic in alarm["end_points"]["MQTT"]["subscribe"] } # Subscribe to topics if broker == SERVICE_BROKER_PORT["ip"]: for topic in topics: self.service.subscribe(topic) print(f"[{time.ctime()}] SUBSCRIBED to : {topic}") # Connect to external brokers else: if broker in self._mqtt_client: # Connection already made for topic in topics: self._mqtt_client[broker].subscribe(topic) print(f"[{time.ctime()}] SUBSCRIBED to : {topic}") else: # Connect to the broker self._mqtt_client[broker] = Client( f"EmailAlarmService{randrange(1, 1000000)}") self._mqtt_client[broker].on_message = self.my_on_message self._mqtt_client[broker].connect( host=broker, port=self._broker_port[broker]) for topic in topics: self._mqtt_client[broker].subscribe(topic) print(f"[{time.ctime()}] SUBSCRIBED to : {topic}") def _clear(self): """ Clear all the data stored """ # Unsubscribe from all topics self.unsubscribe(self._service_list.copy()) # Disconnect from all the external brokers for broker in self._mqtt_client: self._mqtt_client[broker].disconnect() self._mqtt_client[broker].loop_stop() print(f"[{time.ctime()}] DISCONNECTED from broker: {broker}") # Clear self._mqtt_client.clear() self._service_list.clear() self._broker.clear() self._broker_port.clear() self._topic.clear() def stop(self): """ Stop the service """ # Stop the schedule self._update_thread.cancel() # Wait for the threads to finish if self._update_thread.is_alive(): self._update_thread.join() # Disconnect the clients print(f"[{time.ctime()}] SHUTTING DOWN") for client in self._mqtt_client: self._mqtt_client[client].disconnect() self._mqtt_client[client].loop_stop() # Disconnect the service self.service.disconnect() print(f"[{time.ctime()}] EXIT")