def mqtt_handler(): global mqtt_client Client.connected_flag = False mqtt_client = Client() # set mosquitto broker password and username mqtt_client.username_pw_set(username=USERNAME, password=PASSWORD) # set TLS cert for the client mqtt_client.tls_set(ca_certs=TLS_CERT) mqtt_client.tls_insecure_set(True) mqtt_client.on_connect = on_connect mqtt_client.on_message = on_message mqtt_client.loop_start() mqtt_client.connect(host=MQTT_ADDR, port=MQTT_PRT) while not mqtt_client.connected_flag: # wait in loop print("In wait loop") time.sleep(1) mqtt_client.subscribe(topic='%s/+' % ORDER_STATUS) mqtt_client.loop_forever() mqtt_client.disconnect()
def _get_client(self, tls_cafile: Optional[str] = None, tls_certfile: Optional[str] = None, tls_keyfile: Optional[str] = None, tls_version: Optional[str] = None, tls_ciphers: Optional[str] = None, tls_insecure: Optional[bool] = None, username: Optional[str] = None, password: Optional[str] = None): from paho.mqtt.client import Client tls_cafile = self._expandpath(tls_cafile or self.tls_cafile) tls_certfile = self._expandpath(tls_certfile or self.tls_certfile) tls_keyfile = self._expandpath(tls_keyfile or self.tls_keyfile) tls_ciphers = tls_ciphers or self.tls_ciphers username = username or self.username password = password or self.password tls_version = tls_version or self.tls_version if tls_version: tls_version = self.get_tls_version(tls_version) if tls_insecure is None: tls_insecure = self.tls_insecure client = Client() if username and password: client.username_pw_set(username, password) if tls_cafile: client.tls_set(ca_certs=tls_cafile, certfile=tls_certfile, keyfile=tls_keyfile, tls_version=tls_version, ciphers=tls_ciphers) client.tls_insecure_set(tls_insecure) return client
class MqttConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() self.__log = log self.config = config self.__connector_type = connector_type self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__gateway = gateway self.__broker = config.get('broker') self.__mapping = config.get('mapping') self.__server_side_rpc = config.get('serverSideRpc') self.__service_config = { "connectRequests": None, "disconnectRequests": None } self.__attribute_updates = [] self.__get_service_config(config) self.__sub_topics = {} client_id = ''.join( random.choice(string.ascii_lowercase) for _ in range(23)) self._client = Client(client_id) self.setName( config.get( "name", self.__broker.get( "name", 'Mqtt Broker ' + ''.join( random.choice(string.ascii_lowercase) for _ in range(5))))) if "username" in self.__broker["security"]: self._client.username_pw_set(self.__broker["security"]["username"], self.__broker["security"]["password"]) if "caCert" in self.__broker["security"] or self.__broker[ "security"].get("type", "none").lower() == "tls": ca_cert = self.__broker["security"].get("caCert") private_key = self.__broker["security"].get("privateKey") cert = self.__broker["security"].get("cert") if ca_cert is None: self._client.tls_set_context( ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) else: try: self._client.tls_set(ca_certs=ca_cert, certfile=cert, keyfile=private_key, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) except Exception as e: self.__log.error( "Cannot setup connection to broker %s using SSL. Please check your configuration.\nError: %s", self.get_name(), e) self._client.tls_insecure_set(False) self._client.on_connect = self._on_connect self._client.on_message = self._on_message self._client.on_subscribe = self._on_subscribe self.__subscribes_sent = {} # For logging the subscriptions self._client.on_disconnect = self._on_disconnect self._client.on_log = self._on_log self._connected = False self.__stopped = False self.daemon = True def is_connected(self): return self._connected def open(self): self.__stopped = False self.start() def run(self): try: while not self._connected and not self.__stopped: try: self._client.connect(self.__broker['host'], self.__broker.get('port', 1883)) self._client.loop_start() if not self._connected: time.sleep(1) except Exception as e: self.__log.exception(e) time.sleep(10) except Exception as e: self.__log.exception(e) try: self.close() except Exception as e: self.__log.exception(e) while True: if self.__stopped: break else: time.sleep(1) def close(self): self.__stopped = True try: self._client.disconnect() except Exception as e: log.exception(e) self._client.loop_stop() self.__log.info('%s has been stopped.', self.get_name()) def get_name(self): return self.name def __subscribe(self, topic): message = self._client.subscribe(topic) try: self.__subscribes_sent[message[1]] = topic except Exception as e: self.__log.exception(e) def _on_connect(self, client, userdata, flags, rc, *extra_params): result_codes = { 1: "incorrect protocol version", 2: "invalid client identifier", 3: "server unavailable", 4: "bad username or password", 5: "not authorised", } if rc == 0: self._connected = True self.__log.info('%s connected to %s:%s - successfully.', self.get_name(), self.__broker["host"], self.__broker.get("port", "1883")) for mapping in self.__mapping: try: converter = None if mapping["converter"]["type"] == "custom": try: module = TBUtility.check_and_import( self.__connector_type, mapping["converter"]["extension"]) if module is not None: self.__log.debug( 'Custom converter for topic %s - found!', mapping["topicFilter"]) converter = module(mapping) else: self.__log.error( "\n\nCannot find extension module for %s topic.\n\Please check your configuration.\n", mapping["topicFilter"]) except Exception as e: self.__log.exception(e) else: converter = JsonMqttUplinkConverter(mapping) if converter is not None: regex_topic = TBUtility.topic_to_regex( mapping.get("topicFilter")) if not self.__sub_topics.get(regex_topic): self.__sub_topics[regex_topic] = [] self.__sub_topics[regex_topic].append( {converter: None}) # self._client.subscribe(TBUtility.regex_to_topic(regex_topic)) self.__subscribe(mapping["topicFilter"]) self.__log.info('Connector "%s" subscribe to %s', self.get_name(), TBUtility.regex_to_topic(regex_topic)) else: self.__log.error("Cannot find converter for %s topic", mapping["topicFilter"]) except Exception as e: self.__log.exception(e) try: for request in self.__service_config: if self.__service_config.get(request) is not None: for request_config in self.__service_config.get( request): self.__subscribe(request_config["topicFilter"]) except Exception as e: self.__log.error(e) else: if rc in result_codes: self.__log.error("%s connection FAIL with error %s %s!", self.get_name(), rc, result_codes[rc]) else: self.__log.error("%s connection FAIL with unknown error!", self.get_name()) def _on_disconnect(self, *args): self.__log.debug('"%s" was disconnected.', self.get_name()) def _on_log(self, *args): self.__log.debug(args) # pass def _on_subscribe(self, client, userdata, mid, granted_qos): try: if granted_qos[0] == 128: self.__log.error( '"%s" subscription failed to topic %s subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) else: self.__log.info( '"%s" subscription success to topic %s, subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) if self.__subscribes_sent.get(mid) is not None: del self.__subscribes_sent[mid] except Exception as e: self.__log.exception(e) def __get_service_config(self, config): for service_config in self.__service_config: if service_config != "attributeUpdates" and config.get( service_config): self.__service_config[service_config] = config[service_config] else: self.__attribute_updates = config[service_config] def _on_message(self, client, userdata, message): self.statistics['MessagesReceived'] += 1 content = TBUtility.decode(message) regex_topic = [ regex for regex in self.__sub_topics if fullmatch(regex, message.topic) ] if regex_topic: try: for regex in regex_topic: if self.__sub_topics.get(regex): for converter_value in range( len(self.__sub_topics.get(regex))): if self.__sub_topics[regex][converter_value]: for converter in self.__sub_topics.get( regex)[converter_value]: converted_content = converter.convert( message.topic, content) if converted_content: try: self.__sub_topics[regex][ converter_value][ converter] = converted_content except Exception as e: self.__log.exception(e) self.__gateway.send_to_storage( self.name, converted_content) self.statistics['MessagesSent'] += 1 else: continue else: self.__log.error( 'Cannot find converter for topic:"%s"!', message.topic) return except Exception as e: log.exception(e) return elif self.__service_config.get("connectRequests"): connect_requests = [ connect_request for connect_request in self.__service_config.get("connectRequests") ] if connect_requests: for request in connect_requests: if request.get("topicFilter"): if message.topic in request.get("topicFilter") or\ (request.get("deviceNameTopicExpression") is not None and search(request.get("deviceNameTopicExpression"), message.topic)): founded_device_name = None if request.get("deviceNameJsonExpression"): founded_device_name = TBUtility.get_value( request["deviceNameJsonExpression"], content) if request.get("deviceNameTopicExpression"): device_name_expression = request[ "deviceNameTopicExpression"] founded_device_name = search( device_name_expression, message.topic) if founded_device_name is not None and founded_device_name not in self.__gateway.get_devices( ): self.__gateway.add_device( founded_device_name, {"connector": self}) else: self.__log.error( "Cannot find connect request for device from message from topic: %s and with data: %s", message.topic, content) else: self.__log.error( "\"topicFilter\" in connect requests config not found." ) else: self.__log.error("Connection requests in config not found.") elif self.__service_config.get("disconnectRequests") is not None: disconnect_requests = [ disconnect_request for disconnect_request in self.__service_config.get("disconnectRequests") ] if disconnect_requests: for request in disconnect_requests: if request.get("topicFilter") is not None: if message.topic in request.get("topicFilter") or\ (request.get("deviceNameTopicExpression") is not None and search(request.get("deviceNameTopicExpression"), message.topic)): founded_device_name = None if request.get("deviceNameJsonExpression"): founded_device_name = TBUtility.get_value( request["deviceNameJsonExpression"], content) if request.get("deviceNameTopicExpression"): device_name_expression = request[ "deviceNameTopicExpression"] founded_device_name = search( device_name_expression, message.topic) if founded_device_name is not None and founded_device_name in self.__gateway.get_devices( ): self.__gateway.del_device(founded_device_name) else: self.__log.error( "Cannot find connect request for device from message from topic: %s and with data: %s", message.topic, content) else: self.__log.error( "\"topicFilter\" in connect requests config not found." ) else: self.__log.error("Disconnection requests in config not found.") elif message.topic in self.__gateway.rpc_requests_in_progress: self.__gateway.rpc_with_reply_processing(message.topic, content) else: self.__log.debug( "Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"", message.topic, content) def on_attributes_update(self, content): attribute_updates_config = [ update for update in self.__attribute_updates ] if attribute_updates_config: for attribute_update in attribute_updates_config: if match(attribute_update["deviceNameFilter"], content["device"]) and \ content["data"].get(attribute_update["attributeFilter"]): topic = attribute_update["topicExpression"]\ .replace("${deviceName}", content["device"])\ .replace("${attributeKey}", attribute_update["attributeFilter"])\ .replace("${attributeValue}", content["data"][attribute_update["attributeFilter"]]) data = '' try: data = attribute_update["valueExpression"]\ .replace("${attributeKey}", attribute_update["attributeFilter"])\ .replace("${attributeValue}", content["data"][attribute_update["attributeFilter"]]) except Exception as e: self.__log.error(e) self._client.publish(topic, data).wait_for_publish() self.__log.debug( "Attribute Update data: %s for device %s to topic: %s", data, content["device"], topic) else: self.__log.error( "Not found deviceName by filter in message or attributeFilter in message with data: %s", content) else: self.__log.error("Attribute updates config not found.") def server_side_rpc_handler(self, content): for rpc_config in self.__server_side_rpc: if search(rpc_config["deviceNameFilter"], content["device"]) \ and search(rpc_config["methodFilter"], content["data"]["method"]) is not None: # Subscribe to RPC response topic if rpc_config.get("responseTopicExpression"): topic_for_subscribe = rpc_config["responseTopicExpression"] \ .replace("${deviceName}", content["device"]) \ .replace("${methodName}", content["data"]["method"]) \ .replace("${requestId}", str(content["data"]["id"])) \ .replace("${params}", content["data"]["params"]) if rpc_config.get("responseTimeout"): timeout = time.time() * 1000 + rpc_config.get( "responseTimeout") self.__gateway.register_rpc_request_timeout( content, timeout, topic_for_subscribe, self.rpc_cancel_processing) # Maybe we need to wait for the command to execute successfully before publishing the request. self._client.subscribe(topic_for_subscribe) else: self.__log.error( "Not found RPC response timeout in config, sending without waiting for response" ) # Publish RPC request if rpc_config.get("requestTopicExpression") is not None\ and rpc_config.get("valueExpression"): topic = rpc_config.get("requestTopicExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) data_to_send = rpc_config.get("valueExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) try: self._client.publish(topic, data_to_send) self.__log.debug( "Send RPC with no response request to topic: %s with data %s", topic, data_to_send) if rpc_config.get("responseTopicExpression") is None: self.__gateway.send_rpc_reply( device=content["device"], req_id=content["data"]["id"], success_sent=True) except Exception as e: self.__log.exception(e) def rpc_cancel_processing(self, topic): self._client.unsubscribe(topic)
class MqttConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() self.__gateway = gateway # Reference to TB Gateway self._connector_type = connector_type # Should be "mqtt" self.config = config # mqtt.json contents self.__log = log self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__subscribes_sent = {} # Extract main sections from configuration --------------------------------------------------------------------- self.__broker = config.get('broker') self.__mapping = [] self.__server_side_rpc = [] self.__connect_requests = [] self.__disconnect_requests = [] self.__attribute_requests = [] self.__attribute_updates = [] mandatory_keys = { "mapping": ['topicFilter', 'converter'], "serverSideRpc": [ 'deviceNameFilter', 'methodFilter', 'requestTopicExpression', 'valueExpression' ], "connectRequests": ['topicFilter'], "disconnectRequests": ['topicFilter'], "attributeRequests": ['topicFilter', 'topicExpression', 'valueExpression'], "attributeUpdates": [ 'deviceNameFilter', 'attributeFilter', 'topicExpression', 'valueExpression' ] } # Mappings, i.e., telemetry/attributes-push handlers provided by user via configuration file self.load_handlers('mapping', mandatory_keys['mapping'], self.__mapping) # RPCs, i.e., remote procedure calls (ThingsBoard towards devices) handlers self.load_handlers('serverSideRpc', mandatory_keys['serverSideRpc'], self.__server_side_rpc) # Connect requests, i.e., telling ThingsBoard that a device is online even if it does not post telemetry self.load_handlers('connectRequests', mandatory_keys['connectRequests'], self.__connect_requests) # Disconnect requests, i.e., telling ThingsBoard that a device is offline even if keep-alive has not expired yet self.load_handlers('disconnectRequests', mandatory_keys['disconnectRequests'], self.__disconnect_requests) # Shared attributes direct requests, i.e., asking ThingsBoard for some shared attribute value self.load_handlers('attributeRequests', mandatory_keys['attributeRequests'], self.__attribute_requests) # Attributes updates requests, i.e., asking ThingsBoard to send updates about an attribute self.load_handlers('attributeUpdates', mandatory_keys['attributeUpdates'], self.__attribute_updates) # Setup topic substitution lists for each class of handlers ---------------------------------------------------- self.__mapping_sub_topics = {} self.__connect_requests_sub_topics = {} self.__disconnect_requests_sub_topics = {} self.__attribute_requests_sub_topics = {} # Set up external MQTT broker connection ----------------------------------------------------------------------- client_id = self.__broker.get( "clientId", ''.join(random.choice(string.ascii_lowercase) for _ in range(23))) self._client = Client(client_id) self.setName( config.get( "name", self.__broker.get( "name", 'Mqtt Broker ' + ''.join( random.choice(string.ascii_lowercase) for _ in range(5))))) if "username" in self.__broker["security"]: self._client.username_pw_set(self.__broker["security"]["username"], self.__broker["security"]["password"]) if "caCert" in self.__broker["security"] \ or self.__broker["security"].get("type", "none").lower() == "tls": ca_cert = self.__broker["security"].get("caCert") private_key = self.__broker["security"].get("privateKey") cert = self.__broker["security"].get("cert") if ca_cert is None: self._client.tls_set_context( ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) else: try: self._client.tls_set(ca_certs=ca_cert, certfile=cert, keyfile=private_key, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) except Exception as e: self.__log.error( "Cannot setup connection to broker %s using SSL. " "Please check your configuration.\nError: ", self.get_name()) self.__log.exception(e) if self.__broker["security"].get("insecure", False): self._client.tls_insecure_set(True) else: self._client.tls_insecure_set(False) # Set up external MQTT broker callbacks ------------------------------------------------------------------------ self._client.on_connect = self._on_connect self._client.on_message = self._on_message self._client.on_subscribe = self._on_subscribe self._client.on_disconnect = self._on_disconnect # self._client.on_log = self._on_log # Set up lifecycle flags --------------------------------------------------------------------------------------- self._connected = False self.__stopped = False self.daemon = True self.__msg_queue = Queue() self.__workers_thread_pool = [] self.__max_msg_number_for_worker = config.get( 'maxMessageNumberPerWorker', 10) self.__max_number_of_workers = config.get('maxNumberOfWorkers', 100) def load_handlers(self, handler_flavor, mandatory_keys, accepted_handlers_list): if handler_flavor not in self.config: self.__log.error("'%s' section missing from configuration", handler_flavor) else: for handler in self.config.get(handler_flavor): discard = False for key in mandatory_keys: if key not in handler: # Will report all missing fields to user before discarding the entry => no break here discard = True self.__log.error( "Mandatory key '%s' missing from %s handler: %s", key, handler_flavor, simplejson.dumps(handler)) else: self.__log.debug( "Mandatory key '%s' found in %s handler: %s", key, handler_flavor, simplejson.dumps(handler)) if discard: self.__log.error( "%s handler is missing some mandatory keys => rejected: %s", handler_flavor, simplejson.dumps(handler)) else: accepted_handlers_list.append(handler) self.__log.debug( "%s handler has all mandatory keys => accepted: %s", handler_flavor, simplejson.dumps(handler)) self.__log.info("Number of accepted %s handlers: %d", handler_flavor, len(accepted_handlers_list)) self.__log.info( "Number of rejected %s handlers: %d", handler_flavor, len(self.config.get(handler_flavor)) - len(accepted_handlers_list)) def is_connected(self): return self._connected def open(self): self.__stopped = False self.start() def run(self): try: self.__connect() except Exception as e: self.__log.exception(e) try: self.close() except Exception as e: self.__log.exception(e) while True: if self.__stopped: break elif not self._connected: self.__connect() self.__threads_manager() sleep(.2) def __connect(self): while not self._connected and not self.__stopped: try: self._client.connect(self.__broker['host'], self.__broker.get('port', 1883)) self._client.loop_start() if not self._connected: sleep(1) except ConnectionRefusedError as e: self.__log.error(e) sleep(10) def close(self): self.__stopped = True try: self._client.disconnect() except Exception as e: log.exception(e) self._client.loop_stop() self.__log.info('%s has been stopped.', self.get_name()) def get_name(self): return self.name def __subscribe(self, topic, qos): message = self._client.subscribe(topic, qos) try: self.__subscribes_sent[message[1]] = topic except Exception as e: self.__log.exception(e) def _on_connect(self, client, userdata, flags, result_code, *extra_params): result_codes = { 1: "incorrect protocol version", 2: "invalid client identifier", 3: "server unavailable", 4: "bad username or password", 5: "not authorised", } if result_code == 0: self._connected = True self.__log.info('%s connected to %s:%s - successfully.', self.get_name(), self.__broker["host"], self.__broker.get("port", "1883")) self.__log.debug( "Client %s, userdata %s, flags %s, extra_params %s", str(client), str(userdata), str(flags), extra_params) self.__mapping_sub_topics = {} # Setup data upload requests handling ---------------------------------------------------------------------- for mapping in self.__mapping: try: # Load converter for this mapping entry ------------------------------------------------------------ # mappings are guaranteed to have topicFilter and converter fields. See __init__ default_converter_class_name = "JsonMqttUplinkConverter" # Get converter class from "extension" parameter or default converter converter_class_name = mapping["converter"].get( "extension", default_converter_class_name) # Find and load required class module = TBModuleLoader.import_module( self._connector_type, converter_class_name) if module: self.__log.debug('Converter %s for topic %s - found!', converter_class_name, mapping["topicFilter"]) converter = module(mapping) else: self.__log.error("Cannot find converter for %s topic", mapping["topicFilter"]) continue # Setup regexp topic acceptance list --------------------------------------------------------------- regex_topic = TBUtility.topic_to_regex( mapping["topicFilter"]) # There may be more than one converter per topic, so I'm using vectors if not self.__mapping_sub_topics.get(regex_topic): self.__mapping_sub_topics[regex_topic] = [] self.__mapping_sub_topics[regex_topic].append(converter) # Subscribe to appropriate topic ------------------------------------------------------------------- self.__subscribe(mapping["topicFilter"], mapping.get("subscriptionQos", 1)) self.__log.info('Connector "%s" subscribe to %s', self.get_name(), TBUtility.regex_to_topic(regex_topic)) except Exception as e: self.__log.exception(e) # Setup connection requests handling ----------------------------------------------------------------------- for request in [ entry for entry in self.__connect_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 1)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__connect_requests_sub_topics[topic_filter] = request # Setup disconnection requests handling -------------------------------------------------------------------- for request in [ entry for entry in self.__disconnect_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 1)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__disconnect_requests_sub_topics[topic_filter] = request # Setup attributes requests handling ----------------------------------------------------------------------- for request in [ entry for entry in self.__attribute_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 1)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__attribute_requests_sub_topics[topic_filter] = request else: if result_code in result_codes: self.__log.error("%s connection FAIL with error %s %s!", self.get_name(), result_code, result_codes[result_code]) else: self.__log.error("%s connection FAIL with unknown error!", self.get_name()) def _on_disconnect(self, *args): self._connected = False self.__log.debug('"%s" was disconnected. %s', self.get_name(), str(args)) def _on_log(self, *args): self.__log.debug(args) def _on_subscribe(self, _, __, mid, granted_qos, *args): log.info(args) try: if granted_qos[0] == 128: self.__log.error( '"%s" subscription failed to topic %s subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) else: self.__log.info( '"%s" subscription success to topic %s, subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) except Exception as e: self.__log.exception(e) # Success or not, remove this topic from the list of pending subscription requests if self.__subscribes_sent.get(mid) is not None: del self.__subscribes_sent[mid] def put_data_to_convert(self, converter, message, content) -> bool: if not self.__msg_queue.full(): self.__msg_queue.put((converter.convert, message.topic, content), True, 100) return True return False def _save_converted_msg(self, topic, data): self.__gateway.send_to_storage(self.name, data) self.statistics['MessagesSent'] += 1 self.__log.debug("Successfully converted message from topic %s", topic) def __threads_manager(self): if len(self.__workers_thread_pool) == 0: worker = MqttConnector.ConverterWorker("Main", self.__msg_queue, self._save_converted_msg) self.__workers_thread_pool.append(worker) worker.start() number_of_needed_threads = round( self.__msg_queue.qsize() / self.__max_msg_number_for_worker, 0) threads_count = len(self.__workers_thread_pool) if number_of_needed_threads > threads_count < self.__max_number_of_workers: thread = MqttConnector.ConverterWorker( "Worker " + ''.join( random.choice(string.ascii_lowercase) for _ in range(5)), self.__msg_queue, self._save_converted_msg) self.__workers_thread_pool.append(thread) thread.start() elif number_of_needed_threads < threads_count and threads_count > 1: worker: MqttConnector.ConverterWorker = self.__workers_thread_pool[ -1] if not worker.in_progress: worker.stopped = True self.__workers_thread_pool.remove(worker) def _on_message(self, client, userdata, message): self.statistics['MessagesReceived'] += 1 content = TBUtility.decode(message) # Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" --------------------------- topic_handlers = [ regex for regex in self.__mapping_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: # Note: every topic may be associated to one or more converter. This means that a single MQTT message # may produce more than one message towards ThingsBoard. This also means that I cannot return after # the first successful conversion: I got to use all the available ones. # I will use a flag to understand whether at least one converter succeeded request_handled = False for topic in topic_handlers: available_converters = self.__mapping_sub_topics[topic] for converter in available_converters: try: if isinstance(content, list): for item in content: request_handled = self.put_data_to_convert( converter, message, item) if not request_handled: self.__log.error( 'Cannot find converter for the topic:"%s"! Client: %s, User data: %s', message.topic, str(client), str(userdata)) else: request_handled = self.put_data_to_convert( converter, message, content) except Exception as e: log.exception(e) if not request_handled: self.__log.error( 'Cannot find converter for the topic:"%s"! Client: %s, User data: %s', message.topic, str(client), str(userdata)) # Note: if I'm in this branch, this was for sure a telemetry/attribute push message # => Execution must end here both in case of failure and success return None # Check if message topic exists in connection handlers "i.e., I'm connecting a device" ------------------------- topic_handlers = [ regex for regex in self.__connect_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: for topic in topic_handlers: handler = self.__connect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search( handler["deviceTypeTopicExpression"], message.topic) found_device_type = device_type_match.group( 0) if device_type_match is not None else handler[ "deviceTypeTopicExpression"] elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from connection request") continue # Note: device must be added even if it is already known locally: else ThingsBoard # will not send RPCs and attribute updates self.__log.info("Connecting device %s of type %s", found_device_name, found_device_type) self.__gateway.add_device(found_device_name, {"connector": self}, device_type=found_device_type) # Note: if I'm in this branch, this was for sure a connection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in disconnection handlers "i.e., I'm disconnecting a device" ------------------- topic_handlers = [ regex for regex in self.__disconnect_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: for topic in topic_handlers: handler = self.__disconnect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search( handler["deviceTypeTopicExpression"], message.topic) if device_type_match is not None: found_device_type = device_type_match.group(0) elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from disconnection request") continue if found_device_name in self.__gateway.get_devices(): self.__log.info("Disconnecting device %s of type %s", found_device_name, found_device_type) self.__gateway.del_device(found_device_name) else: self.__log.info("Device %s was not connected", found_device_name) break # Note: if I'm in this branch, this was for sure a disconnection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in attribute request handlers "i.e., I'm asking for a shared attribute" -------- topic_handlers = [ regex for regex in self.__attribute_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: try: for topic in topic_handlers: handler = self.__attribute_requests_sub_topics[topic] found_device_name = None found_attribute_name = None # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get attribute name, either from topic or from content if handler.get("attributeNameTopicExpression"): attribute_name_match = search( handler["attributeNameTopicExpression"], message.topic) if attribute_name_match is not None: found_attribute_name = attribute_name_match.group( 0) elif handler.get("attributeNameJsonExpression"): found_attribute_name = TBUtility.get_value( handler["attributeNameJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from attribute request") continue if found_attribute_name is None: self.__log.error( "Attribute name missing from attribute request") continue self.__log.info("Will retrieve attribute %s of %s", found_attribute_name, found_device_name) self.__gateway.tb_client.client.gw_request_shared_attributes( found_device_name, [found_attribute_name], lambda data, *args: self.notify_attribute( data, found_attribute_name, handler.get("topicExpression"), handler.get("valueExpression"), handler.get('retain', False))) break except Exception as e: log.exception(e) # Note: if I'm in this branch, this was for sure an attribute request message # => Execution must end here both in case of failure and success return None # Check if message topic exists in RPC handlers ---------------------------------------------------------------- # The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is if self.__gateway.is_rpc_in_progress(message.topic): log.info("RPC response arrived. Forwarding it to thingsboard.") self.__gateway.rpc_with_reply_processing(message.topic, content) return None self.__log.debug( "Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"", message.topic, content) def notify_attribute(self, incoming_data, attribute_name, topic_expression, value_expression, retain): if incoming_data.get("device") is None or incoming_data.get( "value") is None: return device_name = incoming_data.get("device") attribute_value = incoming_data.get("value") topic = topic_expression \ .replace("${deviceName}", str(device_name)) \ .replace("${attributeKey}", str(attribute_name)) data = value_expression.replace("${attributeKey}", str(attribute_name)) \ .replace("${attributeValue}", str(attribute_value)) self._client.publish(topic, data, retain=retain).wait_for_publish() def on_attributes_update(self, content): if self.__attribute_updates: for attribute_update in self.__attribute_updates: if match(attribute_update["deviceNameFilter"], content["device"]): for attribute_key in content["data"]: if match(attribute_update["attributeFilter"], attribute_key): try: topic = attribute_update["topicExpression"] \ .replace("${deviceName}", str(content["device"])) \ .replace("${attributeKey}", str(attribute_key)) \ .replace("${attributeValue}", str(content["data"][attribute_key])) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e try: data = attribute_update["valueExpression"] \ .replace("${attributeKey}", str(attribute_key)) \ .replace("${attributeValue}", str(content["data"][attribute_key])) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e self._client.publish( topic, data, retain=attribute_update.get( 'retain', False)).wait_for_publish() self.__log.debug( "Attribute Update data: %s for device %s to topic: %s", data, content["device"], topic) else: self.__log.error( "Cannot find attributeName by filter in message with data: %s", content) else: self.__log.error( "Cannot find deviceName by filter in message with data: %s", content) else: self.__log.error("Attribute updates config not found.") def server_side_rpc_handler(self, content): self.__log.info("Incoming server-side RPC: %s", content) # Check whether one of my RPC handlers can handle this request for rpc_config in self.__server_side_rpc: if search(rpc_config["deviceNameFilter"], content["device"]) \ and search(rpc_config["methodFilter"], content["data"]["method"]) is not None: # This handler seems able to handle the request self.__log.info("Candidate RPC handler found") expects_response = rpc_config.get("responseTopicExpression") defines_timeout = rpc_config.get("responseTimeout") # 2-way RPC setup if expects_response and defines_timeout: expected_response_topic = rpc_config["responseTopicExpression"] \ .replace("${deviceName}", str(content["device"])) \ .replace("${methodName}", str(content["data"]["method"])) \ .replace("${requestId}", str(content["data"]["id"])) expected_response_topic = TBUtility.replace_params_tags( expected_response_topic, content) timeout = time() * 1000 + rpc_config.get("responseTimeout") # Start listenting on the response topic self.__log.info("Subscribing to: %s", expected_response_topic) self.__subscribe(expected_response_topic, rpc_config.get("responseTopicQoS", 1)) # Wait for subscription to be carried out sub_response_timeout = 10 while expected_response_topic in self.__subscribes_sent.values( ): sub_response_timeout -= 1 sleep(0.1) if sub_response_timeout == 0: break # Ask the gateway to enqueue this as an RPC response self.__gateway.register_rpc_request_timeout( content, timeout, expected_response_topic, self.rpc_cancel_processing) # Wait for RPC to be successfully enqueued, which never fails. while self.__gateway.is_rpc_in_progress( expected_response_topic): sleep(0.1) elif expects_response and not defines_timeout: self.__log.info( "2-way RPC without timeout: treating as 1-way") # Actually reach out for the device request_topic: str = rpc_config.get("requestTopicExpression") \ .replace("${deviceName}", str(content["device"])) \ .replace("${methodName}", str(content["data"]["method"])) \ .replace("${requestId}", str(content["data"]["id"])) request_topic = TBUtility.replace_params_tags( request_topic, content) data_to_send_tags = TBUtility.get_values( rpc_config.get('valueExpression'), content['data'], 'params', get_tag=True) data_to_send_values = TBUtility.get_values( rpc_config.get('valueExpression'), content['data'], 'params', expression_instead_none=True) data_to_send = rpc_config.get('valueExpression') for (tag, value) in zip(data_to_send_tags, data_to_send_values): data_to_send = data_to_send.replace( '${' + tag + '}', str(value)) try: self.__log.info("Publishing to: %s with data %s", request_topic, data_to_send) self._client.publish(request_topic, data_to_send, retain=rpc_config.get( 'retain', False)) if not expects_response or not defines_timeout: self.__log.info( "One-way RPC: sending ack to ThingsBoard immediately" ) self.__gateway.send_rpc_reply( device=content["device"], req_id=content["data"]["id"], success_sent=True) # Everything went out smoothly: RPC is served return except Exception as e: self.__log.exception(e) self.__log.error("RPC not handled: %s", content) def rpc_cancel_processing(self, topic): log.info("RPC canceled or terminated. Unsubscribing from %s", topic) self._client.unsubscribe(topic) class ConverterWorker(Thread): def __init__(self, name, incoming_queue, send_result): super().__init__() self.stopped = False self.setName(name) self.setDaemon(True) self.__msg_queue = incoming_queue self.in_progress = False self.__send_result = send_result def run(self): while not self.stopped: if not self.__msg_queue.empty(): self.in_progress = True convert_function, config, incoming_data = self.__msg_queue.get( True, 100) converted_data = convert_function(config, incoming_data) log.debug(converted_data) self.__send_result(config, converted_data) self.in_progress = False else: sleep(.2)
def on_Soc(client, userdata, message): try: val = json.loads(message.payload) print("SOC = %s" % val["value"]) except: print("SOC Exception") reconnect() def on_connect(client, userdata, rc, *args): client.subscribe(bat_topic) client.subscribe(soc_topic) #client.subscribe("N/%s/+/+/ProductId" % portal_id) #client.subscribe("N/%s/#" % portal_id) client = Client("P1") client.tls_set(cert_reqs=ssl.CERT_NONE) client.tls_insecure_set(True) client.username_pw_set(username, password=password) client.connect(mqtt_broker, port=8883) client.on_connect = on_connect client.on_message = on_message client.message_callback_add(bat_topic, on_BatVoltage) client.message_callback_add(soc_topic, on_Soc) client.loop_start() while True: sleep(1)
class Mqtt(): """Main Mqtt class. :param app: flask application object :param connect_async: if True then connect_aync will be used to connect to MQTT broker :param mqtt_logging: if True then messages from MQTT client will be logged """ def __init__(self, app=None, connect_async=False, mqtt_logging=False): # type: (Flask, bool, bool) -> None self.app = app self._connect_async = connect_async # type: bool self._connect_handler = None # type: Optional[Callable] self._disconnect_handler = None # type: Optional[Callable] self.topics = {} # type: Dict[str, TopicQos] self.connected = False self.client = Client() if mqtt_logging: self.client.enable_logger(logger) if app is not None: self.init_app(app) def init_app(self, app): # type: (Flask) -> None """Init the Flask-MQTT addon.""" self.client_id = app.config.get("MQTT_CLIENT_ID", "") self.clean_session = app.config.get("MQTT_CLEAN_SESSION", True) if isinstance(self.client_id, unicode): self.client._client_id = self.client_id.encode('utf-8') else: self.client._client_id = self.client_id self.client._clean_session = self.clean_session self.client._transport = app.config.get("MQTT_TRANSPORT", "tcp").lower() self.client._protocol = app.config.get("MQTT_PROTOCOL_VERSION", MQTTv311) self.client.on_connect = self._handle_connect self.client.on_disconnect = self._handle_disconnect self.username = app.config.get("MQTT_USERNAME") self.password = app.config.get("MQTT_PASSWORD") self.broker_url = app.config.get("MQTT_BROKER_URL", "localhost") self.broker_port = app.config.get("MQTT_BROKER_PORT", 1883) self.tls_enabled = app.config.get("MQTT_TLS_ENABLED", False) self.keepalive = app.config.get("MQTT_KEEPALIVE", 60) self.last_will_topic = app.config.get("MQTT_LAST_WILL_TOPIC") self.last_will_message = app.config.get("MQTT_LAST_WILL_MESSAGE") self.last_will_qos = app.config.get("MQTT_LAST_WILL_QOS", 0) self.last_will_retain = app.config.get("MQTT_LAST_WILL_RETAIN", False) if self.tls_enabled: self.tls_ca_certs = app.config["MQTT_TLS_CA_CERTS"] self.tls_certfile = app.config.get("MQTT_TLS_CERTFILE") self.tls_keyfile = app.config.get("MQTT_TLS_KEYFILE") self.tls_cert_reqs = app.config.get("MQTT_TLS_CERT_REQS", ssl.CERT_REQUIRED) self.tls_version = app.config.get("MQTT_TLS_VERSION", ssl.PROTOCOL_TLSv1) self.tls_ciphers = app.config.get("MQTT_TLS_CIPHERS") self.tls_insecure = app.config.get("MQTT_TLS_INSECURE", False) # set last will message if self.last_will_topic is not None: self.client.will_set( self.last_will_topic, self.last_will_message, self.last_will_qos, self.last_will_retain, ) self._connect() def _connect(self): # type: () -> None if self.username is not None: self.client.username_pw_set(self.username, self.password) # security if self.tls_enabled: self.client.tls_set( ca_certs=self.tls_ca_certs, certfile=self.tls_certfile, keyfile=self.tls_keyfile, cert_reqs=self.tls_cert_reqs, tls_version=self.tls_version, ciphers=self.tls_ciphers, ) if self.tls_insecure: self.client.tls_insecure_set(self.tls_insecure) if self._connect_async: # if connect_async is used self.client.connect_async(self.broker_url, self.broker_port, keepalive=self.keepalive) else: res = self.client.connect(self.broker_url, self.broker_port, keepalive=self.keepalive) if res == 0: logger.debug("Connected client '{0}' to broker {1}:{2}".format( self.client_id, self.broker_url, self.broker_port)) else: logger.error( "Could not connect to MQTT Broker, Error Code: {0}".format( res)) self.client.loop_start() def _disconnect(self): # type: () -> None self.client.loop_stop() self.client.disconnect() logger.debug('Disconnected from Broker') def _handle_connect(self, client, userdata, flags, rc): # type: (Client, Any, Dict, int) -> None if rc == MQTT_ERR_SUCCESS: self.connected = True for key, item in self.topics.items(): self.client.subscribe(topic=item.topic, qos=item.qos) if self._connect_handler is not None: self._connect_handler(client, userdata, flags, rc) def _handle_disconnect(self, client, userdata, rc): # type: (str, Any, int) -> None self.connected = False if self._disconnect_handler is not None: self._disconnect_handler() def on_topic(self, topic): # type: (str) -> Callable """Decorator. Decorator to add a callback function that is called when a certain topic has been published. The callback function is expected to have the following form: `handle_topic(client, userdata, message)` :parameter topic: a string specifying the subscription topic to subscribe to The topic still needs to be subscribed via mqtt.subscribe() before the callback function can be used to handle a certain topic. This way it is possible to subscribe and unsubscribe during runtime. **Example usage:**:: app = Flask(__name__) mqtt = Mqtt(app) mqtt.subscribe('home/mytopic') @mqtt.on_topic('home/mytopic') def handle_mytopic(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): # type: (Callable[[str], None]) -> Callable[[str], None] self.client.message_callback_add(topic, handler) return handler return decorator def subscribe(self, topic, qos=0): # type: (str, int) -> Tuple[int, int] """ Subscribe to a certain topic. :param topic: a string specifying the subscription topic to subscribe to. :param qos: the desired quality of service level for the subscription. Defaults to 0. :rtype: (int, int) :result: (result, mid) A topic is a UTF-8 string, which is used by the broker to filter messages for each connected client. A topic consists of one or more topic levels. Each topic level is separated by a forward slash (topic level separator). The function returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not currently connected. mid is the message ID for the subscribe request. The mid value can be used to track the subscribe request by checking against the mid argument in the on_subscribe() callback if it is defined. **Topic example:** `myhome/groundfloor/livingroom/temperature` """ # TODO: add support for list of topics # don't subscribe if already subscribed # try to subscribe result, mid = self.client.subscribe(topic=topic, qos=qos) # if successful add to topics if result == MQTT_ERR_SUCCESS: self.topics[topic] = TopicQos(topic=topic, qos=qos) logger.debug('Subscribed to topic: {0}, qos: {1}'.format( topic, qos)) else: logger.error('Error {0} subscribing to topic: {1}'.format( result, topic)) return (result, mid) def unsubscribe(self, topic): # type: (str) -> Optional[Tuple[int, int]] """ Unsubscribe from a single topic. :param topic: a single string that is the subscription topic to unsubscribe from :rtype: (int, int) :result: (result, mid) Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not currently connected. mid is the message ID for the unsubscribe request. The mid value can be used to track the unsubscribe request by checking against the mid argument in the on_unsubscribe() callback if it is defined. """ # don't unsubscribe if not in topics if topic in self.topics: result, mid = self.client.unsubscribe(topic) if result == MQTT_ERR_SUCCESS: self.topics.pop(topic) logger.debug('Unsubscribed from topic: {0}'.format(topic)) else: logger.debug('Error {0} unsubscribing from topic: {1}'.format( result, topic)) # if successful remove from topics return result, mid return None def unsubscribe_all(self): # type: () -> None """Unsubscribe from all topics.""" topics = list(self.topics.keys()) for topic in topics: self.unsubscribe(topic) def publish(self, topic, payload=None, qos=0, retain=False): # type: (str, bytes, int, bool) -> Tuple[int, int] """ Send a message to the broker. :param topic: the topic that the message should be published on :param payload: the actual message to send. If not given, or set to None a zero length message will be used. Passing an int or float will result in the payload being converted to a string representing that number. If you wish to send a true int/float, use struct.pack() to create the payload you require. :param qos: the quality of service level to use :param retain: if set to True, the message will be set as the "last known good"/retained message for the topic :returns: Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN if the client is not currently connected. mid is the message ID for the publish request. """ if not self.connected: self.client.reconnect() result, mid = self.client.publish(topic, payload, qos, retain) if result == MQTT_ERR_SUCCESS: logger.debug('Published topic {0}: {1}'.format(topic, payload)) else: logger.error('Error {0} publishing topic {1}'.format( result, topic)) return (result, mid) def on_connect(self): # type: () -> Callable """Decorator. Decorator to handle the event when the broker responds to a connection request. Only the last decorated function will be called. """ def decorator(handler): # type: (Callable) -> Callable self._connect_handler = handler return handler return decorator def on_disconnect(self): # type: () -> Callable """Decorator. Decorator to handle the event when client disconnects from broker. Only the last decorated function will be called. """ def decorator(handler): # type: (Callable) -> Callable self._disconnect_handler = handler return handler return decorator def on_message(self): # type: () -> Callable """Decorator. Decorator to handle all messages that have been subscribed and that are not handled via the `on_message` decorator. **Note:** Unlike as written in the paho mqtt documentation this callback will not be called if there exists an topic-specific callback added by the `on_topic` decorator. **Example Usage:**:: @mqtt.on_message() def handle_messages(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_message = handler return handler return decorator def on_publish(self): # type: () -> Callable """Decorator. Decorator to handle all messages that have been published by the client. **Example Usage:**:: @mqtt.on_publish() def handle_publish(client, userdata, mid): print('Published message with mid {}.' .format(mid)) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_publish = handler return handler return decorator def on_subscribe(self): # type: () -> Callable """Decorate a callback function to handle subscritions. **Usage:**:: @mqtt.on_subscribe() def handle_subscribe(client, userdata, mid, granted_qos): print('Subscription id {} granted with qos {}.' .format(mid, granted_qos)) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_subscribe = handler return handler return decorator def on_unsubscribe(self): # type: () -> Callable """Decorate a callback funtion to handle unsubscribtions. **Usage:**:: @mqtt.unsubscribe() def handle_unsubscribe(client, userdata, mid) print('Unsubscribed from topic (id: {})' .format(mid)') """ def decorator(handler): # type: (Callable) -> Callable self.client.on_unsubscribe = handler return handler return decorator def on_log(self): # type: () -> Callable """Decorate a callback function to handle MQTT logging. **Example Usage:** :: @mqtt.on_log() def handle_logging(client, userdata, level, buf): print(client, userdata, level, buf) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_log = handler return handler return decorator
class Mqtt(): def __init__(self, app=None): # type: (Flask) -> None self.app = app self.client = Client() self.client.on_connect = self._handle_connect self.client.on_disconnect = self._handle_disconnect self.topics = [] # type: List[str] self.connected = False if app is not None: self.init_app(app) def init_app(self, app): # type: (Flask) -> None self.username = app.config.get('MQTT_USERNAME') self.password = app.config.get('MQTT_PASSWORD') self.broker_url = app.config.get('MQTT_BROKER_URL', 'localhost') self.broker_port = app.config.get('MQTT_BROKER_PORT', 1883) self.tls_enabled = app.config.get('MQTT_TLS_ENABLED', False) self.keepalive = app.config.get('MQTT_KEEPALIVE', 60) self.last_will_topic = app.config.get('MQTT_LAST_WILL_TOPIC') self.last_will_message = app.config.get('MQTT_LAST_WILL_MESSAGE') self.last_will_qos = app.config.get('MQTT_LAST_WILL_QOS', 0) self.last_will_retain = app.config.get('MQTT_LAST_WILL_RETAIN', False) if self.tls_enabled: self.tls_ca_certs = app.config['MQTT_TLS_CA_CERTS'] self.tls_certfile = app.config.get('MQTT_TLS_CERTFILE') self.tls_keyfile = app.config.get('MQTT_TLS_KEYFILE') self.tls_cert_reqs = app.config.get('MQTT_TLS_CERT_REQS', ssl.CERT_REQUIRED) self.tls_version = app.config.get('MQTT_TLS_VERSION', ssl.PROTOCOL_TLSv1) self.tls_ciphers = app.config.get('MQTT_TLS_CIPHERS') self.tls_insecure = app.config.get('MQTT_TLS_INSECURE', False) # set last will message if self.last_will_topic is not None: self.client.will_set(self.last_will_topic, self.last_will_message, self.last_will_qos, self.last_will_retain) self.app = app self._connect() def _connect(self): # type: () -> None if self.username is not None: self.client.username_pw_set(self.username, self.password) # security if self.tls_enabled: if self.tls_insecure: self.client.tls_insecure_set(self.tls_insecure) self.client.tls_set( ca_certs=self.tls_ca_certs, certfile=self.tls_certfile, keyfile=self.tls_keyfile, cert_reqs=self.tls_cert_reqs, tls_version=self.tls_version, ciphers=self.tls_ciphers, ) self.client.loop_start() res = self.client.connect(self.broker_url, self.broker_port, keepalive=self.keepalive) def _disconnect(self): # type: () -> None self.client.loop_stop() self.client.disconnect() def _handle_connect(self, client, userdata, flags, rc): # type: (Client, Any, Dict, int) -> None if rc == MQTT_ERR_SUCCESS: self.connected = True for topic in self.topics: self.client.subscribe(topic) def _handle_disconnect(self, client, userdata, rc): # type: (str, Any, int) -> None self.connected = False def on_topic(self, topic): # type: (str) -> Callable """ Decorator to add a callback function that is called when a certain topic has been published. The callback function is expected to have the following form: `handle_topic(client, userdata, message)` :parameter topic: a string specifying the subscription topic to subscribe to The topic still needs to be subscribed via mqtt.subscribe() before the callback function can be used to handle a certain topic. This way it is possible to subscribe and unsubscribe during runtime. **Example usage:**:: app = Flask(__name__) mqtt = Mqtt(app) mqtt.subscribe('home/mytopic') @mqtt.on_topic('home/mytopic') def handle_mytopic(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): # type: (Callable[[str], None]) -> Callable[[str], None] self.client.message_callback_add(topic, handler) return handler return decorator def subscribe(self, topic, qos=0): # type: (str, int) -> tuple(int, int) """ Subscribe to a certain topic. :param topic: a string specifying the subscription topic to subscribe to. :param qos: the desired quality of service level for the subscription. Defaults to 0. :rtype: (int, int) :result: (result, mid) A topic is a UTF-8 string, which is used by the broker to filter messages for each connected client. A topic consists of one or more topic levels. Each topic level is separated by a forward slash (topic level separator). The function returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not currently connected. mid is the message ID for the subscribe request. The mid value can be used to track the subscribe request by checking against the mid argument in the on_subscribe() callback if it is defined. **Topic example:** `myhome/groundfloor/livingroom/temperature` """ # TODO: add support for list of topics # don't subscribe if already subscribed # try to subscribe result, mid = self.client.subscribe(topic, qos) # if successful add to topics if result == MQTT_ERR_SUCCESS: if topic not in self.topics: self.topics.append(topic) return (result, mid) def unsubscribe(self, topic): # type: (str) -> tuple(int, int) """ Unsubscribe from a single topic. :param topic: a single string that is the subscription topic to unsubscribe from :rtype: (int, int) :result: (result, mid) Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or (MQTT_ERR_NO_CONN, None) if the client is not currently connected. mid is the message ID for the unsubscribe request. The mid value can be used to track the unsubscribe request by checking against the mid argument in the on_unsubscribe() callback if it is defined. """ # don't unsubscribe if not in topics if topic not in self.topics: return result, mid = self.client.unsubscribe(topic) # if successful remove from topics if result == MQTT_ERR_SUCCESS: self.topics.remove(topic) return result, mid def unsubscribe_all(self): # type: () -> None """ Unsubscribe from all topics. """ topics = self.topics[:] for topic in topics: self.unsubscribe(topic) def publish(self, topic, payload=None, qos=0, retain=False): # type: (str, bytes, int, bool) -> Tuple[int, int] """ Send a message to the broker. :param topic: the topic that the message should be published on :param payload: the actual message to send. If not given, or set to None a zero length message will be used. Passing an int or float will result in the payload being converted to a string representing that number. If you wish to send a true int/float, use struct.pack() to create the payload you require. :param qos: the quality of service level to use :param retain: if set to True, the message will be set as the "last known good"/retained message for the topic :returns: Returns a tuple (result, mid), where result is MQTT_ERR_SUCCESS to indicate success or MQTT_ERR_NO_CONN if the client is not currently connected. mid is the message ID for the publish request. """ if not self.connected: self.client.reconnect() return self.client.publish(topic, payload, qos, retain) def on_message(self): # type: () -> Callable """ Decorator to handle all messages that have been subscribed and that are not handled via the `on_message` decorator. **Note:** Unlike as written in the paho mqtt documentation this callback will not be called if there exists an topic-specific callback added by the `on_topic` decorator. **Example Usage:**:: @mqtt.on_message() def handle_messages(client, userdata, message): print('Received message on topic {}: {}' .format(message.topic, message.payload.decode())) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_message = handler return handler return decorator def on_publish(self): """ Decorator to handle all messages that have been published by the client. **Example Usage:**:: @mqtt.on_publish() def handle_publish(client, userdata, mid): print('Published message with mid {}.' .format(mid)) """ def decorator(handler): self.client.on_publish = handler return handler return decorator def on_subscribe(self): """ Decorator to handle subscribe callbacks. **Usage:**:: @mqtt.on_subscribe() def handle_subscribe(client, userdata, mid, granted_qos): print('Subscription id {} granted with qos {}.' .format(mid, granted_qos)) """ def decorator(handler): self.client.on_subscribe = handler return handler return decorator def on_unsubscribe(self): """ Decorator to handle unsubscribe callbacks. **Usage:**:: @mqtt.unsubscribe() def handle_unsubscribe(client, userdata, mid) print('Unsubscribed from topic (id: {})' .format(mid)') """ def decorator(handler): self.client.on_unsubscribe = handler return handler return decorator def on_log(self): # type: () -> Callable """ Decorator to handle MQTT logging. **Example Usage:** :: @mqtt.on_log() def handle_logging(client, userdata, level, buf): print(client, userdata, level, buf) """ def decorator(handler): # type: (Callable) -> Callable self.client.on_log = handler return handler return decorator
class MQTTConnection(): client: Client _instance = None @classmethod def get_instance(cls) -> "MQTTConnection": if cls._instance is None: cls._instance = MQTTConnection() return cls._instance def __init__(self): self.client = Client( "pai" + os.urandom(8).hex(), protocol=protocol_map.get(str(cfg.MQTT_PROTOCOL), MQTTv311), transport=cfg.MQTT_TRANSPORT, ) self._last_pai_status = "unknown" self.pai_status_topic = "{}/{}/{}".format(cfg.MQTT_BASE_TOPIC, cfg.MQTT_INTERFACE_TOPIC, "pai_status") self.availability_topic = "{}/{}/{}".format(cfg.MQTT_BASE_TOPIC, cfg.MQTT_INTERFACE_TOPIC, "availability") self.client.on_connect = self._on_connect_cb self.client.on_disconnect = self._on_disconnect_cb self.state = ConnectionState.NEW # self.client.enable_logger(logger) # self.client.on_subscribe = lambda client, userdata, mid, granted_qos: logger.debug("Subscribed: %s" %(mid)) # self.client.on_message = lambda client, userdata, message: logger.debug("Message received: %s" % str(message)) # self.client.on_publish = lambda client, userdata, mid: logger.debug("Message published: %s" % str(mid)) ps.subscribe(self.on_run_state_change, "run-state") self.registrars = [] if cfg.MQTT_USERNAME is not None and cfg.MQTT_PASSWORD is not None: self.client.username_pw_set(username=cfg.MQTT_USERNAME, password=cfg.MQTT_PASSWORD) if cfg.MQTT_TLS_CERT_PATH is not None: self.client.tls_set( ca_certs=cfg.MQTT_TLS_CERT_PATH, certfile=None, keyfile=None, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None, ) self.client.tls_insecure_set(False) self.client.will_set(self.availability_topic, "offline", 0, retain=True) self.client.on_log = self.on_client_log def on_client_log(self, client, userdata, level, buf): level_std = LOGGING_LEVEL[level] exc_info = None type_, exc, trace = sys.exc_info() if exc: # Can be (socket.error, OSError, WebsocketConnectionError, ...) if hasattr(exc, "errno"): exc_msg = f"{os.strerror(exc.errno)}({exc.errno})" if exc.errno in [22, 49]: level_std = logging.ERROR buf = f"{buf}: Please check MQTT connection settings. Especially MQTT_BIND_ADDRESS and MQTT_BIND_PORT" else: exc_msg = str(exc) buf = f"{buf}: {exc_msg}" if "Connection failed" in buf: level_std = logging.WARNING if level_std > logging.DEBUG: logger.log(level_std, buf, exc_info=exc_info) def on_run_state_change(self, state: RunState): v = RUN_STATE_2_PAYLOAD.get(state, "unknown") self._report_pai_status(v) def start(self): if self.state == ConnectionState.NEW: self.client.loop_start() # TODO: Some initial connection retry mechanism required try: self.client.connect_async( host=cfg.MQTT_HOST, port=cfg.MQTT_PORT, keepalive=cfg.MQTT_KEEPALIVE, bind_address=cfg.MQTT_BIND_ADDRESS, bind_port=cfg.MQTT_BIND_PORT, ) self.state = ConnectionState.CONNECTING logger.info("MQTT loop started") except socket.gaierror: logger.exception("Failed to connect to MQTT (%s:%d)", cfg.MQTT_HOST, cfg.MQTT_PORT) def stop(self): if self.state in [ ConnectionState.CONNECTING, ConnectionState.CONNECTED ]: self.disconnect() self.client.loop_stop() logger.info("MQTT loop stopped") def publish(self, topic, payload=None, *args, **kwargs): logger.debug("MQTT: {}={}".format(topic, payload)) self.client.publish(topic, payload, *args, **kwargs) def _call_registars(self, method, *args, **kwargs): for r in self.registrars: try: if hasattr(r, method) and isinstance(getattr(r, method), typing.Callable): getattr(r, method)(*args, **kwargs) except: logger.exception('Failed to call "%s" on "%s"', method, r.__class__.__name__) def register(self, cls): self.registrars.append(cls) self.start() def unregister(self, cls): self.registrars.remove(cls) if len(self.registrars) == 0: self.stop() @property def connected(self): return self.state == ConnectionState.CONNECTED def _report_pai_status(self, status): self._last_pai_status = status self.publish(self.pai_status_topic, status, qos=cfg.MQTT_QOS, retain=True) self.publish( self.availability_topic, "online" if status in ["online", "paused"] else "offline", qos=cfg.MQTT_QOS, retain=True, ) def _on_connect_cb(self, client, userdata, flags, result, properties=None): # called on Thread-6 if result == MQTT_ERR_SUCCESS: logger.info("MQTT Broker Connected") self.state = ConnectionState.CONNECTED self._report_pai_status(self._last_pai_status) self._call_registars("on_connect", client, userdata, flags, result) else: logger.error( f"Failed to connect to MQTT: {connack_string(result)} ({result})" ) def _on_disconnect_cb(self, client, userdata, rc): # called on Thread-6 if rc == MQTT_ERR_SUCCESS: logger.info("MQTT Broker Disconnected") else: logger.error(f"MQTT Broker unexpectedly disconnected. Code: {rc}") self.state = ConnectionState.NEW self._call_registars("on_disconnect", client, userdata, rc) def disconnect(self, reasoncode=None, properties=None): self.state = ConnectionState.DISCONNECTING self._report_pai_status("offline") self.client.disconnect() def message_callback_add(self, *args, **kwargs): self.client.message_callback_add(*args, **kwargs) def subscribe(self, *args, **kwargs): self.client.subscribe(*args, **kwargs)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--POD-CONFIG", required=True) parser.add_argument("--MQTT-SERVER", required=True, default=None, nargs="?") parser.add_argument("--MQTT-PORT", required=False, default="1881", nargs="?") parser.add_argument("--MQTT-SSL", required=False, default="", nargs="?") parser.add_argument("--MQTT-CLIENTID", required=True, default="", nargs="?") parser.add_argument("--MQTT-TOPIC", required=True, default="", nargs="?") parser.add_argument("--LOG-LEVEL", required=False, default="DEBUG", nargs="?") parser.add_argument("--LOG-FILE", required=False, default=None, nargs="?") args = parser.parse_args() formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") ch = logging.StreamHandler(sys.stdout) ch.setFormatter(formatter) ch.setLevel(level=args.LOG_LEVEL) logging.basicConfig(level=args.LOG_LEVEL, handlers=[ch]) if args.LOG_FILE: fh = logging.FileHandler(filename=args.LOG_FILE) fh.setFormatter(formatter) fh.setLevel(level=args.LOG_LEVEL) logging.getLogger().addHandler(fh) pod = Pod.Load(args.POD_CONFIG) mqtt_client = Client(client_id=args.MQTT_CLIENTID, clean_session=False, protocol=MQTTv311, transport="tcp") if args.MQTT_SSL != "": mqtt_client.tls_set(certfile=None, keyfile=None, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) mqtt_client.tls_insecure_set(True) mqtt_client.reconnect_delay_set(min_delay=5, max_delay=120) mqtt_client.retry_first_connection = True pdm = Pdm(pod) processor = Processor(mqtt_client, args.MQTT_TOPIC, pdm) processor.start(args.MQTT_SERVER, int(args.MQTT_PORT), 30) try: while not exit_event.wait(timeout=10): pass except KeyboardInterrupt: pass processor.stop()
def publish(self, topic: str, msg: Any, host: Optional[str] = None, port: int = 1883, reply_topic: Optional[str] = None, timeout: int = 60, tls_cafile: Optional[str] = None, tls_certfile: Optional[str] = None, tls_keyfile: Optional[str] = None, tls_version: Optional[str] = None, tls_ciphers: Optional[str] = None, tls_insecure: Optional[bool] = None, username: Optional[str] = None, password: Optional[str] = None): """ Sends a message to a topic. :param topic: Topic/channel where the message will be delivered :param msg: Message to be sent. It can be a list, a dict, or a Message object. :param host: MQTT broker hostname/IP. :param port: MQTT broker port (default: 1883). :param reply_topic: If a ``reply_topic`` is specified, then the action will wait for a response on this topic. :param timeout: If ``reply_topic`` is set, use this parameter to specify the maximum amount of time to wait for a response (default: 60 seconds). :param tls_cafile: If TLS/SSL is enabled on the MQTT server and the certificate requires a certificate authority to authenticate it, `ssl_cafile` will point to the provided ca.crt file (default: None). :param tls_certfile: If TLS/SSL is enabled on the MQTT server and a client certificate it required, specify it here (default: None). :param tls_keyfile: If TLS/SSL is enabled on the MQTT server and a client certificate key it required, specify it here (default: None). :param tls_version: If TLS/SSL is enabled on the MQTT server and it requires a certain TLS version, specify it here (default: None). Supported versions: ``tls`` (automatic), ``tlsv1``, ``tlsv1.1``, ``tlsv1.2``. :param tls_insecure: Set to True to ignore TLS insecure warnings (default: False). :param tls_ciphers: If TLS/SSL is enabled on the MQTT server and an explicit list of supported ciphers is required, specify it here (default: None). :param username: Specify it if the MQTT server requires authentication (default: None). :param password: Specify it if the MQTT server requires authentication (default: None). """ from paho.mqtt.client import Client if not host and not self.host: raise RuntimeError( 'No host specified and no default host configured') if not host: host = self.host port = self.port tls_cafile = self.tls_cafile tls_certfile = self.tls_certfile tls_keyfile = self.tls_keyfile tls_version = self.tls_version tls_ciphers = self.tls_ciphers tls_insecure = self.tls_insecure username = self.username password = self.password else: tls_cafile = self._expandpath(tls_cafile) tls_certfile = self._expandpath(tls_certfile) tls_keyfile = self._expandpath(tls_keyfile) if tls_version: tls_version = self.get_tls_version(tls_version) if tls_insecure is None: tls_insecure = self.tls_insecure client = Client() if username and password: client.username_pw_set(username, password) if tls_cafile: client.tls_set(ca_certs=tls_cafile, certfile=tls_certfile, keyfile=tls_keyfile, tls_version=tls_version, ciphers=tls_ciphers) client.tls_insecure_set(tls_insecure) # Try to parse it as a platypush message or dump it to JSON from a dict/list if isinstance(msg, (dict, list)): msg = json.dumps(msg) # noinspection PyBroadException try: msg = Message.build(json.loads(msg)) except: pass client.connect(host, port, keepalive=timeout) response_buffer = io.BytesIO() try: response_received = threading.Event() if reply_topic: client.on_message = self._response_callback( reply_topic=reply_topic, event=response_received, buffer=response_buffer) client.subscribe(reply_topic) client.publish(topic, str(msg)) if not reply_topic: return client.loop_start() ok = response_received.wait(timeout=timeout) if not ok: raise TimeoutError('Response timed out') return response_buffer.getvalue() finally: response_buffer.close() # noinspection PyBroadException try: client.loop_stop() except: pass client.disconnect()
else: mqtt_client.connected_flag = False def on_message(client, userdata, message): info = simplejson.loads(decrypt(MESSAGE_DECRYPT_KEY, message.payload)) print('Customer Order:') info['Order_Status'] = 'Confirmed' print(info) client.publish(topic='%s/%s' % (ORDER_STATUS, info['Room']), payload=cipher(MESSAGE_DECRYPT_KEY, simplejson.dumps(info))) mqtt_client.on_connect = on_connect mqtt_client.on_message = on_message # set mosquitto broker password and username mqtt_client.username_pw_set(username=USERNAME, password=PASSWORD) # set TLS cert for the client mqtt_client.tls_set(ca_certs=TLS_CERT) mqtt_client.tls_insecure_set(True) mqtt_client.loop_start() mqtt_client.connect(host=MQTT_ADDR, port=MQTT_PRT) while not mqtt_client.connected_flag: # wait in loop print("In wait loop") time.sleep(1) mqtt_client.subscribe(topic='%s/+' % FD_TOPIC) mqtt_client.loop_forever() mqtt_client.disconnect()
class MqttServer(Thread): """Server fuer die Uebertragung des Prozessabbilds per MQTT.""" def __init__(self, basetopic, sendinterval, broker_address, port=1883, tls_set=False, username="", password=None, client_id="", send_events=False, write_outputs=False, replace_ios=None): """Init MqttServer class. @param basetopic Basis-Topic fuer Datenaustausch @param sendinterval Prozessabbild alle n Sekunden senden / 0 = aus @param broker_address Adresse <class 'str'> des MQTT-Servers @param port Portnummer <class 'int'> des MQTT-Servers @param tls_set TLS fuer Verbindung zum MQTT-Server verwenden @param username Optional Benutzername fuer MQTT-Server @param password Optional Password fuer MQTT-Server @param client_id MQTT ClientID, wenn leer automatisch random erzeugung @param send_events Sendet Werte bei IO Wertaenderung @param write_outputs Per MQTT auch Outputs schreiben @param replace_ios Replace IOs of RevPiModIO """ if not isinstance(basetopic, str): raise ValueError("parameter topic must be <class 'str'>") if not (isinstance(sendinterval, int) and sendinterval >= 0): raise ValueError( "parameter sendinterval must be <class 'int'> and >= 0") if not (isinstance(broker_address, str) and broker_address != ""): raise ValueError( "parameter broker_address must be <class 'str'> and not empty") if not (isinstance(port, int) and 0 < port < 65535): raise ValueError( "parameter sendinterval must be <class 'int'> and 1 - 65535") if not isinstance(tls_set, bool): raise ValueError("parameter tls_set must be <class 'bool'>") if not isinstance(username, str): raise ValueError("parameter username must be <class 'str'>") if not (password is None or isinstance(password, str)): raise ValueError("parameter password must be <class 'str'>") if not isinstance(client_id, str): raise ValueError("parameter client_id must be <class 'str'>") if not isinstance(send_events, bool): raise ValueError("parameter send_events must be <class 'bool'>") if not isinstance(write_outputs, bool): raise ValueError("parameter write_outputs must be <class 'bool'>") if not (replace_ios is None or isinstance(replace_ios, str)): raise ValueError("parameter replace_ios must be <class 'str'>") super().__init__() # Klassenvariablen self.__exit = False self._evt_data = Event() self._exported_ios = [] self._broker_address = broker_address self._port = port self._reloadmodio = False self._replace_ios = replace_ios self._rpi = None self._send_events = send_events self._sendinterval = sendinterval self._write_outputs = write_outputs # RevPiModIO laden oder mit Exception aussteigen self._loadrevpimodio() # Topics konfigurieren self._mqtt_evt_io = join(basetopic, "event/{0}") self._mqtt_got_io = join(basetopic, "got/{0}") self._mqtt_io = join(basetopic, "io/{0}") self._mqtt_ioget = join(basetopic, "get/#") self._mqtt_ioset = join(basetopic, "set/#") self._mqtt_ioreset = join(basetopic, "reset/#") self._mqtt_pictory = join(basetopic, "pictory") self._mqtt_senddata = join(basetopic, "get") self._mqtt_sendpictory = join(basetopic, "needpictory") self._mq = Client(client_id) if username != "": self._mq.username_pw_set(username, password) if tls_set: self._mq.tls_set(cert_reqs=CERT_NONE) self._mq.tls_insecure_set(True) # Handler konfigurieren self._mq.on_connect = self._on_connect self._mq.on_message = self._on_message def _evt_io(self, name, value, requested=False): """Sendet Daten aus Events. @param name IO-Name @param value IO-Value @param requested Wenn True, wird 'got' Topic verwendet """ if requested: topic = self._mqtt_got_io.format(name) else: topic = self._mqtt_evt_io.format(name) if isinstance(value, bytes): value = int.from_bytes(value, "little") self._mq.publish(topic, int(value)) def _loadrevpimodio(self): """Instantiiert das RevPiModIO Modul. @return None or Exception""" self._reloadmodio = False self._exported_ios = [] # RevPiModIO-Modul Instantiieren if self._rpi is not None: self._rpi.cleanup() proginit.logger.debug("create revpimodio2 object for MQTT") try: # Vollzugriff und Eventüberwachung self._rpi = revpimodio2.RevPiModIO( autorefresh=self._send_events, monitoring=not self._write_outputs, configrsc=proginit.pargs.configrsc, procimg=proginit.pargs.procimg, replace_io_file=self._replace_ios, shared_procimg=True, ) self._rpi.debug = -1 if self._replace_ios: proginit.logger.info("loaded replace_ios to MQTT") except Exception as e: try: # Lesend und Eventüberwachung self._rpi = revpimodio2.RevPiModIO( autorefresh=self._send_events, monitoring=not self._write_outputs, configrsc=proginit.pargs.configrsc, procimg=proginit.pargs.procimg, shared_procimg=True, ) self._rpi.debug = -1 proginit.logger.warning( "replace_ios_file not loadable for MQTT - using " "defaults now | {0}".format(e)) except Exception as e: self._rpi = None proginit.logger.error( "piCtory configuration not loadable for MQTT | " "{0}".format(e)) raise e # Exportierte IOs laden for dev in self._rpi.device: for io in dev.get_allios(export=True): io.reg_event(self._evt_io) self._exported_ios.append(io) # CoreIOs prüfen und zu export hinzufügen lst_coreio = [] if self._rpi.core: if self._rpi.core.a1green.export: lst_coreio.append(self._rpi.core.a1green) if self._rpi.core.a1red.export: lst_coreio.append(self._rpi.core.a1red) if self._rpi.core.a2green.export: lst_coreio.append(self._rpi.core.a2green) if self._rpi.core.a2red.export: lst_coreio.append(self._rpi.core.a2red) if self._rpi.core.wd.export: lst_coreio.append(self._rpi.core.wd) # Connect-IOs anhängen if type(self._rpi.core) == revpimodio2.device.Connect: if self._rpi.core.a3green.export: lst_coreio.append(self._rpi.core.a3green) if self._rpi.core.a3red.export: lst_coreio.append(self._rpi.core.a3red) if self._rpi.core.x2in.export: lst_coreio.append(self._rpi.core.x2in) if self._rpi.core.x2out.export: lst_coreio.append(self._rpi.core.x2out) # IOs exportieren und Events anmelden for io in lst_coreio: io.reg_event(self._evt_io) self._exported_ios.append(io) proginit.logger.debug("created revpimodio2 object") def _on_connect(self, client, userdata, flags, rc): """Verbindung zu MQTT Broker.""" proginit.logger.debug("enter MqttServer._on_connect()") if rc > 0: proginit.logger.warning( "can not connect to mqtt broker '{0}' - error '{1}' - " "will retry".format(self._broker_address, connack_string(rc))) else: # Subscribe piCtory Anforderung client.subscribe(self._mqtt_ioget) client.subscribe(self._mqtt_senddata) client.subscribe(self._mqtt_sendpictory) if self._write_outputs: client.subscribe(self._mqtt_ioset) client.subscribe(self._mqtt_ioreset) proginit.logger.debug("leave MqttServer._on_connect()") def _on_disconnect(self, client, userdata, rc): """Wertet Verbindungsabbruch aus.""" proginit.logger.debug("enter MqttServer._on_disconnect()") if rc != 0: proginit.logger.warning( "unexpected disconnection from mqtt broker - " "will try to reconnect") proginit.logger.debug("leave MqttServer._on_disconnect()") def _on_message(self, client, userdata, msg): """Sendet piCtory Konfiguration.""" if msg.topic == self._mqtt_pictory: # piCtory Konfiguration senden self._send_pictory_conf() elif msg.topic == self._mqtt_senddata: # Alle zyklischen Daten senden self._evt_data.set() else: lst_topic = msg.topic.split("/") if len(lst_topic) < 2: proginit.logger.info( "wrong topic format - need ./get/ioname or ./set/ioname") return # Aktion und IO auswerten ioget = lst_topic[-2].lower() == "get" ioset = lst_topic[-2].lower() == "set" ioreset = lst_topic[-2].lower() == "reset" ioname = lst_topic[-1] coreio = ioname.find(".") != -1 try: # IO holen if coreio: coreio = ioname.split(".")[-1] io = getattr(self._rpi.core, coreio) if not isinstance(io, revpimodio2.io.IOBase): raise RuntimeError() else: io = self._rpi.io[ioname] io_needbytes = type(io.value) == bytes except Exception: proginit.logger.error( "can not find io '{0}' for MQTT".format(ioname)) return # Aktion verarbeiten if not io.export: proginit.logger.error( "io '{0}' is not marked as export in piCtory for MQTT use" "".format(ioname)) elif ioget: # Werte laden, wenn nicht autorefresh if not self._send_events: io._parentdevice.readprocimg() # Publish Wert von IO self._evt_io(io.name, io.value, requested=True) elif ioset and io.type != revpimodio2.OUT: proginit.logger.error("can not write to inputs with MQTT") elif ioset: # Convert MQTT Payload to valid Output-Value value = msg.payload.decode("utf8") if value.isdecimal(): value = int(value) # Muss eine Byteumwandlung vorgenommen werden? if io_needbytes: try: value = value.to_bytes(io.length, io.byteorder) except OverflowError: proginit.logger.error( "can not convert value '{0}' to fitting bytes" "".format(value)) return elif value == "false" and not io_needbytes: value = 0 elif value == "true" and not io_needbytes: value = 1 else: proginit.logger.error( "can not convert value '{0}' for output '{1}'" "".format(value, ioname)) return # Write Value to RevPi try: io.value = value except Exception: proginit.logger.error( "could not write '{0}' to Output '{1}'" "".format(value, ioname)) elif ioreset: # Counter zurücksetzen if not isinstance(io, revpimodio2.io.IntIOCounter): proginit.logger.warning("this io has no counter") else: io.reset() else: # Aktion nicht erkennbar proginit.logger.warning( "can not see get/set in topic '{0}'".format(msg.topic)) def _send_pictory_conf(self): """Sendet piCtory Konfiguration per MQTT.""" try: fh = open(proginit.pargs.configrsc, "rb") self._mq.publish(self._mqtt_pictory, fh.read()) fh.close() except Exception: proginit.logger.error( "can not read and publish piCtory config '{0}'" "".format(proginit.pargs.configrsc)) def newlogfile(self): """Konfiguriert die FileHandler auf neue Logdatei.""" pass def reload_revpimodio(self): """Fuehrt im naechsten Zyklus zum Reload.""" proginit.logger.debug("enter MqttServer.reload_revpimodio()") self._reloadmodio = True self._evt_data.set() proginit.logger.debug("leave MqttServer.reload_revpimodio()") def run(self): """Startet die Uebertragung per MQTT.""" proginit.logger.debug("enter MqttServer.run()") # MQTT verbinden proginit.logger.info("connecting to mqtt broker {0}".format( self._broker_address)) try: self._mq.connect(self._broker_address, self._port, keepalive=60) except Exception: self._on_connect(self._mq, None, None, 3) self._mq.connect_async(self._broker_address, self._port, keepalive=60) self._mq.loop_start() # Eventüberwachung starten if self._send_events: proginit.logger.debug("start non blocking mainloop of revpimodio") self._rpi.mainloop(blocking=False) # mainloop send_cycledata = self._sendinterval > 0 while not self.__exit: self._evt_data.clear() # RevPiModIO neu laden if self._reloadmodio: proginit.logger.info("reload revpimodio for mqtt") self._loadrevpimodio() # Eventüberwachung erneut starten if self._send_events: proginit.logger.debug( "start non blocking mainloop of revpimodio") self._rpi.mainloop(blocking=False) if send_cycledata: # Werte laden, wenn nicht autorefresh if not self._send_events: self._rpi.readprocimg() # Exportierte IOs übertragen for io in self._exported_ios: value = io.value if isinstance(value, bytes): value = int.from_bytes(value, "little") self._mq.publish(self._mqtt_io.format(io.name), int(value)) self._evt_data.wait( 10 if not send_cycledata else self._sendinterval) # MQTT trennen proginit.logger.info("disconnecting from mqtt broker {0}".format( self._broker_address)) # NOTE: dies gab dead-locks: self._mq.loop_stop() self._mq.disconnect() # RevPiModIO aufräumen self._rpi.cleanup() proginit.logger.debug("leave MqttServer.run()") def stop(self): """Stoppt die Uebertragung per MQTT.""" proginit.logger.debug("enter MqttServer.stop()") self.__exit = True self._evt_data.set() proginit.logger.debug("leave MqttServer.stop()")
class MqttConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() self.__log = log self.config = config self._connector_type = connector_type self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__gateway = gateway self.__broker = config.get('broker') self.__mapping = config.get('mapping') self.__server_side_rpc = config.get('serverSideRpc', []) self.__service_config = { "connectRequests": [], "disconnectRequests": [] } self.__attribute_updates = config.get("attributeUpdates") self.__get_service_config(config) self.__sub_topics = {} client_id = ''.join( random.choice(string.ascii_lowercase) for _ in range(23)) self._client = Client(client_id) self.setName( config.get( "name", self.__broker.get( "name", 'Mqtt Broker ' + ''.join( random.choice(string.ascii_lowercase) for _ in range(5))))) if "username" in self.__broker["security"]: self._client.username_pw_set(self.__broker["security"]["username"], self.__broker["security"]["password"]) if "caCert" in self.__broker["security"] or self.__broker[ "security"].get("type", "none").lower() == "tls": ca_cert = self.__broker["security"].get("caCert") private_key = self.__broker["security"].get("privateKey") cert = self.__broker["security"].get("cert") if ca_cert is None: self._client.tls_set_context( ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) else: try: self._client.tls_set(ca_certs=ca_cert, certfile=cert, keyfile=private_key, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) except Exception as e: self.__log.error( "Cannot setup connection to broker %s using SSL. Please check your configuration.\nError: ", self.get_name()) self.__log.exception(e) self._client.tls_insecure_set(False) self._client.on_connect = self._on_connect self._client.on_message = self._on_message self._client.on_subscribe = self._on_subscribe self.__subscribes_sent = {} # For logging the subscriptions self._client.on_disconnect = self._on_disconnect # self._client.on_log = self._on_log self._connected = False self.__stopped = False self.daemon = True def is_connected(self): return self._connected def open(self): self.__stopped = False self.start() def run(self): try: while not self._connected and not self.__stopped: try: self._client.connect(self.__broker['host'], self.__broker.get('port', 1883)) self._client.loop_start() if not self._connected: time.sleep(1) except Exception as e: self.__log.exception(e) time.sleep(10) except Exception as e: self.__log.exception(e) try: self.close() except Exception as e: self.__log.exception(e) while True: if self.__stopped: break time.sleep(.01) def close(self): self.__stopped = True try: self._client.disconnect() except Exception as e: log.exception(e) self._client.loop_stop() self.__log.info('%s has been stopped.', self.get_name()) def get_name(self): return self.name def __subscribe(self, topic): message = self._client.subscribe(topic) try: self.__subscribes_sent[message[1]] = topic except Exception as e: self.__log.exception(e) def _on_connect(self, client, userdata, flags, result_code, *extra_params): result_codes = { 1: "incorrect protocol version", 2: "invalid client identifier", 3: "server unavailable", 4: "bad username or password", 5: "not authorised", } if result_code == 0: self._connected = True self.__log.info('%s connected to %s:%s - successfully.', self.get_name(), self.__broker["host"], self.__broker.get("port", "1883")) self.__log.debug( "Client %s, userdata %s, flags %s, extra_params %s", str(client), str(userdata), str(flags), extra_params) for mapping in self.__mapping: try: converter = None if mapping["converter"]["type"] == "custom": module = TBUtility.check_and_import( self._connector_type, mapping["converter"]["extension"]) if module is not None: self.__log.debug( 'Custom converter for topic %s - found!', mapping["topicFilter"]) converter = module(mapping) else: self.__log.error( "\n\nCannot find extension module for %s topic.\nPlease check your configuration.\n", mapping["topicFilter"]) else: converter = JsonMqttUplinkConverter(mapping) if converter is not None: regex_topic = TBUtility.topic_to_regex( mapping.get("topicFilter")) if not self.__sub_topics.get(regex_topic): self.__sub_topics[regex_topic] = [] self.__sub_topics[regex_topic].append( {converter: None}) # self._client.subscribe(TBUtility.regex_to_topic(regex_topic)) self.__subscribe(mapping["topicFilter"]) self.__log.info('Connector "%s" subscribe to %s', self.get_name(), TBUtility.regex_to_topic(regex_topic)) else: self.__log.error("Cannot find converter for %s topic", mapping["topicFilter"]) except Exception as e: self.__log.exception(e) try: for request in self.__service_config: if self.__service_config.get(request) is not None: for request_config in self.__service_config.get( request): self.__subscribe(request_config["topicFilter"]) except Exception as e: self.__log.error(e) else: if result_code in result_codes: self.__log.error("%s connection FAIL with error %s %s!", self.get_name(), result_code, result_codes[result_code]) else: self.__log.error("%s connection FAIL with unknown error!", self.get_name()) def _on_disconnect(self, *args): self.__log.debug('"%s" was disconnected. %s', self.get_name(), str(args)) def _on_log(self, *args): self.__log.debug(args) def _on_subscribe(self, _, __, mid, granted_qos): try: if granted_qos[0] == 128: self.__log.error( '"%s" subscription failed to topic %s subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) else: self.__log.info( '"%s" subscription success to topic %s, subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) if self.__subscribes_sent.get(mid) is not None: del self.__subscribes_sent[mid] except Exception as e: self.__log.exception(e) def __get_service_config(self, config): for service_config in self.__service_config: if config.get(service_config): self.__service_config[service_config] = config[service_config] def _on_message(self, client, userdata, message): self.statistics['MessagesReceived'] += 1 content = TBUtility.decode(message) # Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" regex_topic = [ regex for regex in self.__sub_topics if fullmatch(regex, message.topic) ] if regex_topic: try: for regex in regex_topic: if self.__sub_topics.get(regex): for converter_value in range( len(self.__sub_topics.get(regex))): if self.__sub_topics[regex][converter_value]: for converter in self.__sub_topics.get( regex)[converter_value]: converted_content = converter.convert( message.topic, content) if converted_content: try: self.__sub_topics[regex][ converter_value][ converter] = converted_content except Exception as e: self.__log.exception(e) self.__gateway.send_to_storage( self.name, converted_content) self.statistics['MessagesSent'] += 1 else: continue else: self.__log.error( 'Cannot find converter for the topic:"%s"! Client: %s, User data: %s', message.topic, str(client), str(userdata)) return None except Exception as e: log.exception(e) return None # Check if message topic is matched by an existing connection request handler if self.__service_config.get("connectRequests"): for request in self.__service_config["connectRequests"]: # Check that the current connection request handler defines a topic filter (mandatory) if request.get("topicFilter") is None: continue found_device_name = None found_device_type = 'default' # Extract device name and type from regexps, if any. # This cannot be postponed because message topic may contain wildcards if request.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( request["deviceNameJsonExpression"], content) if request.get("deviceNameTopicExpression"): device_name_expression = request[ "deviceNameTopicExpression"] device_name_match = search(device_name_expression, message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) if request.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( request["deviceTypeJsonExpression"], content) if request.get("deviceTypeTopicExpression"): device_type_expression = request[ "deviceTypeTopicExpression"] found_device_type = search(device_type_expression, message.topic) # Check if request topic matches with message topic before of after regexp substitution if message.topic not in request.get("topicFilter"): sub_topic = message.topic # Substitute device name (if defined) in topic if found_device_name is not None: sub_topic = sub(found_device_name, "+", sub_topic) # Substitute device type in topic sub_topic = sub(found_device_type, "+", sub_topic) # If topic still not matches, this is not the correct handler if sub_topic not in request.get("topicFilter"): continue # I'm now sure that this message must be handled by this connection request handler if found_device_name is None: self.__log.error( "Device name missing from connection request") return None # Note: device must be added even if it is already known locally: else ThingsBoard # will not send RPCs and attribute updates self.__gateway.add_device(found_device_name, {"connector": self}, device_type=found_device_type) return None # Check if message topic is matched by an existing disconnection request handler if self.__service_config.get("disconnectRequests"): for request in self.__service_config["disconnectRequests"]: # Check that the current disconnection request handler defines a topic filter (mandatory) if request.get("topicFilter") is None: continue found_device_name = None found_device_type = 'default' # Extract device name and type from regexps, if any. # This cannot be postponed because message topic may contain wildcards if request.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( request["deviceNameJsonExpression"], content) if request.get("deviceNameTopicExpression"): device_name_expression = request[ "deviceNameTopicExpression"] device_name_match = search(device_name_expression, message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) if request.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( request["deviceTypeJsonExpression"], content) if request.get("deviceTypeTopicExpression"): device_type_expression = request[ "deviceTypeTopicExpression"] found_device_type = search(device_type_expression, message.topic) # Check if request topic matches with message topic before of after regexp substitution if message.topic not in request.get("topicFilter"): sub_topic = message.topic # Substitute device name (if defined) in topic if found_device_name is not None: sub_topic = sub(found_device_name, "+", sub_topic) # Substitute device type in topic sub_topic = sub(found_device_type, "+", sub_topic) # If topic still not matches, this is not the correct handler if sub_topic not in request.get("topicFilter"): continue # I'm now sure that this message must be handled by this connection request handler if found_device_name is None: self.__log.error( "Device name missing from disconnection request") return None if found_device_name in self.__gateway.get_devices(): self.__log.info("Device %s of type %s disconnected", found_device_name, found_device_type) self.__gateway.del_device(found_device_name) else: self.__log.info("Device %s is already disconnected", found_device_name) return None if message.topic in self.__gateway.rpc_requests_in_progress: self.__gateway.rpc_with_reply_processing(message.topic, content) else: self.__log.debug( "Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"", message.topic, content) def on_attributes_update(self, content): if self.__attribute_updates: for attribute_update in self.__attribute_updates: if match(attribute_update["deviceNameFilter"], content["device"]): for attribute_key in content["data"]: if match(attribute_update["attributeFilter"], attribute_key): try: topic = attribute_update["topicExpression"]\ .replace("${deviceName}", content["device"])\ .replace("${attributeKey}", attribute_key)\ .replace("${attributeValue}", content["data"][attribute_key]) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e try: data = attribute_update["valueExpression"]\ .replace("${attributeKey}", attribute_key)\ .replace("${attributeValue}", content["data"][attribute_key]) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e self._client.publish(topic, data).wait_for_publish() self.__log.debug( "Attribute Update data: %s for device %s to topic: %s", data, content["device"], topic) else: self.__log.error( "Cannot find attributeName by filter in message with data: %s", content) else: self.__log.error( "Cannot find deviceName by filter in message with data: %s", content) else: self.__log.error("Attribute updates config not found.") def server_side_rpc_handler(self, content): for rpc_config in self.__server_side_rpc: if search(rpc_config["deviceNameFilter"], content["device"]) \ and search(rpc_config["methodFilter"], content["data"]["method"]) is not None: # Subscribe to RPC response topic if rpc_config.get("responseTopicExpression"): topic_for_subscribe = rpc_config["responseTopicExpression"] \ .replace("${deviceName}", content["device"]) \ .replace("${methodName}", content["data"]["method"]) \ .replace("${requestId}", str(content["data"]["id"])) \ .replace("${params}", content["data"]["params"]) if rpc_config.get("responseTimeout"): timeout = time.time() * 1000 + rpc_config.get( "responseTimeout") self.__gateway.register_rpc_request_timeout( content, timeout, topic_for_subscribe, self.rpc_cancel_processing) # Maybe we need to wait for the command to execute successfully before publishing the request. self._client.subscribe(topic_for_subscribe) else: self.__log.error( "Not found RPC response timeout in config, sending without waiting for response" ) # Publish RPC request if rpc_config.get("requestTopicExpression") is not None\ and rpc_config.get("valueExpression"): topic = rpc_config.get("requestTopicExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) data_to_send = rpc_config.get("valueExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) try: self._client.publish(topic, data_to_send) self.__log.debug( "Send RPC with no response request to topic: %s with data %s", topic, data_to_send) if rpc_config.get("responseTopicExpression") is None: self.__gateway.send_rpc_reply( device=content["device"], req_id=content["data"]["id"], success_sent=True) except Exception as e: self.__log.exception(e) def rpc_cancel_processing(self, topic): self._client.unsubscribe(topic)
(ADDR, BLID, PASS) = roomba_get_password.read_info() # If missing config. Get it while ADDR is None or BLID is None or PASS is None: roomba_get_password.main() (ADDR, BLID, PASS) = roomba_get_password.read_info() # Open the data file data_file = open(args.out_file, 'w') data_file.write("Time,X,Y,RSSI\n") # ssl TLS context ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) # MQTT client c = Client(client_id=BLID, userdata=data_file) c.on_message = on_message c.tls_set_context(ssl_context) c.tls_insecure_set(True) c.username_pw_set(username=BLID, password=PASS) c.connect(ADDR, 8883) print "Listening..." try: c.loop_forever() except KeyboardInterrupt: pass data_file.close() c.disconnect()
class MqttConnector(Connector, Thread): def __init__(self, gateway, config, connector_type): super().__init__() self.__gateway = gateway # Reference to TB Gateway self._connector_type = connector_type # Should be "mqtt" self.config = config # mqtt.json contents self.__log = log self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0} self.__subscribes_sent = {} # Extract main sections from configuration --------------------------------------------------------------------- self.__broker = config.get('broker') self.__mapping = [] self.__server_side_rpc = [] self.__connect_requests = [] self.__disconnect_requests = [] self.__attribute_requests = [] self.__attribute_updates = [] mandatory_keys = { "mapping": ['topicFilter', 'converter'], "serverSideRpc": ['deviceNameFilter', 'methodFilter'], "connectRequests": ['topicFilter'], "disconnectRequests": ['topicFilter'], "attributeRequests": ['topicFilter', 'topicExpression', 'valueExpression'], "attributeUpdates": [ 'deviceNameFilter', 'attributeFilter', 'topicExpression', 'valueExpression' ] } # Mappings, i.e., telemetry/attributes-push handlers provided by user via configuration file self.load_handlers('mapping', mandatory_keys['mapping'], self.__mapping) # RPCs, i.e., remote procedure calls (ThingsBoard towards devices) handlers self.load_handlers('serverSideRpc', mandatory_keys['serverSideRpc'], self.__server_side_rpc) # Connect requests, i.e., telling ThingsBoard that a device is online even if it does not post telemetry self.load_handlers('connectRequests', mandatory_keys['connectRequests'], self.__connect_requests) # Disconnect requests, i.e., telling ThingsBoard that a device is offline even if keep-alive has not expired yet self.load_handlers('disconnectRequests', mandatory_keys['disconnectRequests'], self.__disconnect_requests) # Shared attributes direct requests, i.e., asking ThingsBoard for some shared attribute value self.load_handlers('attributeRequests', mandatory_keys['attributeRequests'], self.__attribute_requests) # Attributes updates requests, i.e., asking ThingsBoard to send updates about an attribute self.load_handlers('attributeUpdates', mandatory_keys['attributeUpdates'], self.__attribute_updates) # Setup topic substitution lists for each class of handlers ---------------------------------------------------- self.__mapping_sub_topics = {} self.__connect_requests_sub_topics = {} self.__disconnect_requests_sub_topics = {} self.__attribute_requests_sub_topics = {} # Set up external MQTT broker connection ----------------------------------------------------------------------- client_id = ''.join( random.choice(string.ascii_lowercase) for _ in range(23)) self._client = Client(client_id) self.setName( config.get( "name", self.__broker.get( "name", 'Mqtt Broker ' + ''.join( random.choice(string.ascii_lowercase) for _ in range(5))))) if "username" in self.__broker["security"]: self._client.username_pw_set(self.__broker["security"]["username"], self.__broker["security"]["password"]) if "caCert" in self.__broker["security"] \ or self.__broker["security"].get("type", "none").lower() == "tls": ca_cert = self.__broker["security"].get("caCert") private_key = self.__broker["security"].get("privateKey") cert = self.__broker["security"].get("cert") if ca_cert is None: self._client.tls_set_context( ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) else: try: self._client.tls_set(ca_certs=ca_cert, certfile=cert, keyfile=private_key, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLSv1_2, ciphers=None) except Exception as e: self.__log.error( "Cannot setup connection to broker %s using SSL. " "Please check your configuration.\nError: ", self.get_name()) self.__log.exception(e) self._client.tls_insecure_set(False) # Set up external MQTT broker callbacks ------------------------------------------------------------------------ self._client.on_connect = self._on_connect self._client.on_message = self._on_message self._client.on_subscribe = self._on_subscribe self._client.on_disconnect = self._on_disconnect # self._client.on_log = self._on_log # Set up lifecycle flags --------------------------------------------------------------------------------------- self._connected = False self.__stopped = False self.daemon = True def load_handlers(self, handler_flavor, mandatory_keys, accepted_handlers_list): if handler_flavor not in self.config: self.__log.error("'%s' section missing from configuration", handler_flavor) else: for handler in self.config.get(handler_flavor): discard = False for key in mandatory_keys: if key not in handler: # Will report all missing fields to user before discarding the entry => no break here discard = True self.__log.error( "Mandatory key '%s' missing from %s handler: %s", key, handler_flavor, json.dumps(handler)) else: self.__log.debug( "Mandatory key '%s' found in %s handler: %s", key, handler_flavor, json.dumps(handler)) if discard: self.__log.error( "%s handler is missing some mandatory keys => rejected: %s", handler_flavor, json.dumps(handler)) else: accepted_handlers_list.append(handler) self.__log.debug( "%s handler has all mandatory keys => accepted: %s", handler_flavor, json.dumps(handler)) self.__log.info("Number of accepted %s handlers: %d", handler_flavor, len(accepted_handlers_list)) self.__log.info( "Number of rejected %s handlers: %d", handler_flavor, len(self.config.get(handler_flavor)) - len(accepted_handlers_list)) def is_connected(self): return self._connected def open(self): self.__stopped = False self.start() def run(self): try: while not self._connected and not self.__stopped: try: self._client.connect(self.__broker['host'], self.__broker.get('port', 1883)) self._client.loop_start() if not self._connected: time.sleep(1) except Exception as e: self.__log.exception(e) time.sleep(10) except Exception as e: self.__log.exception(e) try: self.close() except Exception as e: self.__log.exception(e) while True: if self.__stopped: break time.sleep(.01) def close(self): self.__stopped = True try: self._client.disconnect() except Exception as e: log.exception(e) self._client.loop_stop() self.__log.info('%s has been stopped.', self.get_name()) def get_name(self): return self.name def __subscribe(self, topic, qos): message = self._client.subscribe(topic, qos) try: self.__subscribes_sent[message[1]] = topic except Exception as e: self.__log.exception(e) def _on_connect(self, client, userdata, flags, result_code, *extra_params): result_codes = { 1: "incorrect protocol version", 2: "invalid client identifier", 3: "server unavailable", 4: "bad username or password", 5: "not authorised", } if result_code == 0: self._connected = True self.__log.info('%s connected to %s:%s - successfully.', self.get_name(), self.__broker["host"], self.__broker.get("port", "1883")) self.__log.debug( "Client %s, userdata %s, flags %s, extra_params %s", str(client), str(userdata), str(flags), extra_params) # Setup data upload requests handling ---------------------------------------------------------------------- for mapping in self.__mapping: try: converter = None # Load converter for this mapping entry ------------------------------------------------------------ # mappings are guaranteed to have topicFilter and converter fields. See __init__ converter_type = mapping["converter"]["type"] converter_extension = mapping["converter"]["extension"] if converter_type: if converter_extension: module = TBUtility.check_and_import( self._connector_type, converter_extension) if module: self.__log.debug( 'Custom converter for topic %s - found!', mapping["topicFilter"]) converter = module(mapping) else: self.__log.error( "\n\nCannot find extension module for %s topic." "\nPlease check your configuration.\n", mapping["topicFilter"]) else: converter = JsonMqttUplinkConverter(mapping) if converter is None: self.__log.error("Cannot find converter for %s topic", mapping["topicFilter"]) continue # Setup regexp topic acceptance list --------------------------------------------------------------- regex_topic = TBUtility.topic_to_regex( mapping["topicFilter"]) # There may be more than one converter per topic, so I'm using vectors if not self.__mapping_sub_topics.get(regex_topic): self.__mapping_sub_topics[regex_topic] = [] self.__mapping_sub_topics[regex_topic].append(converter) # Subscribe to appropriate topic ------------------------------------------------------------------- self.__subscribe(mapping["topicFilter"], mapping.get("subscriptionQos", 0)) self.__log.info('Connector "%s" subscribe to %s', self.get_name(), TBUtility.regex_to_topic(regex_topic)) except Exception as e: self.__log.exception(e) # Setup connection requests handling ----------------------------------------------------------------------- for request in [ entry for entry in self.__connect_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 0)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__connect_requests_sub_topics[topic_filter] = request # Setup disconnection requests handling -------------------------------------------------------------------- for request in [ entry for entry in self.__disconnect_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 0)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__disconnect_requests_sub_topics[topic_filter] = request # Setup attributes requests handling ----------------------------------------------------------------------- for request in [ entry for entry in self.__attribute_requests if entry is not None ]: # requests are guaranteed to have topicFilter field. See __init__ self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 0)) topic_filter = TBUtility.topic_to_regex( request.get("topicFilter")) self.__attribute_requests_sub_topics[topic_filter] = request else: if result_code in result_codes: self.__log.error("%s connection FAIL with error %s %s!", self.get_name(), result_code, result_codes[result_code]) else: self.__log.error("%s connection FAIL with unknown error!", self.get_name()) def _on_disconnect(self, *args): self.__log.debug('"%s" was disconnected. %s', self.get_name(), str(args)) def _on_log(self, *args): self.__log.debug(args) def _on_subscribe(self, _, __, mid, granted_qos): try: if granted_qos[0] == 128: self.__log.error( '"%s" subscription failed to topic %s subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) else: self.__log.info( '"%s" subscription success to topic %s, subscription message id = %i', self.get_name(), self.__subscribes_sent.get(mid), mid) if self.__subscribes_sent.get(mid) is not None: del self.__subscribes_sent[mid] except Exception as e: self.__log.exception(e) def _on_message(self, client, userdata, message): self.statistics['MessagesReceived'] += 1 content = TBUtility.decode(message) # Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" --------------------------- topic_handlers = [ regex for regex in self.__mapping_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: # Note: every topic may be associated to one or more converter. This means that a single MQTT message # may produce more than one message towards ThingsBoard. This also means that I cannot return after # the first successful conversion: I got to use all the available ones. # I will use a flag to understand whether at least one converter succeeded request_handled = False for topic in topic_handlers: available_converters = self.__mapping_sub_topics[topic] for converter in available_converters: converted_content = converter.convert( message.topic, content) if converted_content: request_handled = True self.__gateway.send_to_storage(self.name, converted_content) self.statistics['MessagesSent'] += 1 self.__log.info( "Successfully converted message from topic %s", message.topic) else: continue if not request_handled: self.__log.error( 'Cannot find converter for the topic:"%s"! Client: %s, User data: %s', message.topic, str(client), str(userdata)) # Note: if I'm in this branch, this was for sure a telemetry/attribute push message # => Execution must end here both in case of failure and success return None # Check if message topic exists in connection handlers "i.e., I'm connecting a device" ------------------------- topic_handlers = [ regex for regex in self.__connect_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: for topic in topic_handlers: handler = self.__connect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search( handler["deviceTypeTopicExpression"], message.topic) if device_type_match is not None: found_device_type = device_type_match.group(0) elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from connection request") continue # Note: device must be added even if it is already known locally: else ThingsBoard # will not send RPCs and attribute updates self.__log.info("Connecting device %s of type %s", found_device_name, found_device_type) self.__gateway.add_device(found_device_name, {"connector": self}, device_type=found_device_type) # Note: if I'm in this branch, this was for sure a connection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in disconnection handlers "i.e., I'm disconnecting a device" ------------------- topic_handlers = [ regex for regex in self.__disconnect_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: for topic in topic_handlers: handler = self.__disconnect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search( handler["deviceTypeTopicExpression"], message.topic) if device_type_match is not None: found_device_type = device_type_match.group(0) elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value( handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from disconnection request") continue if found_device_name in self.__gateway.get_devices(): self.__log.info("Disconnecting device %s of type %s", found_device_name, found_device_type) self.__gateway.del_device(found_device_name) else: self.__log.info("Device %s was not connected", found_device_name) break # Note: if I'm in this branch, this was for sure a disconnection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in attribute request handlers "i.e., I'm asking for a shared attribute" -------- topic_handlers = [ regex for regex in self.__attribute_requests_sub_topics if fullmatch(regex, message.topic) ] if topic_handlers: try: for topic in topic_handlers: handler = self.__attribute_requests_sub_topics[topic] found_device_name = None found_attribute_name = None # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search( handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value( handler["deviceNameJsonExpression"], content) # Get attribute name, either from topic or from content if handler.get("attributeNameTopicExpression"): attribute_name_match = search( handler["attributeNameTopicExpression"], message.topic) if attribute_name_match is not None: found_attribute_name = attribute_name_match.group( 0) elif handler.get("attributeNameJsonExpression"): found_attribute_name = TBUtility.get_value( handler["attributeNameJsonExpression"], content) if found_device_name is None: self.__log.error( "Device name missing from attribute request") continue if found_attribute_name is None: self.__log.error( "Attribute name missing from attribute request") continue self.__log.info("Will retrieve attribute %s of %s", found_attribute_name, found_device_name) self.__gateway.tb_client.client.gw_request_shared_attributes( found_device_name, [found_attribute_name], lambda data, *args: self.notify_attribute( data, found_attribute_name, handler.get("topicExpression"), handler.get("valueExpression"))) break except Exception as e: log.exception(e) # Note: if I'm in this branch, this was for sure an attribute request message # => Execution must end here both in case of failure and success return None # Check if message topic exists in RPC handlers ---------------------------------------------------------------- # The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is if message.topic in self.__gateway.rpc_requests_in_progress: self.__gateway.rpc_with_reply_processing(message.topic, content) return None self.__log.debug( "Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"", message.topic, content) def notify_attribute(self, incoming_data, attribute_name, topic_expression, value_expression): if incoming_data.get("device") is None or incoming_data.get( "value") is None: return device_name = incoming_data.get("device") attribute_value = incoming_data.get("value") topic = topic_expression \ .replace("${deviceName}", device_name) \ .replace("${attributeKey}", attribute_name) data = value_expression.replace("${attributeKey}", attribute_name) \ .replace("${attributeValue}", attribute_value) self._client.publish(topic, data).wait_for_publish() def on_attributes_update(self, content): if self.__attribute_updates: for attribute_update in self.__attribute_updates: if match(attribute_update["deviceNameFilter"], content["device"]): for attribute_key in content["data"]: if match(attribute_update["attributeFilter"], attribute_key): try: topic = attribute_update["topicExpression"]\ .replace("${deviceName}", content["device"])\ .replace("${attributeKey}", attribute_key)\ .replace("${attributeValue}", content["data"][attribute_key]) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e try: data = attribute_update["valueExpression"]\ .replace("${attributeKey}", attribute_key)\ .replace("${attributeValue}", content["data"][attribute_key]) except KeyError as e: log.exception( "Cannot form topic, key %s - not found", e) raise e self._client.publish(topic, data).wait_for_publish() self.__log.debug( "Attribute Update data: %s for device %s to topic: %s", data, content["device"], topic) else: self.__log.error( "Cannot find attributeName by filter in message with data: %s", content) else: self.__log.error( "Cannot find deviceName by filter in message with data: %s", content) else: self.__log.error("Attribute updates config not found.") def server_side_rpc_handler(self, content): for rpc_config in self.__server_side_rpc: if search(rpc_config["deviceNameFilter"], content["device"]) \ and search(rpc_config["methodFilter"], content["data"]["method"]) is not None: # Subscribe to RPC response topic if rpc_config.get("responseTopicExpression"): topic_for_subscribe = rpc_config["responseTopicExpression"] \ .replace("${deviceName}", content["device"]) \ .replace("${methodName}", content["data"]["method"]) \ .replace("${requestId}", str(content["data"]["id"])) \ .replace("${params}", content["data"]["params"]) if rpc_config.get("responseTimeout"): timeout = time.time() * 1000 + rpc_config.get( "responseTimeout") self.__gateway.register_rpc_request_timeout( content, timeout, topic_for_subscribe, self.rpc_cancel_processing) # Maybe we need to wait for the command to execute successfully before publishing the request. self._client.subscribe(topic_for_subscribe) else: self.__log.error( "Not found RPC response timeout in config, sending without waiting for response" ) # Publish RPC request if rpc_config.get("requestTopicExpression") is not None\ and rpc_config.get("valueExpression"): topic = rpc_config.get("requestTopicExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) data_to_send = rpc_config.get("valueExpression")\ .replace("${deviceName}", content["device"])\ .replace("${methodName}", content["data"]["method"])\ .replace("${requestId}", str(content["data"]["id"]))\ .replace("${params}", content["data"]["params"]) try: self._client.publish(topic, data_to_send) self.__log.debug( "Send RPC with no response request to topic: %s with data %s", topic, data_to_send) if rpc_config.get("responseTopicExpression") is None: self.__gateway.send_rpc_reply( device=content["device"], req_id=content["data"]["id"], success_sent=True) except Exception as e: self.__log.exception(e) def rpc_cancel_processing(self, topic): self._client.unsubscribe(topic)