def convert(self, _, body): try: data = body["data"]["value"] self.dict_result["deviceName"] = TBUtility.get_value( self.__config.get("deviceNameJsonExpression"), body, expression_instead_none=True) self.dict_result["deviceType"] = TBUtility.get_value( self.__config.get("deviceTypeJsonExpression"), body, expression_instead_none=True) self.dict_result["attributes"] = [] self.dict_result["telemetry"] = [] converted_bytes = bytearray.fromhex(data) if self.__config.get("extension-config") is not None: for telemetry_key in self.__config["extension-config"]: value = None byteorder = telemetry_key.get("byteorder", "big").lower() signed = telemetry_key.get("signed", True) if telemetry_key.get("byteAddress") is None: interest_bytes = converted_bytes[ telemetry_key["fromByte"]:telemetry_key["toByte"]] if telemetry_key["type"] == "float": value = struct.unpack( ">f" if byteorder == "big" else "<f", interest_bytes) value = value[0] if isinstance(value, list) else None if telemetry_key["type"] == "int": value = int.from_bytes(interest_bytes, byteorder=byteorder, signed=signed) else: interest_byte = converted_bytes[ telemetry_key["byteAddress"]] bits = "{0:{fill}8b}".format(interest_byte, fill='0') bits = bits if byteorder == "big" else bits[::-1] value = int( bits[::-1][telemetry_key.get("fromBit"): telemetry_key.get("toBit")][::-1], 2) if value is not None: value = value * telemetry_key.get("multiplier", 1) telemetry_to_send = { telemetry_key["key"]: value } # creating telemetry data for sending into Thingsboard # current_byte_position += self.__config["extension-config"][telemetry_key] self.dict_result["telemetry"].append( telemetry_to_send ) # adding data to telemetry array else: self.dict_result["telemetry"] = { "data": int(body, 0) } # if no specific configuration in config file - just send data which received return self.dict_result except Exception as e: log.error( 'Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), body) log.exception(e)
def add_device(self, data): if self.__devices_address_name.get(data["address"]) is None: for device in self.__config_devices: if device["address"] == data["address"]: try: config_address = Address(device["address"]) device_name_tag = TBUtility.get_value( device["deviceName"], get_tag=True) device_name = device["deviceName"].replace( "${" + device_name_tag + "}", data.pop("name")) device_information = { **data, **self.__get_requests_configs(device), "type": device["deviceType"], "config": device, "attributes": device.get("attributes", []), "telemetry": device.get("timeseries", []), "poll_period": device.get("pollPeriod", 5000), "deviceName": device_name, } if config_address == data["address"] or \ (config_address, GlobalBroadcast) or \ (isinstance(config_address, LocalBroadcast) and isinstance(device["address"], LocalStation)) or \ (isinstance(config_address, (LocalStation, RemoteStation)) and isinstance(data["address"], ( LocalStation, RemoteStation))): self.__devices_address_name[data[ "address"]] = device_information["deviceName"] self.__devices.append(device_information) log.debug(data["address"].addrType) except Exception as e: log.exception(e)
def _on_message(self, client, userdata, message): self.statistics['MessagesReceived'] += 1 content = TBUtility.decode(message) #print("[MqttConnector] content", content) # Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" --------------------------- topic_handlers = [regex for regex in self.__mapping_sub_topics if fullmatch(regex, message.topic)] # for regex in self.__mapping_sub_topics: # print("[MqttConnector] self.__mapping_sub_topics",regex) if topic_handlers: # Note: every topic may be associated to one or more converter. This means that a single MQTT message # may produce more than one message towards ThingsBoard. This also means that I cannot return after # the first successful conversion: I got to use all the available ones. # I will use a flag to understand whether at least one converter succeeded request_handled = False for topic in topic_handlers: available_converters = self.__mapping_sub_topics[topic] for converter in available_converters: try: converted_content = converter.convert(message.topic, content) #print("[MqttConnector] converted_content", converted_content) if converted_content: request_handled = True self.__gateway.send_to_storage(self.name, converted_content) self.statistics['MessagesSent'] += 1 self.__log.debug("Successfully converted message from topic %s", message.topic) else: continue except Exception as e: log.exception(e) if not request_handled: self.__log.error('Cannot find converter for the topic:"%s"! Client: %s, User data: %s', message.topic, str(client), str(userdata)) # Note: if I'm in this branch, this was for sure a telemetry/attribute push message # => Execution must end here both in case of failure and success return None # Check if message topic exists in connection handlers "i.e., I'm connecting a device" ------------------------- topic_handlers = [regex for regex in self.__connect_requests_sub_topics if fullmatch(regex, message.topic)] if topic_handlers: for topic in topic_handlers: handler = self.__connect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search(handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value(handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search(handler["deviceTypeTopicExpression"], message.topic) found_device_type = device_type_match.group(0) if device_type_match is not None else handler["deviceTypeTopicExpression"] elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value(handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error("Device name missing from connection request") continue # Note: device must be added even if it is already known locally: else ThingsBoard # will not send RPCs and attribute updates self.__log.info("Connecting device %s of type %s", found_device_name, found_device_type) self.__gateway.add_device(found_device_name, {"connector": self}, device_type=found_device_type) # Note: if I'm in this branch, this was for sure a connection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in disconnection handlers "i.e., I'm disconnecting a device" ------------------- topic_handlers = [regex for regex in self.__disconnect_requests_sub_topics if fullmatch(regex, message.topic)] if topic_handlers: for topic in topic_handlers: handler = self.__disconnect_requests_sub_topics[topic] found_device_name = None found_device_type = 'default' # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search(handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value(handler["deviceNameJsonExpression"], content) # Get device type (if any), either from topic or from content if handler.get("deviceTypeTopicExpression"): device_type_match = search(handler["deviceTypeTopicExpression"], message.topic) if device_type_match is not None: found_device_type = device_type_match.group(0) elif handler.get("deviceTypeJsonExpression"): found_device_type = TBUtility.get_value(handler["deviceTypeJsonExpression"], content) if found_device_name is None: self.__log.error("Device name missing from disconnection request") continue if found_device_name in self.__gateway.get_devices(): self.__log.info("Disconnecting device %s of type %s", found_device_name, found_device_type) self.__gateway.del_device(found_device_name) else: self.__log.info("Device %s was not connected", found_device_name) break # Note: if I'm in this branch, this was for sure a disconnection message # => Execution must end here both in case of failure and success return None # Check if message topic exists in attribute request handlers "i.e., I'm asking for a shared attribute" -------- topic_handlers = [regex for regex in self.__attribute_requests_sub_topics if fullmatch(regex, message.topic)] if topic_handlers: try: for topic in topic_handlers: handler = self.__attribute_requests_sub_topics[topic] found_device_name = None found_attribute_name = None # Get device name, either from topic or from content if handler.get("deviceNameTopicExpression"): device_name_match = search(handler["deviceNameTopicExpression"], message.topic) if device_name_match is not None: found_device_name = device_name_match.group(0) elif handler.get("deviceNameJsonExpression"): found_device_name = TBUtility.get_value(handler["deviceNameJsonExpression"], content) # Get attribute name, either from topic or from content if handler.get("attributeNameTopicExpression"): attribute_name_match = search(handler["attributeNameTopicExpression"], message.topic) if attribute_name_match is not None: found_attribute_name = attribute_name_match.group(0) elif handler.get("attributeNameJsonExpression"): found_attribute_name = TBUtility.get_value(handler["attributeNameJsonExpression"], content) if found_device_name is None: self.__log.error("Device name missing from attribute request") continue if found_attribute_name is None: self.__log.error("Attribute name missing from attribute request") continue self.__log.info("Will retrieve attribute %s of %s", found_attribute_name, found_device_name) self.__gateway.tb_client.client.gw_request_shared_attributes( found_device_name, [found_attribute_name], lambda data, *args: self.notify_attribute( data, found_attribute_name, handler.get("topicExpression"), handler.get("valueExpression"))) break except Exception as e: log.exception(e) # Note: if I'm in this branch, this was for sure an attribute request message # => Execution must end here both in case of failure and success return None # Check if message topic exists in RPC handlers ---------------------------------------------------------------- # The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is if message.topic in self.__gateway.rpc_requests_in_progress: self.__gateway.rpc_with_reply_processing(message.topic, content) return None self.__log.debug("Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"", message.topic, content)
def __search_general_info(self, device): result = [] match_devices = [] self.__search_node(self.__opcua_nodes["root"], TBUtility.get_value(device["deviceNodePattern"], get_tag=True), result=match_devices) for device_node in match_devices: if device_node is not None: result_device_dict = { "deviceName": None, "deviceType": None, "deviceNode": device_node, "configuration": deepcopy(device) } name_pattern_config = device["deviceNamePattern"] name_expression = TBUtility.get_value(name_pattern_config, get_tag=True) if "${" in name_pattern_config and "}" in name_pattern_config: log.debug("Looking for device name") device_name_from_node = "" if name_expression == "$DisplayName": device_name_from_node = device_node.get_display_name( ).Text elif name_expression == "$BrowseName": device_name_from_node = device_node.get_browse_name( ).Name elif name_expression == "$NodeId.Identifier": device_name_from_node = str( device_node.nodeid.Identifier) else: name_path = self._check_path(name_expression, device_node) device_name_node = [] self.__search_node(device_node, name_path, result=device_name_node) device_name_node = device_name_node[0] if device_name_node is not None: device_name_from_node = device_name_node.get_value( ) if device_name_from_node == "": log.error( "Device name node not found with expression: %s", name_expression) return None full_device_name = name_pattern_config.replace( "${" + name_expression + "}", str(device_name_from_node)).replace( name_expression, str(device_name_from_node)) else: full_device_name = name_expression result_device_dict["deviceName"] = full_device_name log.debug("Device name: %s", full_device_name) if device.get("deviceTypePattern"): device_type_expression = TBUtility.get_value( device["deviceTypePattern"], get_tag=True) if "${" in device_type_expression and "}" in device_type_expression: type_path = self._check_path(device_type_expression, device_node) device_type_node = [] self.__search_node(device_node, type_path, result=device_type_node) device_type_node = device_type_node[0] if device_type_node is not None: device_type = device_type_node.get_value() full_device_type = device_type_expression.replace( "${" + device_type_expression + "}", device_type).replace(device_type_expression, device_type) else: log.error( "Device type node not found with expression: %s", device_type_expression) full_device_type = "default" else: full_device_type = device_type_expression result_device_dict["deviceType"] = full_device_type log.debug("Device type: %s", full_device_type) else: result_device_dict["deviceType"] = "default" result.append(result_device_dict) else: log.error( "Device node not found with expression: %s", TBUtility.get_value(device["deviceNodePattern"], get_tag=True)) return result
def __search_nodes_and_subscribe(self, device_info): sub_nodes = [] information_types = { "attributes": "attributes", "timeseries": "telemetry" } for information_type in information_types: for information in device_info["configuration"][information_type]: information_key = information["key"] config_path = TBUtility.get_value(information["path"], get_tag=True) information_path = self._check_path(config_path, device_info["deviceNode"]) information["path"] = '${%s}' % information_path information_nodes = [] self.__search_node(device_info["deviceNode"], information_path, result=information_nodes) for information_node in information_nodes: if information_node is not None: try: information_value = information_node.get_value() except: log.error("Err get_value: %s", str(information_node)) continue log.debug( "Node for %s \"%s\" with path: %s - FOUND! Current values is: %s", information_type, information_key, information_path, str(information_value)) if device_info.get("uplink_converter") is None: configuration = { **device_info["configuration"], "deviceName": device_info["deviceName"], "deviceType": device_info["deviceType"] } if device_info["configuration"].get( 'converter') is None: converter = OpcUaUplinkConverter(configuration) else: converter = TBModuleLoader.import_module( self._connector_type, configuration) device_info["uplink_converter"] = converter else: converter = device_info["uplink_converter"] self.subscribed[information_node] = { "converter": converter, "path": information_path, "config_path": config_path } if not device_info.get( information_types[information_type]): device_info[ information_types[information_type]] = [] converted_data = converter.convert( (config_path, information_path), information_value) self.statistics['MessagesReceived'] = self.statistics[ 'MessagesReceived'] + 1 self.data_to_send.append(converted_data) self.statistics['MessagesSent'] = self.statistics[ 'MessagesSent'] + 1 log.debug("Data to ThingsBoard: %s", converted_data) if not self.__server_conf.get("disableSubscriptions", False): sub_nodes.append(information_node) else: log.error( "Node for %s \"%s\" with path %s - NOT FOUND!", information_type, information_key, information_path) if not self.__server_conf.get("disableSubscriptions", False): if self.__sub is None: self.__sub = self.client.create_subscription( self.__server_conf.get("subCheckPeriodInMillis", 500), self.__sub_handler) if sub_nodes: self.__sub.subscribe_data_change(sub_nodes) log.debug("Added subscription to nodes: %s", str(sub_nodes))
def convert(self, config, data): #print("[JsonMqttUplinkConverter]",config,data) #print("[JsonMqttUplinkConverter] self.__config",self.__config) datatypes = {"attributes": "attributes", "timeseries": "telemetry"} dict_result = { "deviceName": None, "deviceType": None, "attributes": [], "telemetry": [] } try: if self.__config.get("deviceNameJsonExpression") is not None: dict_result["deviceName"] = TBUtility.get_value( self.__config.get("deviceNameJsonExpression"), data, expression_instead_none=True) elif self.__config.get("deviceNameTopicExpression") is not None: search_result = search( self.__config["deviceNameTopicExpression"], config) if search_result is not None: dict_result["deviceName"] = search_result.group(0) else: log.debug( "Regular expression result is None. deviceNameTopicExpression parameter will be interpreted as a deviceName\n Topic: %s\nRegex: %s", config, self.__config.get("deviceNameTopicExpression")) dict_result["deviceName"] = self.__config.get( "deviceNameTopicExpression") else: log.error( "The expression for looking \"deviceName\" not found in config %s", dumps(self.__config)) if self.__config.get("deviceTypeJsonExpression") is not None: dict_result["deviceType"] = TBUtility.get_value( self.__config.get("deviceTypeJsonExpression"), data, expression_instead_none=True) elif self.__config.get("deviceTypeTopicExpression") is not None: search_result = search( self.__config["deviceTypeTopicExpression"], config) if search_result is not None: dict_result["deviceType"] = search_result.group(0) else: log.debug( "Regular expression result is None. deviceTypeTopicExpression will be interpreted as a deviceType\n Topic: %s\nRegex: %s", config, self.__config.get("deviceTypeTopicExpression")) dict_result["deviceType"] = self.__config.get( "deviceTypeTopicExpression") else: log.error( "The expression for looking \"deviceType\" not found in config %s", dumps(self.__config)) if self.__config.get("udoi") is not None: dict_result["udoi"] = TBUtility.get_value( self.__config.get("udoi"), data, expression_instead_none=True) else: log.error( "The expression for looking \"udoi\" not found in config %s", dumps(self.__config['converter'])) if self.__config.get("schemaId") is not None: dict_result["schemaId"] = TBUtility.get_value( self.__config.get("schemaId"), data, expression_instead_none=True) else: log.error( "The expression for looking \"schemaId\" not found in config %s", dumps(self.__config['converter'])) except Exception as e: log.error( 'Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), data) log.exception(e) try: for datatype in datatypes: #print("datatype",datatype) dict_result[datatypes[datatype]] = [] for datatype_config in self.__config.get(datatype, []): #print("[JsonMqttUplinkConverter] datatype_config",datatype_config) value = TBUtility.get_value(datatype_config["value"], data, datatype_config["type"], expression_instead_none=True) #print("[JsonMqttUplinkConverter]",value) value_tag = TBUtility.get_value(datatype_config["value"], data, datatype_config["type"], get_tag=True) #print("[JsonMqttUplinkConverter]",value_tag) key = TBUtility.get_value(datatype_config["key"], data, datatype_config["type"], expression_instead_none=True) #print("[JsonMqttUplinkConverter]",key) key_tag = TBUtility.get_value(datatype_config["key"], data, get_tag=True) #print("[JsonMqttUplinkConverter]",key_tag) #print("[JsonMqttUplinkConverter]",value,value_tag,key,key_tag) if ("${" not in str(value) and "}" not in str(value)) \ and ("${" not in str(key) and "}" not in str(key)): is_valid_key = isinstance( key, str) and "${" in datatype_config[ "key"] and "}" in datatype_config["key"] is_valid_value = isinstance( value, str) and "${" in datatype_config[ "value"] and "}" in datatype_config["value"] full_key = datatype_config["key"].replace( '${' + str(key_tag) + '}', str(key)) if is_valid_key else key_tag full_value = datatype_config["value"].replace( '${' + str(value_tag) + '}', str(value)) if is_valid_value else value #print("[JsonMqttUplinkConverter]",is_valid_key,is_valid_value,full_key,full_value) if datatype == 'timeseries' and ( data.get("ts") is not None or data.get("timestamp") is not None): dict_result[datatypes[datatype]].append({ "ts": data.get('ts', data.get('timestamp', int(time()))), 'values': { full_key: full_value } }) else: dict_result[datatypes[datatype]].append( {full_key: full_value}) except Exception as e: log.error( 'Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), str(data)) log.exception(e) return dict_result
def convert(self, config, data): datatypes = {"attributes": "attributes", "timeseries": "telemetry"} dict_result = { "deviceName": None, "deviceType": None, "attributes": [], "telemetry": [] } try: if self.__config.get("deviceNameExpression") is not None: dict_result["deviceName"] = TBUtility.get_value( self.__config.get("deviceNameExpression"), data, expression_instead_none=True) else: log.error( "The expression for looking \"deviceName\" not found in config %s", dumps(self.__config)) if self.__config.get("deviceTypeExpression") is not None: dict_result["deviceType"] = TBUtility.get_value( self.__config.get("deviceTypeExpression"), data, expression_instead_none=True) else: log.error( "The expression for looking \"deviceType\" not found in config %s", dumps(self.__config)) except Exception as e: log.error( 'Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), data) log.exception(e) try: for datatype in datatypes: dict_result[datatypes[datatype]] = [] for datatype_config in self.__config.get(datatype, []): value = TBUtility.get_value(datatype_config["value"], data, datatype_config["type"], expression_instead_none=True) value_tag = TBUtility.get_value(datatype_config["value"], data, datatype_config["type"], get_tag=True) key = TBUtility.get_value(datatype_config["key"], data, datatype_config["type"], expression_instead_none=True) key_tag = TBUtility.get_value(datatype_config["key"], data, get_tag=True) if ("${" not in str(value) and "}" not in str(value)) \ and ("${" not in str(key) and "}" not in str(key)): is_valid_key = isinstance( key, str) and "${" in datatype_config[ "key"] and "}" in datatype_config["key"] is_valid_value = isinstance( value, str) and "${" in datatype_config[ "value"] and "}" in datatype_config["value"] full_key = datatype_config["key"].replace( '${' + str(key_tag) + '}', str(key)) if is_valid_key else key full_value = datatype_config["value"].replace( '${' + value_tag + '}', value) if is_valid_value else value if datatype == 'timeseries' and ( data.get("ts") is not None or data.get("timestamp") is not None): dict_result[datatypes[datatype]].append({ "ts": data.get('ts', data.get('timestamp', int(time()))), 'values': { full_key: full_value } }) else: dict_result[datatypes[datatype]].append( {full_key: full_value}) except Exception as e: log.error( 'Error in converter, for config: \n%s\n and message: \n%s\n', dumps(self.__config), str(data)) log.exception(e) return dict_result
def convert(self, config, data): #print("[JsonRequestUplinkConverter convert]", self.__config, data) if isinstance(data, (bytes, str)): data = loads(data) datatypes = {"attributes": "attributes", "timeseries": "telemetry"} dict_result = { "deviceName": None, "deviceType": None, "udoi": None, "schemaId": None, "attributes": [], "telemetry": [] } try: if self.__config['converter'].get( "deviceNameJsonExpression") is not None: dict_result["deviceName"] = TBUtility.get_value( self.__config['converter'].get("deviceNameJsonExpression"), data, expression_instead_none=True) else: log.error( "The expression for looking \"deviceName\" not found in config %s", dumps(self.__config['converter'])) if self.__config['converter'].get( "deviceTypeJsonExpression") is not None: dict_result["deviceType"] = TBUtility.get_value( self.__config['converter'].get("deviceTypeJsonExpression"), data, expression_instead_none=True) else: log.error( "The expression for looking \"deviceType\" not found in config %s", dumps(self.__config['converter'])) if self.__config['converter'].get("udoi") is not None: dict_result["udoi"] = TBUtility.get_value( self.__config['converter'].get("udoi"), data, expression_instead_none=True) else: log.error( "The expression for looking \"udoi\" not found in config %s", dumps(self.__config['converter'])) if self.__config['converter'].get("schemaId") is not None: dict_result["schemaId"] = TBUtility.get_value( self.__config['converter'].get("schemaId"), data, expression_instead_none=True) else: log.error( "The expression for looking \"schemaId\" not found in config %s", dumps(self.__config['converter'])) except Exception as e: log.exception(e) try: for datatype in self.__datatypes: current_datatype = self.__datatypes[datatype] for datatype_object_config in self.__config["converter"].get( datatype, []): #print("[JsonRequestUplinkConverter convert datatype_object_config]",datatype_object_config) datatype_object_config_key = TBUtility.get_value( datatype_object_config["key"], data, datatype_object_config["type"], expression_instead_none=True) datatype_object_config_value = TBUtility.get_value( datatype_object_config["value"], data, datatype_object_config["type"], expression_instead_none=True) #print("[JsonRequestUplinkConverter convert]",datatype_object_config_key,datatype_object_config_value) if datatype_object_config_key is not None and datatype_object_config_value is not None: dict_result[current_datatype].append({ datatype_object_config_key: datatype_object_config_value }) else: error_string = "Cannot find the key in the input data" if datatype_object_config_key is None else "Cannot find the value from the input data" log.error(error_string) except Exception as e: log.exception(e) #print("[JsonRequestUplinkConverter convert dict_result]",dict_result) return dict_result