def convert(self, _, body):
try:
data = body["data"]["value"]
self.dict_result["deviceName"] = TBUtility.get_value(
self.__config.get("deviceNameJsonExpression"),
body,
expression_instead_none=True)
self.dict_result["deviceType"] = TBUtility.get_value(
self.__config.get("deviceTypeJsonExpression"),
body,
expression_instead_none=True)
self.dict_result["attributes"] = []
self.dict_result["telemetry"] = []
converted_bytes = bytearray.fromhex(data)
if self.__config.get("extension-config") is not None:
for telemetry_key in self.__config["extension-config"]:
value = None
byteorder = telemetry_key.get("byteorder", "big").lower()
signed = telemetry_key.get("signed", True)
if telemetry_key.get("byteAddress") is None:
interest_bytes = converted_bytes[
telemetry_key["fromByte"]:telemetry_key["toByte"]]
if telemetry_key["type"] == "float":
value = struct.unpack(
">f" if byteorder == "big" else "<f",
interest_bytes)
value = value[0] if isinstance(value,
list) else None
if telemetry_key["type"] == "int":
value = int.from_bytes(interest_bytes,
byteorder=byteorder,
signed=signed)
else:
interest_byte = converted_bytes[
telemetry_key["byteAddress"]]
bits = "{0:{fill}8b}".format(interest_byte, fill='0')
bits = bits if byteorder == "big" else bits[::-1]
value = int(
bits[::-1][telemetry_key.get("fromBit"):
telemetry_key.get("toBit")][::-1], 2)
if value is not None:
value = value * telemetry_key.get("multiplier", 1)
telemetry_to_send = {
telemetry_key["key"]: value
} # creating telemetry data for sending into Thingsboard
# current_byte_position += self.__config["extension-config"][telemetry_key]
self.dict_result["telemetry"].append(
telemetry_to_send
) # adding data to telemetry array
else:
self.dict_result["telemetry"] = {
"data": int(body, 0)
} # if no specific configuration in config file - just send data which received
return self.dict_result
except Exception as e:
log.error(
'Error in converter, for config: \n%s\n and message: \n%s\n',
dumps(self.__config), body)
log.exception(e)
def __get_converter(self, config, need_uplink):
if config is None:
return BytesCanUplinkConverter(
) if need_uplink else BytesCanDownlinkConverter()
else:
if need_uplink:
uplink = config.get("uplink")
return BytesCanUplinkConverter() if uplink is None \
else TBUtility.check_and_import(self.__connector_type, uplink)
else:
downlink = config.get("downlink")
return BytesCanDownlinkConverter() if downlink is None \
else TBUtility.check_and_import(self.__connector_type, downlink)
def __prepare_connectors_configuration(self, input_connector_config):
try:
self.__gateway.connectors_configs = {}
for connector in input_connector_config['thingsboard'][
'connectors']:
for input_connector in input_connector_config[
connector['type']]:
if input_connector['name'] == connector['name']:
if not self.__gateway.connectors_configs.get(
connector['type']):
self.__gateway.connectors_configs[
connector['type']] = []
self.__gateway.connectors_configs[
connector['type']].append({
"name": connector["name"],
"config": {
connector['configuration']:
input_connector["config"]
}
})
connector_class = TBUtility.check_and_import(
connector["type"],
self.__gateway._default_connectors.get(
connector["type"], connector.get("class")))
self.__gateway._implemented_connectors[
connector["type"]] = connector_class
except Exception as e:
LOG.exception(e)
def update(self):
if self.__version["latest_version"] != self.__version[
"current_version"]:
result = TBUtility.install_package(
"thingsboard-gateway", self.__version["latest_version"])
else:
result = "Congratulations! You have the latest version."
return result
def __init__(self, gateway, config, connector_type):
self.__connector_type = connector_type
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
super().__init__()
self.__config = config
self.setName(
config.get(
'name',
'BACnet ' + ''.join(choice(ascii_lowercase)
for _ in range(5))))
self.__devices = []
self.__device_indexes = {}
self.__devices_address_name = {}
self.__gateway = gateway
self._application = TBBACnetApplication(self, self.__config)
self.__bacnet_core_thread = Thread(target=run,
name="BACnet core thread")
self.__bacnet_core_thread.start()
self.__stopped = False
self.__config_devices = self.__config["devices"]
self.default_converters = {
"uplink_converter":
TBUtility.check_and_import(self.__connector_type,
"BACnetUplinkConverter"),
"downlink_converter":
TBUtility.check_and_import(self.__connector_type,
"BACnetDownlinkConverter")
}
self.__request_functions = {
"writeProperty": self._application.do_write_property,
"readProperty": self._application.do_read_property
}
self.__available_object_resources = {}
self.rpc_requests_in_progress = {}
self.__connected = False
self.daemon = True
def __load_converters(self, device):
datatypes = [
"attributes", "telemetry", "attribute_updates", "server_side_rpc"
]
for datatype in datatypes:
for datatype_config in device.get(datatype, []):
try:
for converter_type in self.default_converters:
converter_object = self.default_converters[
converter_type] if datatype_config.get(
"class"
) is None else TBUtility.check_and_import(
self.__connector_type, device.get("class"))
datatype_config[converter_type] = converter_object(
device)
except Exception as e:
log.exception(e)
def send_to_storage(self, connector_name, data):
#print("[TBGatewayService send_to_storage]",connector_name,data)
if not connector_name == self.name:
if not TBUtility.validate_converted_data(data):
log.error("Data from %s connector is invalid.", connector_name)
return None
if data["deviceName"] not in self.get_devices():
self.add_device(
data["deviceName"],
{"connector": self.available_connectors[connector_name]},
wait_for_publish=True,
device_type=data["deviceType"])
if not self.__connector_incoming_messages.get(connector_name):
self.__connector_incoming_messages[connector_name] = 0
else:
self.__connector_incoming_messages[connector_name] += 1
else:
data["deviceName"] = "currentThingsBoardGateway"
telemetry = {}
telemetry_with_ts = []
for item in data["telemetry"]:
if item.get("ts") is None:
telemetry = {**telemetry, **item}
else:
telemetry_with_ts.append({
"ts": item["ts"],
"values": {
**item["values"]
}
})
if telemetry_with_ts:
data["telemetry"] = telemetry_with_ts
else:
data["telemetry"] = {"ts": int(time() * 1000), "values": telemetry}
json_data = dumps(data)
#print("[TBGatewayService send_to_storage json_data]",json_data)
save_result = self._event_storage.put(json_data)
if not save_result:
log.error(
'Data from the device "%s" cannot be saved, connector name is %s.',
data["deviceName"], connector_name)
def add_device(self, data):
if self.__devices_address_name.get(data["address"]) is None:
for device in self.__config_devices:
if device["address"] == data["address"]:
try:
config_address = Address(device["address"])
device_name_tag = TBUtility.get_value(
device["deviceName"], get_tag=True)
device_name = device["deviceName"].replace(
"${" + device_name_tag + "}", data.pop("name"))
device_information = {
**data,
**self.__get_requests_configs(device),
"type":
device["deviceType"],
"config":
device,
"attributes":
device.get("attributes", []),
"telemetry":
device.get("timeseries", []),
"poll_period":
device.get("pollPeriod", 5000),
"deviceName":
device_name,
}
if config_address == data["address"] or \
(config_address, GlobalBroadcast) or \
(isinstance(config_address, LocalBroadcast) and isinstance(device["address"], LocalStation)) or \
(isinstance(config_address, (LocalStation, RemoteStation)) and isinstance(data["address"], (
LocalStation, RemoteStation))):
self.__devices_address_name[data[
"address"]] = device_information["deviceName"]
self.__devices.append(device_information)
log.debug(data["address"].addrType)
except Exception as e:
log.exception(e)
def _on_message(self, client, userdata, message):
self.statistics['MessagesReceived'] += 1
content = TBUtility.decode(message)
#print("[MqttConnector] content", content)
# Check if message topic exists in mappings "i.e., I'm posting telemetry/attributes" ---------------------------
topic_handlers = [regex for regex in self.__mapping_sub_topics if fullmatch(regex, message.topic)]
# for regex in self.__mapping_sub_topics:
# print("[MqttConnector] self.__mapping_sub_topics",regex)
if topic_handlers:
# Note: every topic may be associated to one or more converter. This means that a single MQTT message
# may produce more than one message towards ThingsBoard. This also means that I cannot return after
# the first successful conversion: I got to use all the available ones.
# I will use a flag to understand whether at least one converter succeeded
request_handled = False
for topic in topic_handlers:
available_converters = self.__mapping_sub_topics[topic]
for converter in available_converters:
try:
converted_content = converter.convert(message.topic, content)
#print("[MqttConnector] converted_content", converted_content)
if converted_content:
request_handled = True
self.__gateway.send_to_storage(self.name, converted_content)
self.statistics['MessagesSent'] += 1
self.__log.debug("Successfully converted message from topic %s", message.topic)
else:
continue
except Exception as e:
log.exception(e)
if not request_handled:
self.__log.error('Cannot find converter for the topic:"%s"! Client: %s, User data: %s',
message.topic,
str(client),
str(userdata))
# Note: if I'm in this branch, this was for sure a telemetry/attribute push message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in connection handlers "i.e., I'm connecting a device" -------------------------
topic_handlers = [regex for regex in self.__connect_requests_sub_topics if fullmatch(regex, message.topic)]
if topic_handlers:
for topic in topic_handlers:
handler = self.__connect_requests_sub_topics[topic]
found_device_name = None
found_device_type = 'default'
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(handler["deviceNameTopicExpression"], message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(handler["deviceNameJsonExpression"], content)
# Get device type (if any), either from topic or from content
if handler.get("deviceTypeTopicExpression"):
device_type_match = search(handler["deviceTypeTopicExpression"], message.topic)
found_device_type = device_type_match.group(0) if device_type_match is not None else handler["deviceTypeTopicExpression"]
elif handler.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(handler["deviceTypeJsonExpression"], content)
if found_device_name is None:
self.__log.error("Device name missing from connection request")
continue
# Note: device must be added even if it is already known locally: else ThingsBoard
# will not send RPCs and attribute updates
self.__log.info("Connecting device %s of type %s", found_device_name, found_device_type)
self.__gateway.add_device(found_device_name, {"connector": self}, device_type=found_device_type)
# Note: if I'm in this branch, this was for sure a connection message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in disconnection handlers "i.e., I'm disconnecting a device" -------------------
topic_handlers = [regex for regex in self.__disconnect_requests_sub_topics if fullmatch(regex, message.topic)]
if topic_handlers:
for topic in topic_handlers:
handler = self.__disconnect_requests_sub_topics[topic]
found_device_name = None
found_device_type = 'default'
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(handler["deviceNameTopicExpression"], message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(handler["deviceNameJsonExpression"], content)
# Get device type (if any), either from topic or from content
if handler.get("deviceTypeTopicExpression"):
device_type_match = search(handler["deviceTypeTopicExpression"], message.topic)
if device_type_match is not None:
found_device_type = device_type_match.group(0)
elif handler.get("deviceTypeJsonExpression"):
found_device_type = TBUtility.get_value(handler["deviceTypeJsonExpression"], content)
if found_device_name is None:
self.__log.error("Device name missing from disconnection request")
continue
if found_device_name in self.__gateway.get_devices():
self.__log.info("Disconnecting device %s of type %s", found_device_name, found_device_type)
self.__gateway.del_device(found_device_name)
else:
self.__log.info("Device %s was not connected", found_device_name)
break
# Note: if I'm in this branch, this was for sure a disconnection message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in attribute request handlers "i.e., I'm asking for a shared attribute" --------
topic_handlers = [regex for regex in self.__attribute_requests_sub_topics if fullmatch(regex, message.topic)]
if topic_handlers:
try:
for topic in topic_handlers:
handler = self.__attribute_requests_sub_topics[topic]
found_device_name = None
found_attribute_name = None
# Get device name, either from topic or from content
if handler.get("deviceNameTopicExpression"):
device_name_match = search(handler["deviceNameTopicExpression"], message.topic)
if device_name_match is not None:
found_device_name = device_name_match.group(0)
elif handler.get("deviceNameJsonExpression"):
found_device_name = TBUtility.get_value(handler["deviceNameJsonExpression"], content)
# Get attribute name, either from topic or from content
if handler.get("attributeNameTopicExpression"):
attribute_name_match = search(handler["attributeNameTopicExpression"], message.topic)
if attribute_name_match is not None:
found_attribute_name = attribute_name_match.group(0)
elif handler.get("attributeNameJsonExpression"):
found_attribute_name = TBUtility.get_value(handler["attributeNameJsonExpression"], content)
if found_device_name is None:
self.__log.error("Device name missing from attribute request")
continue
if found_attribute_name is None:
self.__log.error("Attribute name missing from attribute request")
continue
self.__log.info("Will retrieve attribute %s of %s", found_attribute_name, found_device_name)
self.__gateway.tb_client.client.gw_request_shared_attributes(
found_device_name,
[found_attribute_name],
lambda data, *args: self.notify_attribute(
data,
found_attribute_name,
handler.get("topicExpression"),
handler.get("valueExpression")))
break
except Exception as e:
log.exception(e)
# Note: if I'm in this branch, this was for sure an attribute request message
# => Execution must end here both in case of failure and success
return None
# Check if message topic exists in RPC handlers ----------------------------------------------------------------
# The gateway is expecting for this message => no wildcards here, the topic must be evaluated as is
if message.topic in self.__gateway.rpc_requests_in_progress:
self.__gateway.rpc_with_reply_processing(message.topic, content)
return None
self.__log.debug("Received message to topic \"%s\" with unknown interpreter data: \n\n\"%s\"",
message.topic,
content)
def _on_connect(self, client, userdata, flags, result_code, *extra_params):
result_codes = {
1: "incorrect protocol version",
2: "invalid client identifier",
3: "server unavailable",
4: "bad username or password",
5: "not authorised",
}
if result_code == 0:
self._connected = True
self.__log.info('%s connected to %s:%s - successfully.',
self.get_name(),
self.__broker["host"],
self.__broker.get("port", "1883"))
self.__log.debug("Client %s, userdata %s, flags %s, extra_params %s",
str(client),
str(userdata),
str(flags),
extra_params)
self.__mapping_sub_topics = {}
# Setup data upload requests handling ----------------------------------------------------------------------
for mapping in self.__mapping:
try:
# Load converter for this mapping entry ------------------------------------------------------------
# mappings are guaranteed to have topicFilter and converter fields. See __init__
default_converter_class_name = "JsonMqttUplinkConverter"
# Get converter class from "extension" parameter or default converter
converter_class_name = mapping["converter"].get("extension", default_converter_class_name)
# Find and load required class
module = TBModuleLoader.import_module(self._connector_type, converter_class_name)
if module:
self.__log.debug('Converter %s for topic %s - found!', converter_class_name, mapping["topicFilter"])
converter = module(mapping)
else:
self.__log.error("Cannot find converter for %s topic", mapping["topicFilter"])
continue
# Setup regexp topic acceptance list ---------------------------------------------------------------
regex_topic = TBUtility.topic_to_regex(mapping["topicFilter"])
# There may be more than one converter per topic, so I'm using vectors
if not self.__mapping_sub_topics.get(regex_topic):
self.__mapping_sub_topics[regex_topic] = []
self.__mapping_sub_topics[regex_topic].append(converter)
# Subscribe to appropriate topic -------------------------------------------------------------------
self.__subscribe(mapping["topicFilter"], mapping.get("subscriptionQos", 1))
self.__log.info('Connector "%s" subscribe to %s',
self.get_name(),
TBUtility.regex_to_topic(regex_topic))
except Exception as e:
self.__log.exception(e)
# Setup connection requests handling -----------------------------------------------------------------------
for request in [entry for entry in self.__connect_requests if entry is not None]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(request.get("topicFilter"))
self.__connect_requests_sub_topics[topic_filter] = request
# Setup disconnection requests handling --------------------------------------------------------------------
for request in [entry for entry in self.__disconnect_requests if entry is not None]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(request.get("topicFilter"))
self.__disconnect_requests_sub_topics[topic_filter] = request
# Setup attributes requests handling -----------------------------------------------------------------------
for request in [entry for entry in self.__attribute_requests if entry is not None]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"], request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(request.get("topicFilter"))
self.__attribute_requests_sub_topics[topic_filter] = request
else:
if result_code in result_codes:
self.__log.error("%s connection FAIL with error %s %s!", self.get_name(), result_code, result_codes[result_code])
else:
self.__log.error("%s connection FAIL with unknown error!", self.get_name())
def __search_general_info(self, device):
result = []
match_devices = []
self.__search_node(self.__opcua_nodes["root"],
TBUtility.get_value(device["deviceNodePattern"],
get_tag=True),
result=match_devices)
for device_node in match_devices:
if device_node is not None:
result_device_dict = {
"deviceName": None,
"deviceType": None,
"deviceNode": device_node,
"configuration": deepcopy(device)
}
name_pattern_config = device["deviceNamePattern"]
name_expression = TBUtility.get_value(name_pattern_config,
get_tag=True)
if "${" in name_pattern_config and "}" in name_pattern_config:
log.debug("Looking for device name")
device_name_from_node = ""
if name_expression == "$DisplayName":
device_name_from_node = device_node.get_display_name(
).Text
elif name_expression == "$BrowseName":
device_name_from_node = device_node.get_browse_name(
).Name
elif name_expression == "$NodeId.Identifier":
device_name_from_node = str(
device_node.nodeid.Identifier)
else:
name_path = self._check_path(name_expression,
device_node)
device_name_node = []
self.__search_node(device_node,
name_path,
result=device_name_node)
device_name_node = device_name_node[0]
if device_name_node is not None:
device_name_from_node = device_name_node.get_value(
)
if device_name_from_node == "":
log.error(
"Device name node not found with expression: %s",
name_expression)
return None
full_device_name = name_pattern_config.replace(
"${" + name_expression + "}",
str(device_name_from_node)).replace(
name_expression, str(device_name_from_node))
else:
full_device_name = name_expression
result_device_dict["deviceName"] = full_device_name
log.debug("Device name: %s", full_device_name)
if device.get("deviceTypePattern"):
device_type_expression = TBUtility.get_value(
device["deviceTypePattern"], get_tag=True)
if "${" in device_type_expression and "}" in device_type_expression:
type_path = self._check_path(device_type_expression,
device_node)
device_type_node = []
self.__search_node(device_node,
type_path,
result=device_type_node)
device_type_node = device_type_node[0]
if device_type_node is not None:
device_type = device_type_node.get_value()
full_device_type = device_type_expression.replace(
"${" + device_type_expression + "}",
device_type).replace(device_type_expression,
device_type)
else:
log.error(
"Device type node not found with expression: %s",
device_type_expression)
full_device_type = "default"
else:
full_device_type = device_type_expression
result_device_dict["deviceType"] = full_device_type
log.debug("Device type: %s", full_device_type)
else:
result_device_dict["deviceType"] = "default"
result.append(result_device_dict)
else:
log.error(
"Device node not found with expression: %s",
TBUtility.get_value(device["deviceNodePattern"],
get_tag=True))
return result
def __search_nodes_and_subscribe(self, device_info):
sub_nodes = []
information_types = {
"attributes": "attributes",
"timeseries": "telemetry"
}
for information_type in information_types:
for information in device_info["configuration"][information_type]:
information_key = information["key"]
config_path = TBUtility.get_value(information["path"],
get_tag=True)
information_path = self._check_path(config_path,
device_info["deviceNode"])
information["path"] = '${%s}' % information_path
information_nodes = []
self.__search_node(device_info["deviceNode"],
information_path,
result=information_nodes)
for information_node in information_nodes:
if information_node is not None:
try:
information_value = information_node.get_value()
except:
log.error("Err get_value: %s",
str(information_node))
continue
log.debug(
"Node for %s \"%s\" with path: %s - FOUND! Current values is: %s",
information_type, information_key,
information_path, str(information_value))
if device_info.get("uplink_converter") is None:
configuration = {
**device_info["configuration"], "deviceName":
device_info["deviceName"],
"deviceType":
device_info["deviceType"]
}
if device_info["configuration"].get(
'converter') is None:
converter = OpcUaUplinkConverter(configuration)
else:
converter = TBModuleLoader.import_module(
self._connector_type, configuration)
device_info["uplink_converter"] = converter
else:
converter = device_info["uplink_converter"]
self.subscribed[information_node] = {
"converter": converter,
"path": information_path,
"config_path": config_path
}
if not device_info.get(
information_types[information_type]):
device_info[
information_types[information_type]] = []
converted_data = converter.convert(
(config_path, information_path), information_value)
self.statistics['MessagesReceived'] = self.statistics[
'MessagesReceived'] + 1
self.data_to_send.append(converted_data)
self.statistics['MessagesSent'] = self.statistics[
'MessagesSent'] + 1
log.debug("Data to ThingsBoard: %s", converted_data)
if not self.__server_conf.get("disableSubscriptions",
False):
sub_nodes.append(information_node)
else:
log.error(
"Node for %s \"%s\" with path %s - NOT FOUND!",
information_type, information_key,
information_path)
if not self.__server_conf.get("disableSubscriptions", False):
if self.__sub is None:
self.__sub = self.client.create_subscription(
self.__server_conf.get("subCheckPeriodInMillis", 500),
self.__sub_handler)
if sub_nodes:
self.__sub.subscribe_data_change(sub_nodes)
log.debug("Added subscription to nodes: %s", str(sub_nodes))
def convert(self, config, data):
#print("[JsonRequestUplinkConverter convert]", self.__config, data)
if isinstance(data, (bytes, str)):
data = loads(data)
datatypes = {"attributes": "attributes", "timeseries": "telemetry"}
dict_result = {
"deviceName": None,
"deviceType": None,
"udoi": None,
"schemaId": None,
"attributes": [],
"telemetry": []
}
try:
if self.__config['converter'].get(
"deviceNameJsonExpression") is not None:
dict_result["deviceName"] = TBUtility.get_value(
self.__config['converter'].get("deviceNameJsonExpression"),
data,
expression_instead_none=True)
else:
log.error(
"The expression for looking \"deviceName\" not found in config %s",
dumps(self.__config['converter']))
if self.__config['converter'].get(
"deviceTypeJsonExpression") is not None:
dict_result["deviceType"] = TBUtility.get_value(
self.__config['converter'].get("deviceTypeJsonExpression"),
data,
expression_instead_none=True)
else:
log.error(
"The expression for looking \"deviceType\" not found in config %s",
dumps(self.__config['converter']))
if self.__config['converter'].get("udoi") is not None:
dict_result["udoi"] = TBUtility.get_value(
self.__config['converter'].get("udoi"),
data,
expression_instead_none=True)
else:
log.error(
"The expression for looking \"udoi\" not found in config %s",
dumps(self.__config['converter']))
if self.__config['converter'].get("schemaId") is not None:
dict_result["schemaId"] = TBUtility.get_value(
self.__config['converter'].get("schemaId"),
data,
expression_instead_none=True)
else:
log.error(
"The expression for looking \"schemaId\" not found in config %s",
dumps(self.__config['converter']))
except Exception as e:
log.exception(e)
try:
for datatype in self.__datatypes:
current_datatype = self.__datatypes[datatype]
for datatype_object_config in self.__config["converter"].get(
datatype, []):
#print("[JsonRequestUplinkConverter convert datatype_object_config]",datatype_object_config)
datatype_object_config_key = TBUtility.get_value(
datatype_object_config["key"],
data,
datatype_object_config["type"],
expression_instead_none=True)
datatype_object_config_value = TBUtility.get_value(
datatype_object_config["value"],
data,
datatype_object_config["type"],
expression_instead_none=True)
#print("[JsonRequestUplinkConverter convert]",datatype_object_config_key,datatype_object_config_value)
if datatype_object_config_key is not None and datatype_object_config_value is not None:
dict_result[current_datatype].append({
datatype_object_config_key:
datatype_object_config_value
})
else:
error_string = "Cannot find the key in the input data" if datatype_object_config_key is None else "Cannot find the value from the input data"
log.error(error_string)
except Exception as e:
log.exception(e)
#print("[JsonRequestUplinkConverter convert dict_result]",dict_result)
return dict_result
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from random import choice
from threading import Thread
from time import time, sleep
from string import ascii_lowercase
from tb_utility.tb_utility import TBUtility
try:
from bacpypes.core import run, stop
except ImportError:
print("BACnet library not found - installing...")
TBUtility.install_package("bacpypes", ">=0.18.0")
from bacpypes.core import run, stop
from bacpypes.pdu import Address, GlobalBroadcast, LocalBroadcast, LocalStation, RemoteStation
from connectors.connector import Connector, log
from connectors.bacnet.bacnet_utilities.tb_gateway_bacnet_application import TBBACnetApplication
class BACnetConnector(Thread, Connector):
def __init__(self, gateway, config, connector_type):
self.__connector_type = connector_type
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
super().__init__()
self.__config = config
self.setName(
from threading import Thread
from time import sleep, time
from random import choice
from string import ascii_lowercase
from socket import gethostbyname
from re import search
from connectors.connector import Connector, log
from tb_utility.tb_loader import TBModuleLoader
from tb_utility.tb_utility import TBUtility
try:
import puresnmp
except ImportError:
TBUtility.install_package("puresnmp")
import puresnmp
from puresnmp.exc import Timeout as SNMPTimeoutException
class SNMPConnector(Connector, Thread):
def __init__(self, gateway, config, connector_type):
super().__init__()
self.daemon = True
self.__gateway = gateway
self._connected = False
self.__stopped = False
self._connector_type = connector_type
self.__config = config
self.__devices = self.__config["devices"]
# limitations under the License.
import time
import threading
from random import choice
from string import ascii_lowercase
from tb_utility.tb_loader import TBModuleLoader
from tb_utility.tb_utility import TBUtility
# Try import Pymodbus library or install it and import
try:
from pymodbus.constants import Defaults
except ImportError:
print("Modbus library not found - installing...")
TBUtility.install_package("pymodbus", ">=2.3.0")
TBUtility.install_package('pyserial')
from pymodbus.constants import Defaults
from pymodbus.client.sync import ModbusTcpClient, ModbusUdpClient, ModbusSerialClient, ModbusRtuFramer, ModbusSocketFramer
from pymodbus.bit_write_message import WriteSingleCoilResponse, WriteMultipleCoilsResponse
from pymodbus.register_write_message import WriteMultipleRegistersResponse, WriteSingleRegisterResponse
from pymodbus.register_read_message import ReadRegistersResponseBase
from pymodbus.bit_read_message import ReadBitsResponseBase
from pymodbus.exceptions import ConnectionException
from connectors.connector import Connector, log
from connectors.modbus.constants import *
from connectors.modbus.bytes_modbus_uplink_converter import BytesModbusUplinkConverter
from connectors.modbus.bytes_modbus_downlink_converter import BytesModbusDownlinkConverter
import re
import sched
import time
from threading import Thread
from copy import copy
from random import choice
from string import ascii_lowercase
from tb_utility.tb_utility import TBUtility
try:
from can import Notifier, BufferedReader, Message, CanError, ThreadSafeBus
except ImportError:
print("CAN library not found - installing...")
TBUtility.install_package("python-can")
from can import Notifier, BufferedReader, Message, CanError, ThreadSafeBus
from connectors.can.bytes_can_downlink_converter import BytesCanDownlinkConverter
from connectors.can.bytes_can_uplink_converter import BytesCanUplinkConverter
from connectors.connector import Connector, log
class CanConnector(Connector, Thread):
CMD_REGEX = r"^(\d{1,2}):(\d{1,2}):?(big|little)?:(\d+)$"
VALUE_REGEX = r"^(\d{1,2}):((?:-1)?|\d{1,2}):?(big|little)?:(bool|boolean|int|long|float|double|string|raw):?([0-9A-Za-z-_]+)?$"
NO_CMD_ID = "no_cmd"
UNKNOWN_ARBITRATION_ID = -1
DEFAULT_RECONNECT_PERIOD = 30.0
def convert(self, config, data):
datatypes = {"attributes": "attributes", "timeseries": "telemetry"}
dict_result = {
"deviceName": None,
"deviceType": None,
"attributes": [],
"telemetry": []
}
try:
if self.__config.get("deviceNameExpression") is not None:
dict_result["deviceName"] = TBUtility.get_value(
self.__config.get("deviceNameExpression"),
data,
expression_instead_none=True)
else:
log.error(
"The expression for looking \"deviceName\" not found in config %s",
dumps(self.__config))
if self.__config.get("deviceTypeExpression") is not None:
dict_result["deviceType"] = TBUtility.get_value(
self.__config.get("deviceTypeExpression"),
data,
expression_instead_none=True)
else:
log.error(
"The expression for looking \"deviceType\" not found in config %s",
dumps(self.__config))
except Exception as e:
log.error(
'Error in converter, for config: \n%s\n and message: \n%s\n',
dumps(self.__config), data)
log.exception(e)
try:
for datatype in datatypes:
dict_result[datatypes[datatype]] = []
for datatype_config in self.__config.get(datatype, []):
value = TBUtility.get_value(datatype_config["value"],
data,
datatype_config["type"],
expression_instead_none=True)
value_tag = TBUtility.get_value(datatype_config["value"],
data,
datatype_config["type"],
get_tag=True)
key = TBUtility.get_value(datatype_config["key"],
data,
datatype_config["type"],
expression_instead_none=True)
key_tag = TBUtility.get_value(datatype_config["key"],
data,
get_tag=True)
if ("${" not in str(value) and "}" not in str(value)) \
and ("${" not in str(key) and "}" not in str(key)):
is_valid_key = isinstance(
key, str) and "${" in datatype_config[
"key"] and "}" in datatype_config["key"]
is_valid_value = isinstance(
value, str) and "${" in datatype_config[
"value"] and "}" in datatype_config["value"]
full_key = datatype_config["key"].replace(
'${' + str(key_tag) +
'}', str(key)) if is_valid_key else key
full_value = datatype_config["value"].replace(
'${' + value_tag +
'}', value) if is_valid_value else value
if datatype == 'timeseries' and (
data.get("ts") is not None
or data.get("timestamp") is not None):
dict_result[datatypes[datatype]].append({
"ts":
data.get('ts',
data.get('timestamp', int(time()))),
'values': {
full_key: full_value
}
})
else:
dict_result[datatypes[datatype]].append(
{full_key: full_value})
except Exception as e:
log.error(
'Error in converter, for config: \n%s\n and message: \n%s\n',
dumps(self.__config), str(data))
log.exception(e)
return dict_result
from os import path
from pathlib import Path
from time import sleep
from random import choice
from string import ascii_lowercase
from threading import Thread
from simplejson import dumps, load
from tb_utility.tb_loader import TBModuleLoader
from tb_utility.tb_utility import TBUtility
try:
import pyodbc
except ImportError:
print("ODBC library not found - installing...")
TBUtility.install_package("pyodbc")
import pyodbc
from connectors.odbc.odbc_uplink_converter import OdbcUplinkConverter
from connectors.connector import Connector, log
class OdbcConnector(Connector, Thread):
DEFAULT_SEND_IF_CHANGED = False
DEFAULT_RECONNECT_STATE = True
DEFAULT_SAVE_ITERATOR = False
DEFAULT_RECONNECT_PERIOD = 60
DEFAULT_POLL_PERIOD = 60
DEFAULT_ENABLE_UNKNOWN_RPC = False
DEFAULT_OVERRIDE_RPC_PARAMS = False
def _on_message(self, client, userdata, message):
content = TBUtility.decode(message)
TBDeviceMqttClient._on_decoded_message(self, content, message)
self._on_decoded_message(content, message)
from threading import Thread
from string import ascii_lowercase
from random import choice
from time import time
from re import fullmatch
from queue import Queue
from simplejson import loads, JSONDecodeError
from tb_utility.tb_loader import TBModuleLoader
from tb_utility.tb_utility import TBUtility
try:
from requests import Timeout, request as regular_request
except ImportError:
print("Requests library not found - installing...")
TBUtility.install_package("requests")
from requests import Timeout, request as regular_request
import requests
from requests.auth import HTTPBasicAuth as HTTPBasicAuthRequest
from requests.exceptions import RequestException
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += ':ADH-AES128-SHA256'
try:
from flask import Flask, jsonify, request
except ImportError:
print("Flask library not found - installing...")
TBUtility.install_package("flask")
from flask import Flask, jsonify, request
try:
from flask_restful import reqparse, abort, Api, Resource
except ImportError:
from queue import Queue
from random import choice
from string import ascii_lowercase
from time import sleep, time
from re import fullmatch
from json import JSONDecodeError
import json
from tb_utility.tb_loader import TBModuleLoader
from tb_utility.tb_utility import TBUtility
try:
from requests import Timeout, request
except ImportError:
print("Requests library not found - installing...")
TBUtility.install_package("requests")
from requests import Timeout, request
import requests
from requests.auth import HTTPBasicAuth
from requests.exceptions import RequestException
from connectors.connector import Connector, log
from connectors.request.json_request_uplink_converter import JsonRequestUplinkConverter
from connectors.request.json_request_downlink_converter import JsonRequestDownlinkConverter
# pylint: disable=E1101
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += ':ADH-AES128-SHA256'
class RequestConnector(Connector, Thread):
def __init__(self, gateway, config, connector_type):
def convert(self, config, data):
#print("[JsonMqttUplinkConverter]",config,data)
#print("[JsonMqttUplinkConverter] self.__config",self.__config)
datatypes = {"attributes": "attributes", "timeseries": "telemetry"}
dict_result = {
"deviceName": None,
"deviceType": None,
"attributes": [],
"telemetry": []
}
try:
if self.__config.get("deviceNameJsonExpression") is not None:
dict_result["deviceName"] = TBUtility.get_value(
self.__config.get("deviceNameJsonExpression"),
data,
expression_instead_none=True)
elif self.__config.get("deviceNameTopicExpression") is not None:
search_result = search(
self.__config["deviceNameTopicExpression"], config)
if search_result is not None:
dict_result["deviceName"] = search_result.group(0)
else:
log.debug(
"Regular expression result is None. deviceNameTopicExpression parameter will be interpreted as a deviceName\n Topic: %s\nRegex: %s",
config, self.__config.get("deviceNameTopicExpression"))
dict_result["deviceName"] = self.__config.get(
"deviceNameTopicExpression")
else:
log.error(
"The expression for looking \"deviceName\" not found in config %s",
dumps(self.__config))
if self.__config.get("deviceTypeJsonExpression") is not None:
dict_result["deviceType"] = TBUtility.get_value(
self.__config.get("deviceTypeJsonExpression"),
data,
expression_instead_none=True)
elif self.__config.get("deviceTypeTopicExpression") is not None:
search_result = search(
self.__config["deviceTypeTopicExpression"], config)
if search_result is not None:
dict_result["deviceType"] = search_result.group(0)
else:
log.debug(
"Regular expression result is None. deviceTypeTopicExpression will be interpreted as a deviceType\n Topic: %s\nRegex: %s",
config, self.__config.get("deviceTypeTopicExpression"))
dict_result["deviceType"] = self.__config.get(
"deviceTypeTopicExpression")
else:
log.error(
"The expression for looking \"deviceType\" not found in config %s",
dumps(self.__config))
if self.__config.get("udoi") is not None:
dict_result["udoi"] = TBUtility.get_value(
self.__config.get("udoi"),
data,
expression_instead_none=True)
else:
log.error(
"The expression for looking \"udoi\" not found in config %s",
dumps(self.__config['converter']))
if self.__config.get("schemaId") is not None:
dict_result["schemaId"] = TBUtility.get_value(
self.__config.get("schemaId"),
data,
expression_instead_none=True)
else:
log.error(
"The expression for looking \"schemaId\" not found in config %s",
dumps(self.__config['converter']))
except Exception as e:
log.error(
'Error in converter, for config: \n%s\n and message: \n%s\n',
dumps(self.__config), data)
log.exception(e)
try:
for datatype in datatypes:
#print("datatype",datatype)
dict_result[datatypes[datatype]] = []
for datatype_config in self.__config.get(datatype, []):
#print("[JsonMqttUplinkConverter] datatype_config",datatype_config)
value = TBUtility.get_value(datatype_config["value"],
data,
datatype_config["type"],
expression_instead_none=True)
#print("[JsonMqttUplinkConverter]",value)
value_tag = TBUtility.get_value(datatype_config["value"],
data,
datatype_config["type"],
get_tag=True)
#print("[JsonMqttUplinkConverter]",value_tag)
key = TBUtility.get_value(datatype_config["key"],
data,
datatype_config["type"],
expression_instead_none=True)
#print("[JsonMqttUplinkConverter]",key)
key_tag = TBUtility.get_value(datatype_config["key"],
data,
get_tag=True)
#print("[JsonMqttUplinkConverter]",key_tag)
#print("[JsonMqttUplinkConverter]",value,value_tag,key,key_tag)
if ("${" not in str(value) and "}" not in str(value)) \
and ("${" not in str(key) and "}" not in str(key)):
is_valid_key = isinstance(
key, str) and "${" in datatype_config[
"key"] and "}" in datatype_config["key"]
is_valid_value = isinstance(
value, str) and "${" in datatype_config[
"value"] and "}" in datatype_config["value"]
full_key = datatype_config["key"].replace(
'${' + str(key_tag) +
'}', str(key)) if is_valid_key else key_tag
full_value = datatype_config["value"].replace(
'${' + str(value_tag) +
'}', str(value)) if is_valid_value else value
#print("[JsonMqttUplinkConverter]",is_valid_key,is_valid_value,full_key,full_value)
if datatype == 'timeseries' and (
data.get("ts") is not None
or data.get("timestamp") is not None):
dict_result[datatypes[datatype]].append({
"ts":
data.get('ts',
data.get('timestamp', int(time()))),
'values': {
full_key: full_value
}
})
else:
dict_result[datatypes[datatype]].append(
{full_key: full_value})
except Exception as e:
log.error(
'Error in converter, for config: \n%s\n and message: \n%s\n',
dumps(self.__config), str(data))
log.exception(e)
return dict_result
from concurrent.futures import CancelledError, TimeoutError as FuturesTimeoutError
from copy import deepcopy
from random import choice
from threading import Thread
from string import ascii_lowercase
import regex
from simplejson import dumps
from tb_utility.tb_loader import TBModuleLoader
from tb_utility.tb_utility import TBUtility
try:
from opcua import Client, ua
except ImportError:
print("OPC-UA library not found")
TBUtility.install_package("opcua")
from opcua import Client, ua
from connectors.connector import Connector, log
from connectors.opcua.opcua_uplink_converter import OpcUaUplinkConverter
class OpcUaConnector(Thread, Connector):
def __init__(self, gateway, config, connector_type):
self._connector_type = connector_type
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
super().__init__()
self.__gateway = gateway
self.__server_conf = config.get("server")
self.__interest_nodes = []
self.__available_object_resources = {}
self.__show_map = self.__server_conf.get("showMap", False)
def _on_message(self, client, userdata, message):
content = TBUtility.decode(message)
super()._on_decoded_message(content, message)
self._on_decoded_message(content, message)
def __fill_interest_devices(self):
if self.__config.get('devices') is None:
log.error(
'Devices not found in configuration file. BLE Connector stopped.'
)
self._connected = False
return None
for interest_device in self.__config.get('devices'):
keys_in_config = ['attributes', 'telemetry']
if interest_device.get('MACAddress') is not None:
default_converter = BytesBLEUplinkConverter(interest_device)
interest_uuid = {}
for key_type in keys_in_config:
for type_section in interest_device.get(key_type):
if type_section.get("characteristicUUID") is not None:
converter = None
if type_section.get('converter') is not None:
try:
module = TBUtility.check_and_import(
self.__connector_type,
type_section['converter'])
if module is not None:
log.debug(
'Custom converter for device %s - found!',
interest_device['MACAddress'])
converter = module(interest_device)
else:
log.error(
"\n\nCannot find extension module for device %s .\nPlease check your configuration.\n",
interest_device['MACAddress'])
except Exception as e:
log.exception(e)
else:
converter = default_converter
if converter is not None:
if interest_uuid.get(
type_section["characteristicUUID"].
upper()) is None:
interest_uuid[type_section[
"characteristicUUID"].upper()] = [{
'section_config':
type_section,
'type':
key_type,
'converter':
converter
}]
else:
interest_uuid[type_section[
"characteristicUUID"].upper()].append({
'section_config':
type_section,
'type':
key_type,
'converter':
converter
})
else:
log.error(
"No characteristicUUID found in configuration section for %s:\n%s\n",
key_type, pformat(type_section))
if self.__devices_around.get(
interest_device['MACAddress'].upper()) is None:
self.__devices_around[
interest_device['MACAddress'].upper()] = {}
self.__devices_around[interest_device['MACAddress'].upper(
)]['device_config'] = interest_device
self.__devices_around[interest_device['MACAddress'].upper(
)]['interest_uuid'] = interest_uuid
else:
log.error(
"Device address not found, please check your settings.")