def __load_converters(self):
try:
for device in self.__config["devices"]:
if self.__config.get("converter") is not None:
converter = TBModuleLoader.import_module(
self._connector_type,
self.__config["converter"])(device)
else:
converter = BytesModbusUplinkConverter(device)
if self.__config.get("downlink_converter") is not None:
downlink_converter = TBModuleLoader.import_module(
self._connector_type,
self.__config["downlink_converter"])(device)
else:
downlink_converter = BytesModbusDownlinkConverter(device)
if device.get(
'deviceName') not in self.__gateway.get_devices():
self.__gateway.add_device(
device.get('deviceName'), {"connector": self},
device_type=device.get("deviceType"))
self.__devices[device["deviceName"]] = {
"config": device,
"converter": converter,
"downlink_converter": downlink_converter,
"next_attributes_check": 0,
"next_timeseries_check": 0,
"telemetry": {},
"attributes": {},
"last_telemetry": {},
"last_attributes": {}
}
except Exception as e:
log.exception(e)
def __init__(self, gateway, config, connector_type):
self._connector_type = connector_type
self.statistics = {'MessagesReceived': 0,
'MessagesSent': 0}
super().__init__()
self.__config = config
self.setName(config.get('name', 'BACnet ' + ''.join(choice(ascii_lowercase) for _ in range(5))))
self.__devices = []
self.__device_indexes = {}
self.__devices_address_name = {}
self.__gateway = gateway
self._application = TBBACnetApplication(self, self.__config)
self.__bacnet_core_thread = Thread(target=run, name="BACnet core thread", daemon=True,
kwargs={"sigterm": None, "sigusr1": None})
self.__bacnet_core_thread.start()
self.__stopped = False
self.__config_devices = self.__config["devices"]
self.default_converters = {
"uplink_converter": TBModuleLoader.import_module(self._connector_type, "BACnetUplinkConverter"),
"downlink_converter": TBModuleLoader.import_module(self._connector_type, "BACnetDownlinkConverter")}
self.__request_functions = {"writeProperty": self._application.do_write_property,
"readProperty": self._application.do_read_property,
"risingEdge": self._application.do_binary_rising_edge}
self.__available_object_resources = {}
self.rpc_requests_in_progress = {}
self.__connected = False
self.daemon = True
self.__convert_and_save_data_queue = Queue()
def __get_converter(self, config, need_uplink):
if config is None:
return BytesCanUplinkConverter() if need_uplink else BytesCanDownlinkConverter()
else:
if need_uplink:
uplink = config.get("uplink")
return BytesCanUplinkConverter() if uplink is None \
else TBModuleLoader.import_module(self.__connector_type, uplink)
else:
downlink = config.get("downlink")
return BytesCanDownlinkConverter() if downlink is None \
else TBModuleLoader.import_module(self.__connector_type, downlink)
def __fill_converters(self):
try:
for device in self.__devices:
device["uplink_converter"] = TBModuleLoader.import_module(
"snmp",
device.get('converter',
self._default_converters["uplink"]))(device)
device["downlink_converter"] = TBModuleLoader.import_module(
"snmp",
device.get('converter',
self._default_converters["downlink"]))(device)
except Exception as e:
log.exception(e)
def __prepare_connectors_configuration(self, input_connector_config):
try:
self.__gateway.connectors_configs = {}
for connector in input_connector_config['thingsboard'][
'connectors']:
for input_connector in input_connector_config[
connector['type']]:
if input_connector['name'] == connector['name']:
if not self.__gateway.connectors_configs.get(
connector['type']):
self.__gateway.connectors_configs[
connector['type']] = []
self.__gateway.connectors_configs[
connector['type']].append({
"name": connector["name"],
"config": {
connector['configuration']:
input_connector["config"]
}
})
connector_class = TBModuleLoader.import_module(
connector["type"],
self.__gateway._default_connectors.get(
connector["type"], connector.get("class")))
self.__gateway._implemented_connectors[
connector["type"]] = connector_class
except Exception as e:
LOG.exception(e)
def _load_connectors(self):
self.connectors_configs = {}
if self.__config.get("connectors"):
for connector in self.__config['connectors']:
try:
connector_class = TBModuleLoader.import_module(
connector["type"],
self._default_connectors.get(connector["type"],
connector.get("class")))
self._implemented_connectors[
connector["type"]] = connector_class
with open(self._config_dir + connector['configuration'],
'r',
encoding="UTF-8") as conf_file:
connector_conf = load(conf_file)
if not self.connectors_configs.get(connector['type']):
self.connectors_configs[connector['type']] = []
connector_conf["name"] = connector["name"]
self.connectors_configs[connector['type']].append({
"name":
connector["name"],
"config": {
connector['configuration']: connector_conf
}
})
except Exception as e:
log.error("Error on loading connector:")
log.exception(e)
else:
log.error("Connectors - not found! Check your configuration!")
self.__init_remote_configuration(force=True)
log.info("Remote configuration is enabled forcibly!")
def __init__(self, gateway, config, connector_type):
super().__init__()
self.daemon = True
self.setName(
config.get(
"name", 'ODBC Connector ' +
''.join(choice(ascii_lowercase) for _ in range(5))))
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
self.__gateway = gateway
self._connector_type = connector_type
self.__config = config
self.__stopped = False
self.__config_dir = self.__gateway.get_config_path(
) + "odbc" + path.sep
self.__connection = None
self.__cursor = None
self.__rpc_cursor = None
self.__iterator = None
self.__iterator_file_name = ""
self.__devices = {}
self.__column_names = []
self.__attribute_columns = []
self.__timeseries_columns = []
self.__converter = OdbcUplinkConverter() if not self.__config.get("converter", "") else \
TBModuleLoader.import_module(self._connector_type, self.__config["converter"])
self.__configure_pyodbc()
self.__parse_rpc_config()
def __load_converters(
self,
connector_type): # Function for search a converter and save it.
devices_config = self.__config.get('devices')
try:
if devices_config is not None:
for device_config in devices_config:
if device_config.get('converter') is not None:
converter = TBModuleLoader.import_module(
connector_type, device_config['converter'])
self.__devices[device_config['name']] = {
'converter': converter(device_config),
'device_config': device_config
}
else:
log.error(
'Converter configuration for the custom connector %s -- not found, please check your configuration file.',
self.get_name())
else:
log.error(
'Section "devices" in the configuration not found. A custom connector %s has being stopped.',
self.get_name())
self.close()
except Exception as e:
log.exception(e)
def __fill_requests(self):
log.debug(self.__config["mapping"])
for endpoint in self.__config["mapping"]:
try:
log.debug(endpoint)
converter = None
if endpoint["converter"]["type"] == "custom":
module = TBModuleLoader.import_module(
self.__connector_type,
endpoint["converter"]["extension"])
if module is not None:
log.debug('Custom converter for url %s - found!',
endpoint["url"])
converter = module(endpoint)
else:
log.error(
"\n\nCannot find extension module for %s url.\nPlease check your configuration.\n",
endpoint["url"])
else:
converter = JsonRequestUplinkConverter(endpoint)
self.__requests_in_progress.append({
"config": endpoint,
"converter": converter,
"next_time": time(),
"request": request
})
except Exception as e:
log.exception(e)
def __fill_rpc_requests(self):
for rpc_request in self.__config.get("serverSideRpc", []):
if rpc_request.get("converter") is not None:
converter = TBModuleLoader.import_module("request", rpc_request["converter"])(rpc_request)
else:
converter = JsonRequestDownlinkConverter(rpc_request)
rpc_request_dict = {**rpc_request, "converter": converter}
self.__rpc_requests.append(rpc_request_dict)
def __fill_attribute_updates(self):
for attribute_request in self.__config.get("attributeUpdates", []):
if attribute_request.get("converter") is not None:
converter = TBModuleLoader.import_module("request", attribute_request["converter"])(attribute_request)
else:
converter = JsonRequestDownlinkConverter(attribute_request)
attribute_request_dict = {**attribute_request, "converter": converter}
self.__attribute_updates.append(attribute_request_dict)
def __load_converter(self, device):
converter_class_name = device.get('converter', DEFAULT_UPLINK_CONVERTER)
module = TBModuleLoader.import_module(self._connector_type, converter_class_name)
if module:
log.debug('Converter %s for device %s - found!', converter_class_name, self.name)
return module
log.error("Cannot find converter for %s device", self.name)
return None
def __load_converter(self):
converter_class_name = self.config['extension']
module = TBModuleLoader.import_module(self.__connector_type, converter_class_name)
if module:
log.debug('Converter %s for device %s - found!', converter_class_name, self.name)
self.__converter = module
else:
log.error("Cannot find converter for %s device", self.name)
self.stopped = True
def __search_nodes_and_subscribe(self, device_info):
sub_nodes = []
information_types = {"attributes": "attributes", "timeseries": "telemetry"}
for information_type in information_types:
for information in device_info["configuration"][information_type]:
information_key = information["key"]
config_path = TBUtility.get_value(information["path"], get_tag=True)
information_path = self._check_path(config_path, device_info["deviceNode"])
information["path"] = '${%s}' % information_path
information_nodes = []
self.__search_node(device_info["deviceNode"], information_path, result=information_nodes)
for information_node in information_nodes:
if information_node is not None:
try:
information_value = information_node.get_value()
except:
log.error("Err get_value: %s", str(information_node))
continue
log.debug("Node for %s \"%s\" with path: %s - FOUND! Current values is: %s",
information_type,
information_key,
information_path,
str(information_value))
if device_info.get("uplink_converter") is None:
configuration = {**device_info["configuration"],
"deviceName": device_info["deviceName"],
"deviceType": device_info["deviceType"]}
if device_info["configuration"].get('converter') is None:
converter = OpcUaUplinkConverter(configuration)
else:
converter = TBModuleLoader.import_module(self._connector_type, configuration)
device_info["uplink_converter"] = converter
else:
converter = device_info["uplink_converter"]
self.subscribed[information_node] = {"converter": converter,
"path": information_path,
"config_path": config_path}
if not device_info.get(information_types[information_type]):
device_info[information_types[information_type]] = []
converted_data = converter.convert((config_path, information_path), information_value)
self.statistics['MessagesReceived'] = self.statistics['MessagesReceived'] + 1
self.data_to_send.append(converted_data)
self.statistics['MessagesSent'] = self.statistics['MessagesSent'] + 1
log.debug("Data to ThingsBoard: %s", converted_data)
if not self.__server_conf.get("disableSubscriptions", False):
sub_nodes.append(information_node)
else:
log.error("Node for %s \"%s\" with path %s - NOT FOUND!", information_type, information_key, information_path)
if not self.__server_conf.get("disableSubscriptions", False):
if self.__sub is None:
self.__sub = self.client.create_subscription(self.__server_conf.get("subCheckPeriodInMillis", 500),
self.__sub_handler)
if sub_nodes:
self.__sub.subscribe_data_change(sub_nodes)
log.debug("Added subscription to nodes: %s", str(sub_nodes))
def __load_converters(self, device):
datatypes = ["attributes", "telemetry", "attribute_updates", "server_side_rpc"]
for datatype in datatypes:
for datatype_config in device.get(datatype, []):
try:
for converter_type in self.default_converters:
converter_object = self.default_converters[converter_type] if datatype_config.get(
"class") is None else TBModuleLoader.import_module(self._connector_type,
device.get("class"))
datatype_config[converter_type] = converter_object(device)
except Exception as e:
log.exception(e)
def __load_converters(self):
try:
for device in self.__config[CONFIG_DEVICES_SECTION_PARAMETER]:
if self.__config.get(UPLINK_PREFIX +
CONVERTER_PARAMETER) is not None:
converter = TBModuleLoader.import_module(
self._connector_type,
self.__config[UPLINK_PREFIX +
CONVERTER_PARAMETER])(device)
else:
converter = BytesModbusUplinkConverter(device)
if self.__config.get(DOWNLINK_PREFIX +
CONVERTER_PARAMETER) is not None:
downlink_converter = TBModuleLoader.import_module(
self._connector_type,
self.__config[DOWNLINK_PREFIX +
CONVERTER_PARAMETER])(device)
else:
downlink_converter = BytesModbusDownlinkConverter(device)
if device.get(DEVICE_NAME_PARAMETER
) not in self.__gateway.get_devices():
self.__gateway.add_device(
device.get(DEVICE_NAME_PARAMETER),
{CONNECTOR_PARAMETER: self},
device_type=device.get(DEVICE_TYPE_PARAMETER))
self.__devices[device[DEVICE_NAME_PARAMETER]] = {
CONFIG_SECTION_PARAMETER: device,
UPLINK_PREFIX + CONVERTER_PARAMETER: converter,
DOWNLINK_PREFIX + CONVERTER_PARAMETER: downlink_converter,
NEXT_PREFIX + ATTRIBUTES_PARAMETER + CHECK_POSTFIX: 0,
NEXT_PREFIX + TIMESERIES_PARAMETER + CHECK_POSTFIX: 0,
TELEMETRY_PARAMETER: {},
ATTRIBUTES_PARAMETER: {},
LAST_PREFIX + TELEMETRY_PARAMETER: {},
LAST_PREFIX + ATTRIBUTES_PARAMETER: {},
CONNECTION_ATTEMPT_PARAMETER: 0
}
except Exception as e:
log.exception(e)
def load_endpoints(self):
endpoints = {}
for mapping in self.__config.get("mapping"):
converter = TBModuleLoader.import_module(
self._connector_type,
mapping.get("extension", self._default_converters["uplink"]))
endpoints.update({
mapping['endpoint']: {
"config": mapping,
"converter": converter
}
})
return endpoints
def __load_converters(self, connector, gateway):
try:
if self.config.get(UPLINK_PREFIX + CONVERTER_PARAMETER) is not None:
converter = TBModuleLoader.import_module(connector.connector_type,
self.config[UPLINK_PREFIX + CONVERTER_PARAMETER])(self)
else:
converter = BytesModbusUplinkConverter({**self.config, 'deviceName': self.name})
if self.config.get(DOWNLINK_PREFIX + CONVERTER_PARAMETER) is not None:
downlink_converter = TBModuleLoader.import_module(connector.connector_type, self.config[
DOWNLINK_PREFIX + CONVERTER_PARAMETER])(self)
else:
downlink_converter = BytesModbusDownlinkConverter(self.config)
if self.name not in gateway.get_devices():
gateway.add_device(self.name, {CONNECTOR_PARAMETER: connector},
device_type=self.config.get(DEVICE_TYPE_PARAMETER))
self.config[UPLINK_PREFIX + CONVERTER_PARAMETER] = converter
self.config[DOWNLINK_PREFIX + CONVERTER_PARAMETER] = downlink_converter
except Exception as e:
log.exception(e)
def __fill_requests_from_TB(self):
requests_from_tb = {
"attributeUpdates": self.__attribute_updates,
"serverSideRpc": self.__rpc_requests,
}
for request_section in requests_from_tb:
for request_config_object in self.__config.get(
request_section, []):
uplink_converter = TBModuleLoader.import_module(
self._connector_type,
request_config_object.get(
"extension", self._default_converters["uplink"]))(
request_config_object)
downlink_converter = TBModuleLoader.import_module(
self._connector_type,
request_config_object.get(
"extension", self._default_converters["downlink"]))(
request_config_object)
request_dict = {
**request_config_object,
"uplink_converter": uplink_converter,
"downlink_converter": downlink_converter,
}
requests_from_tb[request_section].append(request_dict)
def __prepare_connectors_configuration(self, input_connector_config):
try:
self.__gateway.connectors_configs = {}
for connector in input_connector_config['thingsboard'][
'connectors']:
for input_connector in input_connector_config[
connector['type']]:
if input_connector['name'] == connector['name']:
if not self.__gateway.connectors_configs.get(
connector['type']):
self.__gateway.connectors_configs[
connector['type']] = []
config_file_path = self.__gateway.get_config_path + connector[
'configuration']
# Create the configuration json file if not exists
open(config_file_path, 'w')
self.__gateway.connectors_configs[
connector['type']].append({
"name":
connector["name"],
"config": {
connector['configuration']:
input_connector["config"]
},
"config_updated":
stat(config_file_path),
"config_file_path":
config_file_path
})
connector_class = TBModuleLoader.import_module(
connector["type"],
self.__gateway._default_connectors.get(
connector["type"], connector.get("class")))
self.__gateway._implemented_connectors[
connector["type"]] = connector_class
except Exception as e:
LOG.exception(e)
def _on_connect(self, client, userdata, flags, result_code, *extra_params):
result_codes = {
1: "incorrect protocol version",
2: "invalid client identifier",
3: "server unavailable",
4: "bad username or password",
5: "not authorised",
}
if result_code == 0:
self._connected = True
self.__log.info('%s connected to %s:%s - successfully.',
self.get_name(), self.__broker["host"],
self.__broker.get("port", "1883"))
self.__log.debug(
"Client %s, userdata %s, flags %s, extra_params %s",
str(client), str(userdata), str(flags), extra_params)
self.__mapping_sub_topics = {}
# Setup data upload requests handling ----------------------------------------------------------------------
for mapping in self.__mapping:
try:
# Load converter for this mapping entry ------------------------------------------------------------
# mappings are guaranteed to have topicFilter and converter fields. See __init__
default_converter_class_name = "JsonMqttUplinkConverter"
# Get converter class from "extension" parameter or default converter
converter_class_name = mapping["converter"].get(
"extension", default_converter_class_name)
# Find and load required class
module = TBModuleLoader.import_module(
self._connector_type, converter_class_name)
if module:
self.__log.debug('Converter %s for topic %s - found!',
converter_class_name,
mapping["topicFilter"])
converter = module(mapping)
else:
self.__log.error("Cannot find converter for %s topic",
mapping["topicFilter"])
continue
# Setup regexp topic acceptance list ---------------------------------------------------------------
regex_topic = TBUtility.topic_to_regex(
mapping["topicFilter"])
# There may be more than one converter per topic, so I'm using vectors
if not self.__mapping_sub_topics.get(regex_topic):
self.__mapping_sub_topics[regex_topic] = []
self.__mapping_sub_topics[regex_topic].append(converter)
# Subscribe to appropriate topic -------------------------------------------------------------------
self.__subscribe(mapping["topicFilter"],
mapping.get("subscriptionQos", 1))
self.__log.info('Connector "%s" subscribe to %s',
self.get_name(),
TBUtility.regex_to_topic(regex_topic))
except Exception as e:
self.__log.exception(e)
# Setup connection requests handling -----------------------------------------------------------------------
for request in [
entry for entry in self.__connect_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__connect_requests_sub_topics[topic_filter] = request
# Setup disconnection requests handling --------------------------------------------------------------------
for request in [
entry for entry in self.__disconnect_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__disconnect_requests_sub_topics[topic_filter] = request
# Setup attributes requests handling -----------------------------------------------------------------------
for request in [
entry for entry in self.__attribute_requests
if entry is not None
]:
# requests are guaranteed to have topicFilter field. See __init__
self.__subscribe(request["topicFilter"],
request.get("subscriptionQos", 1))
topic_filter = TBUtility.topic_to_regex(
request.get("topicFilter"))
self.__attribute_requests_sub_topics[topic_filter] = request
else:
if result_code in result_codes:
self.__log.error("%s connection FAIL with error %s %s!",
self.get_name(), result_code,
result_codes[result_code])
else:
self.__log.error("%s connection FAIL with unknown error!",
self.get_name())
def __fill_interest_devices(self):
if self.__config.get('devices') is None:
log.error(
'Devices not found in configuration file. BLE Connector stopped.'
)
self._connected = False
return None
for interest_device in self.__config.get('devices'):
keys_in_config = ['attributes', 'telemetry']
if interest_device.get('MACAddress') is not None:
default_converter = BytesBLEUplinkConverter(interest_device)
interest_uuid = {}
for key_type in keys_in_config:
for type_section in interest_device.get(key_type):
if type_section.get("characteristicUUID") is not None:
converter = None
if type_section.get('converter') is not None:
try:
module = TBModuleLoader.import_module(
self.__connector_type,
type_section['converter'])
if module is not None:
log.debug(
'Custom converter for device %s - found!',
interest_device['MACAddress'])
converter = module(interest_device)
else:
log.error(
"\n\nCannot find extension module for device %s .\nPlease check your configuration.\n",
interest_device['MACAddress'])
except Exception as e:
log.exception(e)
else:
converter = default_converter
if converter is not None:
if interest_uuid.get(
type_section["characteristicUUID"].
upper()) is None:
interest_uuid[type_section[
"characteristicUUID"].upper()] = [{
'section_config':
type_section,
'type':
key_type,
'converter':
converter
}]
else:
interest_uuid[type_section[
"characteristicUUID"].upper()].append({
'section_config':
type_section,
'type':
key_type,
'converter':
converter
})
else:
log.error(
"No characteristicUUID found in configuration section for %s:\n%s\n",
key_type, pformat(type_section))
if self.__devices_around.get(
interest_device['MACAddress'].upper()) is None:
self.__devices_around[
interest_device['MACAddress'].upper()] = {}
self.__devices_around[interest_device['MACAddress'].upper(
)]['device_config'] = interest_device
self.__devices_around[interest_device['MACAddress'].upper(
)]['interest_uuid'] = interest_uuid
else:
log.error(
"Device address not found, please check your settings.")
