def __connect_to_devices(self): # Function for opening connection and connecting to devices
for device in self.__devices:
try: # Start error handler
connection_start = time.time()
if self.__devices[device].get("serial") is None \
or self.__devices[device]["serial"] is None \
or not self.__devices[device]["serial"].isOpen(): # Connect only if serial not available earlier or it is closed.
self.__devices[device]["serial"] = None
while self.__devices[device]["serial"] is None or not self.__devices[device]["serial"].isOpen(): # Try connect
# connection to serial port with parameters from configuration file or default
self.__devices[device]["serial"] = serial.Serial(port=self.__config.get('port', '/dev/ttyUSB0'),
baudrate=self.__config.get('baudrate', 9600),
bytesize=self.__config.get('bytesize', serial.EIGHTBITS),
parity=self.__config.get('parity', serial.PARITY_NONE),
stopbits=self.__config.get('stopbits', serial.STOPBITS_ONE),
timeout=self.__config.get('timeout', 1),
xonxoff=self.__config.get('xonxoff', False),
rtscts=self.__config.get('rtscts', False),
write_timeout=self.__config.get('write_timeout', None),
dsrdtr=self.__config.get('dsrdtr', False),
inter_byte_timeout=self.__config.get('inter_byte_timeout', None),
exclusive=self.__config.get('exclusive', None))
time.sleep(.1)
if time.time() - connection_start > 10: # Break connection try if it setting up for 10 seconds
log.error("Connection refused per timeout for device %s", self.__devices[device]["device_config"].get("name"))
break
except serial.serialutil.SerialException:
log.error("Port %s for device %s - not found", self.__config.get('port', '/dev/ttyUSB0'), device)
except Exception as e:
log.exception(e)
else: # if no exception handled - add device and change connection state
self.__gateway.add_device(self.__devices[device]["device_config"]["name"], {"connector": self}, self.__devices[device]["device_config"]["type"])
self.__connected = True
def __fill_requests(self):
log.debug(self.__config["mapping"])
for endpoint in self.__config["mapping"]:
try:
log.debug(endpoint)
converter = None
if endpoint["converter"]["type"] == "custom":
module = TBModuleLoader.import_module(
self.__connector_type,
endpoint["converter"]["extension"])
if module is not None:
log.debug('Custom converter for url %s - found!',
endpoint["url"])
converter = module(endpoint)
else:
log.error(
"\n\nCannot find extension module for %s url.\nPlease check your configuration.\n",
endpoint["url"])
else:
converter = JsonRequestUplinkConverter(endpoint)
self.__requests_in_progress.append({
"config": endpoint,
"converter": converter,
"next_time": time(),
"request": request
})
except Exception as e:
log.exception(e)
def __search_attribute_update_variables(self, device_info):
try:
if device_info["configuration"].get("attributes_updates", []):
node = device_info["deviceNode"]
device_name = device_info["deviceName"]
if self.__available_object_resources.get(device_name) is None:
self.__available_object_resources[device_name] = {}
if self.__available_object_resources[device_name].get(
"variables") is None:
self.__available_object_resources[device_name][
"variables"] = []
for attribute_update in device_info["configuration"][
"attributes_updates"]:
attribute_path = self._check_path(
attribute_update["attributeOnDevice"], node)
attribute_nodes = []
self.__search_node(node,
attribute_path,
result=attribute_nodes)
for attribute_node in attribute_nodes:
if attribute_node is not None:
self.__available_object_resources[device_name][
"variables"].append({
attribute_update["attributeOnThingsBoard"]:
attribute_node
})
else:
log.error(
"Attribute update node with path \"%s\" - NOT FOUND!",
attribute_path)
except Exception as e:
log.exception(e)
def send_data_to_bus(self,
data,
config,
data_check=True,
raise_exception=False):
try:
self.__bus.send(
Message(arbitration_id=config["nodeId"],
is_extended_id=config.get(
"isExtendedId", self.DEFAULT_EXTENDED_ID_FLAG),
is_fd=config.get("isFd", self.DEFAULT_FD_FLAG),
bitrate_switch=config.get(
"bitrateSwitch", self.DEFAULT_BITRATE_SWITCH_FLAG),
data=data,
check=data_check))
return True
except (ValueError, TypeError) as e:
log.error("[%s] Wrong CAN message data: %s", self.get_name(),
str(e))
except CanError as e:
log.error("[%s] Failed to send CAN message: %s", self.get_name(),
str(e))
if raise_exception:
raise e
else:
self.__on_bus_error(e)
return False
def on_attributes_update(self, content):
for attr_name, attr_value in content["data"].items():
attr_config = self.__shared_attributes.get(content["device"],
{}).get(attr_name)
if attr_config is None:
log.warning(
"[%s] No configuration for '%s' attribute, ignore its update",
self.get_name(), attr_name)
return
log.debug(
"[%s] Processing attribute update for '%s' device: attribute=%s,value=%s",
self.get_name(), content["device"], attr_name, attr_value)
# Converter expects dictionary as the second parameter so pack an attribute value to a dictionary
data = self.__converters[content["device"]]["downlink"].convert(
attr_config, {"value": attr_value})
if data is None:
log.error(
"[%s] Failed to update '%s' attribute for '%s' device: data conversion failure",
self.get_name(), attr_name, content["device"])
return
done = self.send_data_to_bus(data, attr_config, data_check=True)
if done:
log.debug("[%s] Updated '%s' attribute for '%s' device",
self.get_name(), attr_name, content["device"])
else:
log.error(
"[%s] Failed to update '%s' attribute for '%s' device",
self.get_name(), attr_name, content["device"])
def __general_cb(self, iocb, callback_params=None, value=None):
try:
if callback_params is None:
callback_params = self.requests_in_progress[iocb]
if iocb.ioResponse:
apdu = iocb.ioResponse
if isinstance(apdu, SimpleAckPDU):
log.debug("Write to %s - successfully.",
str(apdu.pduSource))
else:
log.debug("Received response: %r", apdu)
elif iocb.ioError:
log.exception(iocb.ioError)
else:
log.error("There are no data in response and no errors.")
if isinstance(callback_params,
dict) and callback_params.get("callback"):
try:
callback_params["callback"](iocb, callback_params)
except TypeError:
callback_params["callback"](iocb)
except Exception as e:
log.exception(
"During processing callback, exception has been raised:")
log.exception(e)
if self.requests_in_progress.get(iocb) is not None:
del self.requests_in_progress[iocb]
def __function_to_device(self, config, unit_id):
function_code = config.get(FUNCTION_CODE_PARAMETER)
result = None
if function_code in (1, 2, 3, 4):
result = self.__available_functions[function_code](
address=config[ADDRESS_PARAMETER],
count=config.get(
OBJECTS_COUNT_PARAMETER,
config.get("registersCount",
config.get("registerCount", 1))),
unit=unit_id)
elif function_code in (5, 6):
result = self.__available_functions[function_code](
address=config[ADDRESS_PARAMETER],
value=config[PAYLOAD_PARAMETER],
unit=unit_id)
elif function_code in (15, 16):
result = self.__available_functions[function_code](
address=config[ADDRESS_PARAMETER],
values=config[PAYLOAD_PARAMETER],
unit=unit_id)
else:
log.error("Unknown Modbus function with code: %i", function_code)
log.debug("With result %s", str(result))
if "Exception" in str(result):
log.exception(result)
return result
def on_attributes_update(self, content):
try:
log.debug('Recieved Attribute Update Request: %r', str(content))
for device in self.__devices:
if device["deviceName"] == content["device"]:
for request in device["attribute_updates"]:
if request["config"].get("requestType") is not None:
for attribute in content["data"]:
if attribute == request["key"]:
request["iocb"][1]["config"].update({
"propertyValue":
content["data"][attribute]
})
kwargs = request["iocb"][1]
iocb = request["iocb"][0](device, **kwargs)
self.__request_functions[
request["config"]["requestType"]](iocb)
return
else:
log.error(
"\"requestType\" not found in request configuration for key %s device: %s",
request.get("key", "[KEY IS EMPTY]"),
device["deviceName"])
except Exception as e:
log.exception(e)
def __service_processing(self, device, characteristic_processing_conf):
for service in self.__devices_around[device]['services']:
characteristic_uuid_from_config = characteristic_processing_conf.get(
'characteristicUUID')
if self.__devices_around[device]['services'][service].get(
characteristic_uuid_from_config.upper()) is None:
continue
characteristic = self.__devices_around[device]['services'][
service][characteristic_uuid_from_config]['characteristic']
self.__check_and_reconnect(device)
data = None
if characteristic_processing_conf.get(
'method', '_').upper().split()[0] == "READ":
if characteristic.supportsRead():
self.__check_and_reconnect(device)
data = characteristic.read()
log.debug(data)
else:
log.error(
'This characteristic doesn\'t support "READ" method.')
if characteristic_processing_conf.get(
'method', '_').upper().split()[0] == "NOTIFY":
self.__check_and_reconnect(device)
descriptor = characteristic.getDescriptors(forUUID=0x2902)[0]
handle = descriptor.handle
if self.__notify_delegators.get(device) is None:
self.__notify_delegators[device] = {}
if self.__notify_delegators[device].get(handle) is None:
self.__notify_delegators[device][handle] = {
'function':
self.__notify_handler,
'args': (self.__devices_around[device], handle,
self.__notify_delegators[device].get(handle)),
'delegate':
None
}
self.__notify_delegators[device][handle][
'delegate'] = self.__notify_delegators[device][handle][
'function'](*self.__notify_delegators[device]
[handle]['args'])
data = self.__notify_delegators[device][handle][
'delegate'].data
else:
self.__notify_delegators[device][handle]['args'] = (
self.__devices_around[device], handle,
self.__notify_delegators[device][handle]['delegate'])
self.__notify_delegators[device][handle][
'delegate'] = self.__notify_delegators[device][handle][
'function'](*self.__notify_delegators[device]
[handle]['args'])
data = self.__notify_delegators[device][handle][
'delegate'].data
if data is None:
log.error('Cannot process characteristic: %s with config:\n%s',
str(characteristic.uuid).upper(),
pformat(characteristic_processing_conf))
else:
log.debug('data: %s', data)
return data
def __save_iterator_config(self):
try:
Path(self.__config_dir).mkdir(exist_ok=True)
with Path(self.__config_dir +
self.__iterator_file_name).open("w") as iterator_file:
iterator_file.write(
dumps(self.__iterator, indent=2, sort_keys=True))
log.debug("[%s] Saved iterator configuration to %s",
self.get_name(), self.__iterator_file_name)
except Exception as e:
log.error("[%s] Failed to save iterator configuration to %s: %s",
self.get_name(), self.__iterator_file_name, str(e))
def server_side_rpc_handler(self, content):
try:
rpc_method = content["data"].get("method")
for method in self.__available_object_resources[
content["device"]]['methods']:
if rpc_method is not None and method.get(
rpc_method) is not None:
arguments_from_config = method["arguments"]
arguments = content["data"].get(
"params") if content["data"].get(
"params") is not None else arguments_from_config
try:
if isinstance(arguments, list):
result = method["node"].call_method(
method[rpc_method], *arguments)
elif arguments is not None:
try:
result = method["node"].call_method(
method[rpc_method], arguments)
except ua.UaStatusCodeError as e:
if "BadTypeMismatch" in str(e) and isinstance(
arguments, int):
result = method["node"].call_method(
method[rpc_method], float(arguments))
else:
result = method["node"].call_method(
method[rpc_method])
self.__gateway.send_rpc_reply(
content["device"], content["data"]["id"], {
content["data"]["method"]: result,
"code": 200
})
log.debug("method %s result is: %s",
method[rpc_method], result)
except Exception as e:
log.exception(e)
self.__gateway.send_rpc_reply(content["device"],
content["data"]["id"], {
"error": str(e),
"code": 500
})
else:
log.error("Method %s not found for device %s", rpc_method,
content["device"])
self.__gateway.send_rpc_reply(
content["device"], content["data"]["id"], {
"error": "%s - Method not found" % (rpc_method),
"code": 404
})
except Exception as e:
log.exception(e)
def __init_iterator(self):
save_iterator = self.DEFAULT_SAVE_ITERATOR
if "persistent" not in self.__config["polling"]["iterator"]:
self.__config["polling"]["iterator"]["persistent"] = save_iterator
else:
save_iterator = self.__config["polling"]["iterator"]["persistent"]
log.info("[%s] Iterator saving %s", self.get_name(),
"enabled" if save_iterator else "disabled")
if save_iterator and self.__load_iterator_config():
log.info(
"[%s] Init iterator from file '%s': column=%s, start_value=%s",
self.get_name(), self.__iterator_file_name,
self.__iterator["name"], self.__iterator["value"])
return True
self.__iterator = {
"name": self.__config["polling"]["iterator"]["column"],
"total": 0
}
if "value" in self.__config["polling"]["iterator"]:
self.__iterator["value"] = self.__config["polling"]["iterator"][
"value"]
log.info(
"[%s] Init iterator from configuration: column=%s, start_value=%s",
self.get_name(), self.__iterator["name"],
self.__iterator["value"])
elif "query" in self.__config["polling"]["iterator"]:
try:
self.__iterator["value"] = \
self.__cursor.execute(self.__config["polling"]["iterator"]["query"]).fetchone()[0]
log.info(
"[%s] Init iterator from database: column=%s, start_value=%s",
self.get_name(), self.__iterator["name"],
self.__iterator["value"])
except pyodbc.Warning as w:
log.warning("[%s] Warning on init iterator from database: %s",
self.get_name(), str(w))
except pyodbc.Error as e:
log.error("[%s] Failed to init iterator from database: %s",
self.get_name(), str(e))
else:
log.error(
"[%s] Failed to init iterator: value/query param is absent",
self.get_name())
return "value" in self.__iterator
def server_side_rpc_handler(self, server_rpc_request):
try:
if server_rpc_request.get(DEVICE_SECTION_PARAMETER) is not None:
log.debug(
"Modbus connector received rpc request for %s with server_rpc_request: %s",
server_rpc_request[DEVICE_SECTION_PARAMETER],
server_rpc_request)
if isinstance(
self.__devices[
server_rpc_request[DEVICE_SECTION_PARAMETER]]
[CONFIG_SECTION_PARAMETER][RPC_SECTION], dict):
rpc_command_config = self.__devices[
server_rpc_request[DEVICE_SECTION_PARAMETER]][
CONFIG_SECTION_PARAMETER][RPC_SECTION].get(
server_rpc_request[DATA_PARAMETER]
[RPC_METHOD_PARAMETER])
if rpc_command_config is not None:
self.__process_rpc_request(server_rpc_request,
rpc_command_config)
elif isinstance(
self.__devices[
server_rpc_request[DEVICE_SECTION_PARAMETER]]
[CONFIG_SECTION_PARAMETER][RPC_SECTION], list):
for rpc_command_config in self.__devices[
server_rpc_request[DEVICE_SECTION_PARAMETER]][
CONFIG_SECTION_PARAMETER][RPC_SECTION]:
if rpc_command_config[
TAG_PARAMETER] == server_rpc_request[
DATA_PARAMETER][RPC_METHOD_PARAMETER]:
self.__process_rpc_request(server_rpc_request,
rpc_command_config)
break
else:
log.error(
"Received rpc request, but method %s not found in config for %s.",
server_rpc_request[DATA_PARAMETER].get(
RPC_METHOD_PARAMETER), self.get_name())
self.__gateway.send_rpc_reply(
server_rpc_request[DEVICE_SECTION_PARAMETER],
server_rpc_request[DATA_PARAMETER][RPC_ID_PARAMETER], {
server_rpc_request[DATA_PARAMETER][RPC_METHOD_PARAMETER]:
"METHOD NOT FOUND!"
})
else:
log.debug("Received RPC to connector: %r", server_rpc_request)
except Exception as e:
log.exception(e)
def __load_converters(self, connector_type): # Function for search a converter and save it.
devices_config = self.__config.get('devices')
try:
if devices_config is not None:
for device_config in devices_config:
if device_config.get('converter') is not None:
converter = TBModuleLoader.import_module(connector_type, device_config['converter'])
self.__devices[device_config['name']] = {'converter': converter(device_config),
'device_config': device_config}
else:
log.error('Converter configuration for the custom connector %s -- not found, please check your configuration file.', self.get_name())
else:
log.error('Section "devices" in the configuration not found. A custom connector %s has being stopped.', self.get_name())
self.close()
except Exception as e:
log.exception(e)
def form_iocb(device, config=None, request_type="readProperty"):
config = config if config is not None else device
address = device["address"] if isinstance(
device["address"], Address) else Address(device["address"])
object_id = ObjectIdentifier(config["objectId"])
property_id = config.get("propertyId")
value = config.get("propertyValue")
property_index = config.get("propertyIndex")
priority = config.get("priority")
vendor = device.get("vendor", config.get("vendorId", 0))
request = None
iocb = None
if request_type == "readProperty":
try:
request = ReadPropertyRequest(objectIdentifier=object_id,
propertyIdentifier=property_id)
request.pduDestination = address
if property_index is not None:
request.propertyArrayIndex = int(property_index)
iocb = IOCB(request)
except Exception as e:
log.exception(e)
elif request_type == "writeProperty":
datatype = get_datatype(object_id.value[0], property_id, vendor)
if (isinstance(value, str)
and value.lower() == 'null') or value is None:
value = Null()
request = WritePropertyRequest(objectIdentifier=object_id,
propertyIdentifier=property_id)
request.pduDestination = address
request.propertyValue = Any()
try:
value = datatype(value)
request.propertyValue = Any(value)
except AttributeError as e:
log.debug(e)
except Exception as error:
log.exception("WriteProperty cast error: %r", error)
if property_index is not None:
request.propertyArrayIndex = property_index
if priority is not None:
request.priority = priority
iocb = IOCB(request)
else:
log.error("Request type is not found or not implemented")
return iocb
def server_side_rpc_handler(self, content):
try:
log.debug('Recieved RPC Request: %r', str(content))
for device in self.__devices:
if device["deviceName"] == content["device"]:
method_found = False
for request in device["server_side_rpc"]:
if request["config"].get("requestType") is not None:
if content["data"]["method"] == request["method"]:
method_found = True
kwargs = request["iocb"][1]
timeout = time() * 1000 + request[
"config"].get("requestTimeout", 200)
if content["data"].get("params") is not None:
kwargs["config"].update({
"propertyValue":
content["data"]["params"]
})
iocb = request["iocb"][0](device, **kwargs)
self.__request_functions[
request["config"]["requestType"]](
device=iocb,
callback=self.__rpc_response_cb)
self.rpc_requests_in_progress[iocb] = {
"content": content,
"uplink_converter":
request["uplink_converter"]
}
# self.__gateway.register_rpc_request_timeout(content,
# timeout,
# iocb,
# self.__rpc_cancel_processing)
else:
log.error(
"\"requestType\" not found in request configuration for key %s device: %s",
request.get("key", "[KEY IS EMPTY]"),
device["deviceName"])
if not method_found:
log.error("RPC method %s not found in configuration",
content["data"]["method"])
self.__gateway.send_rpc_reply(content["device"],
content["data"]["id"],
success_sent=False)
except Exception as e:
log.exception(e)
def __init_connection(self):
try:
log.debug("[%s] Opening connection to database", self.get_name())
connection_config = self.__config["connection"]
self.__connection = pyodbc.connect(
connection_config["str"],
**connection_config.get("attributes", {}))
if connection_config.get("encoding", ""):
log.info("[%s] Setting encoding to %s", self.get_name(),
connection_config["encoding"])
self.__connection.setencoding(connection_config["encoding"])
decoding_config = connection_config.get("decoding")
if decoding_config is not None:
if isinstance(decoding_config, dict):
if decoding_config.get("char", ""):
log.info("[%s] Setting SQL_CHAR decoding to %s",
self.get_name(), decoding_config["char"])
self.__connection.setdecoding(pyodbc.SQL_CHAR,
decoding_config["char"])
if decoding_config.get("wchar", ""):
log.info("[%s] Setting SQL_WCHAR decoding to %s",
self.get_name(), decoding_config["wchar"])
self.__connection.setdecoding(pyodbc.SQL_WCHAR,
decoding_config["wchar"])
if decoding_config.get("metadata", ""):
log.info("[%s] Setting SQL_WMETADATA decoding to %s",
self.get_name(), decoding_config["metadata"])
self.__connection.setdecoding(
pyodbc.SQL_WMETADATA, decoding_config["metadata"])
else:
log.warning(
"[%s] Unknown decoding configuration %s. Read data may be misdecoded",
self.get_name(), decoding_config)
self.__cursor = self.__connection.cursor()
log.info("[%s] Connection to database opened, attributes %s",
self.get_name(), connection_config.get("attributes", {}))
except pyodbc.Error as e:
log.error("[%s] Failed to connect to database: %s",
self.get_name(), str(e))
self.__close()
return self.is_connected()
def on_attributes_update(self, content):
log.debug(content)
for device in self.__devices_around:
if self.__devices_around[device]['device_config'].get(
'name') == content['device']:
for requests in self.__devices_around[device]['device_config'][
"attributeUpdates"]:
for service in self.__devices_around[device]['services']:
if requests[
'characteristicUUID'] in self.__devices_around[
device]['services'][service]:
characteristic = self.__devices_around[device][
'services'][service][requests[
'characteristicUUID']]['characteristic']
if 'WRITE' in characteristic.propertiesToString():
if content['data'].get(
requests['attributeOnThingsBoard']
) is not None:
try:
self.__check_and_reconnect(device)
content_to_write = content['data'][
requests[
'attributeOnThingsBoard']].encode(
'UTF-8')
characteristic.write(
content_to_write, True)
except BTLEDisconnectError:
self.__check_and_reconnect(device)
content_to_write = content['data'][
requests[
'attributeOnThingsBoard']].encode(
'UTF-8')
characteristic.write(
content_to_write, True)
except Exception as e:
log.exception(e)
else:
log.error(
'Cannot process attribute update request for device: %s with data: %s and config: %s',
device, content,
self.__devices_around[device]
['device_config']["attributeUpdates"])
def __scan_ble(self):
log.debug("Scanning for devices...")
try:
self.__scanner.scan(self.__config.get('scanTimeSeconds', 5),
passive=self.__config.get(
'passiveScanMode', False))
except BTLEManagementError as e:
log.error('BLE working only with root user.')
log.error(
'Or you can try this command:\nsudo setcap '
'\'cap_net_raw,cap_net_admin+eip\' %s'
'\n====== Attention! ====== '
'\nCommand above - provided access to ble devices to any user.'
'\n========================',
str(bluepy_path[0] + '/bluepy-helper'))
self._connected = False
raise e
except Exception as e:
log.exception(e)
time.sleep(10)
def __process_data(self, device):
common_parameters = self.__get_common_parameters(device)
converted_data = {}
for datatype in self.__datatypes:
for datatype_config in device[datatype]:
try:
response = None
method = datatype_config.get("method")
if method is None:
log.error("Method not found in configuration: %r",
datatype_config)
continue
else:
method = method.lower()
if method not in self.__methods:
log.error("Unknown method: %s, configuration is: %r",
method, datatype_config)
response = self.__process_methods(method,
common_parameters,
datatype_config)
converted_data.update(
**device["uplink_converter"].convert((
datatype, datatype_config), response))
except SNMPTimeoutException:
log.error(
"Timeout exception on connection to device \"%s\" with ip: \"%s\"",
device["deviceName"], device["ip"])
return
except Exception as e:
log.exception(e)
if isinstance(converted_data,
dict) and (converted_data.get("attributes")
or converted_data.get("telemetry")):
self.collect_statistic_and_send(self.get_name(),
converted_data)
def scan_nodes_from_config(self):
try:
if self.__interest_nodes:
for device_object in self.__interest_nodes:
for current_device in device_object:
try:
device_configuration = device_object[
current_device]
devices_info_array = self.__search_general_info(
device_configuration)
for device_info in devices_info_array:
if device_info is not None and device_info.get(
"deviceNode") is not None:
self.__search_nodes_and_subscribe(
device_info)
self.__save_methods(device_info)
self.__search_attribute_update_variables(
device_info)
else:
log.error(
"Device node is None, please check your configuration."
)
log.debug(
"Current device node is: %s",
str(
device_configuration.get(
"deviceNodePattern")))
break
except BrokenPipeError:
log.debug("Broken Pipe. Connection lost.")
except OSError:
log.debug("Stop on scanning.")
except FuturesTimeoutError:
self.__check_connection()
except Exception as e:
log.exception(e)
log.debug(self.__interest_nodes)
except Exception as e:
log.exception(e)
def __save_methods(self, device_info):
try:
if self.__available_object_resources.get(
device_info["deviceName"]) is None:
self.__available_object_resources[
device_info["deviceName"]] = {}
if self.__available_object_resources[
device_info["deviceName"]].get("methods") is None:
self.__available_object_resources[
device_info["deviceName"]]["methods"] = []
if device_info["configuration"].get("rpc_methods", []):
node = device_info["deviceNode"]
for method_object in device_info["configuration"][
"rpc_methods"]:
method_node_path = self._check_path(
method_object["method"], node)
methods = []
self.__search_node(node,
method_node_path,
True,
result=methods)
for method in methods:
if method is not None:
node_method_name = method.get_display_name().Text
self.__available_object_resources[
device_info["deviceName"]]["methods"].append({
node_method_name:
method,
"node":
node,
"arguments":
method_object.get("arguments")
})
else:
log.error(
"Node for method with path %s - NOT FOUND!",
method_node_path)
except Exception as e:
log.exception(e)
def __rpc_response_cb(self, iocb, callback_params=None):
device = self.rpc_requests_in_progress[iocb]
converter = device["uplink_converter"]
content = device["content"]
if iocb.ioResponse:
apdu = iocb.ioResponse
log.debug("Received callback with Response: %r", apdu)
converted_data = converter.convert(None, apdu)
if converted_data is None:
converted_data = {"success": True}
self.__gateway.send_rpc_reply(content["device"],
content["data"]["id"],
converted_data)
# self.__gateway.rpc_with_reply_processing(iocb, converted_data or {"success": True})
elif iocb.ioError:
log.exception("Received callback with Error: %r", iocb.ioError)
data = {"error": str(iocb.ioError)}
self.__gateway.send_rpc_reply(content["device"],
content["data"]["id"], data)
log.debug(iocb.ioError)
else:
log.error("Received unknown RPC response callback from device: %r",
iocb)
def __load_iterator_config(self):
if not self.__iterator_file_name:
if not self.__resolve_iterator_file():
log.error(
"[%s] Unable to load iterator configuration from file: file name is not resolved",
self.get_name())
return False
try:
iterator_file_path = Path(self.__config_dir +
self.__iterator_file_name)
if not iterator_file_path.exists():
return False
with iterator_file_path.open("r") as iterator_file:
self.__iterator = load(iterator_file)
log.debug("[%s] Loaded iterator configuration from %s",
self.get_name(), self.__iterator_file_name)
except Exception as e:
log.error("[%s] Failed to load iterator configuration from %s: %s",
self.get_name(), self.__iterator_file_name, str(e))
return bool(self.__iterator)
def load_handlers(self):
data_handlers = {
"basic": BasicDataHandler,
"anonymous": AnonymousDataHandler,
}
for mapping in self.__config.get("mapping"):
try:
security_type = "anonymous" if mapping.get(
"security") is None else mapping["security"]["type"].lower(
)
if security_type != "anonymous":
Users.add_user(mapping['endpoint'],
mapping['security']['username'],
mapping['security']['password'])
self._api.add_resource(
data_handlers[security_type],
mapping['endpoint'],
endpoint=mapping['endpoint'],
resource_class_args=(self.collect_statistic_and_send,
self.get_name(),
self.endpoints[mapping["endpoint"]]))
except Exception as e:
log.error("Error on creating handlers - %s", str(e))
def run(self):
while not self.__stopped:
# Initialization phase
if not self.is_connected():
while not self.__stopped and \
not self.__init_connection() and \
self.__config["connection"].get("reconnect", self.DEFAULT_RECONNECT_STATE):
reconnect_period = self.__config["connection"].get(
"reconnectPeriod", self.DEFAULT_RECONNECT_PERIOD)
log.info("[%s] Will reconnect to database in %d second(s)",
self.get_name(), reconnect_period)
sleep(reconnect_period)
if not self.is_connected():
log.error(
"[%s] Cannot connect to database so exit from main loop",
self.get_name())
break
if not self.__init_iterator():
log.error(
"[%s] Cannot init database iterator so exit from main loop",
self.get_name())
break
# Polling phase
try:
self.__poll()
if not self.__stopped:
polling_period = self.__config["polling"].get(
"period", self.DEFAULT_POLL_PERIOD)
log.debug(
"[%s] Next polling iteration will be in %d second(s)",
self.get_name(), polling_period)
sleep(polling_period)
except pyodbc.Warning as w:
log.warning("[%s] Warning while polling database: %s",
self.get_name(), str(w))
except pyodbc.Error as e:
log.error("[%s] Error while polling database: %s",
self.get_name(), str(e))
self.__close()
self.__close()
self.__stopped = False
log.info("[%s] Stopped", self.get_name())
def __search_nodes_and_subscribe(self, device_info):
sub_nodes = []
information_types = {
"attributes": "attributes",
"timeseries": "telemetry"
}
for information_type in information_types:
for information in device_info["configuration"][information_type]:
information_key = information["key"]
config_path = TBUtility.get_value(information["path"],
get_tag=True)
information_path = self._check_path(config_path,
device_info["deviceNode"])
information["path"] = '${%s}' % information_path
information_nodes = []
self.__search_node(device_info["deviceNode"],
information_path,
result=information_nodes)
for information_node in information_nodes:
if information_node is not None:
try:
information_value = information_node.get_value()
except:
log.error("Err get_value: %s",
str(information_node))
continue
log.debug(
"Node for %s \"%s\" with path: %s - FOUND! Current values is: %s",
information_type, information_key,
information_path, str(information_value))
if device_info.get("uplink_converter") is None:
configuration = {
**device_info["configuration"], "deviceName":
device_info["deviceName"],
"deviceType":
device_info["deviceType"]
}
if device_info["configuration"].get(
'converter') is None:
converter = OpcUaUplinkConverter(configuration)
else:
converter = TBModuleLoader.import_module(
self._connector_type, configuration)
device_info["uplink_converter"] = converter
else:
converter = device_info["uplink_converter"]
self.subscribed[information_node] = {
"converter": converter,
"path": information_path,
"config_path": config_path
}
if not device_info.get(
information_types[information_type]):
device_info[
information_types[information_type]] = []
converted_data = converter.convert(
(config_path, information_path), information_value)
self.statistics['MessagesReceived'] = self.statistics[
'MessagesReceived'] + 1
self.data_to_send.append(converted_data)
self.statistics['MessagesSent'] = self.statistics[
'MessagesSent'] + 1
log.debug("Data to ThingsBoard: %s", converted_data)
if not self.__server_conf.get("disableSubscriptions",
False):
sub_nodes.append(information_node)
else:
log.error(
"Node for %s \"%s\" with path %s - NOT FOUND!",
information_type, information_key,
information_path)
if not self.__server_conf.get("disableSubscriptions", False):
if self.__sub is None:
self.__sub = self.client.create_subscription(
self.__server_conf.get("subCheckPeriodInMillis", 500),
self.__sub_handler)
if sub_nodes:
self.__sub.subscribe_data_change(sub_nodes)
log.debug("Added subscription to nodes: %s", str(sub_nodes))
def __search_node(self,
current_node,
fullpath,
search_method=False,
result=None):
if result is None:
result = []
try:
if regex.match(r"ns=\d*;[isgb]=.*", fullpath, regex.IGNORECASE):
if self.__show_map:
log.debug("Looking for node with config")
node = self.client.get_node(fullpath)
if node is None:
log.warning("NODE NOT FOUND - using configuration %s",
fullpath)
else:
log.debug("Found in %s", node)
result.append(node)
else:
fullpath_pattern = regex.compile(fullpath)
full1 = fullpath.replace('\\\\.', '.')
#current_node_path = '\\.'.join(char.split(":")[1] for char in current_node.get_path(200000, True))
current_node_path = self.get_node_path(current_node)
# we are allways the parent
child_node_parent_class = current_node.get_node_class()
new_parent = current_node
for child_node in current_node.get_children():
new_node_class = child_node.get_node_class()
# this will not change you can do it outside th loop
# basis Description of node.get_parent() function, sometime child_node.get_parent() return None
#new_parent = child_node.get_parent()
#if (new_parent is None):
# child_node_parent_class = current_node.get_node_class()
#else:
# child_node_parent_class = child_node.get_parent().get_node_class()
#current_node_path = '\\.'.join(char.split(":")[1] for char in current_node.get_path(200000, True))
#new_node_path = '\\\\.'.join(char.split(":")[1] for char in child_node.get_path(200000, True))
new_node_path = self.get_node_path(child_node)
if child_node_parent_class == ua.NodeClass.View and new_parent is not None:
parent_path = self.get_node_path(new_parent)
#parent_path = '\\.'.join(char.split(":")[1] for char in new_parent.get_path(200000, True))
fullpath = fullpath.replace(current_node_path,
parent_path)
nnp1 = new_node_path.replace('\\\\.', '.')
nnp2 = new_node_path.replace('\\\\', '\\')
if self.__show_map:
log.debug("SHOW MAP: Current node path: %s",
new_node_path)
regex_fullmatch = regex.fullmatch(fullpath_pattern, nnp1) or \
nnp2 == full1 or \
nnp2 == fullpath or \
nnp1 == full1
if regex_fullmatch:
if self.__show_map:
log.debug(
"SHOW MAP: Current node path: %s - NODE FOUND",
nnp2)
result.append(child_node)
else:
regex_search = fullpath_pattern.fullmatch(nnp1, partial=True) or \
nnp2 in full1 or \
nnp1 in full1
if regex_search:
if self.__show_map:
log.debug(
"SHOW MAP: Current node path: %s - NODE FOUND",
new_node_path)
if new_node_class == ua.NodeClass.Object:
if self.__show_map:
log.debug("SHOW MAP: Search in %s",
new_node_path)
self.__search_node(child_node,
fullpath,
result=result)
elif new_node_class == ua.NodeClass.Variable:
log.debug("Found in %s", new_node_path)
result.append(child_node)
elif new_node_class == ua.NodeClass.Method and search_method:
log.debug("Found in %s", new_node_path)
result.append(child_node)
except CancelledError:
log.error(
"Request during search has been canceled by the OPC-UA server."
)
except BrokenPipeError:
log.error("Broken Pipe. Connection lost.")
except OSError:
log.debug("Stop on scanning.")
except Exception as e:
log.exception(e)
def __search_general_info(self, device):
result = []
match_devices = []
self.__search_node(self.__opcua_nodes["root"],
TBUtility.get_value(device["deviceNodePattern"],
get_tag=True),
result=match_devices)
for device_node in match_devices:
if device_node is not None:
result_device_dict = {
"deviceName": None,
"deviceType": None,
"deviceNode": device_node,
"configuration": deepcopy(device)
}
name_pattern_config = device["deviceNamePattern"]
name_expression = TBUtility.get_value(name_pattern_config,
get_tag=True)
if "${" in name_pattern_config and "}" in name_pattern_config:
log.debug("Looking for device name")
device_name_from_node = ""
if name_expression == "$DisplayName":
device_name_from_node = device_node.get_display_name(
).Text
elif name_expression == "$BrowseName":
device_name_from_node = device_node.get_browse_name(
).Name
elif name_expression == "$NodeId.Identifier":
device_name_from_node = str(
device_node.nodeid.Identifier)
else:
name_path = self._check_path(name_expression,
device_node)
device_name_node = []
self.__search_node(device_node,
name_path,
result=device_name_node)
device_name_node = device_name_node[0]
if device_name_node is not None:
device_name_from_node = device_name_node.get_value(
)
if device_name_from_node == "":
log.error(
"Device name node not found with expression: %s",
name_expression)
return None
full_device_name = name_pattern_config.replace(
"${" + name_expression + "}",
str(device_name_from_node)).replace(
name_expression, str(device_name_from_node))
else:
full_device_name = name_expression
result_device_dict["deviceName"] = full_device_name
log.debug("Device name: %s", full_device_name)
if device.get("deviceTypePattern"):
device_type_expression = TBUtility.get_value(
device["deviceTypePattern"], get_tag=True)
if "${" in device_type_expression and "}" in device_type_expression:
type_path = self._check_path(device_type_expression,
device_node)
device_type_node = []
self.__search_node(device_node,
type_path,
result=device_type_node)
device_type_node = device_type_node[0]
if device_type_node is not None:
device_type = device_type_node.get_value()
full_device_type = device_type_expression.replace(
"${" + device_type_expression + "}",
device_type).replace(device_type_expression,
device_type)
else:
log.error(
"Device type node not found with expression: %s",
device_type_expression)
full_device_type = "default"
else:
full_device_type = device_type_expression
result_device_dict["deviceType"] = full_device_type
log.debug("Device type: %s", full_device_type)
else:
result_device_dict["deviceType"] = "default"
result.append(result_device_dict)
else:
log.error(
"Device node not found with expression: %s",
TBUtility.get_value(device["deviceNodePattern"],
get_tag=True))
return result
def __init__(self, gateway, config, connector_type):
self._connector_type = connector_type
self.statistics = {'MessagesReceived': 0, 'MessagesSent': 0}
super().__init__()
self.__gateway = gateway
self.__server_conf = config.get("server")
self.__interest_nodes = []
self.__available_object_resources = {}
self.__show_map = self.__server_conf.get("showMap", False)
self.__previous_scan_time = 0
for mapping in self.__server_conf["mapping"]:
if mapping.get("deviceNodePattern") is not None:
self.__interest_nodes.append(
{mapping["deviceNodePattern"]: mapping})
else:
log.error(
"deviceNodePattern in mapping: %s - not found, add property deviceNodePattern to processing this mapping",
dumps(mapping))
if "opc.tcp" not in self.__server_conf.get("url"):
self.__opcua_url = "opc.tcp://" + self.__server_conf.get("url")
else:
self.__opcua_url = self.__server_conf.get("url")
self.client = Client(
self.__opcua_url,
timeout=self.__server_conf.get("timeoutInMillis", 4000) / 1000)
if self.__server_conf["identity"].get("type") == "cert.PEM":
try:
ca_cert = self.__server_conf["identity"].get("caCert")
private_key = self.__server_conf["identity"].get("privateKey")
cert = self.__server_conf["identity"].get("cert")
security_mode = self.__server_conf["identity"].get(
"mode", "SignAndEncrypt")
policy = self.__server_conf["security"]
if cert is None or private_key is None:
log.exception(
"Error in ssl configuration - cert or privateKey parameter not found"
)
raise RuntimeError(
"Error in ssl configuration - cert or privateKey parameter not found"
)
security_string = policy + ',' + security_mode + ',' + cert + ',' + private_key
if ca_cert is not None:
security_string = security_string + ',' + ca_cert
self.client.set_security_string(security_string)
except Exception as e:
log.exception(e)
if self.__server_conf["identity"].get("username"):
self.client.set_user(
self.__server_conf["identity"].get("username"))
if self.__server_conf["identity"].get("password"):
self.client.set_password(
self.__server_conf["identity"].get("password"))
self.setName(
self.__server_conf.get(
"name", 'OPC-UA ' +
''.join(choice(ascii_lowercase)
for _ in range(5))) + " Connector")
self.__opcua_nodes = {}
self._subscribed = {}
self.__sub = None
self.__sub_handler = SubHandler(self)
self.data_to_send = []
self.__stopped = False
self.__connected = False
self.daemon = True
