class Gateway:
def __init__(self):
argparser = argparse.ArgumentParser()
argparser.add_argument("-d",
"--device",
help="serial device /dev file modem",
default="/dev/ttyACM0")
argparser.add_argument("-r",
"--rate",
help="baudrate for serial device",
type=int,
default=115200)
argparser.add_argument("-v",
"--verbose",
help="verbose",
default=False,
action="store_true")
argparser.add_argument("-t",
"--token",
help="Access token for the TB gateway",
required=True)
argparser.add_argument("-tb",
"--thingsboard",
help="Thingsboard hostname/IP",
default="localhost")
argparser.add_argument("-p",
"--plugin-path",
help="path where plugins are stored",
default="")
argparser.add_argument("-bp",
"--broker-port",
help="mqtt broker port",
default="1883")
argparser.add_argument(
"-l",
"--logfile",
help="specify path if you want to log to file instead of to stdout",
default="")
argparser.add_argument(
"-k",
"--keep-data",
help=
"Save data locally when Thingsboard is disconnected and send it when connection is restored.",
default=True)
argparser.add_argument(
"-b",
"--save-bandwidth",
help="Send data in binary format to save bandwidth",
action="store_true")
argparser.add_argument("-sf",
"--skip-system-files",
help="Do not read system files on boot",
action="store_true")
self.bridge_count = 0
self.next_report = 0
self.config = argparser.parse_args()
self.log = logging.getLogger()
formatter = logging.Formatter(
'%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
if self.config.logfile == "":
handler = logging.StreamHandler()
else:
handler = logging.FileHandler(self.config.logfile)
handler.setFormatter(formatter)
self.log.addHandler(handler)
self.log.setLevel(logging.INFO)
if self.config.verbose:
self.log.setLevel(logging.DEBUG)
self.tb = Thingsboard(self.config.thingsboard,
self.config.token,
self.on_mqtt_message,
persistData=self.config.keep_data)
if self.config.plugin_path != "":
self.load_plugins(self.config.plugin_path)
self.modem = Modem(self.config.device, self.config.rate,
self.on_command_received,
self.config.save_bandwidth)
connected = self.modem.connect()
while not connected:
try:
self.log.warning("Not connected to modem, retrying ...")
time.sleep(1)
connected = self.modem.connect()
except KeyboardInterrupt:
self.log.info("received KeyboardInterrupt... stopping")
self.tb.disconnect()
exit(-1)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
trace = "".join(lines)
self.log.error(
"Exception while connecting modem: \n{}".format(trace))
# switch to continuous foreground scan access profile
self.modem.execute_command(
Command.create_with_write_file_action_system_file(
DllConfigFile(active_access_class=0x01)),
timeout_seconds=1)
if self.config.save_bandwidth:
self.log.info("Running in save bandwidth mode")
if self.config.plugin_path is not "":
self.log.warning(
"Save bandwidth mode is enabled, plugin files will not be used"
)
# update attribute containing git rev so we can track revision at TB platform
git_sha = subprocess.check_output(["git", "describe",
"--always"]).strip()
ip = self.get_ip()
self.tb.sendGwAttributes({
'UID': self.modem.uid,
'git-rev': git_sha,
'IP': ip,
'save bw': str(self.config.save_bandwidth)
})
self.log.info("Running on {} with git rev {} using modem {}".format(
ip, git_sha, self.modem.uid))
# read all system files on the local node to store as attributes on TB
if not self.config.skip_system_files:
self.log.info("Reading all system files ...")
for file in SystemFiles().files.values():
self.modem.execute_command_async(
Command.create_with_read_file_action_system_file(file))
def load_plugins(self, plugin_path):
self.log.info("Searching for plugins in path %s" % plugin_path)
manager = PluginManagerSingleton.get()
manager.setPluginPlaces([plugin_path])
manager.collectPlugins()
for plugin in manager.getAllPlugins():
self.log.info("Loading plugin '%s'" % plugin.name)
def on_command_received(self, cmd):
try:
if self.config.save_bandwidth:
self.log.info("Command received: binary ALP (size {})".format(
len(cmd)))
else:
self.log.info("Command received: {}".format(cmd))
ts = int(round(time.time() * 1000))
# publish raw ALP command to incoming ALP topic, we will not parse the file contents here (since we don't know how)
# so pass it as an opaque BLOB for parsing in backend
if self.config.save_bandwidth:
self.tb.sendGwAttributes({
'alp':
binascii.hexlify(bytearray(cmd)),
'last_seen':
str(datetime.now().strftime("%y-%m-%d %H:%M:%S"))
})
return
self.tb.sendGwAttributes({
'alp':
jsonpickle.encode(cmd),
'last_seen':
str(datetime.now().strftime("%y-%m-%d %H:%M:%S"))
})
node_id = self.modem.uid # overwritten below with remote node ID when received over D7 interface
# parse link budget (when this is received over D7 interface) and publish separately so we can visualize this in TB
if cmd.interface_status != None and cmd.interface_status.operand.interface_id == 0xd7:
interface_status = cmd.interface_status.operand.interface_status
node_id = '{:x}'.format(interface_status.addressee.id)
linkBudget = interface_status.link_budget
rxLevel = interface_status.rx_level
lastConnect = "D7-" + interface_status.get_short_channel_string(
)
self.tb.sendDeviceTelemetry(node_id, ts, {
'lb': linkBudget,
'rx': rxLevel
})
self.tb.sendDeviceAttributes(node_id, {
'last_conn': lastConnect,
'last_gw': self.modem.uid
})
# store returned file data as attribute on the device
for action in cmd.actions:
if type(action.operation) is ReturnFileData:
data = ""
if action.operation.file_data_parsed is not None:
if not self.config.save_bandwidth:
# for known system files we transmit the parsed data
data = jsonpickle.encode(
action.operation.file_data_parsed)
file_id = "File {}".format(
action.operand.offset.id)
self.tb.sendGwAttributes({file_id: data})
else:
# try if plugin can parse this file
parsed_by_plugin = False
if not self.config.save_bandwidth:
for plugin in PluginManagerSingleton.get(
).getAllPlugins():
for name, value, datapoint_type in plugin.plugin_object.parse_file_data(
action.operand.offset,
action.operand.length,
action.operand.data):
parsed_by_plugin = True
if isinstance(value, int) or isinstance(
value, float):
self.tb.sendDeviceTelemetry(
node_id, ts, {name: value})
else:
self.tb.sendDeviceAttributes(
node_id, {name: value})
if not parsed_by_plugin:
# unknown file content, just transmit raw data
data = jsonpickle.encode(action.operand)
filename = "File {}".format(
action.operand.offset.id)
if action.operation.systemfile_type != None:
filename = "File {} ({})".format(
SystemFileIds(
action.operand.offset.id).name,
action.operand.offset.id)
self.tb.sendDeviceAttributes(
node_id, {filename: data})
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
trace = "".join(lines)
self.log.error(
"Exception while processing command: \n{}".format(trace))
def on_mqtt_message(self, client, config, msg):
try:
payload = json.loads(msg.payload)
uid = payload['device']
method = payload['data']['method']
request_id = payload['data']['id']
self.log.info(
"Received RPC command of type {} for {} (request id {})".
format(method, uid, request_id))
# if uid != self.modem.uid:
# self.log.info("RPC command not for this modem ({}), skipping".format(self.modem.uid))
# return
if method == "execute-alp-async":
try:
cmd = payload['data']['params']
self.log.info("Received command through RPC: %s" % cmd)
self.modem.execute_command_async(cmd)
self.log.info("Executed ALP command through RPC")
# TODO when the command is writing local files we could read them again automatically afterwards, to make sure the digital twin is updated
except Exception as e:
self.log.exception("Could not deserialize: %s" % e)
elif method == "alert":
# TODO needs refactoring so different methods can be supported in a plugin, for now this is very specific case as an example
self.log.info("Alert (payload={})".format(msg.payload))
if msg.payload != "true" and msg.payload != "false":
self.log.info("invalid payload, skipping")
return
file_data = 0
if msg.payload == "true":
file_data = 1
self.log.info("writing alert file")
self.modem.execute_command_async(
Command.create_with_write_file_action(
file_id=0x60,
offset=4,
data=[file_data],
interface_type=InterfaceType.D7ASP,
interface_configuration=Configuration(
qos=QoS(resp_mod=ResponseMode.RESP_MODE_ALL),
addressee=Addressee(access_class=0x11,
id_type=IdType.NOID))))
else:
self.log.info("RPC method not supported, skipping")
return
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
trace = "".join(lines)
msg_info = "no msg info (missing __dict__ attribute)" # TODO because of out of date paho??
if hasattr(msg, '__dict__'):
msg_info = str(msg.__dict__)
self.log.error(
"Exception while processing MQTT message: {} callstack:\n{}".
format(msg_info, trace))
def run(self):
self.log.info("Started")
keep_running = True
while keep_running:
try:
if platform.system() == "Windows":
time.sleep(1)
else:
signal.pause()
except KeyboardInterrupt:
self.log.info(
"received KeyboardInterrupt... stopping processing")
self.tb.disconnect()
keep_running = False
self.report_stats()
def keep_stats(self):
self.bridge_count += 1
def report_stats(self):
if self.next_report < time.time():
if self.bridge_count > 0:
self.log.info("bridged %s messages" % str(self.bridge_count))
self.bridge_count = 0
self.next_report = time.time(
) + 15 # report at most every 15 seconds
def get_ip(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# doesn't even have to be reachable
s.connect(('10.255.255.255', 1))
IP = s.getsockname()[0]
except:
IP = '127.0.0.1'
finally:
s.close()
return IP
argparser = argparse.ArgumentParser()
argparser.add_argument("-d", "--device", help="serial device /dev file modem",
default="/dev/ttyUSB0")
argparser.add_argument("-r", "--rate", help="baudrate for serial device", type=int, default=115200)
argparser.add_argument("-v", "--verbose", help="verbose", default=False, action="store_true")
config = argparser.parse_args()
configure_default_logger(config.verbose)
modem = Modem(config.device, config.rate, unsolicited_response_received_callback=received_command_callback)
modem.connect()
logging.info("Executing query...")
modem.execute_command_async(
alp_command=Command.create_with_read_file_action(
file_id=0x40,
length=8,
interface_type=InterfaceType.D7ASP,
interface_configuration=Configuration(
qos=QoS(resp_mod=ResponseMode.RESP_MODE_ALL),
addressee=Addressee(
access_class=0x11,
id_type=IdType.NOID
)
)
)
)
while True:
sleep(5)
class Gateway:
def __init__(self):
argparser = argparse.ArgumentParser()
argparser.add_argument("-d",
"--device",
help="serial device /dev file modem",
default="/dev/ttyACM0")
argparser.add_argument("-r",
"--rate",
help="baudrate for serial device",
type=int,
default=115200)
argparser.add_argument("-v",
"--verbose",
help="verbose",
default=False,
action="store_true")
argparser.add_argument("-b",
"--broker",
help="mqtt broker hostname",
default="localhost")
argparser.add_argument("-bp",
"--broker-port",
help="mqtt broker port",
default="1883")
argparser.add_argument("-p",
"--plugin-path",
help="path where plugins are stored",
default="")
argparser.add_argument(
"-l",
"--logfile",
help="specify path if you want to log to file instead of to stdout",
default="")
self.bridge_count = 0
self.next_report = 0
self.mq = None
self.mqtt_topic_incoming_alp = ""
self.connected_to_mqtt = False
self.config = argparser.parse_args()
self.log = logging.getLogger()
formatter = logging.Formatter(
'%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
if self.config.logfile == "":
handler = logging.StreamHandler()
else:
handler = logging.FileHandler(self.config.logfile)
handler.setFormatter(formatter)
self.log.addHandler(handler)
self.log.setLevel(logging.INFO)
if self.config.verbose:
self.log.setLevel(logging.DEBUG)
if self.config.plugin_path != "":
self.load_plugins(self.config.plugin_path)
self.modem = Modem(self.config.device, self.config.rate,
self.on_command_received)
self.connect_to_mqtt()
# update attribute containing git rev so we can track revision at TB platform
git_sha = subprocess.check_output(["git", "describe",
"--always"]).strip()
ip = self.get_ip()
# TODO ideally this should be associated with the GW device itself, not with the modem in the GW
# not clear how to do this using TB-GW
self.publish_to_topic(
"/gateway-info",
jsonpickle.json.dumps({
"git-rev": git_sha,
"ip": ip,
"device": self.modem.uid
}))
# make sure TB knows the modem device is connected. TB considers the device connected as well when there is regular
# telemetry data. This is fine for remote nodes which will be auto connected an disconnected in this way. But for the
# node in the gateway we do it explicitly to make sure it always is 'online' even when there is no telemetry to be transmitted,
# so that we can reach it over RPC
self.publish_to_topic(
"sensors/connect",
jsonpickle.json.dumps({"serialNumber": self.modem.uid}))
self.log.info("Running on {} with git rev {} using modem {}".format(
ip, git_sha, self.modem.uid))
# read all system files on the local node to store as attributes on TB
self.log.info("Reading all system files ...")
for file in SystemFiles().files.values():
self.modem.execute_command_async(
Command.create_with_read_file_action_system_file(file))
def load_plugins(self, plugin_path):
self.log.info("Searching for plugins in path %s" % plugin_path)
manager = PluginManagerSingleton.get()
manager.setPluginPlaces([plugin_path])
manager.collectPlugins()
for plugin in manager.getAllPlugins():
self.log.info("Loading plugin '%s'" % plugin.name)
def on_command_received(self, cmd):
try:
self.log.info("Command received: {}".format(cmd))
if not self.connected_to_mqtt:
self.log.warning("Not connected to MQTT, skipping")
return
# publish raw ALP command to incoming ALP topic, we will not parse the file contents here (since we don't know how)
# so pass it as an opaque BLOB for parsing in backend
self.publish_to_topic(
self.mqtt_topic_incoming_alp,
jsonpickle.json.dumps({'alp_command': jsonpickle.encode(cmd)}))
node_id = self.modem.uid # overwritten below with remote node ID when received over D7 interface
# parse link budget (when this is received over D7 interface) and publish separately so we can visualize this in TB
if cmd.interface_status != None and cmd.interface_status.operand.interface_id == 0xd7:
interface_status = cmd.interface_status.operand.interface_status
node_id = '{:x}'.format(interface_status.addressee.id)
self.publish_to_topic(
"/parsed/telemetry",
jsonpickle.json.dumps({
"gateway": self.modem.uid,
"device": node_id,
"name": "link_budget",
"value": interface_status.link_budget,
"timestamp": str(datetime.now())
}))
self.publish_to_topic(
"/parsed/telemetry",
jsonpickle.json.dumps({
"gateway": self.modem.uid,
"device": node_id,
"name": "rx_level",
"value": -interface_status.rx_level,
"timestamp": str(datetime.now())
}))
self.publish_to_topic(
"/parsed/attribute",
jsonpickle.json.dumps({
"device":
node_id,
"name":
"last_network_connection",
"value":
"D7-" + interface_status.get_short_channel_string(),
}))
# store returned file data as attribute on the device
for action in cmd.actions:
if type(action.operation) is ReturnFileData:
data = ""
if action.operation.file_data_parsed is not None:
# for known system files we transmit the parsed data
data = jsonpickle.encode(
action.operation.file_data_parsed)
else:
# try if plugin can parse this file
parsed_by_plugin = False
for plugin in PluginManagerSingleton.get(
).getAllPlugins():
for name, value, datapoint_type in plugin.plugin_object.parse_file_data(
action.operand.offset,
action.operand.data):
parsed_by_plugin = True
self.publish_to_topic(
"/parsed/" + datapoint_type.name,
jsonpickle.json.dumps({
"device": node_id,
"name": name,
"value": value,
}))
if not parsed_by_plugin:
# unknown file content, just transmit raw data
data = jsonpickle.encode(action.operand)
filename = "File {}".format(
action.operand.offset.id)
if action.operation.systemfile_type != None:
filename = "File {} ({})".format(
SystemFileIds(
action.operand.offset.id).name,
action.operand.offset.id)
self.publish_to_topic(
"/filecontent",
jsonpickle.json.dumps({
"device": node_id,
"file-id": filename,
"file-data": data
}))
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
trace = "".join(lines)
self.log.error(
"Exception while processing command: \n{}".format(trace))
def connect_to_mqtt(self):
self.log.info("Connecting to MQTT broker on {}".format(
self.config.broker))
self.connected_to_mqtt = False
self.mq = mqtt.Client("", True, None, mqtt.MQTTv31)
self.mqtt_topic_incoming_alp = "/DASH7/incoming/{}".format(
self.modem.uid)
self.mq.on_connect = self.on_mqtt_connect
self.mq.on_message = self.on_mqtt_message
self.mq.connect(self.config.broker, self.config.broker_port, 60)
self.mq.loop_start()
while not self.connected_to_mqtt:
pass # busy wait until connected
self.log.info("Connected to MQTT broker on {}".format(
self.config.broker))
def on_mqtt_connect(self, client, config, flags, rc):
self.mq.subscribe("sensor/#")
self.connected_to_mqtt = True
def on_mqtt_message(self, client, config, msg):
try:
topic_parts = msg.topic.split('/')
method = topic_parts[3]
uid = topic_parts[1]
request_id = topic_parts[4]
self.log.info(
"Received RPC command of type {} for {} (request id {})".
format(method, uid, request_id))
if uid != self.modem.uid:
self.log.info(
"RPC command not for this modem ({}), skipping".format(
self.modem.uid))
return
if method == "execute-alp-async":
try:
cmd = jsonpickle.decode(jsonpickle.json.loads(msg.payload))
self.log.info("Received command through RPC: %s" % cmd)
self.modem.execute_command_async(cmd)
self.log.info("Executed ALP command through RPC")
# TODO when the command is writing local files we could read them again automatically afterwards, to make sure the digital twin is updated
except Exception as e:
self.log.exception("Could not deserialize: %s" % e)
elif method == "alert":
# TODO needs refactoring so different methods can be supported in a plugin, for now this is very specific case as an example
self.log.info("Alert (payload={})".format(msg.payload))
if msg.payload != "true" and msg.payload != "false":
self.log.info("invalid payload, skipping")
return
file_data = 0
if msg.payload == "true":
file_data = 1
self.log.info("writing alert file")
self.modem.execute_command_async(
Command.create_with_write_file_action(
file_id=0x60,
offset=4,
data=[file_data],
interface_type=InterfaceType.D7ASP,
interface_configuration=Configuration(
qos=QoS(resp_mod=ResponseMode.RESP_MODE_ALL),
addressee=Addressee(access_class=0x11,
id_type=IdType.NOID))))
else:
self.log.info("RPC method not supported, skipping")
return
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
trace = "".join(lines)
msg_info = "no msg info (missing __dict__ attribute)" # TODO because of out of date paho??
if hasattr(msg, '__dict__'):
msg_info = str(msg.__dict__)
self.log.error(
"Exception while processing MQTT message: {} callstack:\n{}".
format(msg_info, trace))
def publish_to_topic(self, topic, msg):
if not self.connected_to_mqtt:
self.log.warning("not connected to MQTT, skipping")
return
self.mq.publish(topic, msg)
def __del__(self):
try:
self.mq.loop_stop()
self.mq.disconnect()
except:
pass
def run(self):
self.log.info("Started")
keep_running = True
while keep_running:
try:
if platform.system() == "Windows":
time.sleep(1)
else:
signal.pause()
except serial.SerialException:
time.sleep(1)
self.log.warning("resetting serial connection...")
self.setup_modem()
return
except KeyboardInterrupt:
self.log.info(
"received KeyboardInterrupt... stopping processing")
keep_running = False
self.report_stats()
def keep_stats(self):
self.bridge_count += 1
def report_stats(self):
if self.next_report < time.time():
if self.bridge_count > 0:
self.log.info("bridged %s messages" % str(self.bridge_count))
self.bridge_count = 0
self.next_report = time.time(
) + 15 # report at most every 15 seconds
def get_ip(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# doesn't even have to be reachable
s.connect(('10.255.255.255', 1))
IP = s.getsockname()[0]
except:
IP = '127.0.0.1'
finally:
s.close()
return IP
