def on_connect():
global modem
if modem == None:
modem = Modem(config.device, config.rate, command_received_callback)
modem.connect()
logging.info("modem: " + str(modem.uid))
emit('module_info', {
'uid': modem.uid,
'application_name': modem.firmware_version.application_name,
'git_sha1': modem.firmware_version.git_sha1,
'd7ap_version': modem.firmware_version.d7ap_version
},
broadcast=True)
help="verbose",
default=False,
action="store_true")
argparser.add_argument(
"-f",
"--file_id",
help="File where we're writing the interface configuration",
default=0x1D)
argparser.add_argument("-t", "--timeout", help="timeout", type=int, default=0)
config = argparser.parse_args()
configure_default_logger(config.verbose)
modem = Modem(config.device,
config.rate,
unsolicited_response_received_callback=received_command_callback)
modem.connect()
# D7 Example
interface_file = InterfaceConfigurationFile(
interface_configuration=InterfaceConfiguration(
interface_id=InterfaceType.D7ASP,
interface_configuration=Configuration(
qos=QoS(resp_mod=ResponseMode.RESP_MODE_PREFERRED,
retry_mod=RetryMode.RETRY_MODE_NO,
stop_on_err=False,
record=False),
dorm_to=CT(),
addressee=Addressee(nls_method=NlsMethod.NONE,
id_type=IdType.UID,
access_class=0x01,
id=CT(mant=3, exp=0)))))
class ThroughtPutTest:
def __init__(self):
self.argparser = argparse.ArgumentParser(
fromfile_prefix_chars="@",
description="Test throughput over 2 serial D7 modems")
self.argparser.add_argument("-n",
"--msg-count",
help="number of messages to transmit",
type=int,
default=10)
self.argparser.add_argument(
"-p",
"--payload-size",
help="number of bytes of (appl level) payload to transmit",
type=int,
default=50)
self.argparser.add_argument(
"-sw",
"--serial-transmitter",
help="serial device /dev file transmitter node",
default=None)
self.argparser.add_argument(
"-sr",
"--serial-receiver",
help="serial device /dev file receiver node",
default=None)
self.argparser.add_argument("-r",
"--rate",
help="baudrate for serial device",
type=int,
default=115200)
self.argparser.add_argument(
"-uid",
"--unicast-uid",
help="UID to use for unicast transmission, "
"when not using receiver "
"(in hexstring, for example 0xb57000009151d)",
default=None)
self.argparser.add_argument(
"-to",
"--receiver-timeout",
help="timeout for the receiver (in seconds)",
type=int,
default=10)
self.argparser.add_argument("-v",
"--verbose",
help="verbose",
default=False,
action="store_true")
self.config = self.argparser.parse_args()
configure_default_logger(self.config.verbose)
if self.config.serial_transmitter == None and self.config.serial_receiver == None:
self.argparser.error(
"At least a transmitter or receiver is required.")
if self.config.serial_receiver == None and self.config.unicast_uid == None:
self.argparser.error(
"When running without receiver a --unicast-uid parameter is required."
)
if self.config.serial_transmitter == None:
self.transmitter_modem = None
print("Running without transmitter")
else:
self.transmitter_modem = Modem(self.config.serial_transmitter,
self.config.rate, None)
self.transmitter_modem.connect()
access_profile = AccessProfile(
channel_header=ChannelHeader(
channel_band=ChannelBand.BAND_868,
channel_coding=ChannelCoding.PN9,
channel_class=ChannelClass.NORMAL_RATE),
sub_profiles=[
SubProfile(subband_bitmap=0x00,
scan_automation_period=CT(exp=0, mant=0)),
SubProfile(),
SubProfile(),
SubProfile()
],
sub_bands=[
SubBand(
channel_index_start=0,
channel_index_end=0,
eirp=10,
cca=86 # TODO
)
])
print("Write Access Profile")
write_ap_cmd = Command.create_with_write_file_action_system_file(
file=AccessProfileFile(access_profile=access_profile,
access_specifier=0))
self.transmitter_modem.execute_command(write_ap_cmd,
timeout_seconds=1)
if self.config.serial_receiver == None:
self.receiver_modem = None
print("Running without receiver")
else:
self.receiver_modem = Modem(self.config.serial_receiver,
self.config.rate,
self.receiver_cmd_callback)
self.receiver_modem.connect()
access_profile = AccessProfile(
channel_header=ChannelHeader(
channel_band=ChannelBand.BAND_868,
channel_coding=ChannelCoding.PN9,
channel_class=ChannelClass.NORMAL_RATE),
sub_profiles=[
SubProfile(subband_bitmap=0x01,
scan_automation_period=CT(exp=0, mant=0)),
SubProfile(),
SubProfile(),
SubProfile()
],
sub_bands=[
SubBand(
channel_index_start=0,
channel_index_end=0,
eirp=10,
cca=86 # TODO
)
])
print("Write Access Profile")
write_ap_cmd = Command.create_with_write_file_action_system_file(
file=AccessProfileFile(access_profile=access_profile,
access_specifier=0))
self.receiver_modem.execute_command(write_ap_cmd,
timeout_seconds=1)
self.receiver_modem.execute_command(
Command.create_with_write_file_action_system_file(
DllConfigFile(active_access_class=0x01)),
timeout_seconds=1)
print("Receiver scanning on Access Class = 0x01")
def start(self):
self.received_commands = defaultdict(list)
payload = range(self.config.payload_size)
if self.receiver_modem != None:
addressee_id = int(self.receiver_modem.uid, 16)
else:
addressee_id = int(self.config.unicast_uid, 16)
if self.transmitter_modem != None:
print(
"\n==> broadcast, with QoS, transmitter active access class = 0x01 ===="
)
self.transmitter_modem.execute_command(
Command.create_with_write_file_action_system_file(
DllConfigFile(active_access_class=0x01)),
timeout_seconds=1)
interface_configuration = Configuration(
qos=QoS(resp_mod=ResponseMode.RESP_MODE_ANY),
addressee=Addressee(
access_class=0x01,
id_type=IdType.NBID,
id=CT(exp=0, mant=1) # we expect one responder
))
self.start_transmitting(
interface_configuration=interface_configuration,
payload=payload)
self.wait_for_receiver(payload)
print(
"\n==> broadcast, no QoS, transmitter active access class = 0x01 ===="
)
self.transmitter_modem.execute_command(
Command.create_with_write_file_action_system_file(
DllConfigFile(active_access_class=0x01)),
timeout_seconds=1)
interface_configuration = Configuration(
qos=QoS(resp_mod=ResponseMode.RESP_MODE_NO),
addressee=Addressee(access_class=0x01, id_type=IdType.NOID))
self.start_transmitting(
interface_configuration=interface_configuration,
payload=payload)
self.wait_for_receiver(payload)
print(
"\n==> unicast, with QoS, transmitter active access class = 0x01"
)
interface_configuration = Configuration(
qos=QoS(resp_mod=ResponseMode.RESP_MODE_ANY),
addressee=Addressee(access_class=0x01,
id_type=IdType.UID,
id=addressee_id))
self.start_transmitting(
interface_configuration=interface_configuration,
payload=payload)
self.wait_for_receiver(payload)
print(
"\n==> unicast, no QoS, transmitter active access class = 0x01"
)
interface_configuration = Configuration(
qos=QoS(resp_mod=ResponseMode.RESP_MODE_NO),
addressee=Addressee(access_class=0x01,
id_type=IdType.UID,
id=addressee_id))
self.start_transmitting(
interface_configuration=interface_configuration,
payload=payload)
self.wait_for_receiver(payload)
else:
# receive only
self.wait_for_receiver(payload)
def start_transmitting(self, interface_configuration, payload):
print(
"Running throughput test with payload size {} and interface_configuration {}\n\nrunning ...\n"
.format(len(payload), interface_configuration))
if self.receiver_modem != None:
self.received_commands = defaultdict(list)
command = Command.create_with_return_file_data_action(
file_id=0x40,
data=payload,
interface_type=InterfaceType.D7ASP,
interface_configuration=interface_configuration)
start = time.time()
for i in range(self.config.msg_count):
sys.stdout.write("{}/{}\r".format(i + 1, self.config.msg_count))
sys.stdout.flush()
self.transmitter_modem.execute_command(command,
timeout_seconds=100)
end = time.time()
print("transmitter: sending {} messages completed in: {} s".format(
self.config.msg_count, end - start))
print(
"transmitter: throughput = {} bps with a payload size of {} bytes".
format((self.config.msg_count * self.config.payload_size * 8) /
(end - start), self.config.payload_size))
def wait_for_receiver(self, payload):
if self.receiver_modem == None:
print(
"Running without receiver so we are not waiting for messages to be received ..."
)
else:
start = time.time()
total_recv = 0
while total_recv < self.config.msg_count and time.time(
) - start < self.config.receiver_timeout:
total_recv = sum(
len(v) for v in self.received_commands.values())
time.sleep(2)
print(
"waiting for receiver to finish ... (current nr of recv msgs: {})"
.format(total_recv))
print("finished receiving or timeout")
payload_has_errors = False
for sender_cmd in self.received_commands.values():
for cmd in sender_cmd:
if type(cmd.actions[0].op
) != ReturnFileData and cmd.actions[
0].operand.data != payload:
payload_has_errors = True
print(
"receiver: received unexpected command: {}".format(
cmd))
if payload_has_errors == False and total_recv == self.config.msg_count:
print(
"receiver: OK: received {} messages with correct payload:".
format(total_recv))
for sender, cmds in self.received_commands.items():
print("\t{}: {}".format(sender, len(cmds)))
else:
print(
"receiver: NOK: received messages {}:".format(total_recv))
for sender, cmds in self.received_commands.items():
print("\t{}: {}".format(sender, len(cmds)))
def receiver_cmd_callback(self, cmd):
print("recv cmd: ".format(cmd))
if cmd.interface_status != None:
uid = cmd.interface_status.operand.interface_status.addressee.id
self.received_commands[uid].append(cmd)
else:
print("Unexpected cmd received, reboot?\n\t{}".format(cmd))
class Gateway:
def __init__(self):
argparser = argparse.ArgumentParser()
argparser.add_argument("-d",
"--device",
help="serial device /dev file modem",
default="/dev/ttyACM0")
argparser.add_argument("-r",
"--rate",
help="baudrate for serial device",
type=int,
default=115200)
argparser.add_argument("-v",
"--verbose",
help="verbose",
default=False,
action="store_true")
argparser.add_argument("-t",
"--token",
help="Access token for the TB gateway",
required=True)
argparser.add_argument("-tb",
"--thingsboard",
help="Thingsboard hostname/IP",
default="localhost")
argparser.add_argument("-p",
"--plugin-path",
help="path where plugins are stored",
default="")
argparser.add_argument("-bp",
"--broker-port",
help="mqtt broker port",
default="1883")
argparser.add_argument(
"-l",
"--logfile",
help="specify path if you want to log to file instead of to stdout",
default="")
argparser.add_argument(
"-k",
"--keep-data",
help=
"Save data locally when Thingsboard is disconnected and send it when connection is restored.",
default=True)
argparser.add_argument(
"-b",
"--save-bandwidth",
help="Send data in binary format to save bandwidth",
action="store_true")
argparser.add_argument("-sf",
"--skip-system-files",
help="Do not read system files on boot",
action="store_true")
self.bridge_count = 0
self.next_report = 0
self.config = argparser.parse_args()
self.log = logging.getLogger()
formatter = logging.Formatter(
'%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
if self.config.logfile == "":
handler = logging.StreamHandler()
else:
handler = logging.FileHandler(self.config.logfile)
handler.setFormatter(formatter)
self.log.addHandler(handler)
self.log.setLevel(logging.INFO)
if self.config.verbose:
self.log.setLevel(logging.DEBUG)
self.tb = Thingsboard(self.config.thingsboard,
self.config.token,
self.on_mqtt_message,
persistData=self.config.keep_data)
if self.config.plugin_path != "":
self.load_plugins(self.config.plugin_path)
self.modem = Modem(self.config.device, self.config.rate,
self.on_command_received,
self.config.save_bandwidth)
connected = self.modem.connect()
while not connected:
try:
self.log.warning("Not connected to modem, retrying ...")
time.sleep(1)
connected = self.modem.connect()
except KeyboardInterrupt:
self.log.info("received KeyboardInterrupt... stopping")
self.tb.disconnect()
exit(-1)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
trace = "".join(lines)
self.log.error(
"Exception while connecting modem: \n{}".format(trace))
# switch to continuous foreground scan access profile
self.modem.execute_command(
Command.create_with_write_file_action_system_file(
DllConfigFile(active_access_class=0x01)),
timeout_seconds=1)
if self.config.save_bandwidth:
self.log.info("Running in save bandwidth mode")
if self.config.plugin_path is not "":
self.log.warning(
"Save bandwidth mode is enabled, plugin files will not be used"
)
# update attribute containing git rev so we can track revision at TB platform
git_sha = subprocess.check_output(["git", "describe",
"--always"]).strip()
ip = self.get_ip()
self.tb.sendGwAttributes({
'UID': self.modem.uid,
'git-rev': git_sha,
'IP': ip,
'save bw': str(self.config.save_bandwidth)
})
self.log.info("Running on {} with git rev {} using modem {}".format(
ip, git_sha, self.modem.uid))
# read all system files on the local node to store as attributes on TB
if not self.config.skip_system_files:
self.log.info("Reading all system files ...")
for file in SystemFiles().files.values():
self.modem.execute_command_async(
Command.create_with_read_file_action_system_file(file))
def load_plugins(self, plugin_path):
self.log.info("Searching for plugins in path %s" % plugin_path)
manager = PluginManagerSingleton.get()
manager.setPluginPlaces([plugin_path])
manager.collectPlugins()
for plugin in manager.getAllPlugins():
self.log.info("Loading plugin '%s'" % plugin.name)
def on_command_received(self, cmd):
try:
if self.config.save_bandwidth:
self.log.info("Command received: binary ALP (size {})".format(
len(cmd)))
else:
self.log.info("Command received: {}".format(cmd))
ts = int(round(time.time() * 1000))
# publish raw ALP command to incoming ALP topic, we will not parse the file contents here (since we don't know how)
# so pass it as an opaque BLOB for parsing in backend
if self.config.save_bandwidth:
self.tb.sendGwAttributes({
'alp':
binascii.hexlify(bytearray(cmd)),
'last_seen':
str(datetime.now().strftime("%y-%m-%d %H:%M:%S"))
})
return
self.tb.sendGwAttributes({
'alp':
jsonpickle.encode(cmd),
'last_seen':
str(datetime.now().strftime("%y-%m-%d %H:%M:%S"))
})
node_id = self.modem.uid # overwritten below with remote node ID when received over D7 interface
# parse link budget (when this is received over D7 interface) and publish separately so we can visualize this in TB
if cmd.interface_status != None and cmd.interface_status.operand.interface_id == 0xd7:
interface_status = cmd.interface_status.operand.interface_status
node_id = '{:x}'.format(interface_status.addressee.id)
linkBudget = interface_status.link_budget
rxLevel = interface_status.rx_level
lastConnect = "D7-" + interface_status.get_short_channel_string(
)
self.tb.sendDeviceTelemetry(node_id, ts, {
'lb': linkBudget,
'rx': rxLevel
})
self.tb.sendDeviceAttributes(node_id, {
'last_conn': lastConnect,
'last_gw': self.modem.uid
})
# store returned file data as attribute on the device
for action in cmd.actions:
if type(action.operation) is ReturnFileData:
data = ""
if action.operation.file_data_parsed is not None:
if not self.config.save_bandwidth:
# for known system files we transmit the parsed data
data = jsonpickle.encode(
action.operation.file_data_parsed)
file_id = "File {}".format(
action.operand.offset.id)
self.tb.sendGwAttributes({file_id: data})
else:
# try if plugin can parse this file
parsed_by_plugin = False
if not self.config.save_bandwidth:
for plugin in PluginManagerSingleton.get(
).getAllPlugins():
for name, value, datapoint_type in plugin.plugin_object.parse_file_data(
action.operand.offset,
action.operand.length,
action.operand.data):
parsed_by_plugin = True
if isinstance(value, int) or isinstance(
value, float):
self.tb.sendDeviceTelemetry(
node_id, ts, {name: value})
else:
self.tb.sendDeviceAttributes(
node_id, {name: value})
if not parsed_by_plugin:
# unknown file content, just transmit raw data
data = jsonpickle.encode(action.operand)
filename = "File {}".format(
action.operand.offset.id)
if action.operation.systemfile_type != None:
filename = "File {} ({})".format(
SystemFileIds(
action.operand.offset.id).name,
action.operand.offset.id)
self.tb.sendDeviceAttributes(
node_id, {filename: data})
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
trace = "".join(lines)
self.log.error(
"Exception while processing command: \n{}".format(trace))
def on_mqtt_message(self, client, config, msg):
try:
payload = json.loads(msg.payload)
uid = payload['device']
method = payload['data']['method']
request_id = payload['data']['id']
self.log.info(
"Received RPC command of type {} for {} (request id {})".
format(method, uid, request_id))
# if uid != self.modem.uid:
# self.log.info("RPC command not for this modem ({}), skipping".format(self.modem.uid))
# return
if method == "execute-alp-async":
try:
cmd = payload['data']['params']
self.log.info("Received command through RPC: %s" % cmd)
self.modem.execute_command_async(cmd)
self.log.info("Executed ALP command through RPC")
# TODO when the command is writing local files we could read them again automatically afterwards, to make sure the digital twin is updated
except Exception as e:
self.log.exception("Could not deserialize: %s" % e)
elif method == "alert":
# TODO needs refactoring so different methods can be supported in a plugin, for now this is very specific case as an example
self.log.info("Alert (payload={})".format(msg.payload))
if msg.payload != "true" and msg.payload != "false":
self.log.info("invalid payload, skipping")
return
file_data = 0
if msg.payload == "true":
file_data = 1
self.log.info("writing alert file")
self.modem.execute_command_async(
Command.create_with_write_file_action(
file_id=0x60,
offset=4,
data=[file_data],
interface_type=InterfaceType.D7ASP,
interface_configuration=Configuration(
qos=QoS(resp_mod=ResponseMode.RESP_MODE_ALL),
addressee=Addressee(access_class=0x11,
id_type=IdType.NOID))))
else:
self.log.info("RPC method not supported, skipping")
return
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
trace = "".join(lines)
msg_info = "no msg info (missing __dict__ attribute)" # TODO because of out of date paho??
if hasattr(msg, '__dict__'):
msg_info = str(msg.__dict__)
self.log.error(
"Exception while processing MQTT message: {} callstack:\n{}".
format(msg_info, trace))
def run(self):
self.log.info("Started")
keep_running = True
while keep_running:
try:
if platform.system() == "Windows":
time.sleep(1)
else:
signal.pause()
except KeyboardInterrupt:
self.log.info(
"received KeyboardInterrupt... stopping processing")
self.tb.disconnect()
keep_running = False
self.report_stats()
def keep_stats(self):
self.bridge_count += 1
def report_stats(self):
if self.next_report < time.time():
if self.bridge_count > 0:
self.log.info("bridged %s messages" % str(self.bridge_count))
self.bridge_count = 0
self.next_report = time.time(
) + 15 # report at most every 15 seconds
def get_ip(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
# doesn't even have to be reachable
s.connect(('10.255.255.255', 1))
IP = s.getsockname()[0]
except:
IP = '127.0.0.1'
finally:
s.close()
return IP
class Modem2Mqtt():
def __init__(self):
argparser = argparse.ArgumentParser()
argparser.add_argument("-d", "--device", help="serial device /dev file modem",
default="/dev/ttyACM0")
argparser.add_argument("-di", "--device-id", help="gateway device-id", required=True)
argparser.add_argument("-r", "--rate", help="baudrate for serial device", type=int, default=115200)
argparser.add_argument("-v", "--verbose", help="verbose", default=False, action="store_true")
argparser.add_argument("-b", "--broker", help="mqtt broker hostname",
default="localhost")
argparser.add_argument("-t", "--topic", help="mqtt topic",
default="/gw/{}")
self.serial = None
self.modem_uid = None
self.bridge_count = 0
self.next_report = 0
self.config = argparser.parse_args()
configure_default_logger(self.config.verbose)
self.modem = Modem(self.config.device, self.config.rate, self.on_command_received, skip_alp_parsing=True)
self.modem.connect()
self.connect_to_mqtt()
def connect_to_mqtt(self):
self.connected_to_mqtt = False
self.mq = mqtt.Client("", True, None, mqtt.MQTTv31)
self.mq.on_connect = self.on_mqtt_connect
self.mq.on_message = self.on_mqtt_message
self.mqtt_topic_incoming = self.config.topic.format(self.config.device_id)
#self.mqtt_topic_outgoing = self.config.topic.format(self.modem_uid)
self.mq.connect(self.config.broker, 1883, 60)
self.mq.loop_start()
while not self.connected_to_mqtt: pass # busy wait until connected
logging.info("Connected to MQTT broker on {}, sending to topic {}".format(
self.config.broker,
self.mqtt_topic_incoming
))
def on_mqtt_connect(self, client, config, flags, rc):
#self.mq.subscribe(self.mqtt_topic_outgoing)
self.connected_to_mqtt = True
def on_mqtt_message(self, client, config, msg):
logging.info("on_message") # TODO
# try: self.handle_msg(msg.topic, msg.payload)
# except:
# exc_type, exc_value, exc_traceback = sys.exc_info()
# lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
# trace = "".join(lines)
# self.log("failed to handle incoming message:", msg.payload, trace)
def publish_to_mqtt(self, msg):
self.mq.publish(self.mqtt_topic_incoming, msg)
def __del__(self): # pragma: no cover
try:
self.mq.loop_stop()
self.mq.disconnect()
except: pass
def on_command_received(self, cmd):
try:
logging.info("Command received: binary ALP (size {})".format(len(cmd)))
# publish raw ALP command to incoming ALP topic, we will not parse the file contents here (since we don't know how)
# so pass it as an opaque BLOB for parsing in backend
self.publish_to_mqtt(bytearray(cmd))
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
trace = "".join(lines)
logging.error("Exception while processing command: \n{}".format(trace))
def run(self):
logging.info("Started")
keep_running = True
while keep_running:
try:
if platform.system() == "Windows":
time.sleep(1)
else:
signal.pause()
except KeyboardInterrupt:
logging.info("received KeyboardInterrupt... stopping processing")
keep_running = False
self.report_stats()
def keep_stats(self):
self.bridge_count += 1
def report_stats(self):
if self.next_report < time.time():
if self.bridge_count > 0:
logging.info("bridged %s messages" % str(self.bridge_count))
self.bridge_count = 0
self.next_report = time.time() + 15 # report at most every 15 seconds
