def verify_payload(self, msg):
search = re.search('gateway/(.*)?/*', msg.topic)
if search is None or (search.group(0)[-2:] not in ["rx", "tx"]):
self.log.debug('topic does not include physical payload')
return True # NOT SURE if this should be True or False
# try to decode payload as utf-8
try:
mqtt_messsage = json.loads(msg.payload.decode("utf-8"))
except Exception as e:
self.log.error(f'payload could not be decoded as utf8: {e}')
return False
# looks for phyPayload field in mqtt message
phyPayload = mqtt_messsage.get('phyPayload', None)
if not phyPayload:
self.log.error('No phyPayload field in mqtt message')
return False
# PHYPayload shouldn't exceed 255 bytes by definition. In DB we support 300 bytes
if len(phyPayload) > 300:
return False
# Parse the base64 PHYPayload
try:
phy_parser.setPHYPayload(phyPayload)
return True
except Exception as e:
self.log.error(f'Error parsing physical payload: {e}')
return False
def on_message(client, userdata, msg):
try:
payload = msg.payload.decode("utf-8")
standardPacket = {}
if len(payload) > 0:
mqttMessage = json.loads(payload)
if 'data' not in mqttMessage:
logging.error(
'Received a message without "data" field. Topic: %s. Message: %s'
% (msg.topic, payload))
return
# Pad the base64 string till it is a multiple of 4
mqttMessage['data'] += "=" * (
(4 - len(mqttMessage['data']) % 4) % 4)
# Parse the base64 PHYPayload
standardPacket = phy_parser.setPHYPayload(mqttMessage['data'])
standardPacket['chan'] = mqttMessage.get('chan', None)
standardPacket['stat'] = mqttMessage.get('stat', None)
standardPacket['lsnr'] = mqttMessage.get('lsnr', None)
standardPacket['rssi'] = mqttMessage.get('rssi', None)
standardPacket['tmst'] = mqttMessage.get('tmst', None)
standardPacket['rfch'] = mqttMessage.get('rfch', None)
standardPacket['freq'] = mqttMessage.get('freq', None)
standardPacket['modu'] = mqttMessage.get('modu', None)
standardPacket['datr'] = json.dumps(
parse_datr(mqttMessage.get('datr', None)))
standardPacket['codr'] = mqttMessage.get('codr', None)
standardPacket['size'] = mqttMessage.get('size', None)
standardPacket['data'] = mqttMessage.get('data', None)
# Gateway not provided by this broker
standardPacket['gateway'] = None
# These fields come in the /up topic
standardPacket['seqn'] = mqttMessage.get('seqn', None)
standardPacket['opts'] = mqttMessage.get('opts', None)
standardPacket['port'] = mqttMessage.get('port', None)
# These fields are indepedant from the payload
standardPacket['topic'] = msg.topic
if "/joined" in msg.topic:
standardPacket['m_type'] = "JoinAccept"
standardPacket['date'] = datetime.datetime.now().__str__()
standardPacket['dev_eui'] = getDevEUIFromMQTTTopic(msg.topic)
standardPacket['data_collector_id'] = client.data_collector_id
standardPacket['organization_id'] = client.organization_id
save(json.dumps(standardPacket), client.data_collector_id)
logging.debug('Topic: {0}. Message received: {1}'.format(
msg.topic, msg.payload.decode("utf-8")))
except Exception as e:
logging.error("Error creating Packet in GenericMqttCollector:", e,
"Topic: ", msg.topic, "Message: ",
msg.payload.decode("utf-8"))
def on_message(self, client, userdata, msg):
# We can receive both protobuf messages and JSON messages. We try to parse it as JSON and if it fails and the topic matches to /up, it's a protobuf instead
is_protobuf_message= False
if self.being_tested:
return
if not self.verified:
if not self.verify_message(msg):
self.log.debug("Collector is not yet verified, skipping message")
return
try:
# print("Topic %s Packet %s"%(msg.topic, msg.payload))
# If message cannot be decoded as json, skip it
mqtt_messsage = json.loads(msg.payload.decode("utf-8"))
except Exception as e:
# First, check if we had a prev_packet. If so, first save it
if client.prev_packet is not None:
client.packet_writter_message['packet'] = client.prev_packet
save(client.packet_writter_message, client.data_collector_id)
# Reset vars
client.packet_writter_message = self.init_packet_writter_message()
client.prev_packet = None
# If failed to decode message, then it's probably protobuf. To make sure, check the /up topic
search = re.search('gateway/(.*)?/*', msg.topic)
if search is not None and search.group(0)[-2:] in ["up"]:
try:
uplink= api.UplinkFrame()
uplink.ParseFromString(msg.payload)
mqtt_messsage= json.loads(MessageToJson(uplink))
is_protobuf_message= True
except Exception as e:
self.log.error(f'Error parsing protobuf: {e}. Protobuf message: {msg.payload}')
else:
# Save this message an topic into MQ
client.packet_writter_message['messages'].append(
{
'topic': msg.topic,
'message': msg.payload.decode("utf-8"),
'data_collector_id': client.data_collector_id
}
)
save(client.packet_writter_message, client.data_collector_id)
# Reset packet_writter_message
client.packet_writter_message = self.init_packet_writter_message()
self.log.debug(f'[SKIPPED] Topic: {msg.topic}. Message received: {msg.payload}')
save_parsing_error(collector_id=client.data_collector_id, message=str(e))
return
try:
standard_packet = {}
# If it's a Join message, then associate DevEUI with DevAddr
if msg.topic[-5:] == "/join":
device_info = {'dev_eui': mqtt_messsage.get('devEUI', None)}
client.devices_map[mqtt_messsage['devAddr']] = device_info
# Save this message an topic into MQ
client.packet_writter_message['messages'].append(
{
'topic': msg.topic,
'message': msg.payload.decode("utf-8"),
'data_collector_id': client.data_collector_id
}
)
save(client.packet_writter_message, client.data_collector_id)
# Reset packet_writter_message
client.packet_writter_message = self.init_packet_writter_message()
return
# From topic gateway/gw_id/tx or gateway/gw_id/tx
search = re.search('gateway/(.*)?/*', msg.topic)
if search is not None and search.group(0)[-2:] in ["rx", "tx", "up"]:
if 'phyPayload' in mqtt_messsage:
# PHYPayload shouldn't exceed 255 bytes by definition. In DB we support 300 bytes
if len(mqtt_messsage['phyPayload']) > 300:
return
# Parse the base64 PHYPayload
standard_packet = phy_parser.setPHYPayload(mqtt_messsage.get('phyPayload'))
# Save the PHYPayload
standard_packet['data'] = mqtt_messsage.get('phyPayload')
if is_protobuf_message:
if 'rxInfo' in mqtt_messsage:
x_info = mqtt_messsage.get('rxInfo')
standard_packet['gateway'] = base64.b64decode(x_info.get('gatewayID')).hex()
standard_packet['chan'] = x_info.get('channel')
standard_packet['rfch'] = x_info.get('rfChain')
standard_packet['stat'] = get_crc_status_integer(x_info.get('crcStatus')) # When protobuf is deserialized, this is a string, but we need to send an integer
standard_packet['rssi'] = x_info.get('rssi')
standard_packet['lsnr'] = x_info.get('loRaSNR')
standard_packet['size'] = x_info.get('size')
if 'txInfo' in mqtt_messsage:
x_info = mqtt_messsage.get('txInfo')
standard_packet['freq'] = x_info.get('frequency') / 1000000 if 'frequency' in x_info else None
lora_modulation_info= x_info.get('loRaModulationInfo')
standard_packet['datr'] = json.dumps({"spread_factor": lora_modulation_info.get('spreadingFactor'),
"bandwidth": lora_modulation_info.get('bandwidth')})
standard_packet['codr'] = lora_modulation_info.get('codeRate')
else:
if 'rxInfo' in mqtt_messsage:
x_info = mqtt_messsage.get('rxInfo')
standard_packet['chan'] = x_info.get('channel')
standard_packet['rfch'] = x_info.get('rfChain')
standard_packet['stat'] = x_info.get('crcStatus')
standard_packet['codr'] = x_info.get('codeRate')
standard_packet['rssi'] = x_info.get('rssi')
standard_packet['lsnr'] = x_info.get('loRaSNR')
standard_packet['size'] = x_info.get('size')
if 'txInfo' in mqtt_messsage:
x_info= mqtt_messsage.get('txInfo')
standard_packet['tmst'] = x_info.get('timestamp')
standard_packet['freq'] = x_info.get('frequency') / 1000000 if 'frequency' in x_info else None
standard_packet['gateway'] = x_info.get('mac')
data_rate= x_info.get('dataRate')
standard_packet['modu'] = data_rate.get('modulation')
standard_packet['datr'] = json.dumps({"spread_factor": data_rate.get('spreadFactor'),
"bandwidth": data_rate.get('bandwidth')})
# Add missing fields, independant from type of packet
standard_packet['topic'] = msg.topic
standard_packet['date'] = datetime.now().__str__()
standard_packet['data_collector_id'] = client.data_collector_id
standard_packet['organization_id'] = client.organization_id
# Save prev_packet in case is not empty
if client.prev_packet is not None:
client.packet_writter_message['packet'] = client.prev_packet
save(client.packet_writter_message, client.data_collector_id)
# Reset variables
client.prev_packet = None
client.packet_writter_message = self.init_packet_writter_message()
# Set the dev_eui and other information if available. Otherwise, save packet
if 'dev_addr' in standard_packet:
if standard_packet['dev_addr'] in client.devices_map:
standard_packet['dev_eui'] = client.devices_map[standard_packet['dev_addr']]['dev_eui']
if len(client.devices_map[standard_packet['dev_addr']]) > 1:
standard_packet['app_name'] = client.devices_map[standard_packet['dev_addr']]['app_name']
standard_packet['dev_name'] = client.devices_map[standard_packet['dev_addr']]['dev_name']
else:
# Save this packet for now
client.prev_packet = standard_packet
# Save the message and topic as well
client.packet_writter_message['messages'].append(
{
'topic': msg.topic,
'message': json.dumps(mqtt_messsage) if is_protobuf_message else msg.payload.decode("utf-8"),
'data_collector_id': client.data_collector_id
}
)
else:
self.log.debug('Unhandled situation')
self.last_seen = datetime.now()
# From topic application/*/device/*/rx or application/*/node/*/rx
elif re.search('application/.*?/device/(.*)/rx', msg.topic) is not None or re.search(
'application/.*?/node/(.*)/rx', msg.topic) is not None:
search = re.search('application/.*?/device/(.*)/rx', msg.topic)
if search is None:
search = re.search('application/.*?/node/(.*)/rx', msg.topic)
if client.prev_packet is not None:
standard_packet = client.prev_packet
client.prev_packet = None
if standard_packet['f_count'] == mqtt_messsage.get('fCnt', None):
# Set location and gw name if given
if 'rxInfo' in mqtt_messsage:
location = mqtt_messsage.get('rxInfo', None)[0].get('location', None)
if location:
standard_packet['latitude'] = location.get('latitude', None)
standard_packet['longitude'] = location.get('longitude', None)
standard_packet['altitude'] = location.get('altitude', None)
gw_name= mqtt_messsage.get('rxInfo')[0].get('name')
standard_packet['gw_name'] = gw_name if gw_name else None
# Make sure we've matched the same device
if 'dev_eui' in standard_packet and standard_packet['dev_eui'] is not None and standard_packet[
'dev_eui'] != search.group(1):
self.log.error("There's an error with Chirsptack collector logic")
# Get dev_eui, app_name and dev_name from message
device_info = {'app_name': mqtt_messsage.get('applicationName', None),
'dev_name': mqtt_messsage.get('deviceName', None),
'dev_eui': mqtt_messsage.get('devEUI', None)}
client.devices_map[standard_packet['dev_addr']] = device_info
# Set previous values to current message
standard_packet['dev_eui'] = client.devices_map[standard_packet['dev_addr']]['dev_eui']
if len(client.devices_map[standard_packet['dev_addr']]) > 1:
standard_packet['app_name'] = client.devices_map[standard_packet['dev_addr']]['app_name']
standard_packet['dev_name'] = client.devices_map[standard_packet['dev_addr']]['dev_name']
self.last_seen = datetime.now()
else:
# First, check if we had a prev_packet. If so, first save it
if client.prev_packet is not None and len(standard_packet) == 0:
client.packet_writter_message['packet'] = client.prev_packet
save(client.packet_writter_message, client.data_collector_id)
# Reset vars
client.packet_writter_message = self.init_packet_writter_message()
client.prev_packet = None
# Save SKIPPED MQ message and topic
client.packet_writter_message['messages'].append(
{
'topic': msg.topic,
'message': msg.payload.decode("utf-8"),
'data_collector_id': client.data_collector_id
}
)
save(client.packet_writter_message, client.data_collector_id)
# Reset packet_writter_message
client.packet_writter_message = self.init_packet_writter_message()
return
# Save packet
if client.prev_packet is None and len(standard_packet) > 0:
# Save packet JSON
client.packet_writter_message['packet'] = standard_packet
# Save MQ message and topic
client.packet_writter_message['messages'].append(
{
'topic': msg.topic,
'message': json.dumps(mqtt_messsage) if is_protobuf_message else msg.payload.decode("utf-8"),
'data_collector_id': client.data_collector_id
}
)
save(client.packet_writter_message, client.data_collector_id)
# Reset packet_writter_message obj
client.packet_writter_message = self.init_packet_writter_message()
except Exception as e:
self.log.error("Error creating Packet in Chirpstack collector:", e, "Topic: ", msg.topic, "Message: ",
json.dumps(mqtt_messsage) if is_protobuf_message else msg.payload.decode("utf-8"))
traceback.print_exc(file=sys.stdout)
save_parsing_error(client.data_collector_id, json.dumps(mqtt_messsage) if is_protobuf_message else msg.payload.decode("utf-8"))
def on_message(ws, raw_message):
# The contents of many messages is an 'h'. We don't want to print that.
if len(raw_message)>1:
logging.info("Message: {}".format(raw_message))
# Remove data format stuff
raw_message = raw_message.replace('\\"', '"')
# Save the message that originates the packet
ws.packet_writter_message['messages'].append(
{
'topic': None,
'message': raw_message[0:4096],
'data_collector_id': ws.data_collector_id
}
)
ws.user_data.last_seen = datetime.now()
has_to_parse = False
if 'gateway downlink' in raw_message:
has_to_parse = True
message = raw_message[20:-2]
elif 'gateway uplink' in raw_message:
has_to_parse = True
message = raw_message[18:-2]
elif 'gateway join request' in raw_message:
has_to_parse = True
message = raw_message[24:-2]
elif 'gateway join accept' in raw_message:
has_to_parse = True
message = raw_message[23:-2]
elif 'gateway status' in raw_message and 'location' in raw_message:
# Check if the location is given in this message. If so, save it and add it in subsequent messages
try:
message = raw_message[18:-2].replace('\\"', '"')
status_message = json.loads(message)
ws.location['longitude']= status_message.get('status').get('location').get('longitude')
ws.location['latitude']= status_message.get('status').get('location').get('latitude')
ws.location['altitude']= status_message.get('status').get('location').get('altitude')
except Exception as e:
logging.error("Error when fetching location in TTNCollector:" + str(e) + " Message: " + raw_message)
elif len(raw_message)>1:
message = raw_message.replace('\\"', '"')
else:
return
try:
if has_to_parse:
message = json.loads(message)
packet = phy_parser.setPHYPayload(message.get('payload'))
packet['chan'] = None
packet['stat'] = None
packet['lsnr'] = message.get('snr', None)
packet['rssi'] = message.get('rssi', None)
packet['tmst'] = datetime.timestamp(dateutil.parser.parse(message.get('timestamp', None))) * 1000
packet['rfch'] = message.get('rfch', None)
packet['freq'] = message.get('frequency', None)
packet['modu'] = None
packet['datr'] = None
packet['codr'] = message.get('coding_rate', None)
packet['size'] = None
packet['data'] = message.get('payload')
if len(ws.location)>0:
packet['latitude'] = ws.location['latitude']
packet['longitude'] = ws.location['longitude']
packet['altitude'] = ws.location['altitude']
# Reset location
ws.location={}
packet['app_name'] = None
packet['dev_name'] = None
gw = ws.gateway
packet['gateway'] = gw.replace('eui-', '') if gw else None
packet['seqn'] = None
packet['opts'] = None
packet['port'] = None
packet['date'] = datetime.now().__str__()
packet['dev_eui'] = message.get('dev_eui')
packet['data_collector_id'] = ws.data_collector_id
packet['organization_id'] = ws.organization_id
ws.packet_writter_message['packet']= packet
# Save the packet
save(ws.packet_writter_message, ws.data_collector_id)
logging.debug('Message received from TTN saved in DB: {0}.'.format(ws.packet_writter_message))
# Reset this variable
ws.packet_writter_message = init_packet_writter_message()
except Exception as e:
logging.error("Error creating Packet in TTNCollector:" + str(e) + " Message: " + raw_message)
def listener(client):
udp_listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
udp_listener.bind(('', client.port))
while True:
if client.stop_thread:
break
payload, source_address = udp_listener.recvfrom(65565)
if len(payload)>4:
try:
if chr(payload[4]) == '{':
udp_message= json.loads(payload[4:].decode("utf-8") )
header= payload[0:4]
else:
udp_message= json.loads(payload[12:].decode("utf-8") )
header= payload[0:12]
except Exception as e:
logging.debug('Skipping packet: {0}'.format(payload))
skip_packet= True
else:
logging.debug('Skipping packet: {0}'.format(payload))
skip_packet= True
try:
if not skip_packet:
standardPacket={}
if "stat" in udp_message:
pkt = udp_message.get("stat")
location = {}
if 'lati' in pkt:
location['latitude'] = pkt.get('lati')
if 'long' in pkt:
location['longitude'] = pkt.get('long')
if 'alti' in pkt:
location['altitude'] = pkt.get('alti')
if len(location) > 0:
gateway= get_gateway_id(header)
gateways_location[gateway]=location
if "rxpk" in udp_message or "txpk" in udp_message:
pkt = udp_message.get("rxpk")[0] if "rxpk" in udp_message else udp_message.get("txpk")
standardPacket = phy_parser.setPHYPayload(pkt.get('data'))
standardPacket['chan'] = pkt.get('chan', None)
standardPacket['stat'] = pkt.get('stat', None)
standardPacket['lsnr'] = pkt.get('lsnr', None)
standardPacket['rssi'] = pkt.get('rssi', None)
standardPacket['tmst'] = pkt.get('tmst', None)
standardPacket['rfch'] = pkt.get('rfch', None)
standardPacket['freq'] = pkt.get('freq', None)
standardPacket['modu'] = pkt.get('modu', None)
standardPacket['datr'] = json.dumps(parse_datr(pkt.get('datr', None)))
standardPacket['codr'] = pkt.get('codr', None)
standardPacket['size'] = pkt.get('size', None)
standardPacket['data'] = pkt.get('data', None)
gateway= get_gateway_id(header)
if gateway:
standardPacket['gateway'] = gateway
if gateway in gateways_location:
standardPacket['latitude']= gateways_location[gateway]['latitude']
standardPacket['longitude']= gateways_location[gateway]['longitude']
standardPacket['altitude']= gateways_location[gateway]['altitude']
standardPacket['date'] = datetime.datetime.now().__str__()
standardPacket['data_collector_id'] = client.data_collector_id
standardPacket['organization_id'] = client.organization_id
client.packet_writter_message['packet']= standardPacket
logging.debug('Message received: {0} \n{1}'.format(payload, json.dumps(standardPacket)))
# Save this message an topic into MQ
client.packet_writter_message['messages'].append(
{
'topic':None,
'message':payload.decode("utf-8"),
'data_collector_id': client.data_collector_id
}
)
# Save the packet
save(client.packet_writter_message, client.data_collector_id)
# Reset packet_writter_message
client.packet_writter_message = init_packet_writter_message()
except Exception as e:
logging.error("Error creating Packet in PacketForwarderCollector: {0}. Message: {1}".format(e,payload))
traceback.print_exc(file=sys.stdout)
def on_message(client, userdata, msg):
global prev_packet
global devices_map
try:
# print("Topic %s Packet %s"%(msg.topic, msg.payload))
# If message cannot, be decoded as json, skip it
mqtt_messsage = json.loads(msg.payload.decode("utf-8"))
except Exception as e:
logging.debug('[SKIPPED] Topic: {0}. Message received: {1}'.format(
msg.topic, msg.payload.decode("utf-8")))
return
try:
standard_packet = {}
if msg.topic[-5:] == "/join":
device_info = {'dev_eui': mqtt_messsage.get('devEUI', None)}
devices_map[mqtt_messsage['devAddr']] = device_info
return
#From topic gateway/gw_id/tx or gateway/gw_id/tx
search = re.search('gateway/(.*)?/*', msg.topic)
if search is not None and (search.group(0)[-2:] == "rx"
or search.group(0)[-2:] == "tx"):
if 'phyPayload' in mqtt_messsage:
# PHYPayload shouldn't exceed 255 bytes by definition. In DB we support 300 bytes
if len(mqtt_messsage['phyPayload']) > 300:
return
# Parse the base64 PHYPayload
standard_packet = phy_parser.setPHYPayload(
mqtt_messsage.get('phyPayload', None))
# Save the PHYPayload
standard_packet['data'] = mqtt_messsage.get('phyPayload', None)
if search.group(0)[-2:] == 'rx':
rx_info = mqtt_messsage.get('rxInfo', None)
standard_packet['tmst'] = rx_info.get('timestamp', None)
if rx_info.get('frequency') is not None:
standard_packet['freq'] = rx_info.get('frequency',
None) / 1000000
standard_packet['chan'] = rx_info.get('channel', None)
standard_packet['rfch'] = rx_info.get('rfChain', None)
standard_packet['stat'] = rx_info.get('crcStatus', None)
standard_packet['codr'] = rx_info.get('codeRate', None)
standard_packet['rssi'] = rx_info.get('rssi', None)
standard_packet['lsnr'] = rx_info.get('loRaSNR', None)
standard_packet['size'] = rx_info.get('size', None)
standard_packet['gateway'] = rx_info.get('mac', None)
data_rate = rx_info.get('dataRate', None)
standard_packet['modu'] = data_rate.get('modulation', None)
standard_packet['datr'] = json.dumps({
"spread_factor":
data_rate.get('spreadFactor', None),
"bandwidth":
data_rate.get('bandwidth', None)
})
elif search.group(0)[-2:] == 'tx':
tx_info = mqtt_messsage.get('txInfo', None)
standard_packet['tmst'] = tx_info.get('timestamp', None)
if tx_info.get('frequency') is not None:
standard_packet['freq'] = tx_info.get('frequency',
None) / 1000000
standard_packet['gateway'] = tx_info.get('mac', None)
data_rate = tx_info.get('dataRate', None)
standard_packet['modu'] = data_rate.get('modulation', None)
standard_packet['datr'] = json.dumps({
"spread_factor":
data_rate.get('spreadFactor', None),
"bandwidth":
data_rate.get('bandwidth', None)
})
# Add missing fields, independant from type of packet
standard_packet['topic'] = msg.topic
standard_packet['date'] = datetime.datetime.now().__str__()
standard_packet['data_collector_id'] = client.data_collector_id
standard_packet['organization_id'] = client.organization_id
# Save prev_packet in case is not empty
if prev_packet is not None:
save(json.dumps(prev_packet), client.data_collector_id)
prev_packet = None
# Set the dev_eui and other information if available. Otherwise, save packet
if 'dev_addr' in standard_packet:
if standard_packet['dev_addr'] in devices_map:
standard_packet['dev_eui'] = devices_map[
standard_packet['dev_addr']]['dev_eui']
if len(devices_map[standard_packet['dev_addr']]) > 1:
standard_packet['app_name'] = devices_map[
standard_packet['dev_addr']]['app_name']
standard_packet['dev_name'] = devices_map[
standard_packet['dev_addr']]['dev_name']
else:
prev_packet = standard_packet
logging.debug('Topic: {0}. Message received: {1}'.format(
msg.topic, msg.payload.decode("utf-8")))
# From topic application/*/device/*/rx or application/*/node/*/rx
elif re.search('application/.*?/device/(.*)/rx',
msg.topic) is not None or re.search(
'application/.*?/node/(.*)/rx',
msg.topic) is not None:
search = re.search('application/.*?/device/(.*)/rx', msg.topic)
if search is None:
search = re.search('application/.*?/node/(.*)/rx', msg.topic)
if prev_packet is not None:
standard_packet = prev_packet
prev_packet = None
if standard_packet['f_count'] == mqtt_messsage.get(
'fCnt', None):
# Set location if given
if len(mqtt_messsage.get('rxInfo', None)) > 0:
location = mqtt_messsage.get('rxInfo', None)[0].get(
'location', None)
if location:
standard_packet['latitude'] = location.get(
'latitude', None)
standard_packet['longitude'] = location.get(
'longitude', None)
standard_packet['altitude'] = location.get(
'altitude', None)
# Make sure we've matched the same device
if 'dev_eui' in standard_packet and standard_packet[
'dev_eui'] is not None and standard_packet[
'dev_eui'] != search.group(1):
logging.warning(
"There's an error with LoraServerIODC logic")
exit(0)
# Get dev_eui, app_name and dev_name from message
device_info = {
'app_name': mqtt_messsage.get('applicationName', None),
'dev_name': mqtt_messsage.get('deviceName', None),
'dev_eui': mqtt_messsage.get('devEUI', None)
}
devices_map[standard_packet['dev_addr']] = device_info
# Set previous values to current message
standard_packet['dev_eui'] = devices_map[
standard_packet['dev_addr']]['dev_eui']
if len(devices_map[standard_packet['dev_addr']]) > 1:
standard_packet['app_name'] = devices_map[
standard_packet['dev_addr']]['app_name']
standard_packet['dev_name'] = devices_map[
standard_packet['dev_addr']]['dev_name']
logging.debug('Topic: {0}. Message received: {1}'.format(
msg.topic, msg.payload.decode("utf-8")))
else:
logging.debug('[SKIPPED] Topic: {0}. Message received: {1}'.format(
msg.topic, msg.payload.decode("utf-8")))
if prev_packet is not None and len(standard_packet) == 0:
standard_packet = prev_packet
prev_packet = None
else:
return
# Save packet
if prev_packet is None and len(standard_packet) > 0:
save(json.dumps(standard_packet), client.data_collector_id)
except Exception as e:
logging.error("Error creating Packet in GenericMqttCollector:", e,
"Topic: ", msg.topic, "Message: ",
msg.payload.decode("utf-8"))
traceback.print_exc(file=sys.stdout)
def on_message(self, ws, raw_message):
if self.being_tested:
return
# The contents of many messages is an 'h'. We don't want to print that.
if len(raw_message) > 1:
self.log.debug("Message: {}".format(raw_message))
else:
self.log.debug('Message len <= 1, skipping')
return
# Retry after disconnection. End thread refreshing token before
if '[200,"disconnected"]' in raw_message:
self.log.info(f"DataCollector {self.data_collector_id}: Disconnected by server. Reconnecting.")
ws.close()
ws.is_closed= True
self.log.debug(f"DataCollector {self.data_collector_id}: Joining refresh token thread.")
self.refresh_token_thread.join()
self.log.debug(f"DataCollector {self.data_collector_id}: Refresh token thread joined.")
self.connect()
# Remove data format stuff
message = raw_message.replace('\\"', '"')
origin_message = message
self.has_to_parse = False
if 'gateway downlink' in message:
self.has_to_parse = True
message = message[20:-2]
elif 'gateway uplink' in message:
self.has_to_parse = True
message = message[18:-2]
elif 'gateway join request' in message:
self.has_to_parse = True
message = message[24:-2]
elif 'gateway join accept' in message:
self.has_to_parse = True
message = message[23:-2]
if not self.verified:
# TTN collectors only verify the physical payload, which is only parsed if has_to_parse is True
if not self.verify_message(message):
self.log.debug("Collector is not yet verified, skipping message\n")
return
# message processing
try:
if 'gateway status' in message and 'location' in message:
# Check if the location is given in this message. If so, save it and add it in subsequent messages
message = message[18:-2].replace('\\"', '"')
try:
status_message = json.loads(message)
ws.location['longitude'] = status_message.get('status').get('location').get('longitude')
ws.location['latitude'] = status_message.get('status').get('location').get('latitude')
ws.location['altitude'] = status_message.get('status').get('location').get('altitude')
except Exception as e:
self.log.error(f"Error when fetching location in TTNCollector: {str(e)} Message: {raw_message}" )
message = message.replace('\\"', '"')
# Save the message that originates the packet
ws.packet_writter_message['messages'].append(
{
'topic': None,
'message': origin_message[0:4096],
'data_collector_id': ws.data_collector_id
}
)
self.last_seen = datetime.now()
if self.has_to_parse:
message = json.loads(message)
packet = phy_parser.setPHYPayload(message.get('payload'))
packet['chan'] = None
packet['stat'] = None
packet['lsnr'] = message.get('snr', None)
packet['rssi'] = message.get('rssi', None)
packet['tmst'] = datetime.timestamp(dateutil.parser.parse(message.get('timestamp', None))) * 1000
packet['rfch'] = message.get('rfch', None)
packet['freq'] = message.get('frequency', None)
packet['modu'] = None
packet['datr'] = None
packet['codr'] = message.get('coding_rate', None)
packet['size'] = None
packet['data'] = message.get('payload')
if len(ws.location) > 0:
packet['latitude'] = ws.location['latitude']
packet['longitude'] = ws.location['longitude']
packet['altitude'] = ws.location['altitude']
# Reset location
ws.location = {}
packet['app_name'] = None
packet['dev_name'] = None
gw = ws.gateway
packet['gateway'] = gw.replace('eui-', '') if gw else None
packet['seqn'] = None
packet['opts'] = None
packet['port'] = None
packet['date'] = datetime.now().__str__()
packet['dev_eui'] = message.get('dev_eui')
packet['data_collector_id'] = ws.data_collector_id
packet['organization_id'] = ws.organization_id
ws.packet_writter_message['packet'] = packet
# Save the packet
save(ws.packet_writter_message, ws.data_collector_id)
self.log.debug(f'Message received from TTN saved in DB: {ws.packet_writter_message}.')
# Reset this variable
ws.packet_writter_message = self.init_packet_writter_message()
except Exception as e:
self.log.error(f"Error creating Packet in TTNCollector ID {ws.data_collector_id}: {str(e)} Message: {raw_message}")
save_parsing_error(ws.data_collector_id, raw_message)
def message(self, raw_message):
if self.being_tested:
return
try:
message = json.loads(raw_message)['result']
message_data = message.get('data')
name = message.get('name')
if name == 'events.stream.start':
return
elif name == 'gs.up.receive' or name == 'gs.down.send':
self.has_to_parse = True
else:
self.has_to_parse = False
if not self.verified:
# TTN collectors only verify the physical payload, which is only parsed if has_to_parse is True
if not self.verify_message(message):
self.log.debug(
f'Collector is not yet verified ({self.verified_packets} verified), skipping message\n'
)
return
# Message processing
if name == 'gs.status.receive' and message_data.get(
'antenna_locations', None):
# Check if the location is given in this message. If so, save it and add it in subsequent messages
try:
self.location['longitude'] = message_data.get(
'antenna_locations')[0].get('longitude')
self.location['latitude'] = message_data.get(
'antenna_locations')[0].get('latitude')
self.location['altitude'] = message_data.get(
'antenna_locations')[0].get('altitude')
except Exception as e:
self.log.error(
f'Error when fetching location in TTNCollector: {str(e)} Message: {raw_message}'
)
# Save the message that originates the packet
self.packet_writter_message['messages'].append({
'topic':
None,
'message':
raw_message[0:4096],
'data_collector_id':
self.data_collector_id
})
self.last_seen = datetime.now()
if self.has_to_parse:
packet = phy_parser.setPHYPayload(
message_data.get('raw_payload'))
packet['chan'] = None
packet['stat'] = None
rx_metadata = message_data.get('rx_metadata', None)
if rx_metadata:
packet['lsnr'] = rx_metadata[0].get('snr', None)
packet['rssi'] = rx_metadata[0].get('rssi', None)
else:
packet['lsnr'] = None
packet['rssi'] = None
tmst = message.get('time', None)
if tmst:
packet['tmst'] = datetime.timestamp(
dateutil.parser.parse(tmst))
else:
packet['tmst'] = None
if name == 'gs.up.receive':
settings = message_data.get('settings', None)
if settings:
packet['freq'] = int(settings.get('frequency',
None)) / 1000000
packet['codr'] = settings.get('coding_rate', None)
else:
packet['freq'] = None
packet['codr'] = None
else: # name == 'gs.down.send':
request = message_data.get('request', None)
if request:
# rx1_frequency is stored as freq, rx2_frequency isn't stored
packet['freq'] = int(request.get(
'rx1_frequency', None)) / 1000000
else:
packet['freq'] = None
packet['codr'] = None
packet['rfch'] = None
packet['modu'] = None
packet['datr'] = None
packet['size'] = None
packet['data'] = message_data.get('raw_payload')
if len(self.location) > 0:
packet['latitude'] = self.location['latitude']
packet['longitude'] = self.location['longitude']
packet['altitude'] = self.location['altitude']
# Reset location
self.location = {}
packet['app_name'] = None
packet['dev_name'] = None
identifiers = message.get('identifiers', None)
if identifiers:
packet['gateway'] = identifiers[0]['gateway_ids']['eui']
else:
packet['gateway'] = None
packet['gw_name'] = self.gateway_name
packet['seqn'] = None
packet['opts'] = None
packet['port'] = None
# If dev_eui couldn't be fetched from raw_payload
if packet.get('dev_eui', None) is None:
packet['dev_eui'] = None
packet['date'] = datetime.now().__str__()
packet['data_collector_id'] = self.data_collector_id
packet['organization_id'] = self.organization_id
self.packet_writter_message['packet'] = packet
# Save the packet
save(self.packet_writter_message, self.data_collector_id)
# Reset this variable
self.packet_writter_message = self.init_packet_writter_message()
except Exception as e:
self.log.error(
f'Error creating Packet in TTNCollector ID {self.data_collector_id}: {str(e)} Message: {raw_message}'
)
save_parsing_error(self.data_collector_id, raw_message)