def parse_mcf88(hex_str, port=None):
"""
Parse payload like "0462651527da078e4d8e01a4691527dd078f488e01676d1527e9078d1a8e015d" float values
:param hex_str: MCF88 hex payload
:param port: LoRaWAN port
:return: dict containing float values
"""
if hex_str[:2] == '04':
line = hex_str[2:62]
n = 20
datas = [line[i:i + n] for i in range(0, len(line), n)]
datalines = []
for data in datas:
value = bytes.fromhex(data)
ts = get_timestamp(value) # datetime in UTC (timezone aware)
# print("'{}'".format(data[8:10]))
parsed_data = {
'temp': struct.unpack('<h', bytes.fromhex(data[8:12]))[0] / 100, # °C
'humi': int(data[12:14], 16) / 2,
'pres': struct.unpack('<I', bytes.fromhex(data[14:20] + '00'))[0] / 100 # hPa
}
dataline = create_dataline(ts, parsed_data)
datalines.append(dataline)
return datalines
return None
def parse_ruuvistation_data(data):
"""
:param data: JSON object
:return: dict of parsed Ruuvitag values
"""
# Create object for all tags
# TODO: add gateway data here too
parsed_data = {
'ruuvitags': [],
}
timestamp = parse(data['time'])
fields = {
'batteryLevel': data['batteryLevel'],
'latitude': data['location']['latitude'],
'longitude': data['location']['longitude'],
'accuracy': data['location']['accuracy'],
}
dataline = create_dataline(timestamp, fields)
parsed_data['gateway'] = {
'devid': data['deviceId'],
'datalines': [dataline]
}
fields2save = [
'accelX', 'accelY', 'accelZ', 'humidity', 'measurementSequenceNumber',
'movementCounter', 'pressure', 'rssi', 'temperature', 'voltage'
]
for item in data['tags']:
datalines = []
fields = {}
devid = item.pop('id').upper()
ts_str = item.pop('updateAt')
timestamp = parse(ts_str)
for f in fields2save:
fields[f] = float(item[f])
dataline = create_dataline(timestamp, fields)
datalines.append(dataline)
parsed_data['ruuvitags'].append({
'devid': devid,
'datalines': datalines
})
return parsed_data
def parse_sentilo_data(data):
"""
Extract two data objects from source data:
- 1 minute average
- 60 values for every seconds
:param data: JSON object
:return: dict of parsed dBA values, one for 1 min average and 60 for every second.
"""
# Create object for both 1 min and 1 sec data
parsed_data = {
'LAeq': {'datalines': []},
'LAeq1s': {'datalines': []},
}
for item in data['sensors']:
devid = item['sensor'][0:-2]
parsed_data['LAeq']['devid'] = devid
parsed_data['LAeq1s']['devid'] = devid
ts_str = item['observations'][0].get('timestamp')
if ts_str is not None:
timestamp = parse(item['observations'][0]['timestamp'], dayfirst=True)
else:
timestamp = datetime.datetime.utcnow()
print(timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ data without timestamp!"))
if item['sensor'].endswith('N'):
fields = {'dBA': float(item['observations'][0]['value'])}
dataline = create_dataline(timestamp, fields)
parsed_data['LAeq']['datalines'].append(dataline)
if item['sensor'].endswith('S'):
cnt = 0
secvals = item['observations'][0]['value'].split(';')
secvals.reverse()
for val in secvals:
fields = {'dBA': float(val.split(',')[0])}
new_ts = (timestamp - datetime.timedelta(seconds=cnt))
dataline = create_dataline(new_ts, fields)
parsed_data['LAeq1s']['datalines'].append(dataline)
cnt += 1
return parsed_data
def parse_ruuvicounter_data(data, port, serialised_request):
"""
:param data: JSON object
:return: dict of parsed Ruuvitag values
"""
# Create object for all tags and gateway
lora = json.loads(serialised_request['request.body'].decode("utf-8"))
ruuvicounterdata = parse_ruuvicounter(data, port)
devid = serialised_request['devid']
timestamp = parse(lora['DevEUI_uplink']['Time']).astimezone(pytz.UTC)
dataline = create_dataline(timestamp, ruuvicounterdata['gateway'])
parsed_data = {}
parsed_data['gateway'] = {'devid': devid, 'datalines': [dataline]}
parsed_data['ruuvicounter'] = {'datalines': []}
for item in ruuvicounterdata['tags']:
mac = item.pop('mac')
dataline = create_dataline(timestamp,
item,
extra={'extratags': {
'mac': mac
}})
parsed_data['ruuvicounter']['datalines'].append(dataline)
return parsed_data
def parse_thingpark_request(serialised_request, data):
d = data['DevEUI_uplink']
devid = d['DevEUI']
port = d['FPort']
datalogger, created = get_datalogger(devid=devid, update_activity=False)
timestamp = parse(d['Time'])
timestamp = timestamp.astimezone(pytz.UTC)
payload_hex = d['payload_hex']
rssi = d['LrrRSSI']
# TODO: This may fail, so prepare to handle exception properly
# Test it by configuring wrong decoder for some Datalogger
try:
payload = decode_payload(datalogger, payload_hex, port, serialised_request=serialised_request)
except ValueError as err:
decoder = get_datalogger_decoder(datalogger)
err_msg = f'Failed to parse "{payload_hex}" using "{decoder}" for "{devid}": {err}'
logger.warning(err_msg)
serialised_request['parse_fail'] = {
'error_message': str(err),
'decoder': get_datalogger_decoder(datalogger)
}
save_parse_fail_datalogger_message(devid, data_pack(serialised_request))
return True
logging.debug(payload)
# Some sensors may already return a dict of lists of datalines
if not payload:
return True
if isinstance(payload, dict) and isinstance(payload[list(payload.keys())[0]], dict):
parsed_data = payload
for k in parsed_data.keys():
datalines = parsed_data[k]['datalines']
if len(datalines) > 0:
datalines[-1]['data']['rssi'] = float(rssi) # Add rssi value to the latest dataline
send_to_exchange(devid, datalogger, datalines, override_measurement=k)
else: # Some sensors may already return a list of datalines
if isinstance(payload, list):
datalines = payload # Use payload as datalines (which already have timestamps)
else:
dataline = create_dataline(timestamp, payload) # Create dataline from LoRaWAN timestamp and payload
datalines = [dataline]
datalines[-1]['data']['rssi'] = float(rssi) # Add rssi value to the latest dataline
send_to_exchange(devid, datalogger, datalines)
return True
def handle_request(self, request):
serialised_request = serialize_django_request(request)
ok, body = decode_json_body(serialised_request['request.body'])
if ok is False:
return HttpResponse(f'JSON ERROR: {body}',
status=400,
content_type='text/plain')
key = body.get('key', '')
key_splitted = key.split('/')
if len(key_splitted) == 3:
devid = '/'.join(key_splitted[:2])
else:
return HttpResponse(
f'Key error: key "{key}" is not correctly formed',
status=400,
content_type='text/plain')
datalogger, created = get_datalogger(devid=devid,
update_activity=False,
create=False)
if datalogger is None:
return HttpResponse(f'Datalogger "{devid}" does not exist',
status=400,
content_type='text/plain')
data = body['data']
location = data.pop('location')
data.update(location)
epoch = data.pop('timestamp') / 1000
timestamp = epoch2datetime(epoch)
dataline = create_dataline(timestamp, data)
datalines = [dataline]
parsed_data = create_parsed_data_message(devid, datalines)
config = datalogger_get_config(datalogger, parsed_data)
db_name = config.get('influxdb_database')
measurement_name = config.get('influxdb_measurement')
if db_name is not None and measurement_name is not None:
save_parsed_data2influxdb(db_name, measurement_name, parsed_data)
return HttpResponse('OK', content_type='text/plain')
else:
return HttpResponse(
f'InfluxDB database and measurement are not defined for Datalogger "{devid}"',
status=400,
content_type='text/plain')
def parse_everynet_request(serialised_request, body):
# FIXME: currently this parses only payload_hex from PAXCOUNTER. MUST check BKS too and others
# TODO: create utility function, which extracts all interesting fields (needed here) out from request data
devid = body['meta'].get('device', 'unknown')
payload_base64 = body['params']['payload']
payload_hex = binascii.hexlify(base64.b64decode(payload_base64))
port = body['params']['port']
rssi = body['params']['radio']['hardware']['rssi']
timestamp = datetime.datetime.utcfromtimestamp(body['params']['rx_time'])
timestamp = timestamp.astimezone(pytz.UTC)
# TODO: this may fail if database is offline
datalogger, created = get_datalogger(devid=devid, update_activity=False)
payload = decode_payload(datalogger, payload_hex, port)
payload['rssi'] = rssi
# RabbitMQ part
key = create_routing_key('everynet', devid)
dataline = create_dataline(timestamp, payload)
datalines = [dataline]
message = create_parsed_data_message(devid, datalines=datalines)
packed_message = data_pack(message)
send_message(settings.PARSED_DATA_EXCHANGE, key, packed_message)
return True