def parse_sentilo_data(data):
measurements = []
for item in data['sensors']:
ts_str = item['observations'][0].get('timestamp')
if ts_str is not None:
ts = parse(item['observations'][0]['timestamp'], dayfirst=True)
else:
ts = datetime.datetime.utcnow()
print(ts.strftime("%Y-%m-%dT%H:%M:%S.%fZ data without timestamp!"))
dev_id = item['sensor'][0:-2]
if item['sensor'].endswith('N'):
fields = {'dBA': float(item['observations'][0]['value'])}
measurement = create_influxdb_obj(dev_id,
'LAeq',
fields,
timestamp=ts)
measurements.append(measurement)
if item['sensor'].endswith('S'):
cnt = 0
secvals = item['observations'][0]['value'].split(';')
secvals.reverse()
for val in secvals:
fields = {'dBA': float(val.split(',')[0])}
measurement = create_influxdb_obj(
dev_id,
'LAeq1s',
fields,
timestamp=(ts - datetime.timedelta(seconds=cnt)))
cnt += 1
measurements.append(measurement)
return measurements
def view_func(self, request):
"""
Endpoint requires idcode, sensor and data parameters. Also a valid Django user must exist. Test like this:
export ESPEASY_URL=esp
echo -n "idcode=unique_id_here&sensor=bme280&id=0&data=Temperature=24.84,Humidity=52.05,Pressure=1002.50" | \
http -v --auth user:pass --form POST http://127.0.0.1:8000/esp
"""
uname, passwd, user = basicauth(request)
p = request.POST
if user is None:
return HttpResponse("Authentication failure", status=401)
if request.method == 'GET':
return HttpResponse('ok', status=200)
idcode = p.get('idcode')
sensor = p.get('sensor')
data = p.get('data')
# Reject request if required variables are not set
if None in [idcode, sensor, data]:
err_msg = '[ESPEASY] idcode, sensor and/or data is missing in request form data'
logger.error(err_msg)
return HttpResponse(err_msg, status=400)
data = data.strip()
# Reject request if data is not comma separated key=value pairs or value is not float
try:
fields = dict([tuple(x.split('=')) for x in data.split(',')])
for k in fields.keys():
fields[k] = float(fields[k])
except ValueError as err:
err_msg = '[ESPEASY] data error: {}. Hint: data was "{}".'.format(err, data)
logger.error(err_msg)
response = HttpResponse(err_msg, status=400)
return response
measurement = create_influxdb_obj(idcode, sensor, fields)
measurements = [measurement]
# import json; print(json.dumps(measurement, indent=1)); print(data)
dbname = uname # Use username as database name
try:
save_to_influxdb.delay(dbname, measurements)
except Exception as err:
logger.error(err)
raise
# iclient = get_influxdb_client(database=dbname)
# try:
# iclient.create_database(dbname)
# iclient.write_points(measurements)
# response = HttpResponse("ok")
# except InfluxDBClientError as err:
# err_msg = '[ESPEASY] InfluxDB error: {}'.format(err)
# logger.error(err_msg)
# response = HttpResponse(err_msg, status=500)
response = HttpResponse("ok")
return response
def view_func(self, request):
res = dump_request(request, postfix='democam')
ok_response = HttpResponse("ok")
body_data = ''
if request.method not in ['POST']:
return HttpResponse('Only POST methdod is allowed', status=405)
try:
body_data = request.body
data = json.loads(body_data.decode('utf-8'))
except (ValueError, UnicodeDecodeError) as err:
return invalid_data(body_data,
"Hint: should be UTF-8 json.",
status=400)
# Validate data
if "direction" not in data.keys():
data["direction"] = -1
for k in ["plate", "date", "country", "confidence", "ip", "direction"]:
if k not in data.keys():
return invalid_data(body_data,
"Hint: key '{}' is missing.".format(k),
status=400)
timestamp = parse(data['date'])
plate = Plate(plate=data['plate'],
timestamp=timestamp,
country=data['country'],
confidence=float(data['confidence']),
ip=data['ip'],
direction=int(data['direction']))
plate.save()
idata = {'vehicle': 1}
measurement = create_influxdb_obj('001', 'cnt', idata, timestamp)
measurements = [measurement]
dbname = 'vehicle'
iclient = get_influxdb_client(database=dbname)
iclient.create_database(dbname)
try:
iclient.write_points(measurements)
response = HttpResponse("OK")
except InfluxDBClientError as err:
err_msg = '[PLATECAMERA] InfluxDB error: {}'.format(err)
logger.error(err_msg)
response = HttpResponse(err_msg, status=500)
return response
def parse_tag_data(data):
measurements = []
for tag in data['tags']:
ts = dateutil.parser.parse(tag['updateAt'])
if ts.tzinfo is None or ts.tzinfo.utcoffset(ts) is None:
ts = get_default_timezone().localize(ts)
dev_id = tag['id']
extratags = {}
name = tag.get('name')
if name:
extratags['name'] = name
fields = {}
for key in to_save:
fields[key] = tag.get(key)
measurement = create_influxdb_obj(dev_id,
'ruuvitag',
fields,
timestamp=ts,
extratags=extratags)
measurements.append(measurement)
# print(measurements)
return measurements
def view_func(self, request):
"""
Endpoint requires idcode, sensor and data parameters. Also a valid Django user must exist. Test like this:
export EVERYNET_URL=everynet
"""
ok_response = HttpResponse("ok")
body_data = ''
if request.method not in ['OPTIONS', 'POST']:
return HttpResponse('Only OPTIONS, POST methdods are allowed',
status=405)
if request.method == 'OPTIONS': # FIXME: I don't know is this a correct answer
return HttpResponse('OK', status=200)
try:
body_data = request.body
data = json.loads(body_data.decode('utf-8'))
except (ValueError, UnicodeDecodeError) as err:
return invalid_data(body_data,
"Hint: should be UTF-8 json.",
status=400)
# meta and type keys should be always in request json
try:
device = data['meta']['device']
times = str(data['meta']['time'])
packet_type = data['type']
except KeyError as err:
err_msg = 'Invalid json structure: "{}". Hint: missing key {}.'.format(
body_data, err)
return invalid_data(body_data, err_msg, status=400)
now = timezone.now().astimezone(pytz.utc)
path = os.path.join(settings.MEDIA_ROOT, 'everynet',
now.strftime('%Y-%m-%d'), device)
os.makedirs(path, exist_ok=True)
fpath = os.path.join(path, now.strftime('%Y%m%dT%H%M%S.%fZ.json'))
dl_descr = 'everynet'
if random.randint(
0, 50) == 10: # Save 1/50 of packages for debugging purposes
with open(fpath, 'wt') as destination:
destination.write(json.dumps(data, indent=1))
dump_request(request, postfix='everynet')
if packet_type == 'error':
logger.warning(
'[EVERYNET]: Got error msg, check {} for details.'.format(
fpath))
return ok_response
elif packet_type == 'uplink':
payload = data['params']['payload'].encode()
_type = request.GET.get('type')
if _type == 'paxcounter':
data_str = base64.decodebytes(payload)
idata = handle_paxcounter(data_str)
keys_str = 'wifi-ble'
dl_descr = 'paxcounter'
elif _type == 'keyval': # data should be key=val,key2=val2,... formatted
data_str = base64.decodebytes(payload).decode('utf8')
print(data_str, type(data_str))
idata = handle_keyval(data_str)
try: # convert values to floats in dict
idata = {k: float(v) for k, v in idata.items()}
except ValueError as err:
err_msg = 'Should be base64 encoded key=val pairs, comma separated.'.format(
data_str[:50])
return invalid_data(data_str, err_msg, status=400)
print(idata)
keys = list(idata.keys())
keys.sort()
if len(keys) == 0:
err_msg = 'Should be base64 encoded key=val pairs, comma separated.'
# NOTE: from everynet's point of view this is not error.
return invalid_data(data_str, err_msg, status=200)
keys_str = '_'.join(keys)
dl_descr = 'keyval'
else:
data_str = base64.decodebytes(payload).decode('utf8')
handle_v1(data_str) # TODO
try:
sensordata = json.loads(data_str)
if not isinstance(sensordata, dict):
err_msg = '[EVERYNET] payload is not json: {}'.format(
data_str)
logger.warning(err_msg)
return HttpResponse("OK: dumped data to a file.")
except (ValueError) as err:
return invalid_data(data_str,
"Hint: should be UTF-8 json.",
status=400)
if 'id' in sensordata and 'sensor' in sensordata:
keys = list(sensordata['data'].keys())
idata = sensordata['data']
pass
else: # old method
keys = list(sensordata.keys())
idata = sensordata
keys.sort()
keys_str = '-'.join(keys)
datalogger, created = get_datalogger(device,
description=dl_descr,
update_activity=True)
# TODO: log new devices (created == True), maybe send email to admin?
ts = datetime.datetime.utcfromtimestamp(data['meta']['time'])
ts = pytz.UTC.localize(ts) # Make timestamp timezone aware at UTC
measurement = create_influxdb_obj(device, keys_str, idata, ts)
measurements = [measurement]
dbname = request.GET.get('db')
if dbname is None:
dbname = EVERYNET_DB
iclient = get_influxdb_client(database=dbname)
iclient.create_database(dbname)
try:
iclient.write_points(measurements)
response = HttpResponse("ok")
except InfluxDBClientError as err:
err_msg = '[EVERYNET] InfluxDB error: {}'.format(err)
logger.error(err_msg)
response = HttpResponse(err_msg, status=500)
return response
else:
err_msg = '[EVERYNET] Unknown packet type {}'.format(packet_type)
logger.warning(err_msg)
return ok_response
def view_func(self, request):
"""
Endpoint requires valid Digita formatted JSON payload.
"""
err_msg = ''
status = 200
try:
body_data = request.body
data = json.loads(body_data.decode('utf-8'))
except (json.decoder.JSONDecodeError, UnicodeDecodeError) as err:
log_msg = '[DIGITA] Invalid data: "{}". Hint: should be UTF-8 json.'.format(
body_data[:50])
err_msg = 'Invalid data: "{}"... Hint: should be UTF-8 json.'.format(
body_data[:50])
logger.error(log_msg)
return HttpResponse(err_msg, status=400)
# meta and type keys should be always in request json
try:
d = data['DevEUI_uplink']
device = d['DevEUI']
times = str(d['Time'])
rssi = d['LrrRSSI']
payload_hex = d['payload_hex']
except KeyError as err:
log_msg = '[DIGITA] Invalid json structure: "{}". Missing key: {}.'.format(
body_data, err)
err_msg = 'Invalid json structure: "{}". Hint: missing key {}.'.format(
body_data, err)
logger.error(log_msg)
return HttpResponse(err_msg, status=400)
now = timezone.now().astimezone(pytz.utc)
path = os.path.join(settings.MEDIA_ROOT, 'digita',
now.strftime('%Y-%m-%d'), device)
os.makedirs(path, exist_ok=True)
fpath = os.path.join(path, now.strftime('%Y%m%dT%H%M%S.%fZ.json'))
with open(fpath, 'wt') as destination:
destination.write(json.dumps(data, indent=1))
response = HttpResponse("ok")
# TODO: move this to a function
if len(payload_hex) == 8:
idata = {
'wifi': int(payload_hex[0:4], 16),
'ble': int(payload_hex[4:8], 16)
}
idata['rssi'] = rssi
keys_str = 'wifi-ble'
dl_descr = 'paxcounter'
datalogger, created = get_datalogger(device,
description=dl_descr,
update_activity=True)
ts = dateutil.parser.parse(times)
measurement = create_influxdb_obj(device, keys_str, idata, ts)
measurements = [measurement]
# dbname = request.GET.get('db', DIGITA_DB)
dbname = 'paxcounter'
iclient = get_influxdb_client(database=dbname)
iclient.create_database(dbname)
try:
iclient.write_points(measurements)
except InfluxDBClientError as err:
err_msg = '[DIGITA] InfluxDB error: {}'.format(err)
status = 500
elif payload_hex[:2] == '13':
idata = handle_clickey_tempsens(payload_hex)
idata['rssi'] = rssi
keys_str = 'tempsens'
datalogger, created = get_datalogger(
device,
description='Clickey Tempsens PRO',
update_activity=True)
ts = dateutil.parser.parse(times)
measurement = create_influxdb_obj(device, keys_str, idata, ts)
measurements = [measurement]
# dbname = request.GET.get('db', DIGITA_DB)
dbname = 'digita'
iclient = get_influxdb_client(database=dbname)
iclient.create_database(dbname)
try:
iclient.write_points(measurements)
except InfluxDBClientError as err:
err_msg = '[DIGITA] InfluxDB error: {}'.format(err)
status = 500
elif payload_hex[:2].lower(
) == '2a': # payload_hex[:4].lower() == '2a2a':
idata = handle_aqburk(payload_hex)
idata['rssi'] = rssi
keys_str = 'aqburk'
datalogger, created = get_datalogger(device,
description='FVH AQ burk',
update_activity=True)
ts = dateutil.parser.parse(times)
measurement = create_influxdb_obj(device, keys_str, idata, ts)
measurements = [measurement]
DIGITA_DB = 'aqburk'
dbname = request.GET.get('db', DIGITA_DB)
iclient = get_influxdb_client(database=dbname)
iclient.create_database(dbname)
try:
iclient.write_points(measurements)
except InfluxDBClientError as err:
err_msg = '[DIGITA] InfluxDB error: {}'.format(err)
status = 500
elif len(payload_hex) >= 2: # Assume we have key-val data
try:
idata = handle_keyval(payload_hex)
except (UnicodeDecodeError, IndexError) as err:
err_msg = '[DIGITA] Payload error: {}'.format(err)
status = 400
logger.error(err_msg)
dump_request(request, postfix='digita')
response = HttpResponse(err_msg, status=status)
return response
idata['rssi'] = rssi
ikeys = list(idata.keys())
ikeys.sort()
keys_str = '_'.join(ikeys)
datalogger, created = get_datalogger(device,
description='LoRaWAN device',
update_activity=True)
ts = dateutil.parser.parse(times)
measurement = create_influxdb_obj(device, keys_str, idata, ts)
measurements = [measurement]
# dbname = request.GET.get('db', DIGITA_DB)
dbname = 'digita'
iclient = get_influxdb_client(database=dbname)
iclient.create_database(dbname)
try:
iclient.write_points(measurements)
except InfluxDBClientError as err:
err_msg = '[DIGITA] InfluxDB error: {}'.format(err)
status = 500
else:
err_msg = '[DIGITA] Not handled'
if err_msg != '':
logger.error(err_msg)
dump_request(request, postfix='digita')
response = HttpResponse(err_msg, status=status)
return response