def test_mqtt_to_grafana(machinery, create_influxdb, reset_influxdb,
reset_grafana):
"""
Publish single reading in JSON format to MQTT broker and proof
that a corresponding datasource and a dashboard was created in Grafana.
"""
# Submit a single measurement, without timestamp.
data = {
'temperature': 42.84,
'humidity': 83.1,
}
yield mqtt_json_sensor(settings.mqtt_topic_json, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
yield sleep(PROCESS_DELAY)
yield sleep(PROCESS_DELAY)
# Proof that Grafana is well provisioned.
logger.info('Grafana: Checking datasource')
datasource_names = []
for datasource in grafana.client.datasources.get():
datasource_names.append(datasource['name'])
assert settings.influx_database in datasource_names
logger.info('Grafana: Checking dashboard')
dashboard_name = settings.grafana_dashboards[0]
dashboard = grafana.client.dashboards.db[dashboard_name].get()
target = dashboard['dashboard']['rows'][0]['panels'][0]['targets'][0]
assert target['measurement'] == settings.influx_measurement_sensors
assert 'temperature' in target['query'] or 'humidity' in target['query']
def test_tasmota_ds18b20(machinery, create_influxdb, reset_influxdb):
"""
Publish another single SENSOR reading in Tasmota/JSON format
to MQTT broker, including a timestamp.
Proof that the reading is processed and stored correctly.
https://getkotori.org/docs/handbook/decoders/tasmota.html#submit-example-payload
"""
# Submit a single measurement.
data = {"Time": "2017-02-16T10:13:52", "DS18B20": {"Temperature": 20.6}}
yield mqtt_json_sensor(tasmota_sensor_topic, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Define reference data.
reference = {
u'time': u'2017-02-16T09:13:52Z',
u'DS18B20.Temperature': 20.6,
}
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert record == reference
yield record
def test_timestamp_rfc3339(machinery, create_influxdb, reset_influxdb):
"""
Publish single reading in JSON format to MQTT broker,
using a timestamp in RFC3339 format.
Proof that the timestamp is processed and stored correctly.
"""
# Submit a single measurement, with timestamp.
data = {
'temperature': 42.84,
'humidity': 83.1,
'timestamp': '2020-03-10 03:38:37.937059000+01:00'
}
yield mqtt_json_sensor(settings.mqtt_topic_json, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert record == {
u'time': u'2020-03-10T02:38:37.937059Z',
u'humidity': 83.1,
u'temperature': 42.84
}
yield record
def test_timestamp_microseconds(machinery, create_influxdb, reset_influxdb):
"""
Publish single reading in JSON format to MQTT broker,
using a timestamp as Unix Epoch in microseconds.
Proof that the timestamp is processed and stored correctly.
"""
# Submit a single measurement, with timestamp.
data = {
'temperature': 42.84,
'humidity': 83.1,
'timestamp': 1583810982123456
}
yield mqtt_json_sensor(settings.mqtt_topic_json, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert record == {
u'time': u'2020-03-10T03:29:42.123456Z',
u'humidity': 83.1,
u'temperature': 42.84
}
yield record
def test_tasmota_wemos_dht22(machinery, create_influxdb, reset_influxdb):
"""
Publish a reading from a Wemos multi sensor device.
Proof that the reading is processed and stored correctly.
"""
# Submit a single measurement.
data = {
"Time": "2017-10-05T22:39:55",
"DHT22": {
"Temperature": 25.4,
"Humidity": 45
},
"TempUnit": "C"
}
yield mqtt_json_sensor(tasmota_sensor_topic, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Define reference data.
reference = {
u'time': u'2017-10-05T20:39:55Z',
u'DHT22.Temperature': 25.4,
u'DHT22.Humidity': 45,
}
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert record == reference
yield record
def test_mqtt_to_grafana(machinery_hiveeyes, create_influxdb_hiveeyes,
reset_influxdb_hiveeyes, reset_grafana_hiveeyes):
"""
Publish a single reading in JSON format to MQTT and proof
- it is stored in the InfluxDB database.
- a corresponding datasource and dashboards have been created in Grafana.
"""
# Submit a single measurement, without timestamp.
data = {
'temperature': 42.84,
'weight': 33.33,
}
yield mqtt_json_sensor(settings.mqtt_topic, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Wait for Grafana to create its artefacts.
yield sleep(2)
# Proof that data arrived in InfluxDB.
record = influx.get_first_record()
del record['time']
assert record == {u'temperature': 42.84, u'weight': 33.33}
yield record
# Proof that Grafana is well provisioned.
logger.info('Grafana: Checking datasource')
datasource_names = []
for datasource in grafana.client.datasources.get():
datasource_names.append(datasource['name'])
assert settings.influx_database in datasource_names
logger.info('Grafana: Checking dashboards')
for dashboard_name in settings.grafana_dashboards:
dashboard = grafana.client.dashboards.db[dashboard_name].get(
)['dashboard']
if 'rows' in dashboard:
umbrella = dashboard['rows'][0]
else:
umbrella = dashboard
target = umbrella['panels'][0]['targets'][0]
#assert target['measurement'] == settings.influx_measurement
assert 'temperature' in target['query'] or 'weight' in target['query']
def test_influxdb_tags(machinery, create_influxdb, reset_influxdb):
"""
Publish single reading in JSON format to MQTT broker.
Proof that all special fields are stored in the
InfluxDB database as tags.
"""
# Define field names which are tags.
# FIXME: Synchronize with ``kotori.daq.influx.storage.format_chunk()``.
tag_fields_main = ['geohash', 'latitude', 'longitude']
tag_fields_more = [
'location', 'location_id', 'location_name', 'sensor_id', 'sensor_type'
]
# Submit a single measurement, without timestamp.
data = {
'temperature': 42.84,
'humidity': 83.1,
'geohash': 'aa3434',
'latitude': 42.50,
'longitude': 52.40,
}
for field in tag_fields_more:
data[field] = idgen()
yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_json,
data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Define reference payload.
reference = deepcopy(data)
reference['latitude'] = str(reference['latitude'])
reference['longitude'] = str(reference['longitude'])
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
del record['time']
assert record == reference
yield record
# Proof that data all special fields have been converged to tags.
resultset = influx_sensors.client.query(
'SHOW TAG KEYS FROM {measurement};'.format(
measurement=settings.influx_measurement_sensors))
yield resultset
tag_names = []
for item in resultset[settings.influx_measurement_sensors]:
tag_names.append(item['tagKey'])
assert set(tag_names) == set(tag_fields_main + tag_fields_more)
def test_tasmota_wemos_multi(machinery, create_influxdb, reset_influxdb):
"""
Publish a reading from a Wemos multi sensor device.
Proof that the reading is processed and stored correctly.
"""
# Submit a single measurement.
data = {
"Time": "2017-10-05T22:39:45",
"DS18x20": {
"DS1": {
"Type": "DS18B20",
"Address": "28FF4CBFA41604C4",
"Temperature": 25.37
},
"DS2": {
"Type": "DS18B20",
"Address": "28FF1E7FA116035D",
"Temperature": 30.44
},
"DS3": {
"Type": "DS18B20",
"Address": "28FF1597A41604CE",
"Temperature": 25.81
}
},
"DHT22": {
"Temperature": 33.2,
"Humidity": 30
},
"TempUnit": "C"
}
yield mqtt_json_sensor(tasmota_sensor_topic, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Define reference data.
reference = {
u'time': u'2017-10-05T20:39:45Z',
u'DHT22.Temperature': 33.2,
u'DHT22.Humidity': 30,
u'DS18x20.DS1.Temperature': 25.37,
u'DS18x20.DS2.Temperature': 30.44,
u'DS18x20.DS3.Temperature': 25.81,
}
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
logger.info('record: %s', json.dumps(record))
assert record == reference
yield record
def test_export_csv(machinery, create_influxdb, reset_influxdb):
"""
Submit single reading in JSON format to HTTP API and proof
it can be retrieved back from the HTTP API in different formats.
"""
# Submit a single measurement, with timestamp.
data = {
'time': 1583810982,
'temperature': 25.26,
'humidity': 51.8,
}
yield threads.deferToThread(http_json_sensor, settings.channel_path_data, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data is available via HTTP API.
ts_from = '2020-03-10T00:00:00.000Z'
ts_to = '2020-03-10T23:59:59.000Z'
# CSV format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='csv', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result == 'time,humidity,temperature\n2020-03-10T03:29:42.000000Z,51.8,25.26\n'
# TXT format (same as CSV).
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='txt', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result == 'time,humidity,temperature\n2020-03-10T03:29:42.000000Z,51.8,25.26\n'
# JSON format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='json', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result == '[{"time":"2020-03-10T03:29:42.000Z","humidity":51.8,"temperature":25.26}]'
# XLSX format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='xlsx', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result.startswith(b'PK\x03\x04\x14\x00\x00\x00\x08\x00')
# HTML format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='html', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert \
'<html>' in deferred.result and \
'<th>temperature</th>' in deferred.result and \
'<td>25.26</td>' in deferred.result
def test_event_mqtt(machinery, create_influxdb, reset_influxdb_events):
"""
Publish event in JSON format to MQTT broker
and proof it is stored in the InfluxDB database.
"""
# Submit a single event, without timestamp.
yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_event, event_data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that event arrived in InfluxDB.
record = influx_events.get_first_record()
del record['time']
assert record == event_data
def test_tasmota_state(machinery, create_influxdb, reset_influxdb):
"""
Publish a single STATE reading in Tasmota/JSON format
to MQTT broker, including a timestamp.
Proof that the reading is processed and stored correctly.
https://getkotori.org/docs/handbook/decoders/tasmota.html#submit-example-payload
"""
# Submit a single measurement.
data = {
"Time": "2019-06-02T22:13:07",
"Uptime": "1T18:10:35",
"Vcc": 3.182,
"SleepMode": "Dynamic",
"Sleep": 50,
"LoadAvg": 19,
"Wifi": {
"AP": 1,
"SSId": "{redacted}",
"BSSId": "A0:F3:C1:{redacted}",
"Channel": 1,
"RSSI": 100,
"LinkCount": 1,
"Downtime": "0T00:00:07"
}
}
yield mqtt_json_sensor(tasmota_state_topic, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Define reference data.
reference = {
u'time': u'2019-06-02T20:13:07Z',
u'Device.Vcc': 3.182,
u'Device.Sleep': 50,
u'Device.LoadAvg': 19,
u'Device.Wifi.Channel': 1,
u'Device.Wifi.RSSI': 100,
u'Device.Wifi.LinkCount': 1,
}
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert record == reference
yield record
def test_event_http_urlencoded(machinery, create_influxdb, reset_influxdb):
"""
Submit event in ``x-www-form-urlencoded`` format to HTTP API
and proof it is stored in the InfluxDB database.
"""
# Submit a single event, without timestamp.
yield threads.deferToThread(http_form_sensor, settings.channel_path_event, event_data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that event arrived in InfluxDB.
record = influx_events.get_first_record()
del record['time']
assert record == event_data
yield record
def test_mqtt_to_influxdb_single(machinery, create_influxdb, reset_influxdb):
"""
Publish discrete values to the MQTT broker and
and proof they are stored in the InfluxDB database.
"""
# Submit discrete measurement values, without timestamp.
topic_temperature = settings.mqtt_topic_single + '/temperature'
topic_humidity = settings.mqtt_topic_single + '/humidity'
yield threads.deferToThread(mqtt_sensor, topic_temperature, 42.84)
yield threads.deferToThread(mqtt_sensor, topic_humidity, 83.1)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert 'temperature' in record or 'humidity' in record
def test_airrohr_http_json(machinery, create_influxdb, reset_influxdb):
"""
Submit single reading in Airrohr JSON format to HTTP API
and proof it is stored in the InfluxDB database.
"""
# Submit a single measurement, without timestamp.
yield threads.deferToThread(http_json_sensor,
settings.channel_path_airrohr, data_in)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
del record['time']
assert record == data_out
yield record
def test_mqtt_legacy(machinery, create_influxdb, reset_influxdb):
"""
Publish single reading in JSON format to MQTT broker on legacy suffix
and proof it is stored in the InfluxDB database.
"""
# Submit a single measurement, without timestamp.
data = {
'temperature': 42.84,
'humidity': 83.1,
}
yield threads.deferToThread(mqtt_json_sensor,
settings.mqtt_topic_json_legacy, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert 'temperature' in record or 'humidity' in record
def test_tasmota_sonoff_sc(machinery, create_influxdb, reset_influxdb):
"""
Publish a single SENSOR reading in Tasmota/JSON format
to MQTT broker, including a timestamp.
Proof that the reading is processed and stored correctly.
https://getkotori.org/docs/handbook/decoders/tasmota.html#submit-example-payload
"""
# Submit a single measurement.
data = {
"Time": "2019-06-02T22:13:07",
"SonoffSC": {
"Temperature": 25,
"Humidity": 15,
"Light": 20,
"Noise": 10,
"AirQuality": 90
},
"TempUnit": "C"
}
yield mqtt_json_sensor(tasmota_sensor_topic, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Define reference data.
reference = {
u'time': u'2019-06-02T20:13:07Z',
u'SonoffSC.AirQuality': 90,
u'SonoffSC.Humidity': 15,
u'SonoffSC.Light': 20,
u'SonoffSC.Noise': 10,
u'SonoffSC.Temperature': 25,
}
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert record == reference
yield record
def test_mqtt_homie(machinery, create_influxdb, reset_influxdb):
"""
Publish reading like a Homie device in JSON format to MQTT broker
and proof it is stored in the InfluxDB database.
"""
# Submit a single measurement, without timestamp.
data = {
'temperature': 42.84,
'humidity': 83.1,
}
yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_homie,
data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
del record['time']
assert record == {u'humidity': 83.1, u'temperature': 42.84}
yield record
def test_http_csv(machinery, create_influxdb, reset_influxdb):
"""
Submit single reading in CSV format to HTTP API
and proof it is stored in the InfluxDB database.
"""
# Submit a single measurement, without timestamp.
data = {
'temperature': 25.26,
'humidity': 51.8,
}
yield threads.deferToThread(http_csv_sensor, settings.channel_path_data,
data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
del record['time']
assert record == {u'temperature': 25.26, u'humidity': 51.8}
yield record
def test_weewx_mqtt(machinery, create_influxdb, reset_influxdb):
"""
Publish single reading in JSON format to MQTT broker
and proof it is stored in the InfluxDB database.
"""
# Submit a single measurement, without timestamp.
yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_json, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert record["time"] == '2017-04-17T22:15:00Z'
assert record["outTemp_C"] == 3.55555555556
assert record["windSpeed10_kph"] == 5.78725803977
assert record["cloudbase_meter"] == 773.082217509
assert record["consBatteryVoltage_volt"] == 4.72
yield record
def test_export(machinery, create_influxdb, reset_influxdb):
"""
Submit single reading in JSON format to HTTP API and proof
it can be retrieved back from the HTTP API in different formats.
"""
# Submit a single measurement, with timestamp.
data = {
'time': 1583810982,
'temperature': 25.26,
'humidity': 51.8,
}
yield threads.deferToThread(http_json_sensor, settings.channel_path_data, data)
# Wait for some time to process the message.
yield sleep(PROCESS_DELAY)
# Proof that data is available via HTTP API.
ts_from = '2020-03-10T00:00:00.000Z'
ts_to = '2020-03-10T23:59:59.000Z'
# CSV format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='csv', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result == 'time,humidity,temperature\n2020-03-10T03:29:42.000000Z,51.8,25.26\n'
# TXT format (same as CSV).
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='txt', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result == 'time,humidity,temperature\n2020-03-10T03:29:42.000000Z,51.8,25.26\n'
# JSON format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='json', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result == '[{"time":"2020-03-10T03:29:42.000Z","humidity":51.8,"temperature":25.26}]'
# XLSX format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='xlsx', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result.startswith(b'PK\x03\x04\x14\x00\x00\x00\x08\x00')
# HTML format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='html', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert \
'<html>' in deferred.result and \
'<th>temperature</th>' in deferred.result and \
'<td>25.26</td>' in deferred.result
# HDF5 format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='hdf5', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result.startswith(b'\x89HDF\r\n\x1a\n\x00\x00\x00\x00\x00')
import h5py
from io import BytesIO
from pandas import array
hdf = h5py.File(BytesIO(deferred.result), "r")
assert hdf["/itest_foo_bar/table"].attrs.get("NROWS") == 1
assert hdf["/itest_foo_bar/table"].attrs.get("index_kind") == b"datetime64"
assert hdf["/itest_foo_bar/table"].attrs.get("humidity_dtype") == b"float64"
assert hdf["/itest_foo_bar/table"].attrs.get("temperature_dtype") == b"float64"
assert hdf["/itest_foo_bar/table"].shape == (1,)
assert hdf["/itest_foo_bar/table"][()] == array(
[(1583810982000000000, 51.8, 25.26)],
dtype=[('index', '<i8'), ('humidity', '<f8'), ('temperature', '<f8')])
# NetCDF format.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='nc', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert deferred.result.startswith(b'\x89HDF\r\n\x1a\n\x00\x00\x00\x00\x00')
# Datatables HTML.
deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='dt', ts_from=ts_from, ts_to=ts_to)
yield deferred
assert b"cdn.datatables.net" in deferred.result
