def test_mqtt_to_grafana(machinery, create_influxdb, reset_influxdb, reset_grafana): """ Publish single reading in JSON format to MQTT broker and proof that a corresponding datasource and a dashboard was created in Grafana. """ # Submit a single measurement, without timestamp. data = { 'temperature': 42.84, 'humidity': 83.1, } yield mqtt_json_sensor(settings.mqtt_topic_json, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) yield sleep(PROCESS_DELAY) yield sleep(PROCESS_DELAY) # Proof that Grafana is well provisioned. logger.info('Grafana: Checking datasource') datasource_names = [] for datasource in grafana.client.datasources.get(): datasource_names.append(datasource['name']) assert settings.influx_database in datasource_names logger.info('Grafana: Checking dashboard') dashboard_name = settings.grafana_dashboards[0] dashboard = grafana.client.dashboards.db[dashboard_name].get() target = dashboard['dashboard']['rows'][0]['panels'][0]['targets'][0] assert target['measurement'] == settings.influx_measurement_sensors assert 'temperature' in target['query'] or 'humidity' in target['query']
def test_tasmota_ds18b20(machinery, create_influxdb, reset_influxdb): """ Publish another single SENSOR reading in Tasmota/JSON format to MQTT broker, including a timestamp. Proof that the reading is processed and stored correctly. https://getkotori.org/docs/handbook/decoders/tasmota.html#submit-example-payload """ # Submit a single measurement. data = {"Time": "2017-02-16T10:13:52", "DS18B20": {"Temperature": 20.6}} yield mqtt_json_sensor(tasmota_sensor_topic, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Define reference data. reference = { u'time': u'2017-02-16T09:13:52Z', u'DS18B20.Temperature': 20.6, } # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() assert record == reference yield record
def test_timestamp_rfc3339(machinery, create_influxdb, reset_influxdb): """ Publish single reading in JSON format to MQTT broker, using a timestamp in RFC3339 format. Proof that the timestamp is processed and stored correctly. """ # Submit a single measurement, with timestamp. data = { 'temperature': 42.84, 'humidity': 83.1, 'timestamp': '2020-03-10 03:38:37.937059000+01:00' } yield mqtt_json_sensor(settings.mqtt_topic_json, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() assert record == { u'time': u'2020-03-10T02:38:37.937059Z', u'humidity': 83.1, u'temperature': 42.84 } yield record
def test_timestamp_microseconds(machinery, create_influxdb, reset_influxdb): """ Publish single reading in JSON format to MQTT broker, using a timestamp as Unix Epoch in microseconds. Proof that the timestamp is processed and stored correctly. """ # Submit a single measurement, with timestamp. data = { 'temperature': 42.84, 'humidity': 83.1, 'timestamp': 1583810982123456 } yield mqtt_json_sensor(settings.mqtt_topic_json, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() assert record == { u'time': u'2020-03-10T03:29:42.123456Z', u'humidity': 83.1, u'temperature': 42.84 } yield record
def test_tasmota_wemos_dht22(machinery, create_influxdb, reset_influxdb): """ Publish a reading from a Wemos multi sensor device. Proof that the reading is processed and stored correctly. """ # Submit a single measurement. data = { "Time": "2017-10-05T22:39:55", "DHT22": { "Temperature": 25.4, "Humidity": 45 }, "TempUnit": "C" } yield mqtt_json_sensor(tasmota_sensor_topic, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Define reference data. reference = { u'time': u'2017-10-05T20:39:55Z', u'DHT22.Temperature': 25.4, u'DHT22.Humidity': 45, } # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() assert record == reference yield record
def test_mqtt_to_grafana(machinery_hiveeyes, create_influxdb_hiveeyes, reset_influxdb_hiveeyes, reset_grafana_hiveeyes): """ Publish a single reading in JSON format to MQTT and proof - it is stored in the InfluxDB database. - a corresponding datasource and dashboards have been created in Grafana. """ # Submit a single measurement, without timestamp. data = { 'temperature': 42.84, 'weight': 33.33, } yield mqtt_json_sensor(settings.mqtt_topic, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Wait for Grafana to create its artefacts. yield sleep(2) # Proof that data arrived in InfluxDB. record = influx.get_first_record() del record['time'] assert record == {u'temperature': 42.84, u'weight': 33.33} yield record # Proof that Grafana is well provisioned. logger.info('Grafana: Checking datasource') datasource_names = [] for datasource in grafana.client.datasources.get(): datasource_names.append(datasource['name']) assert settings.influx_database in datasource_names logger.info('Grafana: Checking dashboards') for dashboard_name in settings.grafana_dashboards: dashboard = grafana.client.dashboards.db[dashboard_name].get( )['dashboard'] if 'rows' in dashboard: umbrella = dashboard['rows'][0] else: umbrella = dashboard target = umbrella['panels'][0]['targets'][0] #assert target['measurement'] == settings.influx_measurement assert 'temperature' in target['query'] or 'weight' in target['query']
def test_influxdb_tags(machinery, create_influxdb, reset_influxdb): """ Publish single reading in JSON format to MQTT broker. Proof that all special fields are stored in the InfluxDB database as tags. """ # Define field names which are tags. # FIXME: Synchronize with ``kotori.daq.influx.storage.format_chunk()``. tag_fields_main = ['geohash', 'latitude', 'longitude'] tag_fields_more = [ 'location', 'location_id', 'location_name', 'sensor_id', 'sensor_type' ] # Submit a single measurement, without timestamp. data = { 'temperature': 42.84, 'humidity': 83.1, 'geohash': 'aa3434', 'latitude': 42.50, 'longitude': 52.40, } for field in tag_fields_more: data[field] = idgen() yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_json, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Define reference payload. reference = deepcopy(data) reference['latitude'] = str(reference['latitude']) reference['longitude'] = str(reference['longitude']) # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() del record['time'] assert record == reference yield record # Proof that data all special fields have been converged to tags. resultset = influx_sensors.client.query( 'SHOW TAG KEYS FROM {measurement};'.format( measurement=settings.influx_measurement_sensors)) yield resultset tag_names = [] for item in resultset[settings.influx_measurement_sensors]: tag_names.append(item['tagKey']) assert set(tag_names) == set(tag_fields_main + tag_fields_more)
def test_tasmota_wemos_multi(machinery, create_influxdb, reset_influxdb): """ Publish a reading from a Wemos multi sensor device. Proof that the reading is processed and stored correctly. """ # Submit a single measurement. data = { "Time": "2017-10-05T22:39:45", "DS18x20": { "DS1": { "Type": "DS18B20", "Address": "28FF4CBFA41604C4", "Temperature": 25.37 }, "DS2": { "Type": "DS18B20", "Address": "28FF1E7FA116035D", "Temperature": 30.44 }, "DS3": { "Type": "DS18B20", "Address": "28FF1597A41604CE", "Temperature": 25.81 } }, "DHT22": { "Temperature": 33.2, "Humidity": 30 }, "TempUnit": "C" } yield mqtt_json_sensor(tasmota_sensor_topic, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Define reference data. reference = { u'time': u'2017-10-05T20:39:45Z', u'DHT22.Temperature': 33.2, u'DHT22.Humidity': 30, u'DS18x20.DS1.Temperature': 25.37, u'DS18x20.DS2.Temperature': 30.44, u'DS18x20.DS3.Temperature': 25.81, } # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() logger.info('record: %s', json.dumps(record)) assert record == reference yield record
def test_export_csv(machinery, create_influxdb, reset_influxdb): """ Submit single reading in JSON format to HTTP API and proof it can be retrieved back from the HTTP API in different formats. """ # Submit a single measurement, with timestamp. data = { 'time': 1583810982, 'temperature': 25.26, 'humidity': 51.8, } yield threads.deferToThread(http_json_sensor, settings.channel_path_data, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data is available via HTTP API. ts_from = '2020-03-10T00:00:00.000Z' ts_to = '2020-03-10T23:59:59.000Z' # CSV format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='csv', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result == 'time,humidity,temperature\n2020-03-10T03:29:42.000000Z,51.8,25.26\n' # TXT format (same as CSV). deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='txt', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result == 'time,humidity,temperature\n2020-03-10T03:29:42.000000Z,51.8,25.26\n' # JSON format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='json', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result == '[{"time":"2020-03-10T03:29:42.000Z","humidity":51.8,"temperature":25.26}]' # XLSX format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='xlsx', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result.startswith(b'PK\x03\x04\x14\x00\x00\x00\x08\x00') # HTML format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='html', ts_from=ts_from, ts_to=ts_to) yield deferred assert \ '<html>' in deferred.result and \ '<th>temperature</th>' in deferred.result and \ '<td>25.26</td>' in deferred.result
def test_event_mqtt(machinery, create_influxdb, reset_influxdb_events): """ Publish event in JSON format to MQTT broker and proof it is stored in the InfluxDB database. """ # Submit a single event, without timestamp. yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_event, event_data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that event arrived in InfluxDB. record = influx_events.get_first_record() del record['time'] assert record == event_data
def test_tasmota_state(machinery, create_influxdb, reset_influxdb): """ Publish a single STATE reading in Tasmota/JSON format to MQTT broker, including a timestamp. Proof that the reading is processed and stored correctly. https://getkotori.org/docs/handbook/decoders/tasmota.html#submit-example-payload """ # Submit a single measurement. data = { "Time": "2019-06-02T22:13:07", "Uptime": "1T18:10:35", "Vcc": 3.182, "SleepMode": "Dynamic", "Sleep": 50, "LoadAvg": 19, "Wifi": { "AP": 1, "SSId": "{redacted}", "BSSId": "A0:F3:C1:{redacted}", "Channel": 1, "RSSI": 100, "LinkCount": 1, "Downtime": "0T00:00:07" } } yield mqtt_json_sensor(tasmota_state_topic, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Define reference data. reference = { u'time': u'2019-06-02T20:13:07Z', u'Device.Vcc': 3.182, u'Device.Sleep': 50, u'Device.LoadAvg': 19, u'Device.Wifi.Channel': 1, u'Device.Wifi.RSSI': 100, u'Device.Wifi.LinkCount': 1, } # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() assert record == reference yield record
def test_event_http_urlencoded(machinery, create_influxdb, reset_influxdb): """ Submit event in ``x-www-form-urlencoded`` format to HTTP API and proof it is stored in the InfluxDB database. """ # Submit a single event, without timestamp. yield threads.deferToThread(http_form_sensor, settings.channel_path_event, event_data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that event arrived in InfluxDB. record = influx_events.get_first_record() del record['time'] assert record == event_data yield record
def test_mqtt_to_influxdb_single(machinery, create_influxdb, reset_influxdb): """ Publish discrete values to the MQTT broker and and proof they are stored in the InfluxDB database. """ # Submit discrete measurement values, without timestamp. topic_temperature = settings.mqtt_topic_single + '/temperature' topic_humidity = settings.mqtt_topic_single + '/humidity' yield threads.deferToThread(mqtt_sensor, topic_temperature, 42.84) yield threads.deferToThread(mqtt_sensor, topic_humidity, 83.1) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() assert 'temperature' in record or 'humidity' in record
def test_airrohr_http_json(machinery, create_influxdb, reset_influxdb): """ Submit single reading in Airrohr JSON format to HTTP API and proof it is stored in the InfluxDB database. """ # Submit a single measurement, without timestamp. yield threads.deferToThread(http_json_sensor, settings.channel_path_airrohr, data_in) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() del record['time'] assert record == data_out yield record
def test_mqtt_legacy(machinery, create_influxdb, reset_influxdb): """ Publish single reading in JSON format to MQTT broker on legacy suffix and proof it is stored in the InfluxDB database. """ # Submit a single measurement, without timestamp. data = { 'temperature': 42.84, 'humidity': 83.1, } yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_json_legacy, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() assert 'temperature' in record or 'humidity' in record
def test_tasmota_sonoff_sc(machinery, create_influxdb, reset_influxdb): """ Publish a single SENSOR reading in Tasmota/JSON format to MQTT broker, including a timestamp. Proof that the reading is processed and stored correctly. https://getkotori.org/docs/handbook/decoders/tasmota.html#submit-example-payload """ # Submit a single measurement. data = { "Time": "2019-06-02T22:13:07", "SonoffSC": { "Temperature": 25, "Humidity": 15, "Light": 20, "Noise": 10, "AirQuality": 90 }, "TempUnit": "C" } yield mqtt_json_sensor(tasmota_sensor_topic, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Define reference data. reference = { u'time': u'2019-06-02T20:13:07Z', u'SonoffSC.AirQuality': 90, u'SonoffSC.Humidity': 15, u'SonoffSC.Light': 20, u'SonoffSC.Noise': 10, u'SonoffSC.Temperature': 25, } # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() assert record == reference yield record
def test_mqtt_homie(machinery, create_influxdb, reset_influxdb): """ Publish reading like a Homie device in JSON format to MQTT broker and proof it is stored in the InfluxDB database. """ # Submit a single measurement, without timestamp. data = { 'temperature': 42.84, 'humidity': 83.1, } yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_homie, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() del record['time'] assert record == {u'humidity': 83.1, u'temperature': 42.84} yield record
def test_http_csv(machinery, create_influxdb, reset_influxdb): """ Submit single reading in CSV format to HTTP API and proof it is stored in the InfluxDB database. """ # Submit a single measurement, without timestamp. data = { 'temperature': 25.26, 'humidity': 51.8, } yield threads.deferToThread(http_csv_sensor, settings.channel_path_data, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() del record['time'] assert record == {u'temperature': 25.26, u'humidity': 51.8} yield record
def test_weewx_mqtt(machinery, create_influxdb, reset_influxdb): """ Publish single reading in JSON format to MQTT broker and proof it is stored in the InfluxDB database. """ # Submit a single measurement, without timestamp. yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_json, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data arrived in InfluxDB. record = influx_sensors.get_first_record() assert record["time"] == '2017-04-17T22:15:00Z' assert record["outTemp_C"] == 3.55555555556 assert record["windSpeed10_kph"] == 5.78725803977 assert record["cloudbase_meter"] == 773.082217509 assert record["consBatteryVoltage_volt"] == 4.72 yield record
def test_export(machinery, create_influxdb, reset_influxdb): """ Submit single reading in JSON format to HTTP API and proof it can be retrieved back from the HTTP API in different formats. """ # Submit a single measurement, with timestamp. data = { 'time': 1583810982, 'temperature': 25.26, 'humidity': 51.8, } yield threads.deferToThread(http_json_sensor, settings.channel_path_data, data) # Wait for some time to process the message. yield sleep(PROCESS_DELAY) # Proof that data is available via HTTP API. ts_from = '2020-03-10T00:00:00.000Z' ts_to = '2020-03-10T23:59:59.000Z' # CSV format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='csv', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result == 'time,humidity,temperature\n2020-03-10T03:29:42.000000Z,51.8,25.26\n' # TXT format (same as CSV). deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='txt', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result == 'time,humidity,temperature\n2020-03-10T03:29:42.000000Z,51.8,25.26\n' # JSON format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='json', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result == '[{"time":"2020-03-10T03:29:42.000Z","humidity":51.8,"temperature":25.26}]' # XLSX format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='xlsx', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result.startswith(b'PK\x03\x04\x14\x00\x00\x00\x08\x00') # HTML format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='html', ts_from=ts_from, ts_to=ts_to) yield deferred assert \ '<html>' in deferred.result and \ '<th>temperature</th>' in deferred.result and \ '<td>25.26</td>' in deferred.result # HDF5 format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='hdf5', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result.startswith(b'\x89HDF\r\n\x1a\n\x00\x00\x00\x00\x00') import h5py from io import BytesIO from pandas import array hdf = h5py.File(BytesIO(deferred.result), "r") assert hdf["/itest_foo_bar/table"].attrs.get("NROWS") == 1 assert hdf["/itest_foo_bar/table"].attrs.get("index_kind") == b"datetime64" assert hdf["/itest_foo_bar/table"].attrs.get("humidity_dtype") == b"float64" assert hdf["/itest_foo_bar/table"].attrs.get("temperature_dtype") == b"float64" assert hdf["/itest_foo_bar/table"].shape == (1,) assert hdf["/itest_foo_bar/table"][()] == array( [(1583810982000000000, 51.8, 25.26)], dtype=[('index', '<i8'), ('humidity', '<f8'), ('temperature', '<f8')]) # NetCDF format. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='nc', ts_from=ts_from, ts_to=ts_to) yield deferred assert deferred.result.startswith(b'\x89HDF\r\n\x1a\n\x00\x00\x00\x00\x00') # Datatables HTML. deferred = threads.deferToThread(http_get_data, settings.channel_path_data, format='dt', ts_from=ts_from, ts_to=ts_to) yield deferred assert b"cdn.datatables.net" in deferred.result