def create_measurements_dict(self):
measurements_record = {}
for each_channel, each_measurement in self.measurement.values.items():
measurement = self.device_measurements.filter(
DeviceMeasurements.channel == each_channel).first()
if measurement and 'value' in each_measurement:
conversion = self.conversions.filter(
Conversion.unique_id == measurement.conversion_id).first()
# If a timestamp is passed from the module, use it
if 'timestamp_utc' in each_measurement:
timestamp = each_measurement['timestamp_utc']
else:
timestamp = None
measurements_record = parse_measurement(
conversion,
measurement,
measurements_record,
each_channel,
each_measurement,
timestamp=timestamp)
self.logger.debug(
f"Adding measurements to InfluxDB with ID {self.unique_id}: {measurements_record}")
return measurements_record
def add_measurement_influxdb(self, channel, measurement):
# Convert value/unit is conversion_id present and valid
if self.channels_conversion[channel]:
conversion = db_retrieve_table_daemon(
Conversion,
unique_id=self.channels_measurement[channel].conversion_id)
if conversion:
meas = parse_measurement(
self.channels_conversion[channel],
self.channels_measurement[channel],
measurement,
channel,
measurement[channel],
timestamp=measurement[channel]['timestamp_utc'])
measurement[channel]['measurement'] = meas[channel][
'measurement']
measurement[channel]['unit'] = meas[channel]['unit']
measurement[channel]['value'] = meas[channel]['value']
if measurement:
self.logger.debug(
"Adding measurement to influxdb: {}".format(measurement))
add_measurements_influxdb(self.unique_id,
measurement,
use_same_timestamp=INPUT_INFORMATION[
'measurements_use_same_timestamp'])
def loop(self):
if self.timer_loop > time.time():
return
while self.timer_loop < time.time():
self.timer_loop += self.period
measurements = {}
for channel in self.channels_measurement:
# Original value/unit
measurements[channel] = {}
measurements[channel]['measurement'] = self.channels_measurement[
channel].measurement
measurements[channel]['unit'] = self.channels_measurement[
channel].unit
measurements[channel]['value'] = random.randint(1, 100)
# Convert value/unit is conversion_id present and valid
if self.channels_conversion[channel]:
conversion = db_retrieve_table_daemon(
Conversion,
unique_id=self.channels_measurement[channel].conversion_id)
if conversion:
meas = parse_measurement(
self.channels_conversion[channel],
self.channels_measurement[channel], measurements,
channel, measurements[channel])
measurements[channel]['measurement'] = meas[channel][
'measurement']
measurements[channel]['unit'] = meas[channel]['unit']
measurements[channel]['value'] = meas[channel]['value']
if measurements:
self.logger.debug(
"Adding measurements to influxdb: {}".format(measurements))
add_measurements_influxdb(self.unique_id, measurements)
else:
self.logger.debug("No measurements to add to influxdb.")
def download_data(self):
self.logger.debug("Downloading Data")
# Clear data previously stored in dictionary
self.gadget.loggedDataReadout = {'Temp': {}, 'Humi': {}}
# Download stored data starting from self.gadget.newestTimeStampMs
self.gadget.readLoggedDataInterval(
startMs=self.gadget.newestTimeStampMs)
while self.running:
if (not self.gadget.waitForNotifications(5) or
not self.gadget.isLogReadoutInProgress()):
break # Done reading data
self.logger.debug("Downloaded Data")
self.logger.debug("Parsing/saving data")
list_timestamps_temp = []
list_timestamps_humi = []
# Store logged temperature
self.logger.debug("Storing {} temperatures".format(len(self.gadget.loggedDataReadout['Temp'])))
for each_ts, each_measure in self.gadget.loggedDataReadout['Temp'].items():
if not self.running:
break
if -40 > each_measure or each_measure > 125:
continue # Temperature outside acceptable range
list_timestamps_temp.append(each_ts)
if self.is_enabled(0):
datetime_ts = datetime.datetime.utcfromtimestamp(each_ts / 1000)
measurement_single = {
0: {
'measurement': 'temperature',
'unit': 'C',
'value': each_measure
}
}
measurement_single = parse_measurement(
self.channels_conversion[0],
self.channels_measurement[0],
measurement_single,
self.channels_measurement[0].channel,
measurement_single[0])
write_influxdb_value(
self.unique_id,
measurement_single[0]['unit'],
value=measurement_single[0]['value'],
measure=measurement_single[0]['measurement'],
channel=0,
timestamp=datetime_ts)
# Store logged humidity
self.logger.debug("Storing {} humidities".format(len(self.gadget.loggedDataReadout['Humi'])))
for each_ts, each_measure in self.gadget.loggedDataReadout['Humi'].items():
if not self.running:
break
if 0 >= each_measure or each_measure > 100:
continue # Humidity outside acceptable range
list_timestamps_humi.append(each_ts)
if self.is_enabled(1):
datetime_ts = datetime.datetime.utcfromtimestamp(each_ts / 1000)
measurement_single = {
1: {
'measurement': 'humidity',
'unit': 'percent',
'value': each_measure
}
}
measurement_single = parse_measurement(
self.channels_conversion[1],
self.channels_measurement[1],
measurement_single,
self.channels_measurement[1].channel,
measurement_single[1])
write_influxdb_value(
self.unique_id,
measurement_single[1]['unit'],
value=measurement_single[1]['value'],
measure=measurement_single[1]['measurement'],
channel=1,
timestamp=datetime_ts)
# Find common timestamps from both temperature and humidity lists
list_timestamps_both = list(set(list_timestamps_temp).intersection(list_timestamps_humi))
self.logger.debug("Calculating/storing {} dewpoint and vpd".format(len(list_timestamps_both)))
for each_ts in list_timestamps_both:
if not self.running:
break
temperature = self.gadget.loggedDataReadout['Temp'][each_ts]
humidity = self.gadget.loggedDataReadout['Humi'][each_ts]
if (-200 > temperature or temperature > 200) or (0 > humidity or humidity > 100):
continue # Measurement outside acceptable range
datetime_ts = datetime.datetime.utcfromtimestamp(each_ts / 1000)
# Calculate and store dew point
if self.is_enabled(3) and self.is_enabled(0) and self.is_enabled(1):
dewpoint = calculate_dewpoint(temperature, humidity)
measurement_single = {
3: {
'measurement': 'dewpoint',
'unit': 'C',
'value': dewpoint
}
}
measurement_single = parse_measurement(
self.channels_conversion[3],
self.channels_measurement[3],
measurement_single,
self.channels_measurement[3].channel,
measurement_single[3])
write_influxdb_value(
self.unique_id,
measurement_single[3]['unit'],
value=measurement_single[3]['value'],
measure=measurement_single[3]['measurement'],
channel=3,
timestamp=datetime_ts)
# Calculate and store vapor pressure deficit
if self.is_enabled(4) and self.is_enabled(0) and self.is_enabled(1):
vpd = calculate_vapor_pressure_deficit(temperature, humidity)
measurement_single = {
4: {
'measurement': 'vapor_pressure_deficit',
'unit': 'Pa',
'value': vpd
}
}
measurement_single = parse_measurement(
self.channels_conversion[4],
self.channels_measurement[4],
measurement_single,
self.channels_measurement[4].channel,
measurement_single[4])
write_influxdb_value(
self.unique_id,
measurement_single[4]['unit'],
value=measurement_single[4]['value'],
measure=measurement_single[4]['measurement'],
channel=4,
timestamp=datetime_ts)
# Download successfully finished, set newest timestamp
self.gadget.newestTimeStampMs = self.gadget.tmp_newestTimeStampMs
self.logger.debug("Parsed/saved data")
def get_new_data(self, past_seconds):
# Basic implementation. Future development may use more complex library to access API
endpoint = "https://{app}.data.thethingsnetwork.org/api/v2/query/{dev}?last={time}".format(
app=self.application_id,
dev=self.device_id,
time="{}s".format(int(past_seconds)))
headers = {"Authorization": "key {k}".format(k=self.app_api_key)}
timestamp_format = '%Y-%m-%dT%H:%M:%S.%f'
response = requests.get(endpoint, headers=headers)
try:
response.json()
except ValueError: # No data returned
self.logger.debug(
"Response Error. Response: {}. Likely there is no data to be retrieved on TTN"
.format(response.content))
return
for each_resp in response.json():
if not self.running:
break
try:
datetime_utc = datetime.datetime.strptime(
each_resp['time'][:-7], timestamp_format)
except Exception:
# Sometimes the original timestamp is in milliseconds
# instead of nanoseconds. Therefore, remove 3 less digits
# past the decimal and try again to parse.
try:
datetime_utc = datetime.datetime.strptime(
each_resp['time'][:-4], timestamp_format)
except Exception as e:
self.logger.error(
"Could not parse timestamp '{}': {}".format(
each_resp['time'], e))
continue # Malformed timestamp encountered. Discard measurement.
if (not self.latest_datetime
or self.latest_datetime < datetime_utc):
self.latest_datetime = datetime_utc
measurements = {}
for channel in self.channels_measurement:
if (self.is_enabled(channel)
and self.options_channels['variable_name'][channel]
in each_resp
and each_resp[self.options_channels['variable_name']
[channel]] is not None):
# Original value/unit
measurements[channel] = {}
measurements[channel][
'measurement'] = self.channels_measurement[
channel].measurement
measurements[channel]['unit'] = self.channels_measurement[
channel].unit
measurements[channel]['value'] = each_resp[
self.options_channels['variable_name'][channel]]
measurements[channel]['timestamp_utc'] = datetime_utc
# Convert value/unit is conversion_id present and valid
if self.channels_conversion[channel]:
conversion = db_retrieve_table_daemon(
Conversion,
unique_id=self.channels_measurement[channel].
conversion_id)
if conversion:
meas = parse_measurement(
self.channels_conversion[channel],
self.channels_measurement[channel],
measurements,
channel,
measurements[channel],
timestamp=datetime_utc)
measurements[channel]['measurement'] = meas[
channel]['measurement']
measurements[channel]['unit'] = meas[channel][
'unit']
measurements[channel]['value'] = meas[channel][
'value']
if measurements:
self.logger.debug(
"Adding measurements to influxdb: {}".format(measurements))
add_measurements_influxdb(
self.unique_id,
measurements,
use_same_timestamp=INPUT_INFORMATION[
'measurements_use_same_timestamp'])
else:
self.logger.debug("No measurements to add to influxdb.")
# set datetime to latest timestamp
if self.running:
with session_scope(MYCODO_DB_PATH) as new_session:
mod_input = new_session.query(Input).filter(
Input.unique_id == self.unique_id).first()
if not mod_input.datetime or mod_input.datetime < self.latest_datetime:
mod_input.datetime = self.latest_datetime
new_session.commit()
def get_measurement(self):
"""Gets the temperature and humidity."""
#
# Initialize measurements dictionary
#
measurements = {}
for channel in self.channels_measurement:
if self.is_enabled(channel):
# Initialize channel dictionary
measurements[channel] = {}
#
# Set the measurement and unit
#
measurements[channel][
'measurement'] = self.channels_measurement[
channel].measurement
measurements[channel]['unit'] = self.channels_measurement[
channel].unit
#
# Set the measurement value
#
measurements[channel]['value'] = self.random.randint(50, 70)
self.logger.info(
"Channel {} is enabled and storing a value of {} "
"with measurement {} and unit {}".format(
channel, measurements[channel]['value'],
measurements[channel]['measurement'],
measurements[channel]['unit']))
# Convert value/unit is conversion_id present and valid
if self.channels_conversion[channel]:
conversion = db_retrieve_table_daemon(
Conversion,
unique_id=self.channels_measurement[channel].
conversion_id)
if conversion:
meas = parse_measurement(
self.channels_conversion[channel],
self.channels_measurement[channel], measurements,
channel, measurements[channel])
measurements[channel]['measurement'] = meas[channel][
'measurement']
measurements[channel]['unit'] = meas[channel]['unit']
measurements[channel]['value'] = meas[channel]['value']
if measurements:
self.logger.debug(
"Adding measurements to influxdb: {}".format(measurements))
add_measurements_influxdb(self.unique_id,
measurements,
use_same_timestamp=INPUT_INFORMATION[
'measurements_use_same_timestamp'])
else:
self.logger.debug("No measurements to add to influxdb.")
self.logger.info("This INFO message will always be displayed. "
"self.fan_modulate: {}, "
"self.fan_seconds: {}, "
"self.measure_range: {}.".format(
self.fan_modulate, self.fan_seconds,
self.measure_range))
self.logger.debug("This DEBUG message will only be displayed if the "
"Debug option is enabled.")
return self.return_dict
def get_new_data(self, past_seconds):
try:
seconds = int(past_seconds)
except:
self.logger.error("past_seconds does not represent an integer")
return
endpoint = "https://nam1.cloud.thethings.network" \
"/api/v3/as/applications/{app}/devices/{dev}/packages/storage/uplink_message?" \
"last={time}s&field_mask=up.uplink_message.decoded_payload".format(
app=self.application_id, dev=self.device_id, time=seconds)
headers = {"Authorization": "Bearer {k}".format(k=self.app_api_key)}
self.logger.debug("endpoint: {}".format(endpoint))
self.logger.debug("headers: {}".format(headers))
# Get measurement data from TTN
try:
response = requests.get(endpoint, headers=headers)
except requests.exceptions.ConnectionError as err:
self.logger.error(
"requests.exceptions.ConnectionError: {}".format(err))
return
except Exception as err:
self.logger.error("Exception: {}".format(err))
return
if response.status_code != 200:
self.logger.info("response.status_code != 200: {}".format(
response.reason))
self.logger.debug("response.content: {}".format(response.content))
list_dicts = response.content.decode().split("\n")
self.logger.debug("list_dicts: {}".format(list_dicts))
for each_resp in list_dicts:
measurements = {}
if not self.running:
break
if not each_resp:
continue
self.logger.debug("each_resp: {}".format(each_resp))
try:
resp_json = json.loads(each_resp)
except:
resp_json = {}
self.logger.debug("resp_json: {}".format(resp_json))
try:
datetime_utc = datetime.datetime.strptime(
resp_json['result']['received_at'][:-7],
self.timestamp_format)
except Exception:
# Sometimes the original timestamp is in milliseconds
# instead of nanoseconds. Therefore, remove 3 less digits
# past the decimal and try again to parse.
try:
datetime_utc = datetime.datetime.strptime(
resp_json['result']['received_at'][:-4],
self.timestamp_format)
except Exception as e:
self.logger.error(
"Could not parse timestamp '{}': {}".format(
resp_json['result']['received_at'], e))
continue # Malformed timestamp encountered. Discard measurement.
if ('result' not in resp_json
or 'uplink_message' not in resp_json['result']
or 'decoded_payload'
not in resp_json['result']['uplink_message']
or not resp_json['result']['uplink_message']
['decoded_payload']):
self.logger.debug(
"resp_json empty or malformed: {}".format(resp_json))
continue
payload = resp_json['result']['uplink_message']['decoded_payload']
if (not self.latest_datetime
or self.latest_datetime < datetime_utc):
self.latest_datetime = datetime_utc
for channel in self.channels_measurement:
jmespath_expression = self.options_channels[
'jmespath_expression'][channel]
try:
jmesexpression = self.jmespath.compile(jmespath_expression)
value = jmesexpression.search(payload)
self.logger.debug("Expression: {}, value: {}".format(
jmespath_expression, value))
except Exception as err:
self.logger.error(
"Error in JSON '{}' finding expression '{}': {}".
format(payload, jmespath_expression, err))
continue
if value is None:
continue
# Original value/unit
measurements[channel] = {}
measurements[channel][
'measurement'] = self.channels_measurement[
channel].measurement
measurements[channel]['unit'] = self.channels_measurement[
channel].unit
measurements[channel]['value'] = value
measurements[channel]['timestamp_utc'] = datetime_utc
# Convert value/unit is conversion_id present and valid
if self.channels_conversion[channel]:
conversion = db_retrieve_table_daemon(
Conversion,
unique_id=self.channels_measurement[channel].
conversion_id)
if conversion:
meas = parse_measurement(
self.channels_conversion[channel],
self.channels_measurement[channel],
measurements,
channel,
measurements[channel],
timestamp=datetime_utc)
measurements[channel]['measurement'] = meas[channel][
'measurement']
measurements[channel]['unit'] = meas[channel]['unit']
measurements[channel]['value'] = meas[channel]['value']
if measurements:
self.logger.debug(
"Adding measurements to influxdb: {}".format(measurements))
add_measurements_influxdb(
self.unique_id,
measurements,
use_same_timestamp=INPUT_INFORMATION[
'measurements_use_same_timestamp'])
else:
self.logger.debug("No measurements to add to influxdb.")
# set datetime to latest timestamp
if self.running:
with session_scope(MYCODO_DB_PATH) as new_session:
mod_input = new_session.query(Input).filter(
Input.unique_id == self.unique_id).first()
if not mod_input.datetime or mod_input.datetime < self.latest_datetime:
mod_input.datetime = self.latest_datetime
new_session.commit()
